diff --git a/.github/actions/llvm-linux/action.yml b/.github/actions/llvm-linux/action.yml new file mode 100644 index 00000000..51daca0e --- /dev/null +++ b/.github/actions/llvm-linux/action.yml @@ -0,0 +1,27 @@ +name: Pinned Linux LLVM +description: Reuse the pinned LLVM development distribution across CI jobs +runs: + using: composite + steps: + - uses: actions/cache@v4 + id: llvm + with: + path: ${{ runner.temp }}/llvm-22.1.8 + key: llvm-22.1.8-linux-x64-v1 + - name: Install LLVM system libraries + shell: bash + run: | + sudo apt-get update + sudo apt-get install --yes g++ zlib1g-dev libzstd-dev + - name: Download LLVM + if: steps.llvm.outputs.cache-hit != 'true' + shell: bash + run: | + curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz + tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP" + mv "$RUNNER_TEMP/LLVM-22.1.8-Linux-X64" "$RUNNER_TEMP/llvm-22.1.8" + - name: Select LLVM + shell: bash + run: | + echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV" + echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" diff --git a/.github/actions/native-packages/action.yml b/.github/actions/native-packages/action.yml new file mode 100644 index 00000000..4bab2121 --- /dev/null +++ b/.github/actions/native-packages/action.yml @@ -0,0 +1,60 @@ +name: Native compiler packages +description: Build or restore native packages using their complete source inputs +inputs: + target: + description: Native package suffix + required: true + helper: + description: Build the LLVM helper as well as the runtime pack + default: "true" +runs: + using: composite + steps: + - name: Identify host toolchain + id: host + shell: bash + run: | + # Runner image changes can update the SDK, C++ ABI, and C compiler. + echo "image=${ImageOS}-${ImageVersion}" >> "$GITHUB_OUTPUT" + if test "$RUNNER_OS" = Linux; then + echo "zig=$(zig version)" >> "$GITHUB_OUTPUT" + fi + - uses: actions/cache@v4 + id: helper + if: inputs.helper == 'true' + with: + path: packages/llvm-${{ inputs.target }}/bin + key: helper-v1-${{ inputs.target }}-${{ steps.host.outputs.image }}-${{ steps.host.outputs.zig }}-${{ hashFiles('.node-version', '.github/actions/native-packages/action.yml', '.github/actions/llvm-linux/action.yml', 'native/llvm-codegen/**', 'packages/llvm-*/package.json', 'packages/llvm-*/scripts/**') }} + - uses: actions/cache@v4 + id: runtime + with: + path: | + packages/runtime-${{ inputs.target }}/artifacts + packages/runtime-${{ inputs.target }}/runtime-pack.json + key: runtime-v1-${{ inputs.target }}-${{ steps.host.outputs.image }}-${{ steps.host.outputs.zig }}-${{ hashFiles('.node-version', '.github/actions/native-packages/action.yml', '.github/actions/llvm-linux/action.yml', 'packages/runtime/package.json', 'packages/runtime/LICENSE', 'packages/runtime/src/**', 'packages/runtime/vendor/**', 'packages/runtime-*/package.json', 'packages/runtime-*/scripts/**', 'packages/runtime-*/*.mjs') }} + - name: Install macOS LLVM + if: runner.os == 'macOS' && inputs.helper == 'true' && steps.helper.outputs.cache-hit != 'true' + shell: bash + run: brew install llvm@22 + - name: Build LLVM helper + if: inputs.helper == 'true' && steps.helper.outputs.cache-hit != 'true' + shell: bash + env: + SCRIPTC_PACKAGE_TARGET: ${{ inputs.target }} + run: pnpm --filter "@scriptc/llvm-$SCRIPTC_PACKAGE_TARGET" build:native + - name: Build runtime pack + if: steps.runtime.outputs.cache-hit != 'true' + shell: bash + env: + SCRIPTC_PACKAGE_TARGET: ${{ inputs.target }} + run: | + if test "$RUNNER_OS" = Linux; then + CC=zig AR=zig pnpm --filter "@scriptc/runtime-$SCRIPTC_PACKAGE_TARGET" build:native + else + pnpm --filter "@scriptc/runtime-$SCRIPTC_PACKAGE_TARGET" build:native + fi + - name: Verify runtime content + shell: bash + env: + SCRIPTC_PACKAGE_TARGET: ${{ inputs.target }} + run: node "packages/runtime-$SCRIPTC_PACKAGE_TARGET/scripts/verify.mjs" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d311c3b4..de18dff5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,36 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + native_macos: + name: build (macOS arm64 native packages) + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/native-packages + with: + target: darwin-arm64 + - name: Preserve executable permissions in shared packages + run: >- + tar -czf "$RUNNER_TEMP/native-darwin-arm64.tar.gz" + packages/llvm-darwin-arm64/bin + packages/runtime-darwin-arm64/artifacts + packages/runtime-darwin-arm64/runtime-pack.json + - uses: actions/upload-artifact@v4 + with: + name: native-darwin-arm64 + path: ${{ runner.temp }}/native-darwin-arm64.tar.gz + retention-days: 1 + tests: + needs: native_macos name: test (${{ matrix.flavor }}, ${{ matrix.shard }}/10) # arm64 macOS: same clang/libc the differential + number-formatting # oracles were verified against. @@ -45,10 +74,12 @@ jobs: with: node-version-file: .node-version cache: pnpm - - run: brew install llvm@22 - run: pnpm install --frozen-lockfile - - run: pnpm --filter @scriptc/llvm-darwin-arm64 build:native - - run: pnpm --filter @scriptc/runtime-darwin-arm64 build:native + - uses: actions/download-artifact@v4 + with: + name: native-darwin-arm64 + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-darwin-arm64.tar.gz" - run: pnpm build # Separate vitest invocations because the shard axes must not mix: a file # lands in exactly ONE --shard slice, so an env-sharded file behind @@ -68,7 +99,6 @@ jobs: --exclude=tests/harness/self-hosting.test.ts --exclude=tests/harness/self-hosting-llvm-emitter.test.ts --exclude=tests/harness/self-hosting-native-driver.test.ts - --exclude=tests/harness/self-hosting-bootstrap.test.ts --exclude=tests/harness/fetch.test.ts --exclude=tests/harness/npm-static.test.ts --exclude=tests/harness/surface-manifest.test.ts @@ -137,7 +167,7 @@ jobs: # Full compiler rebuilds dominate the critical path. Start them immediately # after setup, independently of corpus and integration test shards. bootstrap: - name: test (Linux, ${{ matrix.flavor }}, ${{ matrix.compiler }} bootstrap) + name: test (Linux, ${{ matrix.flavor }}, native bootstrap) # These contracts are portable. Standard Linux runners have enough memory # for LLVM optimization alongside the compiler that produces its input. runs-on: ubuntu-24.04 @@ -145,12 +175,6 @@ jobs: fail-fast: false matrix: flavor: [plain, san] - compiler: [frontend, native] - include: - - compiler: frontend - test: tests/harness/self-hosting-bootstrap.test.ts - - compiler: native - test: tests/harness/self-hosting-native-driver.test.ts env: SCRIPTC_TEST_WORKERS: "1" SCRIPTC_SAN: ${{ matrix.flavor == 'san' && '1' || '' }} @@ -163,23 +187,20 @@ jobs: with: node-version-file: .node-version cache: pnpm - - name: Install pinned LLVM development distribution - run: | - sudo apt-get update - sudo apt-get install --yes g++ zlib1g-dev libzstd-dev - curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz - tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP" - mv "$RUNNER_TEMP/LLVM-22.1.8-Linux-X64" "$RUNNER_TEMP/llvm-22.1.8" - echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV" - echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" + - uses: ./.github/actions/llvm-linux - run: pnpm install --frozen-lockfile - - run: pnpm --filter @scriptc/llvm-linux-x64-gnu build:native - uses: vercel-labs/setup-zig@v1 with: version: 0.16.0 - - run: CC=zig AR=zig pnpm --filter @scriptc/runtime-linux-x64-gnu build:native + - uses: ./.github/actions/native-packages + with: + target: linux-x64-gnu + - uses: ./.github/actions/native-packages + with: + target: wasm32-wasi + helper: "false" - run: pnpm build - - run: pnpm test ${{ matrix.test }} + - run: pnpm test tests/harness/self-hosting-native-driver.test.ts # A real host-clang/glibc build, not the zig cross-target lane: catches # Linux feature-test macro omissions at compile time and separate-libm @@ -203,10 +224,11 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm build - - name: Host clang static-build regression + - name: Host clang and sanitizer build regressions run: >- pnpm test packages/compiler/test/cc-driver.test.ts - --testNamePattern "host-native clang static build" + packages/compiler/src/native/sanitizer.test.ts + --testNamePattern "host-native clang static build|sanitized vendor archives" # The TS7 parity sweep exercises the compiler frontend, so it can run # alongside the Linux LLVM/runtime package build instead of following it. @@ -241,21 +263,14 @@ jobs: with: node-version-file: .node-version cache: pnpm - - name: Install pinned LLVM development distribution - run: | - sudo apt-get update - sudo apt-get install --yes g++ zlib1g-dev libzstd-dev - curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz - tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP" - mv "$RUNNER_TEMP/LLVM-22.1.8-Linux-X64" "$RUNNER_TEMP/llvm-22.1.8" - echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV" - echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" + - uses: ./.github/actions/llvm-linux - run: pnpm install --frozen-lockfile - - run: pnpm --filter @scriptc/llvm-linux-x64-gnu build:native - uses: vercel-labs/setup-zig@v1 with: version: 0.16.0 - - run: CC=zig AR=zig pnpm --filter @scriptc/runtime-linux-x64-gnu build:native + - uses: ./.github/actions/native-packages + with: + target: linux-x64-gnu - name: Build and verify Linux x64 musl release packages run: | pnpm --filter @scriptc/llvm-linux-x64-musl build:native @@ -272,7 +287,10 @@ jobs: if (value.default_target !== "x86_64-unknown-linux-musl" || !value.supported_targets?.includes("wasm32-unknown-wasi")) process.exit(1); ' node packages/runtime-linux-x64-musl/scripts/verify.mjs - - run: pnpm --filter @scriptc/runtime-wasm32-wasi build:native + - uses: ./.github/actions/native-packages + with: + target: wasm32-wasi + helper: "false" - run: pnpm build - name: Linux helper object format and no-clang output contract run: | @@ -288,6 +306,8 @@ jobs: run: | node packages/cli/dist/main.js build tests/corpus/001-hello.ts -o "$RUNNER_TEMP/hello" test "$("$RUNNER_TEMP/hello")" = 'hello world' + - name: Linux release runtime reachability and debug payload + run: pnpm test tests/harness/runtime-tree-shaking.test.ts - name: Debian 12 glibc 2.36 runtime-pack smoke run: | docker run --rm --volume "$GITHUB_WORKSPACE:/work:ro" --workdir /work node:24-bookworm-slim sh -ec ' @@ -297,9 +317,10 @@ jobs: test "$(/tmp/hello)" = "hello world" ' - name: WASI helper object/runtime-pack smoke - run: pnpm test packages/cli/test/wasi-runtime-pack.test.ts + run: pnpm test packages/cli/test/wasi-runtime-pack.test.ts tests/harness/wasm-library.test.ts llvm_artifacts_macos_arm64: + needs: native_macos name: test (macOS arm64 LLVM artifacts, no clang, ${{ matrix.shard }}/5) runs-on: macos-15 # Shard 1 additionally runs package/install and link-parity contracts @@ -325,10 +346,12 @@ jobs: with: node-version-file: .node-version cache: pnpm - - run: brew install llvm@22 - run: pnpm install --frozen-lockfile - - run: pnpm --filter @scriptc/llvm-darwin-arm64 build:native - - run: pnpm --filter @scriptc/runtime-darwin-arm64 build:native + - uses: actions/download-artifact@v4 + with: + name: native-darwin-arm64 + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-darwin-arm64.tar.gz" - run: pnpm build - name: No-clang assembly/object contract if: matrix.shard == 1 @@ -353,22 +376,29 @@ jobs: node scripts/verify-llvm-package.mjs "$RUNNER_TEMP/$(basename "$TARBALL")" - name: Packed npm installation smoke if: matrix.shard == 1 + env: + # Release optimization is covered by the compiler bootstrap jobs. + # This lane verifies packaging and installation with a faster seed. + SCRIPTC_NATIVE_OPTIMIZATION: dev run: | + pnpm --filter @scriptc/cli-darwin-arm64 build:native pnpm --dir packages/runtime pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/runtime-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/compiler pack --pack-destination "$RUNNER_TEMP" --silent + pnpm --dir packages/cli-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent pnpm --dir packages/cli pack --pack-destination "$RUNNER_TEMP" --silent PREFIX="$RUNNER_TEMP/installed-scriptc" - npm install --prefix "$PREFIX" --ignore-scripts \ + npm install --prefix "$PREFIX" --no-audit --no-fund \ "$RUNNER_TEMP/scriptc-runtime-$(node -p "require('./packages/runtime/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-runtime-darwin-arm64-$(node -p "require('./packages/runtime-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-llvm-darwin-arm64-$(node -p "require('./packages/llvm-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-compiler-$(node -p "require('./packages/compiler/package.json').version").tgz" \ + "$RUNNER_TEMP/scriptc-cli-darwin-arm64-$(node -p "require('./packages/cli-darwin-arm64/package.json').version").tgz" \ "$RUNNER_TEMP/scriptc-$(node -p "require('./packages/cli/package.json').version").tgz" - "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ + PATH="" "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ --emit=obj -o "$RUNNER_TEMP/installed.o" file "$RUNNER_TEMP/installed.o" | grep 'Mach-O 64-bit object arm64' - "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ + PATH="" "$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \ --print=native-link-info -o "$RUNNER_TEMP/installed-link.o" \ > "$RUNNER_TEMP/installed-link.json" node examples/native-object/link.mjs cc \ @@ -388,10 +418,10 @@ jobs: with: node-version-file: .node-version cache: pnpm - - run: brew install llvm@22 - run: pnpm install --frozen-lockfile - - run: LLVM_DIR=/usr/local/opt/llvm@22/lib/cmake/llvm pnpm --filter @scriptc/llvm-darwin-x64 build:native - - run: pnpm --filter @scriptc/runtime-darwin-x64 build:native + - uses: ./.github/actions/native-packages + with: + target: darwin-x64 - run: pnpm build - name: Intel Mach-O no-clang object contract run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89f18cc6..9a50514e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,37 +61,48 @@ jobs: runner: macos-15 helper: llvm-darwin-arm64 runtime: runtime-darwin-arm64 + cli: cli-darwin-arm64 - platform: darwin-x64 runner: macos-15-intel helper: llvm-darwin-x64 runtime: runtime-darwin-x64 + cli: cli-darwin-x64 - platform: linux-x64 runner: ubuntu-24.04 helper: llvm-linux-x64-gnu runtime: runtime-linux-x64-gnu + cli: cli-linux-x64-gnu llvm_asset: LLVM-22.1.8-Linux-X64 use_zig: true - platform: linux-arm64 runner: ubuntu-24.04-arm helper: llvm-linux-arm64-gnu runtime: runtime-linux-arm64-gnu + cli: cli-linux-arm64-gnu llvm_asset: LLVM-22.1.8-Linux-ARM64 use_zig: true - platform: windows-x64 runner: windows-2022 helper: llvm-win32-x64-msvc runtime: runtime-win32-x64-msvc + cli: cli-win32-x64-msvc use_zig: true - platform: linux-x64-musl runner: ubuntu-24.04 helper: llvm-linux-x64-musl runtime: runtime-linux-x64-musl + cli: cli-linux-x64-musl + seed_helper: llvm-linux-x64-gnu + cli_target: x86_64-linux-musl llvm_asset: LLVM-22.1.8-Linux-X64 use_zig: true - platform: linux-arm64-musl runner: ubuntu-24.04-arm helper: llvm-linux-arm64-musl runtime: runtime-linux-arm64-musl + cli: cli-linux-arm64-musl + seed_helper: llvm-linux-arm64-gnu + cli_target: aarch64-linux-musl llvm_asset: LLVM-22.1.8-Linux-ARM64 use_zig: true - platform: wasm32-wasi @@ -171,12 +182,25 @@ jobs: run: | pnpm --filter @scriptc/${{ matrix.helper }} build:native pnpm --filter @scriptc/${{ matrix.runtime }} build:native - - uses: actions/upload-artifact@v4 + - name: Build host helper for cross-libc bootstrap + if: matrix.seed_helper != '' + run: pnpm --filter @scriptc/${{ matrix.seed_helper }} build:native + - name: Build native CLI distribution + if: matrix.cli != '' + env: + SCRIPTC_TARGET: ${{ matrix.cli_target }} + run: | + pnpm --filter @scriptc/compiler --filter scriptc build + pnpm --filter @scriptc/${{ matrix.cli }} build:native + node scripts/verify-native-cli.mjs packages/${{ matrix.cli }} --run + - name: Upload native packages + uses: actions/upload-artifact@v4 with: name: native-${{ matrix.platform }} path: | packages/${{ matrix.helper }} packages/${{ matrix.runtime }} + ${{ matrix.cli != '' && format('packages/{0}/dist', matrix.cli) || '' }} retention-days: 1 build-mobile-runtime-packs: @@ -266,12 +290,13 @@ jobs: # upload-artifact does not preserve executable bits. Restore the # helper mode before pnpm runs its prepack checks. find packages -type f -path '*/bin/scriptc-llvm-codegen' -exec chmod 755 {} + + find packages -type f \( -path '*/dist/bin/scriptc' -o -path '*/dist/lib/typescript/lib/tsc' -o -path '*/dist/lib/scriptc-comptime' \) -exec chmod 755 {} + pnpm -r build - name: Check version sync run: | VERSION="${{ needs.check-release.outputs.version }}" - for pkg in packages/runtime packages/runtime-darwin-arm64 packages/llvm-darwin-arm64 packages/runtime-linux-x64-gnu packages/llvm-linux-x64-gnu packages/runtime-linux-arm64-gnu packages/llvm-linux-arm64-gnu packages/runtime-linux-x64-musl packages/llvm-linux-x64-musl packages/runtime-linux-arm64-musl packages/llvm-linux-arm64-musl packages/runtime-wasm32-wasi packages/runtime-ios-arm64 packages/runtime-ios-simulator-arm64 packages/runtime-android-arm64 packages/runtime-win32-x64-msvc packages/llvm-win32-x64-msvc packages/compiler packages/cli; do + for pkg in packages/runtime packages/runtime-* packages/llvm-* packages/compiler packages/cli packages/cli-*; do V=$(node -p "require('./$pkg/package.json').version") if [ "$V" != "$VERSION" ]; then echo "Version mismatch: $pkg is $V, expected $VERSION" @@ -356,6 +381,8 @@ jobs: publish_dir packages/runtime publish_dir packages/runtime-darwin-arm64 publish_dir packages/llvm-darwin-arm64 "$HELPER_TARBALL" + publish_dir packages/runtime-darwin-x64 + publish_dir packages/llvm-darwin-x64 publish_dir packages/runtime-linux-x64-gnu publish_dir packages/llvm-linux-x64-gnu publish_dir packages/runtime-linux-arm64-gnu @@ -371,19 +398,13 @@ jobs: publish_dir packages/runtime-win32-x64-msvc publish_dir packages/llvm-win32-x64-msvc publish_dir packages/compiler + for pkg in packages/cli-*; do publish_dir "$pkg"; done publish_dir packages/cli env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # The GitHub release is a tag, notes, and one asset: the surface - # manifest (packages/compiler/surface-manifest.json — the machine- - # readable listing of the surface the static tier compiles at this - # version, regenerated here and verified against the committed file). - # The platform helper ships through its npm package rather than as a GitHub - # release asset, so this job runs AFTER a successful npm publish and never - # gates it. The body is the CHANGELOG.md block between the - # release:start/release:end markers, which RELEASING.md keeps on the - # latest entry only. + # Publish standalone native distributions and the surface manifest after + # npm publishing succeeds. The release body comes from the marked changelog. github-release: name: Create GitHub Release needs: [check-release, publish] @@ -427,6 +448,16 @@ jobs: fi echo "Extracted release notes for $VERSION ($LINES lines)" + - name: Download native distributions and runtime packs + uses: actions/download-artifact@v4 + with: + pattern: native-* + path: packages + merge-multiple: true + + - name: Package standalone compilers + run: node scripts/package-native-cli.mjs packages /tmp/scriptc-release-assets + - name: Create GitHub Release run: | VERSION="${{ needs.check-release.outputs.version }}" @@ -445,5 +476,6 @@ jobs: # Attach the surface manifest (idempotent: --clobber makes # re-runs replace the asset instead of failing). gh release upload "$TAG" packages/compiler/surface-manifest.json --clobber + gh release upload "$TAG" /tmp/scriptc-release-assets/* --clobber env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 880a0b9f..b70ddf10 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ output/ /packages/runtime-darwin-arm64/runtime-pack.json /packages/runtime-darwin-arm64/.runtime-pack-* /packages/llvm-*/bin/ +/packages/cli/bin/ /packages/runtime-*/artifacts/ /packages/runtime-*/runtime-pack.json /packages/runtime-*/.runtime-pack-* diff --git a/README.md b/README.md index 2c8d849e..f92388f0 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ scriptc is experimental and targets macOS, Linux, Windows, and WebAssembly via W ## Installation -The compiler requires Node.js 24 or newer. `--emit=ir|llvm` needs only Node. `--emit=asm|obj` uses the matching optional platform helper installed with scriptc on supported macOS, Linux, and Windows hosts (and for WASI), but needs no compiler, archiver, linker, or SDK. Ordinary LLVM executable builds need a platform linker driver and SDK/sysroot, but use the bundled helper plus precompiled runtime pack rather than compiling generated or runtime C. Set `SCRIPTC_LINKER` to choose that driver. Runtime development with `--sanitize` additionally needs a C compiler. The executables it produces do not require Node. +The installed compiler runs natively on supported macOS, Linux, and Windows hosts. Compilation, compile-time evaluation, and native execution do not require Node. `--emit=ir|llvm|asm|obj` uses the bundled TypeScript checker and LLVM helper without an external compiler, archiver, linker, or SDK. Executable builds additionally need a platform linker driver and SDK/sysroot; precompiled runtime packs supply the C runtime. Set `SCRIPTC_LINKER` to choose that driver. Runtime development with `--sanitize` additionally needs a C compiler. Node.js 24 or newer is needed for npm installation, development from a source checkout, the JavaScript compiler API, and `scriptc run` of WASI modules. ```console $ npm install -g scriptc diff --git a/RELEASING.md b/RELEASING.md index ae9cd4cf..cd86db4b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -14,15 +14,7 @@ To prepare a release: CI (`.github/workflows/release.yml`) compares the version in `packages/cli/package.json` to what `scriptc` has on npm. If it differs, it builds platform packages on matching macOS, Linux, and Windows runners, verifies the version spine, and publishes the runtime, helper, compiler, and CLI packages in dependency order. After the publish succeeds, a separate job creates the git tag `v` and the GitHub release with the marked changelog entry as its body, and attaches `surface-manifest.json` — the machine-readable listing of the surface the static tier compiles at that version (stable per-entry ids, so two releases diff mechanically; see `packages/compiler/src/coverage/surface-manifest.ts` for the schema). The job regenerates the manifest from the tree and fails on any byte difference from the committed file before attaching, so the asset is always the manifest of the code being released. The same file ships inside the `@scriptc/compiler` package as `@scriptc/compiler/surface-manifest.json`. -The release job builds and strips each pinned LLVM helper on its matching host, -then builds the matching precompiled runtime pack before publishing the -constrained platform packages ahead of `@scriptc/compiler`. Ordinary LLVM-tier -executables use the helper for the program object and the platform pack for -runtime objects; the user's toolchain performs only the final platform link. -Explicit C builds, LLVM refusals, and `--sanitize` retain the external C -toolchain path. npm postinstall skips local runtime-cache compilation when the -platform pack is available. The GitHub release remains a tag, release notes, and the -manifest asset; the npm publish never waits on the GitHub release. +The release job builds each pinned LLVM helper, precompiled runtime pack, and native CLI on its matching host. The native CLI bundles its TypeScript checker, LLVM helper, runtime pack, declarations, and comptime evaluator. npm postinstall installs the matching native executable; Node is needed for npm installation but not native compilation or execution. The user's toolchain performs the final platform link. Sanitized builds also use it to instrument program LLVM and compile the C runtime. Platform packages publish ahead of `@scriptc/compiler` and `scriptc`. The GitHub release attaches standalone compiler archives, their SHA-256 checksums, and the surface manifest after npm publication succeeds. Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for diff --git a/docs/src/app/limitations/page.mdx b/docs/src/app/limitations/page.mdx index 6523c811..b0983c1e 100644 --- a/docs/src/app/limitations/page.mdx +++ b/docs/src/app/limitations/page.mdx @@ -69,9 +69,13 @@ const results = await Promise.all(jobs); // number[] — compiles **The `any`/`unknown` boundary** - `any` without `--dynamic` is generally a compile error (SC2011) — use `unknown` and a checked cast, or opt into the engine. Native Map and Set type arguments may use `any`; their slots use the same checked values as `unknown` without embedding an engine. -- `any` and `unknown` ride locals, parameters, and returns; record and tuple fields hold `unknown`. JavaScript instance fields can also hold native checked values without a JavaScript engine: bare fields start as `undefined`, and redeclaring a field with the same checked-value type resets it in initialization order. Explicit TypeScript `any`/`unknown` class fields, static checked-value fields, statically typed array elements, and union arms remain unsupported in static builds. Program class instances can pass through `unknown` and recover at their exact static class type without losing identity, including classes with internal collections. Dynamic property inspection of a class whose own fields cannot cross the checked-value boundary throws a catchable `TypeError`. Supported native handles also retain identity; JSON-shaped records and arrays retain the copy boundary described below. +- `any` and `unknown` ride locals, parameters, and returns; record and tuple fields hold `unknown`. JavaScript instance fields can also hold native checked values without a JavaScript engine: bare fields start as `undefined`, and redeclaring a field with the same checked-value type resets it in initialization order. Explicit TypeScript `any`/`unknown` class fields, static checked-value fields, statically typed array elements, and union arms remain unsupported in static builds. Program class instances can pass through `unknown` and recover at their exact static class type or a base class without losing identity, including classes with internal collections. Dynamic property inspection of a class whose own fields cannot cross the checked-value boundary throws a catchable `TypeError`. Supported native handles also retain identity. JavaScript argument and return conversions preserve native array references; other JSON-shaped record and array conversions retain the copy boundary described below. - Operations on `unknown` beyond the supported surface (truthiness, `typeof` narrowing, property access, `+`, `switch`, `throw`) need a checked cast first. +**Three.js** + +With `--npm-static=three`, published three.js modules support CPU math, geometry, scene graphs, materials, and a subset of raycasting in native and WASI builds without a JavaScript engine. Raycasting covers mesh intersections, sorted results, recursive traversal, layers, distance limits, perspective and orthographic camera rays, lines, and points. Mesh attribute interpolation can still fail when three.js reads components beyond the end of a vertex attribute; numeric fields cannot retain the resulting `undefined`. WebGL rendering, browser DOM integration, and native graphics windows remain unsupported. + **Type surface** scriptc typechecks your program in its own type world: the standard `es2025` lib plus its own ambient declarations. In enumerated places those declarations are deliberately different from stock lib or `@types/node`, typed as what actually compiles — `JSON.parse` returns `unknown` (not `any`), and the Promise executor's reject reason is pinned to `Error`. So "programs that typecheck" means scriptc's type world, and a program clean under its own `tsc` can still be redirected here — but never with a bare type error it can't reproduce: the tightened declarations get a second-chance preflight (a program clean in the project's own type world passes, and the affected sites meet per-site diagnostics with rewrite hints instead), and where the shipped fallback declarations lack a lowering for a stock member (`console.table`, `console.time`, ...), the member is declared anyway so the call lands on SC2020 naming the supported alternative. `console.log`/`info`/`debug`/`error`/`warn` themselves take `unknown` arguments and render with Node's console semantics — strings verbatim, everything else through the static `util.inspect`. @@ -92,7 +96,7 @@ A static-tier program otherwise produces byte-identical stdout and the same exit **A lying cast on dynamic data throws instead of corrupting memory** — the headline divergence, and the point. `JSON.parse(s) as Config` with mismatched data throws a catchable error naming the offending path (`expected number at $.port, got string`) where JS would silently hand you garbage. -**Structural width subtyping copies.** A record flowing into a strict field-subset shape is copied, not aliased: mutations through the narrower reference are invisible to the original. Existing typed records and arrays also cross the checked boundary as snapshots. Fresh dense array literals created directly in checked storage preserve their own identity, and supported class instances and native handles cross by reference. +**Structural width subtyping copies.** A record flowing into a strict field-subset shape is copied, not aliased: mutations through the narrower reference are invisible to the original. Typed records and arrays generally cross the checked-value boundary as copies. JavaScript argument and return conversions preserve native array references, and class instances retain identity at their exact class type or a base class. Fresh dense array literals created directly in checked storage preserve their own identity, and supported native handles cross by reference. **`Object.keys`/`values`/`entries` and `JSON.stringify` report a record's declaration order**, not per-object insertion order. Identical to Node whenever objects are built in declaration order (the overwhelmingly common case). @@ -137,10 +141,14 @@ Use [Native FFI](/ffi#replacing-a-node-api-addon) to call the underlying native ## WASI target limits -The production wasm32-wasi target supports the complete executable language tier through LLVM: async/await, promises, generators, timers and other portable event-loop work, stdin/readline, filesystem callbacks and promises, and the --dynamic island. Portable WASI Preview 1 has no socket, process-spawn, OS-signal, network-interface, or filesystem-notification capabilities, so networking/fetch, child processes, signal APIs, os.networkInterfaces(), and fs.watch are rejected before linking with SC3002. --sanitize, native FFI, and library-mode archive builds are also unavailable. Filesystem access is bounded by the host's preopens; scriptc run exposes the current working directory and /tmp. See [Platform Support](/platforms) for build and run details. +The production wasm32-wasi target supports the complete executable language tier through LLVM: async/await, promises, generators, timers and other portable event-loop work, stdin/readline, filesystem callbacks and promises, and the --dynamic island. Portable WASI Preview 1 has no socket, process-spawn, OS-signal, network-interface, or filesystem-notification capabilities, so networking/fetch, child processes, signal APIs, os.networkInterfaces(), and fs.watch are rejected before linking with SC3002. --sanitize and native FFI are also unavailable. Library mode emits an async-free Wasm reactor with named host imports; native runtime localization and thread instances are refused. Each Wasm instance owns independent state. Filesystem access is bounded by the host's preopens; scriptc run exposes the current working directory and /tmp. See [Platform Support](/platforms) for build and run details. ## Tooling gaps - `scriptc run` does not forward extra CLI arguments to the program — `build` and invoke the binary directly. - Native FFI is a direct, manifest-declared C ABI link surface. Formats 2–5 cover call-scoped callbacks, copied string/byte callback parameters, explicitly released retained callbacks, and asynchronous foreign-thread delivery. Format 6 adds single-precision floats, signed 8-bit and signed/unsigned 16-bit integers, and borrowed writable byte spans. Variadic calls, structs by value, owned pointer/string/byte returns, runtime dynamic-library loading, and library-mode builds remain unsupported. See [Native FFI](/ffi). - Numbers are JS-exact f64 everywhere. Integer inference and ownership analysis — the systems-language performance ceiling — are roadmap, not shipped. + +## Browser embedding + +Wasm reactors need a host-provided WASI Preview 1 adapter. DOM APIs, browser graphics bindings, and three.js WebGLRenderer are not compiled today. Supported three.js CPU operations can execute inside a reactor while an external web application renders their results. Library mode excludes promises, async functions, timers, and the JavaScript island. See [WebAssembly Modules](/wasm) for the embedding contract. diff --git a/docs/src/app/platforms/page.mdx b/docs/src/app/platforms/page.mdx index 89b4797c..7431895c 100644 --- a/docs/src/app/platforms/page.mdx +++ b/docs/src/app/platforms/page.mdx @@ -2,7 +2,7 @@ ## Owned LLVM targets -Source artifacts selected with --emit=ir|llvm need only Node. On supported hosts, --emit=asm|obj uses the version-matched platform helper installed with scriptc and does not invoke a compiler, archiver, linker, or SDK. Supported assembly/object targets are macOS arm64/x64 (macOS 14.0 artifacts; helper needs macOS 15+), Linux x64/arm64 glibc, Windows x64 MSVC, Linux x64/arm64 musl, and WASI Preview 1. Ordinary LLVM executables additionally need the platform linker and SDK/sysroot; the helper emits the program object and the packaged runtime supplies objects/archives, so that driver compiles neither program nor runtime C. Runtime development with --sanitize requires an external LLVM toolchain and a C compiler for instrumentation. Object output retains undefined runtime references and is not a library archive. +The compiler runs natively and bundles its TypeScript checker and LLVM helper. Outputs selected with --emit=ir|llvm|asm|obj need no Node installation or external compiler, archiver, linker, or SDK. Supported assembly/object targets are macOS arm64/x64 (macOS 14.0 artifacts; helper needs macOS 15+), Linux x64/arm64 glibc, Windows x64 MSVC, Linux x64/arm64 musl, and WASI Preview 1. Executables additionally need the platform linker and SDK/sysroot; the helper emits the program object and the packaged runtime supplies objects/archives. Runtime development with --sanitize requires an external LLVM toolchain and a C compiler for instrumentation. Object output retains undefined runtime references and is not a library archive. ## Cross-compilation via zig @@ -57,16 +57,16 @@ The full library-mode feature set applies: profile-declared exports and ABI entr ### WebAssembly (WASI Preview 1) -Set SCRIPTC_TARGET=wasm32-wasi to produce a standalone .wasm module. Without -o, scriptc build hello.ts writes .scriptc/hello.wasm. scriptc run hosts the module with Node's WASI implementation, inherits stdio and environment, preopens the current working directory as /, and maps the host platform's temporary directory to the guest's /tmp. A built module can run in another WASI Preview 1 host instead. +Set SCRIPTC_TARGET=wasm32-wasi to produce a standalone .wasm module. Without -o, scriptc build hello.ts writes .scriptc/hello.wasm. Compilation does not require Node. scriptc run requires Node.js 24 or newer on PATH to host the module with Node's WASI implementation, inherits stdio and environment, preopens the current working directory as /, and maps the host platform's temporary directory to the guest's /tmp. A built module can run in another WASI Preview 1 host instead. WASI is a production LLVM target with the same language tiers as the native targets. Its 32-bit LLVM ABI supports collections, closures, exceptions, classes, checked dynamic values, async/await, promises, synchronous and asynchronous generators, timers, stdin/readline events, process-exit listeners, filesystem callbacks and promises, and the --dynamic QuickJS island. -The remaining executable boundary is host capability, not language coverage. WASI Preview 1 has no portable socket, process-spawn, OS-signal, network-interface, or filesystem-notification APIs. Networking/fetch, child processes, signal APIs, os.networkInterfaces(), and fs.watch therefore fail before linking with diagnostic SC3002. --sanitize, native FFI, and library-mode archive builds are unavailable too. Filesystem behavior is bounded by the host's preopens, and process/OS introspection follows WASI's reduced model. +The remaining executable boundary is host capability, not language coverage. WASI Preview 1 has no portable socket, process-spawn, OS-signal, network-interface, or filesystem-notification APIs. Networking/fetch, child processes, signal APIs, os.networkInterfaces(), and fs.watch therefore fail before linking with diagnostic SC3002. --sanitize and native FFI are unavailable too. Filesystem behavior is bounded by the host's preopens, and process/OS introspection follows WASI's reduced model. ## Cross-target limits - `--sanitize` is a host-build lane. -- Library-mode static archives are native embedder artifacts; wasm32-wasi rejects scriptc build --lib with SC3002. +- On wasm32-wasi, scriptc build --lib --profile emits a callable Wasm reactor instead of a static archive. See [WebAssembly Modules](/wasm) for exports, host imports, initialization, and memory ownership. - Mobile triples are library-mode-only: standalone executable builds reject with SC3002, and only the library-admissible surface (the async-free static tier a library archive links) is supported there. - iOS targets build on macOS hosts only (Mach-O localization uses the host linker); Android archives build from any supported host with Zig. - WASI cannot host APIs for capabilities missing from Preview 1, as described above. diff --git a/docs/src/app/quickstart/page.mdx b/docs/src/app/quickstart/page.mdx index b69c6a28..2e2fc92d 100644 --- a/docs/src/app/quickstart/page.mdx +++ b/docs/src/app/quickstart/page.mdx @@ -4,9 +4,9 @@ Install the CLI from npm and compile your first binary in a couple of minutes. ## Prerequisites -- **macOS arm64** is the primary platform ([Linux and Windows](/platforms) are cross-compilation targets). -- **Node ≥ 24** — to run the compiler. The binaries it produces need no Node at all. -- **clang** — preinstalled with the Xcode Command Line Tools; required for executable builds, but not for --emit=ir|c|llvm|asm|obj. Assembly/object output uses the matching helper installed with scriptc and requires macOS 15+. +- **macOS 15+ arm64/x64, Linux arm64/x64, or Windows x64** — see [platform support](/platforms) for target details. +- **Node ≥ 24 and npm** — for npm installation. The installed compiler and its native output run without Node. Standalone distributions are also available from [GitHub Releases](https://github.com/vercel-labs/scriptc/releases). +- **A platform linker and SDK** — for executable builds. On macOS, install the Xcode Command Line Tools. --emit=ir|llvm|asm|obj uses bundled tools and needs no external linker or SDK. ## Install @@ -14,6 +14,8 @@ Install the CLI from npm and compile your first binary in a couple of minutes. $ npm install -g scriptc ``` +Keep optional dependencies and installation scripts enabled so npm can install the native command for your platform. + To work from a clone instead (`pnpm install && pnpm build` in the repo, then `pnpm scriptc` from the repo directory), see the [repository](https://github.com/vercel-labs/scriptc). ## Your first binary diff --git a/docs/src/app/wasm/layout.tsx b/docs/src/app/wasm/layout.tsx new file mode 100644 index 00000000..04268d9e --- /dev/null +++ b/docs/src/app/wasm/layout.tsx @@ -0,0 +1,7 @@ +import { pageMetadata } from "@/lib/page-metadata"; + +export const metadata = pageMetadata("wasm"); + +export default function Layout({ children }: { children: React.ReactNode }) { + return children; +} diff --git a/docs/src/app/wasm/page.mdx b/docs/src/app/wasm/page.mdx new file mode 100644 index 00000000..9805bb27 --- /dev/null +++ b/docs/src/app/wasm/page.mdx @@ -0,0 +1,103 @@ +# WebAssembly Modules + +Compile a reusable module with SCRIPTC_TARGET=wasm32-wasi and scriptc build --lib --profile. The output is a WASI Preview 1 reactor: a .wasm file with callable exports, named host imports, and its own memory. Application code runs as compiled Wasm, without an embedded JavaScript engine. + +Your web application owns loading, browser events, animation scheduling, and rendering. It supplies the module's imports and calls its exports. scriptc does not generate a web application or a graphics adapter. + +## Define the interface + +The library profile names the entry module and its public functions. A reachable callback call becomes a synchronous import from the scriptc namespace. + +```ts:counter.ts +declare function changed(value: number): void; +let count = 0; +export function step(delta: number): number { + count += delta; + changed(count); + return count; +} +``` + +```json:counter.json +{ + "profile_format": 1, + "name": "counter", + "entry": "counter.ts", + "emission": "llvm", + "abi": { + "prefix": "counter_", + "init_symbol": "counter_init", + "sink_register_symbol": "counter_sink", + "callback_register_symbol": "counter_callback" + }, + "callbacks": [{ "name": "changed", "params": ["f64"], "returns": "void" }], + "exports": [{ "export": "step", "symbol": "counter_step", "params": ["f64"], "returns": "f64" }] +} +``` + +Build with the wasm32-wasi target and select the output with -o counter.wasm. Without -o, the output is .scriptc/counter.wasm beside the profile. The compiler API's compileLibrary() returns this path in archivePath, matching the existing native library API. + +The profile uses the native library schema. sink_register_symbol and callback_register_symbol name native registration functions; a Wasm reactor does not export them. Wasm binds callbacks during instantiation and delivers panics through scriptc.panic. + +Set "optimization": "dev" in the profile for faster builds. Eligible npm dependencies are attempted automatically. A profile may add "npm_static": ["three"] to explicitly attempt compilation from a package's source, including packages without bundled declarations. Unsupported reachable code still produces a compiler diagnostic. + +## Instantiate and initialize + +A browser host needs a WASI Preview 1 adapter as well as the application's callbacks. Browsers do not supply WASI automatically. Inspect WebAssembly.Module.imports(module) to see the exact imports required by your compiled program. Supply real implementations or explicit failures for unsupported operations. + +```js:loader.js +export async function loadCounter(url, wasi) { + const module = await WebAssembly.compileStreaming(fetch(url)); + let api; + const instance = await WebAssembly.instantiate(module, { + ...wasi.getImportObject(), + scriptc: { + changed(value) { + console.log(value); + }, + panic(pointer, length) { + const bytes = new Uint8Array(api.memory.buffer, pointer, length); + throw new Error(new TextDecoder().decode(bytes)); + }, + }, + }); + api = instance.exports; + wasi.initialize(instance); + api.counter_init(); + return api; +} +``` + +This loader assumes an adapter with getImportObject() and initialize(instance) methods, as provided by Node's node:wasi. Browser adapters may use different method names. Initialization must first connect the adapter to the instance's exported memory and invoke _initialize once, then call the profile's init_symbol. Do not call a command-style start(); reactors have no _start export. + +After initialization, counter_step(2) returns 2, and another call returns 4. Calling counter_init() again resets application globals and reevaluates module initialization. Instantiate the module again for independent state and memory. + +## Values and memory ownership + + + + + + + + + + + +
Profile classWasm boundary
f64JavaScript number.
bool, u8, u32, i32Wasm i32; booleans use 0 or 1. u32 values may appear as signed JavaScript numbers; use value >>> 0 to read them unsigned.
i64, u64JavaScript BigInt. The existing library integer checks and proofs restrict values to the exact JavaScript number range.
string, bytes parametersTwo i32 arguments: byte offset and byte length. Strings use UTF-8. Entry parameters are copied into runtime-owned values.
string, bytes resultsTwo extra i32 arguments identify writable four-byte output slots. The function writes the result pointer and byte length there, in little-endian order.
voidNo result.
+ +The native profile's direction restrictions still apply: u8/u32/i32 are export parameters and callback scalars; i64/u64 are export-only. Callback results must be scalars or void. Callback string/byte parameters are borrowed only for the duration of that callback; copy them before retaining them in the host. + +Allocate host input buffers and output slots with scriptc_alloc(size); a zero result indicates allocation failure. Release each allocation exactly once with scriptc_free(pointer). Only free pointers returned by scriptc_alloc. These allocations remain live across application calls, initialization, and collection until freed. + +Result buffers belong to the module. If the profile declares abi.result_reset_symbol, results accumulate until that function is called. Otherwise, the next application export resets the result arena. Initialization and the optional abi.collect_symbol also invalidate results. Allocation and free do not reset the arena. Copy a result into host-owned storage before invalidating it; never pass its pointer to scriptc_free. + +Recreate typed arrays and DataViews from memory.buffer after every Wasm call that may allocate. Memory growth replaces the backing buffer and invalidates older views. Pointers remain offsets into the instance's memory; never use a pointer with another instance. + +## Errors and scheduling + +A compiled trap or escaped exception calls scriptc.panic(pointer, length) with the native library's structured diagnostic bytes, then traps if the callback returns. Discard the instance after a panic, a Wasm trap, or an exception escaping a host import. Recovery uses a new instance; calling the initialization export does not repair a failed instance. + +Host callbacks run synchronously. They must return before calling any application export, initialization, collection, allocation, or free on the same instance. Reentry is rejected. Copy callback data and schedule subsequent work after the outer call returns. The web app can call a frame export from requestAnimationFrame and render its results through Canvas, WebGL, or WebGPU. + +See [Limitations](/limitations) for browser graphics and library-mode boundaries. diff --git a/docs/src/lib/docs-navigation.ts b/docs/src/lib/docs-navigation.ts index 7a507721..ff00b2c9 100644 --- a/docs/src/lib/docs-navigation.ts +++ b/docs/src/lib/docs-navigation.ts @@ -23,6 +23,7 @@ export const navSections: NavSection[] = [ { name: "npm Dependencies", href: "/dependencies" }, { name: "Native FFI", href: "/ffi" }, { name: "Native Program Objects", href: "/native-objects" }, + { name: "WebAssembly Modules", href: "/wasm" }, { name: "Platform Support", href: "/platforms" }, ], }, diff --git a/docs/src/lib/page-titles.ts b/docs/src/lib/page-titles.ts index 5fc93f55..45e9d15b 100644 --- a/docs/src/lib/page-titles.ts +++ b/docs/src/lib/page-titles.ts @@ -7,6 +7,7 @@ export const PAGE_TITLES: Record = { dependencies: "npm Dependencies", ffi: "Native FFI", "native-objects": "Native Program Objects", + wasm: "WebAssembly Modules", platforms: "Platform Support", compatibility: "Node.js 24 Compatibility", "how-it-works": "How It Works", diff --git a/package.json b/package.json index d0f57f78..2208a4b6 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "pnpm -r --filter \"./packages/*\" run build", "build:fresh": "rm -rf packages/compiler/dist packages/cli/dist node_modules/.cache/scriptc-tsc && pnpm build", - "build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-compiler.mts", + "build:native-compiler": "node --max-old-space-size=8192 --import tsx scripts/build-native-cli.mts", "bench:builds": "node scripts/bench-builds.mjs", "test": "vitest run", "test:ts7": "node scripts/test-ts7.mjs", @@ -31,6 +31,7 @@ "effect": "3.22.2", "eslint": "^9.20.0", "js-yaml": "4.3.0", + "rollup": "4.62.2", "three": "0.186.1", "tsx": "^4.19.0", "typescript": "5.9.3", diff --git a/packages/cli-darwin-arm64/package.json b/packages/cli-darwin-arm64/package.json new file mode 100644 index 00000000..3a7afaca --- /dev/null +++ b/packages/cli-darwin-arm64/package.json @@ -0,0 +1,29 @@ +{ + "name": "@scriptc/cli-darwin-arm64", + "version": "0.1.7", + "description": "Native scriptc compiler for darwin-arm64", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-darwin-arm64" + }, + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli-darwin-x64/package.json b/packages/cli-darwin-x64/package.json new file mode 100644 index 00000000..4091cb9d --- /dev/null +++ b/packages/cli-darwin-x64/package.json @@ -0,0 +1,29 @@ +{ + "name": "@scriptc/cli-darwin-x64", + "version": "0.1.7", + "description": "Native scriptc compiler for darwin-x64", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-darwin-x64" + }, + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli-linux-arm64-gnu/package.json b/packages/cli-linux-arm64-gnu/package.json new file mode 100644 index 00000000..4910e637 --- /dev/null +++ b/packages/cli-linux-arm64-gnu/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-arm64-gnu", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-arm64-gnu", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-arm64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "glibc" + ] +} diff --git a/packages/cli-linux-arm64-musl/package.json b/packages/cli-linux-arm64-musl/package.json new file mode 100644 index 00000000..d068d734 --- /dev/null +++ b/packages/cli-linux-arm64-musl/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-arm64-musl", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-arm64-musl", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-arm64-musl" + }, + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "musl" + ] +} diff --git a/packages/cli-linux-x64-gnu/package.json b/packages/cli-linux-x64-gnu/package.json new file mode 100644 index 00000000..26b08f88 --- /dev/null +++ b/packages/cli-linux-x64-gnu/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-x64-gnu", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-x64-gnu", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-x64-gnu" + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "glibc" + ] +} diff --git a/packages/cli-linux-x64-musl/package.json b/packages/cli-linux-x64-musl/package.json new file mode 100644 index 00000000..89861b1b --- /dev/null +++ b/packages/cli-linux-x64-musl/package.json @@ -0,0 +1,32 @@ +{ + "name": "@scriptc/cli-linux-x64-musl", + "version": "0.1.7", + "description": "Native scriptc compiler for linux-x64-musl", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-linux-x64-musl" + }, + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + }, + "libc": [ + "musl" + ] +} diff --git a/packages/cli-win32-x64-msvc/package.json b/packages/cli-win32-x64-msvc/package.json new file mode 100644 index 00000000..4eeb0f5a --- /dev/null +++ b/packages/cli-win32-x64-msvc/package.json @@ -0,0 +1,29 @@ +{ + "name": "@scriptc/cli-win32-x64-msvc", + "version": "0.1.7", + "description": "Native scriptc compiler for win32-x64-msvc", + "license": "Apache-2.0", + "homepage": "https://scriptc.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/vercel-labs/scriptc.git", + "directory": "packages/cli-win32-x64-msvc" + }, + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "files": [ + "dist" + ], + "scripts": { + "build": "node -e \"\"", + "build:native": "node --max-old-space-size=8192 --import tsx ../../scripts/build-native-cli.mts dist", + "prepack": "node ../../scripts/verify-native-cli.mjs ." + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli/README.md b/packages/cli/README.md index 4562650b..d592eed9 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -22,7 +22,7 @@ $ scriptc build fib.ts -o fib && ./fib $ npm install -g scriptc ``` -Requires Node.js 24. Executable builds require a platform linker driver and SDK/sysroot. On supported macOS, Linux, and Windows hosts, LLVM executables use the matching optional helper and precompiled runtime pack, so the driver only links; select that driver with `SCRIPTC_LINKER`. Runtime development with `--sanitize` additionally needs a C compiler. `--emit=ir|llvm` requires only Node, while `--emit=asm|obj` requires neither an external compiler nor a linker. +The installed compiler runs natively on supported macOS, Linux, and Windows hosts. Compilation, compile-time evaluation, and native execution do not require Node. `--emit=ir|llvm|asm|obj` uses the bundled TypeScript checker and LLVM helper without an external compiler, archiver, linker, or SDK. Executable builds additionally need a platform linker driver and SDK/sysroot; precompiled runtime packs supply the C runtime. Set `SCRIPTC_LINKER` to choose that driver. Runtime development with `--sanitize` additionally needs a C compiler. Node.js 24 or newer is needed for npm installation and `scriptc run` of WASI modules. Builds use a bounded persistent cache. Unchanged source can reuse the validated frontend result and LLVM program objects. Library identity getters occupy a separate LLVM module, so an identity change can reuse the large program object. Runtime objects come from the installed pack. Executable cache entries verify their native dependencies; FFI builds relink against current external inputs. Set `SCRIPTC_NO_CACHE=1` to bypass the cache or `SCRIPTC_CACHE_DIR` to select its location. An existing POSIX override must already be private. @@ -38,7 +38,7 @@ For embedder-hosted modules that are not installed npm packages, coverage can ma No annotations, no dialect, no special stdlib: the same TypeScript you run on Node, type-checked by the real TypeScript compiler. Programs outside the static tier can opt into `--dynamic`, which embeds a small JavaScript engine (~620KB) for the parts that can't be static; everything else fails the build with a specific error code and usually a rewrite hint. -WebAssembly is available as a production LLVM target: `SCRIPTC_TARGET=wasm32-wasi scriptc build app.ts`. It emits a WASI Preview 1 `.wasm` module, and `scriptc run` supplies a WASI host. The complete executable language tier—including async, generators, timers, and `--dynamic`—is supported. APIs needing capabilities WASI P1 does not expose (network sockets/fetch, child processes, OS signals, and filesystem watching), sanitizer builds, native FFI, and library-mode archive builds are rejected with `SC3002`. +WebAssembly is available as a production LLVM target: `SCRIPTC_TARGET=wasm32-wasi scriptc build app.ts`. It emits a WASI Preview 1 `.wasm` module, and `scriptc run` supplies a WASI host. The complete executable language tier—including async, generators, timers, and `--dynamic`—is supported. APIs needing capabilities WASI P1 does not expose (network sockets/fetch, child processes, OS signals, and filesystem watching), sanitizer builds, and native FFI are rejected with `SC3002`. With `--lib --profile`, the target produces a callable Wasm reactor with named host imports and per-instance memory. See [WebAssembly Modules](https://scriptc.dev/wasm) for the embedding contract. Native code can be called through an explicit, link-time C ABI manifest: declare the function signature in TypeScript, bind it to a C symbol, and build with `--ffi `. See the [Native FFI guide](https://scriptc.dev/ffi). diff --git a/packages/cli/package.json b/packages/cli/package.json index a3e23250..9307bd9b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "scriptc", "version": "0.1.7", - "description": "Compile ordinary TypeScript and JavaScript to small, fast native executables — no Node, no V8, no JavaScript engine in the binary", + "description": "Compile ordinary TypeScript and JavaScript to small, fast native executables \u2014 no Node, no V8, no JavaScript engine in the binary", "license": "Apache-2.0", "homepage": "https://scriptc.dev", "repository": { @@ -11,19 +11,41 @@ }, "type": "module", "bin": { - "scriptc": "dist/bootstrap.js" + "scriptc": "bin/scriptc.exe" }, "files": [ - "dist", + "bin", "scripts" ], "engines": { "node": ">=24" }, "scripts": { - "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json" + "build": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json", + "prepack": "node scripts/prepare-native.mjs", + "postinstall": "node scripts/install-native.mjs" }, - "dependencies": { + "devDependencies": { "@scriptc/compiler": "workspace:*" + }, + "optionalDependencies": { + "@scriptc/cli-darwin-arm64": "workspace:*", + "@scriptc/cli-darwin-x64": "workspace:*", + "@scriptc/cli-linux-arm64-gnu": "workspace:*", + "@scriptc/cli-linux-arm64-musl": "workspace:*", + "@scriptc/cli-linux-x64-gnu": "workspace:*", + "@scriptc/cli-linux-x64-musl": "workspace:*", + "@scriptc/cli-win32-x64-msvc": "workspace:*", + "@scriptc/runtime-darwin-arm64": "workspace:*", + "@scriptc/runtime-darwin-x64": "workspace:*", + "@scriptc/runtime-linux-x64-gnu": "workspace:*", + "@scriptc/runtime-linux-arm64-gnu": "workspace:*", + "@scriptc/runtime-linux-x64-musl": "workspace:*", + "@scriptc/runtime-linux-arm64-musl": "workspace:*", + "@scriptc/runtime-win32-x64-msvc": "workspace:*", + "@scriptc/runtime-wasm32-wasi": "workspace:*", + "@scriptc/runtime-ios-arm64": "workspace:*", + "@scriptc/runtime-ios-simulator-arm64": "workspace:*", + "@scriptc/runtime-android-arm64": "workspace:*" } } diff --git a/packages/cli/scripts/install-native.mjs b/packages/cli/scripts/install-native.mjs new file mode 100644 index 00000000..79a4c17f --- /dev/null +++ b/packages/cli/scripts/install-native.mjs @@ -0,0 +1,113 @@ +import { constants, copyFileSync, chmodSync, existsSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function nativeCliPackage(platform, architecture, libc) { + const host = `${platform}-${architecture}`; + if (host === "darwin-arm64" || host === "darwin-x64") return `@scriptc/cli-${host}`; + if (host === "win32-x64") return "@scriptc/cli-win32-x64-msvc"; + if (host === "linux-x64" || host === "linux-arm64") { + if (libc !== "glibc" && libc !== "musl") throw new Error("could not identify the Linux C library"); + return `@scriptc/cli-${host}-${libc === "musl" ? "musl" : "gnu"}`; + } + throw new Error(`scriptc has no native command for ${host}`); +} + +function hostLibc() { + if (process.platform !== "linux") return null; + const report = process.report.getReport(); + if (report.header.glibcVersionRuntime) return "glibc"; + if (report.sharedObjects.some((path) => /(?:^|\/)ld-musl-|libc\.musl-/.test(path))) return "musl"; + // A statically linked Node has no loaded libc in its process report. + // Only select musl when its loader is present for this architecture. + const arch = process.arch === "arm64" ? "aarch64" : "x86_64"; + return existsSync(`/lib/ld-musl-${arch}.so.1`) ? "musl" : null; +} + +export function relocateToolchain(manifest, sourceDirectory, destinationDirectory) { + const relocated = { ...manifest }; + const pathFrom = (value) => relative(destinationDirectory, resolve(sourceDirectory, value)); + for (const name of ["ts7", "llvm_package", "runtime_pack", "runtime_sources", "declarations", "comptime", "wasi_node_runner"]) { + if (typeof relocated[name] === "string") relocated[name] = pathFrom(relocated[name]); + } + for (const name of ["linker", "dsymutil", "archiver", "relocatable_linker"]) { + const value = relocated[name]; + if (typeof value === "string" && !isAbsolute(value) && /[/\\]/.test(value)) relocated[name] = pathFrom(value); + } + if (relocated.runtime_packs) relocated.runtime_packs = relocated.runtime_packs.map((pack) => ({ ...pack, path: pathFrom(pack.path) })); + return relocated; +} + +export function installNativeCli(directory, packageName = nativeCliPackage(process.platform, process.arch, hostLibc())) { + directory = realpathSync(directory); + const require = createRequire(join(directory, "package.json")); + const packageManifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + const version = packageManifest.version; + let platformManifest; + try { platformManifest = require.resolve(`${packageName}/package.json`); } + catch { throw new Error(`scriptc requires ${packageName}@${version}; reinstall with optional dependencies enabled`); } + const identity = JSON.parse(readFileSync(platformManifest, "utf8")); + if (identity.name !== packageName || identity.version !== version) throw new Error(`scriptc requires ${packageName}@${version}, found ${identity.version}`); + const sourceDirectory = join(dirname(platformManifest), "dist", "bin"); + const source = join(sourceDirectory, process.platform === "win32" ? "scriptc.exe" : "scriptc"); + const original = JSON.parse(readFileSync(source + ".json", "utf8")); + if (original.schema !== "scriptc.native-toolchain.v1" || original.compiler_version !== version) throw new Error("native compiler toolchain version does not match this installation"); + const bin = join(directory, "bin"); + mkdirSync(bin, { recursive: true }); + const manifest = relocateToolchain(original, sourceDirectory, bin); + const packs = new Map((manifest.runtime_packs ?? []).map((pack) => [pack.target, pack])); + for (const suffix of ["darwin-arm64", "darwin-x64", "linux-x64-gnu", "linux-arm64-gnu", "linux-x64-musl", "linux-arm64-musl", + "win32-x64-msvc", "wasm32-wasi", "ios-arm64", "ios-simulator-arm64", "android-arm64"]) { + const name = `@scriptc/runtime-${suffix}`; + if (packageManifest.optionalDependencies?.[name] === undefined) continue; + let path; + try { path = require.resolve(`${name}/package.json`); } + catch { continue; } + const packIdentity = JSON.parse(readFileSync(path, "utf8")); + if (packIdentity.name !== name || packIdentity.version !== version) throw new Error(`scriptc requires ${name}@${version}`); + const pack = JSON.parse(readFileSync(join(dirname(path), "runtime-pack.json"), "utf8")); + if (pack.schema !== "scriptc.runtime-pack.v1" || pack.package !== name || pack.version !== version) { + throw new Error(`invalid runtime pack installed for ${name}@${version}`); + } + if (!packs.has(pack.target.name)) packs.set(pack.target.name, { target: pack.target.name, path: relative(bin, dirname(path)) }); + } + manifest.runtime_packs = [...packs.values()]; + // npm normalizes modes for payloads outside package bin entries. These + // tools are launched directly by the compiler after installation. + if (process.platform !== "win32") { + for (const path of [manifest.ts7, manifest.comptime, join(manifest.llvm_package, "bin/scriptc-llvm-codegen")]) { + chmodSync(resolve(bin, path), 0o755); + } + } + // The common filename lets npm's Windows shim invoke a native executable. + // POSIX bin links also execute this file directly, without a JS launcher. + const destination = join(bin, "scriptc.exe"); + const staged = destination + `.install-${process.pid}`; + try { + copyFileSync(source, staged, constants.COPYFILE_FICLONE); + chmodSync(staged, 0o755); + writeFileSync(staged + ".json", JSON.stringify(manifest, null, 2) + "\n"); + renameSync(staged + ".json", destination + ".json"); + renameSync(staged, destination); + } finally { + rmSync(staged, { force: true }); + rmSync(staged + ".json", { force: true }); + } + return destination; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const directory = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + // A source checkout builds the native distribution separately. pnpm pack + // replaces workspace ranges with release versions before publication. + const workspace = Object.values(manifest.optionalDependencies ?? {}).some((value) => String(value).startsWith("workspace:")); + if (!workspace) installNativeCli(directory); + } + catch (error) { + process.stderr.write(`scriptc: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/packages/cli/scripts/prepare-native.mjs b/packages/cli/scripts/prepare-native.mjs new file mode 100644 index 00000000..0af4064f --- /dev/null +++ b/packages/cli/scripts/prepare-native.mjs @@ -0,0 +1,18 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function prepareNativeCommand(directory) { + const bin = join(directory, "bin"); + mkdirSync(bin, { recursive: true }); + // npm creates bin links before postinstall. The payload must exist in the + // tarball, and no shebang may pin the Windows shim to an interpreter. + // Installation replaces this placeholder with the platform executable. + writeFileSync(join(bin, "scriptc.exe"), + "echo 'scriptc: native installation is incomplete; enable install scripts and reinstall scriptc' >&2\nexit 1\n", { mode: 0o755 }); + rmSync(join(bin, "scriptc.exe.json"), { force: true }); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + prepareNativeCommand(resolve(dirname(fileURLToPath(import.meta.url)), "..")); +} diff --git a/packages/cli/src/command.ts b/packages/cli/src/command.ts new file mode 100644 index 00000000..1dea65f1 --- /dev/null +++ b/packages/cli/src/command.ts @@ -0,0 +1 @@ +export * from "@scriptc/compiler/cli/command"; diff --git a/packages/cli/src/host.ts b/packages/cli/src/host.ts new file mode 100644 index 00000000..3e612755 --- /dev/null +++ b/packages/cli/src/host.ts @@ -0,0 +1 @@ +export * from "@scriptc/compiler/cli/host"; diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 35ec1cb3..ad77e5e0 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,12 +1,9 @@ import { spawn } from "node:child_process"; -import { existsSync, readFileSync, rmSync, statSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { parseArgs } from "node:util"; -import { analyze, compile, compileLibrary, isExactExternalTypeSpecifier, renderDiagnostics, renderCoverage, resolveProvenanceSources, setProvenanceSources, sourceTargetPlatform, warmNativeCaches, type NativeCacheWarmProfile } from "@scriptc/compiler"; -import { resolveOutputOptions } from "./output-options.js"; -import { selectOutputPaths } from "./paths.js"; -import { CLI_OPTIONS, USAGE } from "./usage.js"; +import { analyze, compile, compileLibrary, resolveProvenanceSources, sourceTargetPlatform, warmNativeCaches } from "@scriptc/compiler"; +import { runCli } from "./command.js"; /** The version of the installed package. Read from the manifest rather than * baked in by the build, so a stamped release and a source checkout answer @@ -19,334 +16,20 @@ function version(): string { return manifest.version ?? "unknown"; } -/* The exit discipline: NEVER process.exit() after writing output. stdout/ - * stderr to a PIPE are async streams — process.exit() drops whatever libuv - * hasn't flushed yet, which truncates large diagnostic renders at the pipe - * buffer (observed: 64KB cut mid-code-frame). Every path sets - * process.exitCode and returns instead; Node exits naturally once the - * streams drain. */ -class CliExit extends Error { - constructor(readonly code: number) { - super(`exit ${code}`); - } -} - -function fail(msg: string): never { - process.stderr.write(msg + "\n"); - throw new CliExit(1); -} - -/** parseArgs, with its throw turned into the CLI's own one-line error. - * Unparseable arguments are a USER error — an unknown flag or a missing - * value used to reach the top level as an uncaught ERR_PARSE_ARGS_* and - * print a Node stack trace over the user's terminal. */ -function parseCli(): ReturnType> { - try { - return parseArgs({ options: CLI_OPTIONS, allowPositionals: true, allowNegative: true }); - } catch (err) { - // parseArgs appends a paragraph about `--` and positionals to the - // unknown-option message; the first sentence is the part that names - // what was wrong, and USAGE below already covers what was meant. - const raw = err instanceof Error ? err.message : String(err); - const msg = raw.split("\n")[0]!.split(". ")[0]!; - fail(`scriptc: ${msg}\n\n${USAGE}`); - } -} - -async function main(): Promise { - const { values, positionals } = parseCli(); - const externalTypeArgs = values["external-types"] ?? []; - - if (values.version) { - process.stdout.write(`${version()}\n`); - return 0; - } - - if (values.help || positionals.length === 0) { - process.stdout.write(USAGE); - return values.help ? 0 : 1; - } - - const [command, inputArg] = positionals; - if (command === "cache") { - if (inputArg !== "warm") fail(`unknown cache command "${inputArg ?? ""}" (supported: warm)\n\n${USAGE}`); - if (values.lib || values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.ffi !== undefined || values.profile !== undefined || values.strip || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || values["provenance-sources"] || externalTypeArgs.length > 0 || values.out !== undefined || values["emit-ir"] || !values["keep-llvm"]) { - fail(`scriptc cache warm takes only native optimization/sanitizer options and profile names\n\n${USAGE}`); - } - const optimization = values.optimization; - if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { - fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); - } - const profileArgs = positionals.slice(2); - const knownProfiles = new Set(["runtime", "tls", "dynamic"]); - for (const profile of profileArgs) { - if (!knownProfiles.has(profile as NativeCacheWarmProfile)) { - fail(`unknown cache warm profile "${profile}" (supported: runtime, tls, dynamic)`); - } - } - let result; - try { - result = await warmNativeCaches({ - ...(optimization === undefined ? {} : { optimization }), - sanitize: values.sanitize, - ...(profileArgs.length === 0 - ? {} - : { profiles: profileArgs as NativeCacheWarmProfile[] }), - }); - } catch (error) { - fail(`scriptc: ${error instanceof Error ? error.message : String(error)}`); - } - process.stdout.write(`${result.cacheRoot}\n`); - for (const profile of result.profiles) { - process.stdout.write(`${profile.profile}\t${Math.round(profile.elapsedMs)}ms\n`); - } - return 0; - } - if (command !== "build" && command !== "run" && command !== "coverage") { - fail(`unknown command "${command}"\n\n${USAGE}`); - } - if (values.lib) { - // LIBRARY mode: the profile names the entry module and pins the - // emission; the executable lane's mode flags have no meaning here - // (library artifacts are static-tier only, and there is no fallback - // concept — bare npm specifiers are static-or-refuse: the npm-static - // eligibility bar runs automatically, eligible packages compile into - // the graph, ineligible ones refuse with SC4013). - if (command !== "build") fail(`--lib is a build mode (scriptc build --lib --profile )\n\n${USAGE}`); - const profileArg = values.profile; - if (!profileArg) fail(`scriptc build --lib needs --profile \n\n${USAGE}`); - if (inputArg) { - fail("scriptc build --lib takes no input positional: the profile names the entry module"); - } - if (values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.optimization !== undefined || values.strip || values.ffi !== undefined || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || externalTypeArgs.length > 0) { - fail( - "scriptc build --lib takes no --dynamic/--backend/--emit/--print/--optimization/--strip/--windows-subsystem/--npm-static/--ffi/--external-types: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, and external type mappings belong to coverage", - ); - } - const profilePath = resolve(profileArg); - const libOutDir = values.out ? dirname(resolve(values.out)) : join(dirname(profilePath), ".scriptc"); - const result = await compileLibrary({ - profilePath, - outDir: libOutDir, - ...(values.out ? { outPath: resolve(values.out) } : {}), - emitIr: values["emit-ir"], - sanitize: values.sanitize, +process.exitCode = await runCli(process.argv.slice(2), { + version, analyze: async (entry, options) => analyze(entry, options), compile, compileLibrary, resolveProvenanceSources, sourceTargetPlatform, warmNativeCaches, + run: (binary) => new Promise((resolveExit) => { + let child; + if (sourceTargetPlatform() === "wasi") { + const builtRunner = fileURLToPath(new URL("./wasi-runner.js", import.meta.url)); + const runner = existsSync(builtRunner) ? builtRunner : fileURLToPath(new URL("./wasi-runner.ts", import.meta.url)); + child = spawn(process.execPath, [...process.execArgv, "--no-warnings", runner, binary], { stdio: "inherit" }); + } else child = spawn(binary, [], { stdio: "inherit" }); + child.on("exit", (code, signal) => { + if (signal) { + process.stderr.write(`scriptc: program killed by ${signal}\n`); + resolveExit(1); + } else resolveExit(code ?? 0); }); - if (!result.ok) { - const color = process.stderr.isTTY ?? false; - process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); - const n = result.diagnostics.length; - process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); - return 1; - } - if (!values["keep-llvm"]) rmSync(result.llvmPath, { force: true }); - process.stdout.write(`${result.archivePath}\n`); - // The contract sidecar rides the same invocation when the profile - // declares one — name it so the embedder's tooling knows where to look. - if (result.sidecarPath !== undefined) process.stdout.write(`${result.sidecarPath}\n`); - return 0; - } - if (values["emit-ir"] && (command === "build" || command === "run")) { - process.stderr.write("scriptc: warning: --emit-ir is deprecated; use --emit=ir for IR as the primary output\n"); - } - if (!inputArg) fail(`missing input file\n\n${USAGE}`); - const input = resolve(inputArg); - if (command === "coverage" && values.emit !== undefined) { - fail(`--emit is a build/run option\n\n${USAGE}`); - } - if (command === "coverage" && values.strip) { - fail(`--strip is a build/run option\n\n${USAGE}`); - } - if (values.print !== undefined && values.print !== "native-link-info") { - fail(`unknown print kind "${values.print}" (supported: native-link-info)\n\n${USAGE}`); - } - const printNativeLinkInfo = values.print === "native-link-info"; - if (printNativeLinkInfo && command !== "build") { - fail(`--print=native-link-info is a build option\n\n${USAGE}`); - } - if (printNativeLinkInfo && values.emit !== undefined && values.emit !== "obj") { - fail(`--print=native-link-info requires --emit=obj\n\n${USAGE}`); - } - if (externalTypeArgs.length > 0 && command !== "coverage") { - fail(`--external-types is a coverage-only option\n\n${USAGE}`); - } - const externalTypes: Record = Object.create(null) as Record; - for (const mapping of externalTypeArgs) { - const equals = mapping.indexOf("="); - if (equals <= 0 || equals === mapping.length - 1) { - fail(`invalid --external-types mapping ${JSON.stringify(mapping)} (expected )`); - } - const specifier = mapping.slice(0, equals).trim(); - const declarationArg = mapping.slice(equals + 1).trim(); - if (!isExactExternalTypeSpecifier(specifier)) { - fail(`invalid --external-types specifier ${JSON.stringify(specifier)} (expected an exact bare package specifier)`); - } - if (!/\.d\.(?:ts|mts|cts)$/.test(declarationArg)) { - fail(`invalid --external-types declaration ${JSON.stringify(declarationArg)} (expected a .d.ts, .d.mts, or .d.cts file)`); - } - if (externalTypes[specifier] !== undefined) { - fail(`duplicate --external-types mapping for ${JSON.stringify(specifier)}`); - } - const declarationPath = resolve(declarationArg); - try { - if (!statSync(declarationPath).isFile()) throw new Error("not a file"); - } catch { - fail(`--external-types declaration does not name a readable file: ${declarationPath}`); - } - externalTypes[specifier] = declarationPath; - } - const ffiProfilePath = values.ffi !== undefined ? resolve(values.ffi) : undefined; - if (values.backend !== undefined && values.backend !== "llvm") { - fail(`unknown backend "${values.backend}" (supported: llvm)\n\n${USAGE}`); - } - const optimization = values.optimization; - if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { - fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); - } - const windowsSubsystem = values["windows-subsystem"]; - if (windowsSubsystem !== undefined && windowsSubsystem !== "console" && windowsSubsystem !== "gui") { - fail(`unknown Windows subsystem "${windowsSubsystem}" (supported: console, gui)\n\n${USAGE}`); - } - if (windowsSubsystem !== undefined && command === "coverage") { - fail(`--windows-subsystem is only supported for executable builds\n\n${USAGE}`); - } - const output = command === "coverage" - ? null - : resolveOutputOptions(command, { - ...(values.emit === undefined && !printNativeLinkInfo - ? {} - : { emit: values.emit ?? "obj" }), - emitIr: values["emit-ir"], - ...(values.backend === undefined ? {} : { backend: values.backend }), - keepLlvm: values["keep-llvm"], - sanitize: values.sanitize, - ...(values.optimization === undefined ? {} : { optimization: values.optimization }), - strip: values.strip, - ...(windowsSubsystem === undefined ? {} : { windowsSubsystem }), - ...(values.ffi === undefined ? {} : { ffi: values.ffi }), - }); - if (output !== null && !output.ok) fail(`${output.message}\n\n${USAGE}`); - const backend = output?.ok ? output.backend : undefined; - - // --npm-static: repeatable and comma-splittable; the literal "auto" - // switches to eligibility-based detection (mixing "auto" with names - // is rejected — the shapes answer different questions). - const npmStaticRaw = (values["npm-static"] ?? []).flatMap((v) => v.split(",")).map((v) => v.trim()).filter((v) => v !== ""); - let npmStatic: string[] | "auto" | undefined; - if (npmStaticRaw.includes("auto")) { - if (npmStaticRaw.length > 1) fail(`--npm-static auto cannot be combined with package names\n\n${USAGE}`); - npmStatic = "auto"; - } else if (npmStaticRaw.length > 0) { - npmStatic = npmStaticRaw; - } - - // --provenance-sources resolves BEFORE the program loads (tsgo needs the - // source "paths" at creation): attestations and source trees fetch (or - // ride the content-addressed cache / the offline manifest), the registry - // installs, and every fallback prints as a note — never a failure. - const provenance = values["provenance-sources"] ? await resolveProvenanceSources(input) : null; - if (provenance !== null) { - setProvenanceSources(provenance); - for (const pkg of provenance.packages) { - process.stderr.write( - `provenance: ${pkg.name}@${pkg.version} ← ${pkg.repo.replace(/^git\+/, "")} @ ${pkg.commit.slice(0, 12)} (source compiles statically)\n`, - ); - } - for (const note of provenance.notes) process.stderr.write(`provenance: ${note}\n`); - } - - if (command === "coverage") { - const { coverage, sourceTexts } = analyze(input, { - dynamic: values.dynamic, - ...(npmStatic !== undefined ? { npmStatic } : {}), - ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), - ...(Object.keys(externalTypes).length > 0 ? { externalTypes } : {}), - }); - const color = process.stdout.isTTY ?? false; - process.stdout.write(renderCoverage(coverage, { color, sourceTexts }) + "\n"); - return coverage.preflightFailed ? 1 : 0; - } - - if (output === null || !output.ok) throw new Error("internal output-option state"); - if (windowsSubsystem !== undefined && sourceTargetPlatform() !== "win32") { - fail(`--windows-subsystem requires a Windows executable target\n\n${USAGE}`); - } - const { outDir, outPath } = selectOutputPaths(input, output.cliOutputKind, values.out); - - let nativeLinkInfo: object | undefined; - const build = async (): Promise => { - const result = await compile(input, { - outPath, - outDir, - outputKind: output.outputKind, - emitIr: output.emitIr, - sanitize: values.sanitize, - dynamic: values.dynamic, - ...(backend !== undefined ? { backend } : {}), - ...(optimization !== undefined ? { optimization } : {}), - ...(values.strip ? { strip: true } : {}), - ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), - ...(npmStatic !== undefined ? { npmStatic } : {}), - ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), - ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), - }); - if (!result.ok) { - const color = process.stderr.isTTY ?? false; - process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); - const n = result.diagnostics.length; - process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); - throw new CliExit(1); - } - if (result.artifact.kind === "exe") { - if (!values["keep-llvm"]) rmSync(result.artifact.translationUnitPath, { force: true }); - } else if (result.artifact.kind === "obj") { - nativeLinkInfo = result.artifact.nativeLinkInfo; - } - return result.artifact.path; - }; - - const binary = await build(); - - if (command === "run") { - return new Promise((resolveExit) => { - let child; - if (sourceTargetPlatform() === "wasi") { - const builtRunner = fileURLToPath(new URL("./wasi-runner.js", import.meta.url)); - const runner = existsSync(builtRunner) - ? builtRunner - : fileURLToPath(new URL("./wasi-runner.ts", import.meta.url)); - child = spawn( - process.execPath, - [...process.execArgv, "--no-warnings", runner, binary], - { stdio: "inherit" }, - ); - } else { - child = spawn(binary, [], { stdio: "inherit" }); - } - child.on("exit", (code, signal) => { - if (signal) { - process.stderr.write(`scriptc: program killed by ${signal}\n`); - resolveExit(1); - } else { - resolveExit(code ?? 0); - } - }); - }); - } - if (printNativeLinkInfo) { - if (nativeLinkInfo === undefined) throw new Error("internal native-link-info state"); - // Keep stdout pure JSON for tooling; the ordinary artifact path is in - // program.object inside the document. - process.stdout.write(`${JSON.stringify(nativeLinkInfo, null, 2)}\n`); - } else { - process.stdout.write(`${binary}\n`); - } - return 0; -} - -try { - process.exitCode = await main(); -} catch (err) { - if (err instanceof CliExit) process.exitCode = err.code; - else throw err; -} + }), +}); diff --git a/packages/cli/src/output-options.ts b/packages/cli/src/output-options.ts index df5fd24e..f9da6296 100644 --- a/packages/cli/src/output-options.ts +++ b/packages/cli/src/output-options.ts @@ -1,91 +1 @@ -import type { CompileOutputKind } from "@scriptc/compiler"; -import type { CliOutputKind } from "./paths.js"; - -export interface OutputOptionValues { - emit?: string; - emitIr: boolean; - backend?: string; - keepLlvm: boolean; - sanitize: boolean; - optimization?: string; - strip?: boolean; - windowsSubsystem?: string; - ffi?: string; -} - -export type OutputOptionResolution = - | { - ok: true; - outputKind: CompileOutputKind; - cliOutputKind: CliOutputKind; - backend?: "llvm"; - emitIr: boolean; - deprecateEmitIr: boolean; - } - | { ok: false; message: string }; - -const SOURCE_KINDS = new Set(["ir", "llvm"]); -const NATIVE_ARTIFACT_KINDS = new Set(["asm", "obj"]); - -/** Pure compatibility/validation matrix for build/run output selection. */ -export function resolveOutputOptions( - command: "build" | "run", - values: OutputOptionValues, -): OutputOptionResolution { - if (values.backend !== undefined && values.backend !== "llvm") { - return { ok: false, message: `unknown backend "${values.backend}" (supported: llvm)` }; - } - const backend = values.backend as "llvm" | undefined; - const rawEmit = values.emit; - if ( - rawEmit !== undefined && rawEmit !== "ir" && rawEmit !== "llvm" && - rawEmit !== "asm" && rawEmit !== "obj" && rawEmit !== "exe" - ) { - return { - ok: false, - message: `unknown emit kind "${rawEmit}" (supported: ir, llvm, asm, obj, exe)`, - }; - } - const emit = (rawEmit ?? "exe") as CliOutputKind; - if (command === "run" && emit !== "exe") { - return { ok: false, message: `scriptc run requires --emit=exe` }; - } - if (values.emitIr && rawEmit !== undefined && emit !== "ir" && emit !== "exe") { - return { ok: false, message: `--emit-ir cannot be combined with --emit=${emit}; use --emit=ir` }; - } - if (values.emitIr && emit === "ir") { - return { ok: false, message: `--emit-ir and --emit=ir select the same output; use --emit=ir` }; - } - if (values.windowsSubsystem !== undefined && emit !== "exe") { - return { ok: false, message: `--windows-subsystem is only supported with --emit=exe` }; - } - if (values.strip && emit !== "exe") { - return { ok: false, message: `--strip is only supported with --emit=exe` }; - } - if (emit === "ir" && backend !== undefined) { - return { ok: false, message: `--emit=ir cannot be combined with --backend; IR is emitted before backend selection` }; - } - if (SOURCE_KINDS.has(emit)) { - if (!values.keepLlvm) { - return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; - } - if (values.sanitize) { - return { ok: false, message: `--sanitize is only meaningful with --emit=exe` }; - } - if (values.optimization !== undefined) { - return { ok: false, message: `--optimization is only meaningful with --emit=exe` }; - } - } - if (NATIVE_ARTIFACT_KINDS.has(emit) && !values.keepLlvm) { - return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; - } - const outputKind = emit as CompileOutputKind; - return { - ok: true, - outputKind, - cliOutputKind: emit, - ...(emit === "ir" && backend === undefined ? {} : { backend: "llvm" as const }), - emitIr: values.emitIr && emit === "exe", - deprecateEmitIr: values.emitIr, - }; -} +export * from "@scriptc/compiler/cli/output-options"; diff --git a/packages/cli/src/paths.ts b/packages/cli/src/paths.ts index a803e2a3..0995a357 100644 --- a/packages/cli/src/paths.ts +++ b/packages/cli/src/paths.ts @@ -1,103 +1 @@ -import { tmpdir } from "node:os"; -import { basename, dirname, join, resolve } from "node:path"; -import { buildTargetPlatform, sourceTargetPlatform, wasiGuestPath } from "@scriptc/compiler"; - -export type CliOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; - -const POSIX_SUFFIXES: Record = { - ir: ".ir.json", - llvm: ".ll", - asm: ".s", - obj: ".o", - exe: "", -}; - -const WINDOWS_SUFFIXES: Record = { - ...POSIX_SUFFIXES, - asm: ".asm", - obj: ".obj", - exe: ".exe", -}; - -export function defaultOutputName( - stem: string, - kind: CliOutputKind, - platform?: string, -): string { - const selectedPlatform = platform ?? (kind === "exe" ? sourceTargetPlatform() : process.platform); - if (kind === "exe" && selectedPlatform === "wasi") return `${stem}.wasm`; - return `${stem}${(selectedPlatform === "win32" ? WINDOWS_SUFFIXES : POSIX_SUFFIXES)[kind]}`; -} - -export interface OutputPaths { - outDir: string; - outPath: string; -} - -/** One authority for explicit and default primary artifact paths. */ -export function selectOutputPaths( - input: string, - kind: CliOutputKind, - explicitOut?: string, - platform?: string, -): OutputPaths { - const absoluteInput = resolve(input); - const outDir = explicitOut === undefined - ? join(dirname(absoluteInput), ".scriptc") - : dirname(resolve(explicitOut)); - const stem = basename(absoluteInput).replace(/\.(ts|mts|cts|js|mjs|cjs|c|ll)$/, ""); - return { - outDir, - outPath: explicitOut === undefined - ? join(outDir, defaultOutputName(stem, kind, platform)) - : resolve(explicitOut), - }; -} - -/** Default executable filename for the build target. Explicit --out paths - * stay exact; only scriptc's generated default needs the Windows PE suffix. */ -export function defaultExecutableName(stem: string, platform: string = buildTargetPlatform()): string { - return defaultOutputName(stem, "exe", platform); -} - -/** Host paths exposed by `scriptc run` to a WASI Preview 1 module. Guest - * `/tmp` maps to the host's real platform temp directory instead of assuming - * the POSIX spelling exists (notably false on Windows). */ -export function wasiPreopens( - cwd: string = process.cwd(), - hostTmp: string = tmpdir(), -): Record { - return { "/": cwd, "/tmp": hostTmp }; -} - -/** Environment inherited by a WASI module. Host-absolute directory values - * must not claim paths outside the guest namespace: `/` is the module's - * capability root/home/cwd and `/tmp` is its writable temporary directory. */ -export function wasiEnvironment( - env: NodeJS.ProcessEnv = process.env, - cwd: string = process.cwd(), - hostTmp: string = tmpdir(), -): Record { - const guest = Object.fromEntries( - Object.entries(env).filter((entry): entry is [string, string] => entry[1] !== undefined), - ); - - guest["PWD"] = "/"; - guest["HOME"] = "/"; - guest["TMPDIR"] = "/tmp"; - if (guest["USERPROFILE"] !== undefined) guest["USERPROFILE"] = "/"; - if (guest["TMP"] !== undefined) guest["TMP"] = "/tmp"; - if (guest["TEMP"] !== undefined) guest["TEMP"] = "/tmp"; - - // These optional shell/package-manager paths retain their meaning only - // when they fall under a capability the runner actually exposes. - for (const key of ["OLDPWD", "INIT_CWD"] as const) { - const value = guest[key]; - if (value === undefined) continue; - const mapped = wasiGuestPath(value, cwd, hostTmp); - if (mapped === null) delete guest[key]; - else guest[key] = mapped; - } - - return guest; -} +export * from "@scriptc/compiler/cli/paths"; diff --git a/packages/cli/src/usage.ts b/packages/cli/src/usage.ts index 135aaced..dadabbc5 100644 --- a/packages/cli/src/usage.ts +++ b/packages/cli/src/usage.ts @@ -1,85 +1 @@ -export const USAGE = `scriptc — TypeScript/JavaScript to native and WebAssembly executables (experimental) - -Usage: - scriptc build [options] compile to an executable or source artifact - scriptc run [options] compile and run - scriptc coverage how much compiles statically, and why not - scriptc coverage --dynamic what a --dynamic build compiles, and what still blocks it - scriptc coverage --external-types - type-resolve an embedder-provided module for analysis - scriptc build --lib --profile library mode: compile the profile's entry - module to a linkable static archive - (.lib.a) exporting the - profile-declared C symbols; a profile - with a sidecar section also gets the - contract sidecar JSON beside the archive - scriptc cache warm [runtime|tls|dynamic…] prebuild expensive native cache families - for the current compiler/SDK/target - -Options: - -o, --out primary output path (default: .scriptc/) - --emit primary output: ir, llvm, asm, obj, or exe - (default: exe). asm/obj use the matching platform helper - --print print machine-readable metadata instead of the output path - (native-link-info implies --emit=obj and never links) - --backend code generator (llvm) - --optimization - native optimization posture (default: release/-O2). dev - uses -O0, source breakpoints, and cached LLVM object shards; - macOS executable builds also produce an adjacent .dSYM - --strip remove symbol/debug payload from the linked executable - for smaller builds (opt in; --emit=exe only) - --windows-subsystem - Windows executable subsystem (default: console). gui - prevents Windows from opening a console window - --keep-llvm keep generated LLVM IR beside the executable (default) - --no-keep-llvm delete generated LLVM IR after compiling - --emit-ir also write IR beside an executable or library archive; - deprecated for executables: use --emit=ir for primary IR - --sanitize build with ASan + runtime RC audit - --dynamic embed the dynamic engine (adds ~620KB; static stays the default) - --ffi bind signature-only TypeScript declarations to native - C symbols and link the manifest's archives/libraries - --npm-static - compile the named npm packages' shipped JS statically as - program modules (repeatable; "auto" opts in every eligible - direct import: own .d.ts, unminified JS, no build-transform - markers). A package preflight refuses falls back to the - island (--dynamic) with a coverage-report note — opt-in, - experimental - --provenance-sources - EXPERIMENTAL: compile npm dependencies from their - provenance-attested SOURCE (fetched at the attested - commit) as static program modules; packages without a - usable attestation keep the island path (a note, never - a failure) - --external-types - coverage only: map an exact bare module specifier to a - local declaration file. The declaration supplies types - for analysis; the host module remains an explicit - external-boundary blocker (repeatable) - -h, --help show this help - -v, --version print the version -`; - -export const CLI_OPTIONS = { - out: { type: "string", short: "o" }, - emit: { type: "string" }, - print: { type: "string" }, - backend: { type: "string" }, - optimization: { type: "string" }, - strip: { type: "boolean", default: false }, - "windows-subsystem": { type: "string" }, - "keep-llvm": { type: "boolean", default: true }, - "emit-ir": { type: "boolean", default: false }, - sanitize: { type: "boolean", default: false }, - dynamic: { type: "boolean", default: false }, - ffi: { type: "string" }, - "npm-static": { type: "string", multiple: true }, - "provenance-sources": { type: "boolean", default: false }, - "external-types": { type: "string", multiple: true }, - lib: { type: "boolean", default: false }, - profile: { type: "string" }, - help: { type: "boolean", short: "h", default: false }, - version: { type: "boolean", short: "v", default: false }, -} as const; +export * from "@scriptc/compiler/cli/usage"; diff --git a/packages/cli/src/wasi-runner.ts b/packages/cli/src/wasi-runner.ts index 932100c4..3a885260 100644 --- a/packages/cli/src/wasi-runner.ts +++ b/packages/cli/src/wasi-runner.ts @@ -1,39 +1,2 @@ #!/usr/bin/env node -/* The subprocess host for `scriptc run` on wasm32-wasi. Keeping the WASI - * instance outside the CLI process preserves native run's exit isolation: - * process.exit(), traps, and signals cannot take the compiler process down. - */ -import { readFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { WASI } from "node:wasi"; -import { wasiGuestPath } from "@scriptc/compiler"; -import { wasiEnvironment, wasiPreopens } from "./paths.js"; - -type WasiInstance = Parameters[0]; -declare const WebAssembly: { - instantiate( - bytes: Uint8Array, - imports: ReturnType, - ): Promise<{ instance: WasiInstance }>; -}; - -const binary = process.argv[2]; -if (binary === undefined) throw new Error("scriptc WASI runner needs a module path"); -const cwd = process.cwd(); -const hostTmp = tmpdir(); - -const wasi = new WASI({ - version: "preview1", - args: [wasiGuestPath(binary, cwd, hostTmp) ?? binary], - env: wasiEnvironment(process.env, cwd, hostTmp), - // A native scriptc executable inherits access to the caller's filesystem. - // WASI is capability-based, so expose the caller's working tree as `/` - // and the platform's actual temporary directory as guest `/tmp`. - preopens: wasiPreopens(cwd, hostTmp), - returnOnExit: true, -}); -const instantiated = await WebAssembly.instantiate( - await readFile(binary), - wasi.getImportObject(), -); -process.exitCode = wasi.start(instantiated.instance); +import "@scriptc/compiler/cli/wasi-runner"; diff --git a/packages/cli/test/native-install.test.ts b/packages/cli/test/native-install.test.ts new file mode 100644 index 00000000..60982102 --- /dev/null +++ b/packages/cli/test/native-install.test.ts @@ -0,0 +1,131 @@ +import { execFileSync } from "node:child_process"; +import { copyFileSync, mkdtempSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { installNativeCli, nativeCliPackage } from "../scripts/install-native.mjs"; +import { prepareNativeCommand } from "../scripts/prepare-native.mjs"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function fixture() { + const root = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-install-")); + roots.push(root); + const directory = join(root, "installation"); + const packageName = "@scriptc/cli-darwin-arm64"; + const platform = join(directory, "node_modules", packageName); + const bin = join(platform, "dist/bin"); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(directory, "package.json"), JSON.stringify({ name: "scriptc", version: "1.2.3" })); + writeFileSync(join(platform, "package.json"), JSON.stringify({ name: packageName, version: "1.2.3" })); + const binary = join(bin, process.platform === "win32" ? "scriptc.exe" : "scriptc"); + writeFileSync(binary, "native compiler payload"); + const toolchain = { + schema: "scriptc.native-toolchain.v1", compiler_version: "1.2.3", target: "macos-arm64", + ts7: "../lib/typescript/lib/tsc", llvm_package: "../lib/llvm", runtime_pack: "../lib/runtime", + runtime_packs: [{ target: "macos-arm64", path: "../lib/runtime" }], + linker: "clang", linker_args: [], dsymutil: "dsymutil", comptime: "../lib/comptime", + wasi_node_runner: "../lib/wasi/cli/wasi-runner.js", + }; + writeFileSync(binary + ".json", JSON.stringify(toolchain)); + const helpers = [toolchain.ts7, toolchain.comptime, join(toolchain.llvm_package, "bin/scriptc-llvm-codegen")] + .map((path) => resolve(bin, path)); + for (const path of helpers) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, "native helper payload", { mode: 0o644 }); + } + return { root, directory, packageName, platform, binary, toolchain, helpers }; +} + +test("selects only supported native host packages, including Linux libc", () => { + expect(nativeCliPackage("darwin", "arm64", null)).toBe("@scriptc/cli-darwin-arm64"); + expect(nativeCliPackage("win32", "x64", null)).toBe("@scriptc/cli-win32-x64-msvc"); + expect(nativeCliPackage("linux", "x64", "glibc")).toBe("@scriptc/cli-linux-x64-gnu"); + expect(nativeCliPackage("linux", "arm64", "musl")).toBe("@scriptc/cli-linux-arm64-musl"); + expect(() => nativeCliPackage("linux", "x64", null)).toThrow("identify"); + expect(() => nativeCliPackage("win32", "arm64", null)).toThrow("no native command"); +}); + +test("installs a direct executable and relocatable references to platform assets", () => { + const f = fixture(); + const command = installNativeCli(f.directory, f.packageName); + expect(readFileSync(command, "utf8")).toBe("native compiler payload"); + if (process.platform !== "win32") expect(statSync(command).mode & 0o111).toBe(0o111); + if (process.platform !== "win32") { + for (const helper of f.helpers) expect(statSync(helper).mode & 0o111).toBe(0o111); + } + const moved = join(f.root, "relocated"); + renameSync(f.directory, moved); + const installed = join(moved, "bin/scriptc.exe"); + const manifest = JSON.parse(readFileSync(installed + ".json", "utf8")); + const platform = join(moved, "node_modules", f.packageName, "dist/lib"); + expect(resolve(dirname(installed), manifest.ts7)).toBe(join(platform, "typescript/lib/tsc")); + expect(resolve(dirname(installed), manifest.wasi_node_runner)).toBe(join(platform, "wasi/cli/wasi-runner.js")); + expect(resolve(dirname(installed), manifest.runtime_packs[0].path)).toBe(join(platform, "runtime")); + expect(manifest.linker).toBe("clang"); + expect(JSON.stringify(manifest)).not.toContain(f.directory); +}); + +test("missing and mismatched packages fail without replacing an installed command", () => { + const f = fixture(); + const command = installNativeCli(f.directory, f.packageName); + writeFileSync(join(f.platform, "package.json"), JSON.stringify({ name: f.packageName, version: "1.2.4" })); + expect(() => installNativeCli(f.directory, f.packageName)).toThrow("found 1.2.4"); + expect(() => installNativeCli(f.directory, "@scriptc/cli-missing")).toThrow("optional dependencies"); + expect(readFileSync(command, "utf8")).toBe("native compiler payload"); +}); + +test("npm links the installed native executable without an interpreter", () => { + const f = fixture(); + // Keep the package payload small while exercising npm's native shim. + // Production command coverage builds the real compiler in the bootstrap gate. + const windows = process.platform === "win32"; + if (windows) copyFileSync(process.execPath, f.binary); + else { + const source = join(f.root, "payload.c"); + writeFileSync(source, '#include \nint main(void) { puts("native compiler"); return 0; }\n'); + execFileSync("clang", [source, "-o", f.binary]); + } + for (const helper of f.helpers) writeFileSync(helper, "#!/bin/sh\nprintf 'native helper\\n'\n"); + const packageName = "scriptc-native-install-test"; + const scripts = join(f.directory, "scripts"); + mkdirSync(scripts); + copyFileSync(join(import.meta.dirname, "../scripts/install-native.mjs"), join(scripts, "install-native.mjs")); + writeFileSync(join(scripts, "fixture-install.mjs"), + `import { installNativeCli } from './install-native.mjs'; installNativeCli(process.cwd(), ${JSON.stringify(f.packageName)});\n`); + const npm = process.platform === "win32" ? "npm.cmd" : "npm"; + const npmEnv = { ...process.env, NODE_PATH: "", npm_config_cache: join(f.root, "npm-cache"), npm_config_update_notifier: "false" }; + const pack = (directory: string): string => { + const result = execFileSync(npm, ["pack", "--json", "--ignore-scripts"], { + cwd: directory, env: npmEnv, encoding: "utf8", shell: process.platform === "win32", + }); + return join(directory, (JSON.parse(result) as { filename: string }[])[0]!.filename); + }; + const platformTarball = pack(f.platform); + writeFileSync(join(f.directory, "package.json"), JSON.stringify({ + name: packageName, version: "1.2.3", type: "module", + bin: { [packageName]: "bin/scriptc.exe" }, files: ["bin", "scripts"], + scripts: { postinstall: "node scripts/fixture-install.mjs" }, + optionalDependencies: { [f.packageName]: "file:" + platformTarball }, + })); + prepareNativeCommand(f.directory); + const tarball = pack(f.directory); + const installed = join(f.root, "npm-install"); + mkdirSync(installed); + execFileSync(npm, ["install", "--offline", "--no-audit", "--no-fund", tarball], { + cwd: installed, env: npmEnv, shell: process.platform === "win32", stdio: "pipe", + }); + const command = join(installed, "node_modules/.bin", packageName + (process.platform === "win32" ? ".cmd" : "")); + const result = execFileSync(command, windows ? ["--version"] : ["native compiler"], { + env: { ...process.env, PATH: "" }, encoding: "utf8", shell: process.platform === "win32", + }); + expect(result.trim()).toBe(windows ? process.version : "native compiler"); + if (process.platform !== "win32") { + const bin = join(installed, "node_modules", packageName, "bin"); + const manifest = JSON.parse(readFileSync(join(bin, "scriptc.exe.json"), "utf8")); + for (const helper of [manifest.ts7, manifest.comptime, join(manifest.llvm_package, "bin/scriptc-llvm-codegen")]) { + expect(execFileSync(resolve(bin, helper), [], { env: { ...process.env, PATH: "" }, encoding: "utf8" }).trim()).toBe("native helper"); + } + } +}, 60_000); diff --git a/packages/cli/test/wasi-runtime-pack.test.ts b/packages/cli/test/wasi-runtime-pack.test.ts index 1a36bfbf..9faaa32a 100644 --- a/packages/cli/test/wasi-runtime-pack.test.ts +++ b/packages/cli/test/wasi-runtime-pack.test.ts @@ -42,7 +42,7 @@ test.runIf(supported)("WASI helper object plus runtime pack builds and runs with })).resolves.toMatchObject({ stdout: "hello world\n" }); }); -test.runIf(supported)("switching WASI LLVM and native C builds preserves both translation units", async () => { +test.runIf(supported)("switching WASI and native builds retains both executables", async () => { const dir = await mkdtemp(join(tmpdir(), "scriptc-wasi-native-output-")); dirs.push(dir); const entry = join(dir, "hello.ts"); @@ -59,10 +59,12 @@ test.runIf(supported)("switching WASI LLVM and native C builds preserves both tr const llvm = await readFile(join(outDir, "hello.ll")); const wasm = await readFile(join(outDir, "hello.wasm")); await build(["--backend=llvm", "--keep-llvm"], nativeEnv); - const c = await readFile(join(outDir, "hello.c")); - expect(await readFile(join(outDir, "hello.ll"))).toEqual(llvm); + const native = await readFile(join(outDir, process.platform === "win32" ? "hello.exe" : "hello")); + const nativeLlvm = await readFile(join(outDir, "hello.ll")); + expect(nativeLlvm.equals(llvm)).toBe(false); expect(await readFile(join(outDir, "hello.wasm"))).toEqual(wasm); await build([], wasiEnv); - expect(await readFile(join(outDir, "hello.c"))).toEqual(c); + expect(await readFile(join(outDir, "hello.ll"))).toEqual(llvm); + expect(await readFile(join(outDir, process.platform === "win32" ? "hello.exe" : "hello"))).toEqual(native); }); diff --git a/packages/compiler/ambient/scriptc-node-fallback.d.ts b/packages/compiler/ambient/scriptc-node-fallback.d.ts index 728fa4c6..838bf934 100644 --- a/packages/compiler/ambient/scriptc-node-fallback.d.ts +++ b/packages/compiler/ambient/scriptc-node-fallback.d.ts @@ -1080,9 +1080,8 @@ declare module "node:fs" { ): void; /* The bare-encoding spelling — the options record's encoding key alone. */ export function writeFileSync(path: string, data: string, encoding: "utf8" | "utf-8"): void; - export function writeFileSync(path: string, data: Uint8Array): void; - export function appendFileSync(path: string, data: string): void; - export function appendFileSync(path: string, data: Uint8Array): void; + export function writeFileSync(path: string, data: string | Uint8Array): void; + export function appendFileSync(path: string, data: string | Uint8Array): void; export function existsSync(path: string): boolean; export function mkdirSync(path: string): void; export function mkdirSync(path: string, options: { recursive?: boolean; mode?: number }): void; diff --git a/packages/compiler/native/comptime.c b/packages/compiler/native/comptime.c new file mode 100644 index 00000000..8b3ff3ef --- /dev/null +++ b/packages/compiler/native/comptime.c @@ -0,0 +1,189 @@ +/* Isolated JavaScript evaluation for comptime callbacks. No filesystem, + * process, module loader, or console bindings enter the guest context. */ +#ifndef _WIN32 +#define _POSIX_C_SOURCE 200809L +#endif +#include "quickjs.h" +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +static uint64_t now_ms(void) { +#ifdef _WIN32 + return GetTickCount64(); +#else + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0; + return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000; +#endif +} + +typedef struct { uint64_t deadline; int interrupted; } Budget; +static int interrupt(JSRuntime *rt, void *opaque) { + (void)rt; + Budget *budget = opaque; + if (now_ms() >= budget->deadline) budget->interrupted = 1; + return budget->interrupted; +} + +typedef struct { + char *data; + size_t length, capacity; + void *active[512]; + size_t depth; +} Output; + +static int append(Output *out, const char *text) { + size_t size = strlen(text); + if (size > 64 * 1024 * 1024 - out->length) return -1; + size_t required = out->length + size + 1; + if (required > out->capacity) { + size_t capacity = out->capacity ? out->capacity : 1024; + while (capacity < required) capacity *= 2; + char *data = realloc(out->data, capacity); + if (!data) return -1; + out->data = data; + out->capacity = capacity; + } + memcpy(out->data + out->length, text, size + 1); + out->length += size; + return 0; +} + +static int quote(JSContext *ctx, Output *out, JSValueConst value) { + size_t length; + const uint16_t *text = JS_ToCStringLenUTF16(ctx, &length, value); + if (!text) return -1; + int status = append(out, "\""); + for (size_t i = 0; status == 0 && i < length; i++) { + char escaped[7]; + unsigned ch = text[i]; + if (ch >= 32 && ch < 127 && ch != '"' && ch != '\\') { + escaped[0] = (char)ch; escaped[1] = 0; + } else snprintf(escaped, sizeof(escaped), "\\u%04x", ch); + status = append(out, escaped); + } + JS_FreeCStringUTF16(ctx, text); + return status == 0 ? append(out, "\"") : status; +} + +/* Serialize through engine APIs, outside the callback's mutable globals. + * The guest cannot replace JSON.stringify or install a toJSON hook that + * changes the value the compiler validates. */ +static int encode(JSContext *ctx, Output *out, JSValueConst value) { + if (JS_IsException(value)) return -1; + if (JS_IsUndefined(value)) return append(out, "[\"undefined\"]"); + if (JS_IsNull(value)) return append(out, "[\"null\"]"); + if (JS_IsBool(value)) return append(out, JS_ToBool(ctx, value) ? "[\"boolean\",true]" : "[\"boolean\",false]"); + if (JS_IsFunction(ctx, value)) return append(out, "[\"function\"]"); + if (JS_IsSymbol(value)) return append(out, "[\"symbol\"]"); + if (JS_IsString(value) || JS_IsNumber(value) || JS_IsBigInt(value)) { + const char *tag = JS_IsString(value) ? "[\"string\"," : JS_IsNumber(value) ? "[\"number\"," : "[\"bigint\","; + if (append(out, tag) != 0) return -1; + double number = 1; + if (JS_IsNumber(value) && JS_ToFloat64(ctx, &number, value) < 0) return -1; + int status = number == 0 && signbit(number) ? append(out, "\"-0\"") : quote(ctx, out, value); + return status == 0 ? append(out, "]") : status; + } + if (out->depth == 512) { JS_ThrowTypeError(ctx, "compile-time result is nested too deeply"); return -1; } + void *identity = JS_VALUE_GET_PTR(value); + for (size_t i = 0; i < out->depth; i++) { + if (out->active[i] == identity) { JS_ThrowTypeError(ctx, "cyclic compile-time result"); return -1; } + } + out->active[out->depth++] = identity; + int status = 0; + if (JS_IsArray(value)) { + int64_t length = 0; + if (JS_GetLength(ctx, value, &length) < 0 || length < 0 || length > 1000000) status = -1; + if (status == 0) status = append(out, "[\"array\",["); + for (int64_t i = 0; status == 0 && i < length; i++) { + if (i != 0) status = append(out, ","); + JSValue item = JS_GetPropertyUint32(ctx, value, (uint32_t)i); + if (status == 0) status = encode(ctx, out, item); + JS_FreeValue(ctx, item); + } + } else { + JSPropertyEnum *properties = NULL; + uint32_t length = 0; + status = JS_GetOwnPropertyNames(ctx, &properties, &length, value, JS_GPN_STRING_MASK | JS_GPN_ENUM_ONLY); + if (status == 0) status = append(out, "[\"object\",["); + for (uint32_t i = 0; status == 0 && i < length; i++) { + if (i != 0) status = append(out, ","); + if (status == 0) status = append(out, "["); + JSValue key = JS_AtomToString(ctx, properties[i].atom); + if (status == 0) status = quote(ctx, out, key); + JS_FreeValue(ctx, key); + if (status == 0) status = append(out, ","); + JSValue item = JS_GetProperty(ctx, value, properties[i].atom); + if (status == 0) status = encode(ctx, out, item); + JS_FreeValue(ctx, item); + if (status == 0) status = append(out, "]"); + } + JS_FreePropertyEnum(ctx, properties, length); + } + out->depth--; + return status == 0 ? append(out, "]]") : status; +} + +int main(int argc, char **argv) { + if (argc != 3) { fputs("usage: scriptc-comptime \n", stderr); return 2; } + char *end = NULL; + errno = 0; + long timeout = strtol(argv[2], &end, 10); + if (errno || !end || *end || timeout < 1 || timeout > 60000) { + fputs("invalid compile-time budget\n", stderr); return 2; + } + FILE *file = fopen(argv[1], "rb"); + if (!file) { perror("compile-time input"); return 2; } + if (fseek(file, 0, SEEK_END) != 0) { fclose(file); return 2; } + long size = ftell(file); + if (size < 0 || size > 16 * 1024 * 1024 || fseek(file, 0, SEEK_SET) != 0) { fclose(file); return 2; } + char *source = malloc((size_t)size + 1); + if (!source) { fclose(file); return 2; } + if (fread(source, 1, (size_t)size, file) != (size_t)size) { free(source); fclose(file); return 2; } + fclose(file); + source[size] = 0; + JSRuntime *rt = JS_NewRuntime(); + if (!rt) { free(source); return 2; } + JS_SetMemoryLimit(rt, 256 * 1024 * 1024); + JS_SetMaxStackSize(rt, 2 * 1024 * 1024); + Budget budget = { now_ms() + (uint64_t)timeout, 0 }; + JS_SetInterruptHandler(rt, interrupt, &budget); + JSContext *ctx = JS_NewContext(rt); + if (!ctx) { JS_FreeRuntime(rt); free(source); return 2; } + JSValue result = JS_Eval(ctx, source, (size_t)size, "comptime", JS_EVAL_TYPE_GLOBAL); + free(source); + int status = 0; + Output out = {0}; + if (JS_IsException(result) || encode(ctx, &out, result) != 0) { + free(out.data); + memset(&out, 0, sizeof(out)); + JSValue exception = JS_GetException(ctx); + if (budget.interrupted) status = append(&out, "[\"error\",\"ERR_SCRIPT_EXECUTION_TIMEOUT\",\"compile-time evaluation timed out\"]"); + else { + JSValue message = JS_GetPropertyStr(ctx, exception, "message"); + JSValue detail = JS_IsUndefined(message) ? JS_IsNull(exception) + ? JS_NewString(ctx, "compile-time result exceeds the output limit") : JS_DupValue(ctx, exception) : JS_DupValue(ctx, message); + status = append(&out, "[\"error\",\"\","); + if (status == 0) status = quote(ctx, &out, detail); + if (status == 0) status = append(&out, "]"); + JS_FreeValue(ctx, detail); + JS_FreeValue(ctx, message); + } + JS_FreeValue(ctx, exception); + } + if (status == 0 && (fwrite(out.data, 1, out.length, stdout) != out.length || fputc('\n', stdout) == EOF)) status = 2; + free(out.data); + JS_FreeValue(ctx, result); + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + return status == 0 ? 0 : 2; +} diff --git a/packages/compiler/native/host.c b/packages/compiler/native/host.c new file mode 100644 index 00000000..75c2dcba --- /dev/null +++ b/packages/compiler/native/host.c @@ -0,0 +1,46 @@ +#ifndef _WIN32 +#define _POSIX_C_SOURCE 200809L +#endif +#include +#include +#include +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#include +#endif + +/* Accept only directories owned by this user with private POSIX access. + * Windows inherits the user's cache ACL; reject reparse points so an + * installed cache directory cannot redirect compiler payloads elsewhere. */ +uint8_t scriptc_native_private_directory(const uint8_t *bytes, size_t size, uint8_t harden) { + if (size == 0 || memchr(bytes, 0, size) != NULL) return 0; +#ifdef _WIN32 + (void)harden; + if (size > INT_MAX) return 0; + int length = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, (const char *)bytes, (int)size, NULL, 0); + if (length <= 0) return 0; + wchar_t *path = malloc(((size_t)length + 1) * sizeof(wchar_t)); + if (!path) return 0; + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, (const char *)bytes, (int)size, path, length) != length) { free(path); return 0; } + path[length] = 0; + DWORD attrs = GetFileAttributesW(path); + free(path); + return attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY) != 0 && (attrs & FILE_ATTRIBUTE_REPARSE_POINT) == 0; +#else + char *path = malloc(size + 1); + if (!path) return 0; + memcpy(path, bytes, size); + path[size] = 0; + struct stat info; + int ok = lstat(path, &info) == 0 && S_ISDIR(info.st_mode) && info.st_uid == getuid(); + if (ok && harden && (info.st_mode & 077) != 0) ok = chmod(path, 0700) == 0; + if (ok) ok = lstat(path, &info) == 0 && S_ISDIR(info.st_mode) && info.st_uid == getuid() && (info.st_mode & 077) == 0; + free(path); + return ok; +#endif +} diff --git a/packages/compiler/native/host.ffi.json b/packages/compiler/native/host.ffi.json new file mode 100644 index 00000000..f2a73805 --- /dev/null +++ b/packages/compiler/native/host.ffi.json @@ -0,0 +1,6 @@ +{ + "ffi_format": 6, + "functions": [ + { "name": "compilerNativePrivateDirectory", "symbol": "scriptc_native_private_directory", "params": ["string", "bool"], "returns": "bool" } + ] +} diff --git a/packages/compiler/package.json b/packages/compiler/package.json index b77894ff..0c7a0ba0 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -18,7 +18,13 @@ "./scriptc.d.ts": "./ambient/scriptc.d.ts", "./scriptc-overrides.d.ts": "./ambient/scriptc-overrides.d.ts", "./scriptc-node-fallback.d.ts": "./ambient/scriptc-node-fallback.d.ts", - "./surface-manifest.json": "./surface-manifest.json" + "./surface-manifest.json": "./surface-manifest.json", + "./cli/command": "./dist/cli/command.js", + "./cli/host": "./dist/cli/host.js", + "./cli/paths": "./dist/cli/paths.js", + "./cli/output-options": "./dist/cli/output-options.js", + "./cli/usage": "./dist/cli/usage.js", + "./cli/wasi-runner": "./dist/cli/wasi-runner.js" }, "files": [ "dist", diff --git a/packages/compiler/src/backend/build-cache.ts b/packages/compiler/src/backend/build-cache.ts index cb4044f9..f81ca81a 100644 --- a/packages/compiler/src/backend/build-cache.ts +++ b/packages/compiler/src/backend/build-cache.ts @@ -1,7 +1,8 @@ +import { resolveBuildCacheRoot } from "./cache-root.js"; +export { resolveBuildCacheRoot } from "./cache-root.js"; import { createHash } from "node:crypto"; import { chmod, copyFile, mkdir, readdir, readFile, rename, rm, stat, unlink, utimes, writeFile } from "node:fs/promises"; -import { homedir } from "node:os"; -import { basename, dirname, join, resolve } from "node:path"; +import { basename, dirname, join } from "node:path"; export async function fileExists(path: string): Promise { return stat(path).then( @@ -44,28 +45,6 @@ export async function installArtifact( } } -/** Resolve the build cache without touching the filesystem. Exported from this - * internal module so its platform and override behavior can be pinned directly. */ -export function resolveBuildCacheRoot( - env: NodeJS.ProcessEnv = process.env, - platform: NodeJS.Platform = process.platform, - userHome: string = homedir(), -): string | null { - if (env["SCRIPTC_NO_CACHE"] === "1") return null; - const configured = env["SCRIPTC_CACHE_DIR"]; - if (configured !== undefined) return configured === "" ? null : resolve(configured); - - const xdg = env["XDG_CACHE_HOME"]; - if (xdg !== undefined && xdg !== "") return resolve(xdg, "scriptc", "build"); - if (platform === "win32") { - const local = env["LOCALAPPDATA"]; - if (local !== undefined && local !== "") return resolve(local, "scriptc", "cache", "build"); - } - return platform === "darwin" - ? resolve(userHome, "Library", "Caches", "scriptc", "build") - : resolve(userHome, ".cache", "scriptc", "build"); -} - /** Shared persistent-cache root for compiler-level tiers. The early library * cache deliberately follows the native cache's activation and hard-disable * contract, while native compilation retains ownership of toolchain safety. */ diff --git a/packages/compiler/src/backend/cache-root.ts b/packages/compiler/src/backend/cache-root.ts new file mode 100644 index 00000000..3d668713 --- /dev/null +++ b/packages/compiler/src/backend/cache-root.ts @@ -0,0 +1,24 @@ +import { homedir } from "node:os"; +import { resolve } from "node:path"; + +/** Resolve the build cache without touching the filesystem. Exported from this + * internal module so its platform and override behavior can be pinned directly. */ +export function resolveBuildCacheRoot( + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, + userHome: string = homedir(), +): string | null { + if (env["SCRIPTC_NO_CACHE"] === "1") return null; + const configured = env["SCRIPTC_CACHE_DIR"]; + if (configured !== undefined) return configured === "" ? null : resolve(configured); + + const xdg = env["XDG_CACHE_HOME"]; + if (xdg !== undefined && xdg !== "") return resolve(xdg, "scriptc", "build"); + if (platform === "win32") { + const local = env["LOCALAPPDATA"]; + if (local !== undefined && local !== "") return resolve(local, "scriptc", "cache", "build"); + } + return platform === "darwin" + ? resolve(userHome, "Library", "Caches", "scriptc", "build") + : resolve(userHome, ".cache", "scriptc", "build"); +} diff --git a/packages/compiler/src/backend/library-pack.ts b/packages/compiler/src/backend/library-pack.ts index 930c51cf..2b711ee5 100644 --- a/packages/compiler/src/backend/library-pack.ts +++ b/packages/compiler/src/backend/library-pack.ts @@ -5,22 +5,23 @@ import { basename, dirname, join } from "node:path"; import { promisify } from "node:util"; import { installBytes } from "../library/cache-primitives.js"; import { emitNativeArtifact } from "./native-codegen.js"; +import { linkNativeExecutable } from "./linker.js"; import type { NativeLinkFeatures } from "./native-link-info.js"; import { isZigDriver, localizeLibraryObjects, type CcDriver, type LibArchiveOptions } from "./native-toolchain.js"; import { loadRuntimePack, RuntimePackError, stageRuntimePackArtifacts } from "./runtime-pack.js"; -import type { NativeTargetSpec } from "./targets.js"; +import { executableOptimizationLinkerArgs, type NativeTargetSpec } from "./targets.js"; const run = promisify(execFile); /** Assemble a library from LLVM program objects and verified runtime objects. * The runtime's library and per-thread modes are compiled when packaging. */ -export async function compilePackedLibrary(options: LibArchiveOptions, target: NativeTargetSpec): Promise { +export async function compilePackedLibrary(options: LibArchiveOptions, target: NativeTargetSpec, wasmExports?: readonly string[]): Promise { const optimization = options.optimization ?? "release"; const features: NativeLinkFeatures = { dynamic: false, regex: options.regex ?? false, copying: options.copying ?? false, textDecoderLegacy: options.textDecoderLegacy ?? false, fileHandle: false, fetch: false, netIsland: false, zlib: options.zlib ?? false, assert: options.assert ?? false, - inspect: options.inspect ?? false, dynInvoke: false, dc: false, dynAsync: false, + inspect: options.inspect ?? false, dynInvoke: options.dynInvoke ?? false, dc: false, dynAsync: false, events: false, emitter: options.emitter ?? false, symbol: options.symbol ?? false, bigint: options.bigint ?? false, searchParams: options.searchParams ?? false, qs: false, parseArgs: false, stream: false, net: false, http: false, http2: false, @@ -30,6 +31,28 @@ export async function compilePackedLibrary(options: LibArchiveOptions, target: N target, features, optimization, mode: options.threadInstances ? "library-thread" : "library", }); + if (target.platform === "wasi") { + if (wasmExports === undefined) throw new Error("Wasm library linking requires explicit exports"); + const buildDir = await mkdtemp(join(tmpdir(), "scriptc-wasm-library-")); + try { + const program = join(buildDir, "program.o"); + await emitNativeArtifact({ + outputPath: program, llvm: await readFile(options.cPath, "utf8"), sourcePath: options.cPath, + outputKind: "obj", optimization: optimization === "dev" ? "0" : "2", target, + }); + await mkdir(dirname(options.outPath), { recursive: true }); + await linkNativeExecutable({ + target, outputPath: options.outPath, + inputs: [program, ...selection.runtimeObjects, ...selection.archives], + systemLibraries: selection.systemLibraries, + driverFlags: ["-target", target.linkerTargetTriple, "-mexec-model=reactor", ...executableOptimizationLinkerArgs(target.platform, optimization), ...wasmExports.map((name) => `-Wl,--export=${name}`)], + dependencyPaths: selection.dependencyPaths, programObjectDependencies: [], runtimePack: selection, + }); + } finally { + await rm(buildDir, { recursive: true, force: true }); + } + return; + } const stage = await stageRuntimePackArtifacts(selection); try { const buildDir = await mkdtemp(join(tmpdir(), "scriptc-library-pack-")); diff --git a/packages/compiler/src/backend/link-trace.ts b/packages/compiler/src/backend/link-trace.ts new file mode 100644 index 00000000..fc576f5c --- /dev/null +++ b/packages/compiler/src/backend/link-trace.ts @@ -0,0 +1,22 @@ +export function linkTraceCandidate(line: string): string[] { + const trimmed = line.trim().replace(/^(?:LOAD|load)\s+/, ""); + if (trimmed === "") return []; + const unquoted = + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ? trimmed.slice(1, -1) + : trimmed; + const candidates = [unquoted]; + const member = unquoted.lastIndexOf("("); + if (member > 0 && unquoted.endsWith(")")) candidates.push(unquoted.slice(0, member)); + return candidates; +} + +export function driverTraceCandidates(line: string): string[] { + const candidates: string[] = []; + for (const match of line.matchAll(/"((?:\\.|[^"\\])*)"|'([^']*)'|(\S+)/g)) { + const token = (match[1] ?? match[2] ?? match[3] ?? "").replace(/\\(["\\])/g, "$1"); + if (token !== "") candidates.push(token); + } + return candidates; +} diff --git a/packages/compiler/src/backend/llvm/dyn.ts b/packages/compiler/src/backend/llvm/dyn.ts index 7c82069c..6fc828aa 100644 --- a/packages/compiler/src/backend/llvm/dyn.ts +++ b/packages/compiler/src/backend/llvm/dyn.ts @@ -72,6 +72,7 @@ export interface DynHost extends WalkerHost { unitInstanceRef(unionId: string, tag: number): string; liveDynRefAdapter(t: IrType): { snapshot: string; commit: string }; isErrorClass(className: string): boolean; + classSubtypes(className: string): readonly string[]; } const FN_ATTRS = "#0"; @@ -326,6 +327,29 @@ export class LlDyn { /* ── dynMatchHelper (walkers.ts, ported) ──────────────────────── */ + /** Native subclass capsules also satisfy a checked base-class slot. + * Class layouts share their base prefix, so unboxing retains the same + * pointer and virtual dispatch still uses the object's original vtable. */ + private typedRefMatches(B: BlockBuilder, t: IrType): string { + this.host.declare(`declare zeroext i1 @scr_dyn_typed_ref_is(ptr, ptr, ${this.S})`); + const types: IrType[] = isDynTypedRefType(t) + ? this.host.classSubtypes(t.className).map((className) => ({ kind: "object", className })) + : [t]; + let matched = "false"; + for (const type of types) { + const key = typeKey(type); + const next = B.tmp(); + B.line(`${next} = call zeroext i1 @scr_dyn_typed_ref_is(ptr %d, ptr ${this.host.cstr(key)}, ${this.S} ${Buffer.byteLength(key, "utf8")})`); + if (matched === "false") matched = next; + else { + const either = B.tmp(); + B.line(`${either} = or i1 ${matched}, ${next}`); + matched = either; + } + } + return matched; + } + /** `sc_dm_(ptr d) -> i1` — does this dyn fit T? Never throws. */ dynMatchHelper(t: IrType): string { const key = typeKey(t); @@ -335,11 +359,7 @@ export class LlDyn { this.dynMatchers.set(key, name); const B = new BlockBuilder(); if (isRefCounted(t) && t.kind !== "dyn") { - this.host.declare(`declare zeroext i1 @scr_dyn_typed_ref_is(ptr, ptr, ${this.S})`); - const matched = B.tmp(); - B.line( - `${matched} = call zeroext i1 @scr_dyn_typed_ref_is(ptr %d, ptr ${this.host.cstr(key)}, ${this.S} ${Buffer.byteLength(key, "utf8")})`, - ); + const matched = this.typedRefMatches(B, t); const lRef = B.newLabel("dm.tr"); const lNext = B.newLabel("dm.nt"); B.condBr(matched, lRef, lNext); @@ -659,12 +679,8 @@ export class LlDyn { return name; } if (isRefCounted(t) && t.kind !== "dyn") { - host.declare(`declare zeroext i1 @scr_dyn_typed_ref_is(ptr, ptr, ${host.sizeType})`); host.declare(`declare ptr @scr_dyn_typed_ref_unbox(ptr)`); - const matched = B.tmp(); - B.line( - `${matched} = call zeroext i1 @scr_dyn_typed_ref_is(ptr %d, ptr ${host.cstr(key)}, ${host.sizeType} ${Buffer.byteLength(key, "utf8")})`, - ); + const matched = this.typedRefMatches(B, t); const lRef = B.newLabel("dc.tr"); const lNext = B.newLabel("dc.nt"); B.condBr(matched, lRef, lNext); diff --git a/packages/compiler/src/backend/llvm/emitter.test.ts b/packages/compiler/src/backend/llvm/emitter.test.ts index 55d161ca..0a288a4b 100644 --- a/packages/compiler/src/backend/llvm/emitter.test.ts +++ b/packages/compiler/src/backend/llvm/emitter.test.ts @@ -103,6 +103,24 @@ test("tag tests read an owned local without adding a temporary owner", () => { expect(llvm).toContain("icmp eq i32"); }); +test("class brand probes borrow a local only across an inert literal key", () => { + const dyn: IrType = { kind: "dyn" }; + const key: IrExpr = { kind: "strLit", value: "object:Example", type: STRING, loc: receiverLoc }; + const probe: IrExpr = { kind: "libCall", fn: "dyn.typedRefIs", args: [ref(dyn), key], type: BOOL, loc: receiverLoc }; + const borrowed = work(probe, dyn); + expect(borrowed).not.toContain("call ptr @scr_dyn_retain_v("); + expect(borrowed).toContain("icmp ne ptr"); + expect(borrowed).toContain("icmp eq i32"); + expect(borrowed).toContain("@scr_dyn_typed_ref_is_key"); + const boxed = work(probe, dyn, true); + expect(boxed).toContain("call ptr @scr_box_get_ref"); + expect(boxed).toContain("call void @scr_dyn_release"); + const computed = work({ ...probe, args: [ref(dyn), { + kind: "toString", operand: ref(dyn), type: STRING, loc: receiverLoc, + }] }, dyn); + expect(computed).toContain("call ptr @scr_dyn_retain_v("); +}); + test("capture boxes keep their ordinary owned-read contract", () => { const llvm = work(text(child(narrow(ref(union)))), union, true); expect(llvm).toContain("call ptr @scr_box_get_ref"); diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 33a74920..5c573512 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -360,9 +360,9 @@ export class LlEmitter { this.wasi = options.wasi === true; this.emitLibraryIdentity = options.emitLibraryIdentity !== false; this.runtimeAbiMarker = options.runtimeAbiMarker === true; - // ScrCycHdr is { ptr trace; ptr free; i32 color; i16 buffered; - // i16 gen; size_t buf_index }. The object follows it, so color is 12 - // bytes behind a wasm32 object and 16 bytes behind a 64-bit object. + // ScrCycHdr.color stays 12 bytes behind a wasm32 object and 16 bytes + // behind a 64-bit object. The wasm32 header pads before color so its + // payload remains double-aligned without changing this ABI offset. this.cycleColorOffset = options.pointerBits === 32 ? 12 : 16; this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []); this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); @@ -400,6 +400,11 @@ export class LlEmitter { unitInstanceRef: (unionId, tag) => this.unitInstanceRef(unionId, tag), liveDynRefAdapter: (type) => this.liveDynRefAdapter(type), isErrorClass: (name) => this.classMeta.get(name)?.root.def.name === "%Error", + classSubtypes: (name) => { + const target = this.classMetaOf(name); + return [...this.classMeta.values()].filter((meta) => + target.pre <= meta.pre && meta.pre <= target.post).map((meta) => meta.def.name); + }, }; this.walkers = new LlWalkers(this.shapeHost); this.dyn = new LlDyn(this.shapeHost); @@ -578,7 +583,7 @@ export class LlEmitter { `}`, ``, ); - defs.push(...dispatchBody); + for (const line of dispatchBody) defs.push(line); this.declare(`declare ptr @scr_ffi_call_new(ptr, ptr, ptr, ${this.sizeType})`); this.declare(`declare void @scr_ffi_post(ptr)`); @@ -1029,7 +1034,7 @@ export class LlEmitter { // finish_top_level initially notes 13. Replace that hint before exit // listeners run when a higher-priority verdict was already selected. lines.push(` call void @scr_exit_code_note(i32 ${exitStatus})`); - lines.push(...exitListenerLines("xp")); + for (const line of exitListenerLines("xp")) lines.push(line); if (tracksIslandExit) { lines.push( ` %tla_exit_version_after = call ${this.sizeType} @scr_island_exit_code_version()`, @@ -1045,13 +1050,17 @@ export class LlEmitter { ` %tla_final_exit = phi i32 [ %tla_listener_exit, %tla_exit_updated ], [ ${exitStatus}, %tla_exit_unchanged ]`, ); } - lines.push(...topPendingReleases); + for (const line of topPendingReleases) lines.push(line); lines.push(` ret i32 ${tracksIslandExit ? "%tla_final_exit" : exitStatus}`); return lines; }; // LIBRARY mode: the runtime entry points the generated library // symbols delegate to — declared before the extern block flushes. if (this.mod.lib !== undefined) { + if (this.wasi) { + this.declare(`declare ptr @malloc(${this.sizeType})`); + this.declare(`declare void @free(ptr)`); + } this.declare(`declare void @scr_library_entry(i1 zeroext, ptr)`); this.declare(`declare void @scr_library_reset()`); this.declare(`declare void @scr_library_check_exc()`); @@ -1145,8 +1154,8 @@ export class LlEmitter { `%ScrIslandModule = type { ptr, ptr, ${this.sizeType}, ${this.sizeType}, i32, ptr, ${this.sizeType}, ${this.sizeType} }`, `%ScrIslandEdge = type { ptr, ptr, ptr, i32 }`, ]; - out.push(...shapes.typeDefs); - out.push(...classShapes.typeDefs); + for (const line of shapes.typeDefs) out.push(line); + for (const line of classShapes.typeDefs) out.push(line); // Thread-instanced library state (abi.instance_per_thread): the // program TU's mutable globals — module globals, run-once guards, the // lazily-compiled regex literal caches — and the runtime globals its @@ -1291,7 +1300,7 @@ export class LlEmitter { } out.push(``); } - out.push(...ffiCallbacks.globals); + for (const line of ffiCallbacks.globals) out.push(line); if (ffiCallbacks.globals.length > 0) out.push(``); for (const g of globals) { const ty = this.llType(g.type); @@ -1300,16 +1309,16 @@ export class LlEmitter { out.push(`@${mangleGlobal(g.id)} = internal ${tl}global ${ty} ${zero}${debug ? `, !dbg ${debug}` : ""} ; ${g.name}`); } if (globals.length > 0) out.push(``); - out.push(...helpers); - out.push(...ffiCallbacks.defs); - out.push(...shapes.defs); - out.push(...classShapes.defs); - out.push(...classObjDefs); - out.push(...this.walkers.defs); - out.push(...this.dyn.defs); - out.push(...wrappers); - out.push(...asyncDefs); - out.push(...this.resolveThunkDefs); + for (const line of helpers) out.push(line); + for (const line of ffiCallbacks.defs) out.push(line); + for (const line of shapes.defs) out.push(line); + for (const line of classShapes.defs) out.push(line); + for (const line of classObjDefs) out.push(line); + for (const line of this.walkers.defs) out.push(line); + for (const line of this.dyn.defs) out.push(line); + for (const line of wrappers) out.push(line); + for (const line of asyncDefs) out.push(line); + for (const line of this.resolveThunkDefs) out.push(line); out.push(fnDefs.join("\n\n"), ``); // main(): scr_init, the program-dependent error-vt interval stamps, @@ -1365,7 +1374,7 @@ export class LlEmitter { if (this.mod.lib !== undefined) { // LIBRARY mode: no @main — the profile-declared external // symbols specified by the library IR instead. - out.push(...this.emitLibDefs(globals, globalReleaseLines, stamps)); + for (const line of this.emitLibDefs(globals, globalReleaseLines, stamps)) out.push(line); out.push(`attributes #0 = { sanitize_address }`); if (this.wasi) out.push(`attributes #1 = { sanitize_address presplitcoroutine }`); if (hasNoInlineRecordClone) out.push(`attributes #2 = { noinline sanitize_address }`); @@ -1542,6 +1551,23 @@ export class LlEmitter { if (lib.collectSymbol !== null) emitSymConst(lib.collectSymbol); for (const e of lib.exports) emitSymConst(e.symbol); out.push(``); + if (this.wasi) { + for (const symbol of ["scriptc_alloc", "scriptc_free"]) emitSymConst(symbol); + out.push( + `define ptr @scriptc_alloc(i32 %size) ${FN_ATTRS} {`, + `entry:`, + ` call void @scr_library_entry(i1 zeroext false, ptr ${symConst("scriptc_alloc")})`, + ` %p = call ptr @malloc(i32 %size)`, + ` ret ptr %p`, + `}`, ``, + `define void @scriptc_free(ptr %p) ${FN_ATTRS} {`, + `entry:`, + ` call void @scr_library_entry(i1 zeroext false, ptr ${symConst("scriptc_free")})`, + ` call void @free(ptr %p)`, + ` ret void`, + `}`, ``, + ); + } // The runtime detected-trap overlay table (scr_runtime.h declares it, // the library trap funnel consults it): flat code/teaching/remediation // triples, one per runtime trap code (SC4013–SC4019) the profile @@ -1642,7 +1668,7 @@ export class LlEmitter { // from the poisoned guard and every runtime touch (ratified), so a // host can read them before init and after a trap. The u64 rides // i64 two's-complement (LLVM integer constants are signed). - out.push(...emitLibraryIdentityLines(lib.identity, FN_ATTRS)); + for (const line of emitLibraryIdentityLines(lib.identity, FN_ATTRS)) out.push(line); } if (lib.resultResetSymbol !== null) { out.push( @@ -1987,7 +2013,7 @@ export class LlEmitter { if (fn.async !== true || fn.generator !== undefined) continue; const { definitions, ret, tr, spawnParams, argPackLines } = this.emitArgPackAndTrampolinePrologue(fn); - out.push(...definitions); + for (const line of definitions) out.push(line); this.declare(`declare ptr @scr_fiber_promise(ptr)`); this.declare(`declare ptr @scr_async_spawn(ptr, ptr)`); this.needOom(); @@ -2039,7 +2065,7 @@ export class LlEmitter { } tr.push(` ret void`, `}`, ``); } - out.push(...tr); + for (const line of tr) out.push(line); // Spawn wrapper: pack the args (+1 moves in), spawn the fiber. const cache = fn.asyncCacheGlobal !== undefined ? mangleGlobal(fn.asyncCacheGlobal) : null; @@ -2109,9 +2135,9 @@ export class LlEmitter { `}`, ``, ); - out.push(...sp); + for (const line of sp) out.push(line); } - out.push(...this.emitGenScaffolding()); + for (const line of this.emitGenScaffolding()) out.push(line); return out; } @@ -2135,7 +2161,7 @@ export class LlEmitter { spawnParams, argPackLines, } = this.emitArgPackAndTrampolinePrologue(fn); - out.push(...definitions); + for (const line of definitions) out.push(line); this.declare(`declare zeroext i1 @scr_exc_genret_pending()`); this.declare(`declare void @scr_exc_clear()`); this.declare(`declare ptr @scr_gen_of_fiber(ptr)`); @@ -2192,7 +2218,7 @@ export class LlEmitter { } tr.push(` br label %done`, `done:`, ` ret void`, `}`, ``); } - out.push(...tr); + for (const line of tr) out.push(line); let settleAsync: string | null = null; if (fn.async) { @@ -2245,7 +2271,7 @@ export class LlEmitter { } }); dr.push(` call void @free(ptr %ap)`, ` ret void`, `}`, ``); - out.push(...dr); + for (const line of dr) out.push(line); // Spawn wrapper: pack the args (+1 moves in), allocate the // SUSPENDED fiber — nothing runs until the first .next(). @@ -2265,7 +2291,7 @@ export class LlEmitter { `}`, ``, ); - out.push(...sp); + for (const line of sp) out.push(line); } return out; } @@ -3789,10 +3815,11 @@ export class LlEmitter { B.line(`store ${this.sizeType} 0, ptr ${integerSlot}`); this.integerLoopBindings.set(integerLoop.localId, integerSlot); } else if (s.init) { - // A multi-declarator head's locals belong to the loop scope, - // not a temporary block that would release them before condition. - if (s.init.kind === "block") for (const init of s.init.body) this.emitStmt(init); - else this.emitStmt(s.init); + // A multi-declarator head shares the loop's scope. Emitting its + // IR block as an ordinary block would release captured/ref locals + // before the first condition. + const initializers = s.init.kind === "block" ? s.init.body : [s.init]; + for (const initializer of initializers) this.emitStmt(initializer); } const lc = B.newLabel("loop.c"); const lb = B.newLabel("loop.b"); @@ -3800,16 +3827,15 @@ export class LlEmitter { // JS `for (let i ...)`: each iteration gets a FRESH binding holding // a copy of the previous one (closures made in iteration k keep // seeing iteration k's value) — only observable, and only emitted, - // when the init variable is captured (boxed). The freshening (and - // the update) live in the continue-target block. + // when a let variable is captured (boxed). Freshen before the first + // condition and again in the continue-target block before updating. const initializers = s.init?.kind === "block" ? s.init.body : s.init ? [s.init] : []; - const captured = initializers.flatMap((init) => { + const capturedLets = initializers.flatMap((init) => { const local = init.kind === "varDecl" ? this.currentLocals.get(init.localId) : undefined; - return local?.boxed === true ? [local] : []; + return local?.boxed && local.mutable ? [local] : []; }); - const freshens = captured.length > 0; - const freshen = (): void => { - for (const local of captured) { + const freshenBindings = (): void => { + for (const local of capturedLets) { const slot = `%${mangleLocal(local.id)}`; const fresh = B.tmp(); const old = B.tmp(); @@ -3822,8 +3848,8 @@ export class LlEmitter { B.line(`store ptr ${fresh}, ptr ${slot}`); } }; - const lu = s.update || freshens ? B.newLabel("loop.u") : lc; - freshen(); + const lu = s.update || capturedLets.length > 0 ? B.newLabel("loop.u") : lc; + freshenBindings(); B.br(lc); B.startBlock(lc); if (integerLoop && integerSlot) { @@ -3854,7 +3880,7 @@ export class LlEmitter { B.br(lu); if (lu !== lc) { B.startBlock(lu); - freshen(); + freshenBindings(); if (integerLoop && integerSlot) { const old = B.tmp(); const next = B.tmp(); @@ -4696,13 +4722,12 @@ export class LlEmitter { throw new InternalCompilerError(`llvm emitter bug: live dyn ref of ${key}`); } const prefix = `sc_ldr_${this.liveDynRefAdapters.size}`; - const adapter = streamTypedRefMaterializeAdapter( + return streamTypedRefMaterializeAdapter( this, t, { prefix, adapters: this.liveDynRefAdapters }, `${prefix}_materialize`, ); - return adapter; } liveDynUnionRefAdapter( diff --git a/packages/compiler/src/backend/llvm/expr-async.ts b/packages/compiler/src/backend/llvm/expr-async.ts index 9217d2c0..27d47c92 100644 --- a/packages/compiler/src/backend/llvm/expr-async.ts +++ b/packages/compiler/src/backend/llvm/expr-async.ts @@ -235,7 +235,7 @@ export function emitSerializationExpr(host: LlvmEmitterContext, e: ExprOf<"jsonS // clamp/truncate rules); the interned re-indenter rewrites the // compact text with Node's gap algorithm. Compact temp stays // frame-owned; the pretty string is a fresh +1. - const indent = (e as { indent?: string }).indent; + const indent = e.indent; if (indent === undefined || indent === "") return compact; const rewriter = host.walkers.jsonIndentHelper(); const t2 = B.tmp(); diff --git a/packages/compiler/src/backend/llvm/expr-calls.ts b/packages/compiler/src/backend/llvm/expr-calls.ts index c723470f..286b6e32 100644 --- a/packages/compiler/src/backend/llvm/expr-calls.ts +++ b/packages/compiler/src/backend/llvm/expr-calls.ts @@ -39,7 +39,8 @@ export function emitCallExpr(host: LlvmEmitterContext, e: ExprOf<"call" | "ffiCa } case "ffiCall": { // LIBRARY mode: every ffiCall is a profile-declared host-callback - // channel (the library lane loads no native-FFI manifest). Fetch + // channel (the library lane loads no native-FFI manifest). Wasm + // calls named imports; native libraries fetch // the slot's registered pointer — scr_library_cb_require delivers // the channel's trap constant through the funnel (SC4025) when the // host never registered — then brackets the typed indirect call, @@ -50,7 +51,7 @@ export function emitCallExpr(host: LlvmEmitterContext, e: ExprOf<"call" | "ffiCa const libCb = host.mod.lib?.callbacks?.find((c) => c.name === e.import); if (libCb !== undefined) { const cbArgs = e.args.map((arg) => host.emitExpr(arg)); - const natTypes: string[] = ["ptr"]; + const natTypes: string[] = []; const natArgs: string[] = []; libCb.params.forEach((cls, i) => { const arg = cbArgs[i]!; @@ -99,10 +100,10 @@ export function emitCallExpr(host: LlvmEmitterContext, e: ExprOf<"call" | "ffiCa const len = B.tmp(); const data = B.tmp(); B.line(`${lenPtr} = getelementptr inbounds %ScrStr, ptr ${arg.name}, i64 0, i32 1`); - B.line(`${len} = load i64, ptr ${lenPtr}`); - B.line(`${data} = getelementptr inbounds i8, ptr ${arg.name}, i64 24`); - natTypes.push("ptr", "i64"); - natArgs.push(`ptr ${data}`, `i64 ${len}`); + B.line(`${len} = load ${host.sizeType}, ptr ${lenPtr}`); + B.line(`${data} = getelementptr inbounds i8, ptr ${arg.name}, i64 ${host.abiOffset(24, 12)}`); + natTypes.push("ptr", host.sizeType); + natArgs.push(`ptr ${data}`, `${host.sizeType} ${len}`); break; } case "bytes": { @@ -110,26 +111,33 @@ export function emitCallExpr(host: LlvmEmitterContext, e: ExprOf<"call" | "ffiCa const len = B.tmp(); const dataPtr = B.tmp(); const data = B.tmp(); - B.line(`${lenPtr} = getelementptr inbounds i8, ptr ${arg.name}, i64 8`); - B.line(`${len} = load i64, ptr ${lenPtr}`); - B.line(`${dataPtr} = getelementptr inbounds i8, ptr ${arg.name}, i64 24`); + B.line(`${lenPtr} = getelementptr inbounds i8, ptr ${arg.name}, i64 ${host.abiOffset(8, 4)}`); + B.line(`${len} = load ${host.sizeType}, ptr ${lenPtr}`); + B.line(`${dataPtr} = getelementptr inbounds i8, ptr ${arg.name}, i64 ${host.abiOffset(24, 12)}`); B.line(`${data} = load ptr, ptr ${dataPtr}`); - natTypes.push("ptr", "i64"); - natArgs.push(`ptr ${data}`, `i64 ${len}`); + natTypes.push("ptr", host.sizeType); + natArgs.push(`ptr ${data}`, `${host.sizeType} ${len}`); break; } } }); - host.declare(`declare ptr @scr_library_cb_require(${host.sizeType}, ptr)`); - host.declare(`declare ptr @scr_library_cb_ctx(${host.sizeType})`); host.declare(`declare void @scr_library_callback_begin()`); host.declare(`declare void @scr_library_callback_end()`); - const fn = B.tmp(); - B.line(`${fn} = call ptr @scr_library_cb_require(${host.sizeType} ${libCb.slot}, ptr @sc_lib_cb_trap_${libCb.slot})`); - const ctx = B.tmp(); - B.line(`${ctx} = call ptr @scr_library_cb_ctx(${host.sizeType} ${libCb.slot})`); const retTy = ffiNativeTypeLl(libCb.returns); - const call = `call ${retTy} ${fn}(${[`ptr ${ctx}`, ...natArgs].join(", ")})`; + let fn: string; + if (host.wasi) { + fn = `@sc_wasm_import_${libCb.slot}`; + host.declare(`declare ${retTy} ${fn}(${natTypes.join(", ")}) "wasm-import-module"="scriptc" "wasm-import-name"="${libCb.name}"`); + } else { + host.declare(`declare ptr @scr_library_cb_require(${host.sizeType}, ptr)`); + host.declare(`declare ptr @scr_library_cb_ctx(${host.sizeType})`); + fn = B.tmp(); + B.line(`${fn} = call ptr @scr_library_cb_require(${host.sizeType} ${libCb.slot}, ptr @sc_lib_cb_trap_${libCb.slot})`); + const ctx = B.tmp(); + B.line(`${ctx} = call ptr @scr_library_cb_ctx(${host.sizeType} ${libCb.slot})`); + natArgs.unshift(`ptr ${ctx}`); + } + const call = `call ${retTy} ${fn}(${natArgs.join(", ")})`; if (libCb.returns === "void") { B.line(`call void @scr_library_callback_begin()`); B.line(call); diff --git a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts index 1ba2dbc1..bfa63f28 100644 --- a/packages/compiler/src/backend/llvm/expr-stream-bridges.ts +++ b/packages/compiler/src/backend/llvm/expr-stream-bridges.ts @@ -188,14 +188,14 @@ export function streamTypedRefCommitAdapter(host: LlvmEmitterContext, } if (meta.def.fields.some((field) => field.name === DYN_CLASS_PROPERTIES)) { host.declare(`declare ptr @scr_dyn_new_obj()`); - host.declare(`declare ptr @scr_dyn_copy_data_properties(ptr, ptr)`); + host.declare(`declare ptr @scr_dyn_copy_property_descriptors(ptr, ptr)`); host.declare(`declare void @scr_dyn_release_v(ptr)`); host.declare(`declare ptr @scr_str_new(ptr, ${host.sizeType})`); host.declare(`declare void @scr_str_release(ptr)`); host.declare(`declare void @scr_dyn_key_delete(ptr, ptr, i1 zeroext)`); lines.push( ` %bag = call ptr @scr_dyn_new_obj()`, - ` %bag_copy = call ptr @scr_dyn_copy_data_properties(ptr %bag, ptr %d)`, + ` %bag_copy = call ptr @scr_dyn_copy_property_descriptors(ptr %bag, ptr %d)`, ` call void @scr_dyn_release_v(ptr %bag_copy)`, ` %bag_pending = call zeroext i1 @scr_exc_pending()`, ` br i1 %bag_pending, label %bag_fail, label %bag_keys`, @@ -559,11 +559,9 @@ export function streamTypedRefMaterializeAdapter(host: LlvmEmitterContext, return adapter; } } - // Reserve both names before generating field conversions: recursive - // class/record conversions can request this adapter while it is in flight. - if (isDynTypedRefType(t) || t.kind === "bytes" || t.kind === "array" || t.kind === "record") { - adapter.commit = `@${snapshot}_commit`; - } + // Publish both symbols before their bodies: class-array and callback + // converters can request this same adapter while emitting the commit. + adapter.commit = `@${snapshot}_commit`; adapter.commit = host.streamTypedRefCommitAdapter(t, snapshot); const B = new BlockBuilder(); @@ -602,13 +600,13 @@ export function streamTypedRefMaterializeAdapter(host: LlvmEmitterContext, B.line(`call void @scr_dyn_obj_set(ptr ${out}, ptr ${host.cstr(field.name)}, ${host.sizeType} ${Buffer.byteLength(field.name, "utf8")}, ptr ${boxed})`); } if (meta.def.fields.some((field) => field.name === DYN_CLASS_PROPERTIES)) { - host.declare(`declare ptr @scr_dyn_copy_data_properties(ptr, ptr)`); + host.declare(`declare ptr @scr_dyn_copy_property_descriptors(ptr, ptr)`); host.declare(`declare void @scr_dyn_release_v(ptr)`); const { index } = classFieldIndex(meta, DYN_CLASS_PROPERTIES); const slot = B.tmp(), bag = B.tmp(), copied = B.tmp(); B.line(`${slot} = getelementptr inbounds %${classStructSym(t.className)}, ptr %p, i64 0, i32 ${index}`); B.line(`${bag} = load ptr, ptr ${slot}`); - B.line(`${copied} = call ptr @scr_dyn_copy_data_properties(ptr ${out}, ptr ${bag})`); + B.line(`${copied} = call ptr @scr_dyn_copy_property_descriptors(ptr ${out}, ptr ${bag})`); B.line(`call void @scr_dyn_release_v(ptr ${copied})`); } B.terminate(`ret ptr ${out}`); @@ -700,6 +698,25 @@ export function streamTypedRefMaterializeAdapter(host: LlvmEmitterContext, const len = B.tmp(); B.line(`${len} = call double @scr_arr_len(ptr %p)`); B.countedLoop(len, (index) => { + host.declare(`declare double @scr_arr_state(ptr, double)`); + host.declare(`declare ptr @scr_dyn_undefined()`); + host.declare(`declare ptr @scr_dyn_retain_v(ptr)`); + const state = B.tmp(); + const present = B.tmp(); + const valueLabel = B.newLabel("live.array.value"); + const absentLabel = B.newLabel("live.array.absent"); + const doneLabel = B.newLabel("live.array.done"); + B.line(`${state} = call double @scr_arr_state(ptr %p, double ${index})`); + B.line(`${present} = fcmp oeq double ${state}, 1.0`); // SCR_ARR_VALUE + B.condBr(present, valueLabel, absentLabel); + B.startBlock(absentLabel); + const undefinedValue = B.tmp(); + const retained = B.tmp(); + B.line(`${undefinedValue} = call ptr @scr_dyn_undefined()`); + B.line(`${retained} = call ptr @scr_dyn_retain_v(ptr ${undefinedValue})`); + B.line(`call void @scr_dyn_arr_push(ptr ${out}, ptr ${retained})`); + B.br(doneLabel); + B.startBlock(valueLabel); let value: string; if (elem.kind === "f64" || elem.kind === "bool") { const valueTy = elem.kind === "f64" ? "double" : "i1"; @@ -718,6 +735,8 @@ export function streamTypedRefMaterializeAdapter(host: LlvmEmitterContext, if (isRefCounted(elem)) { B.line(`call void ${releaseSym(host.shapeHost, elem)}(ptr ${value})`); } + B.br(doneLabel); + B.startBlock(doneLabel); }); B.terminate(`ret ptr ${out}`); } else { diff --git a/packages/compiler/src/backend/llvm/lib-dispatch.ts b/packages/compiler/src/backend/llvm/lib-dispatch.ts index 7ecb1f06..4c4475ed 100644 --- a/packages/compiler/src/backend/llvm/lib-dispatch.ts +++ b/packages/compiler/src/backend/llvm/lib-dispatch.ts @@ -8,6 +8,7 @@ import { MAY_THROW_LIB_FNS } from "../../ir/ir.js"; import { LlvmUnsupportedError } from "./unsupported.js"; import type { LlvmEmitterContext, LibCallExpr, LibCallPrefix, LlValue } from "./expr-context.js"; import { LIB_FN_SYMS, USES_TIMERS_LIB_FNS } from "./lib-shared.js"; +import { DYN_KIND } from "./dyn.js"; export function emitAssertInspectLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; @@ -75,6 +76,39 @@ export function emitIoLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue export function emitGenericLibCall(host: LlvmEmitterContext, e: LibCallExpr): LlValue { const B = host.B; + if (e.fn === "dyn.typedRefIs") { + // A literal brand cannot replace the receiver while being evaluated. + // Generated class dispatch uses these probes repeatedly on one local. + const value = e.args[1]!.kind === "strLit" ? host.emitReadReceiver(e.args[0]!) : host.emitExpr(e.args[0]!); + const key = host.emitExpr(e.args[1]!); + // Expose the capsule tag to LLVM so class dispatch can skip the whole + // chain of brand comparisons for ordinary checked-dynamic values. + // Keep the runtime's null handling and evaluate both operands once. + const slot = B.slot(); + B.entryAllocas.push(`${slot} = alloca i1`); + B.line(`store i1 false, ptr ${slot}`); + const present = B.tmp(); + const inspect = B.newLabel("brand.inspect"); + const capsule = B.newLabel("brand.capsule"); + const done = B.newLabel("brand.done"); + B.line(`${present} = icmp ne ptr ${value.name}, null`); + B.condBr(present, inspect, done); + B.startBlock(inspect); + const kind = host.dynKind(value.name); + const typed = B.tmp(); + B.line(`${typed} = icmp eq i32 ${kind}, ${DYN_KIND.TYPED_REF}`); + B.condBr(typed, capsule, done); + B.startBlock(capsule); + host.declare("declare zeroext i1 @scr_dyn_typed_ref_is_key(ptr, ptr) memory(read)"); + const matches = B.tmp(); + B.line(`${matches} = call zeroext i1 @scr_dyn_typed_ref_is_key(ptr ${value.name}, ptr ${key.name})`); + B.line(`store i1 ${matches}, ptr ${slot}`); + B.br(done); + B.startBlock(done); + const result = B.tmp(); + B.line(`${result} = load i1, ptr ${slot}`); + return { name: result, type: e.type }; + } if (e.fn === "crypto.randomBytesCb" || e.fn === "crypto.pbkdf2Cb") { host.usesTimers = true; const args = e.args.map((arg) => host.emitExpr(arg)); diff --git a/packages/compiler/src/backend/native-link-info-core.ts b/packages/compiler/src/backend/native-link-info-core.ts new file mode 100644 index 00000000..83629213 --- /dev/null +++ b/packages/compiler/src/backend/native-link-info-core.ts @@ -0,0 +1,155 @@ +import { join } from "node:path"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { EXTERNAL_OBJECT_ABI_STABILITY, RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "./runtime-abi.js"; +import { executableLinkInputs } from "./link-plan-core.js"; +import type { RuntimePackArtifact, RuntimePackArtifacts, RuntimePackManifest } from "./runtime-pack-core.js"; +import type { NativeTargetSpec } from "./targets.js"; + +export interface NativeLinkFeatures { + dynamic: boolean; + regex: boolean; + copying: boolean; + textDecoderLegacy: boolean; + fileHandle: boolean; + fetch: boolean; + netIsland: boolean; + zlib: boolean; + assert: boolean; + inspect: boolean; + dynInvoke: boolean; + dc: boolean; + dynAsync: boolean; + events: boolean; + emitter: boolean; + symbol: boolean; + bigint: boolean; + searchParams: boolean; + qs: boolean; + parseArgs: boolean; + stream: boolean; + net: boolean; + http: boolean; + http2: boolean; + dgram: boolean; + watch: boolean; + foreignFfi: boolean; + nodeTest: boolean; + tls: boolean; + tlsCa: boolean; +} + +export interface NativeLinkInfo { + schema: "scriptc.native-link-info.v1"; + format: 1; + compiler_version: string; + object_abi: { + stability: "experimental"; + compatibility: "exact-runtime-version"; + }; + target: { + name: NativeTargetSpec["name"]; + llvm_triple: NativeTargetSpec["llvmTriple"]; + architecture: NativeTargetSpec["architecture"]; + object_format: NativeTargetSpec["objectFormat"]; + minimum_os: NativeTargetSpec["minimumOs"]; + relocation_model: NativeTargetSpec["relocationModel"]; + }; + program: { + object: string; + entry_symbol: "main"; + undefined_runtime_symbol_prefix: "scr_"; + }; + runtime_abi: { + version: typeof RUNTIME_ABI_VERSION; + marker: typeof RUNTIME_ABI_MARKER; + }; + runtime_pack: { + kind: "precompiled"; + package: string; + version: string; + root: string; + path_base: "runtime_pack.root"; + flavor: "release" | "dev"; + objects: RuntimePackArtifact[]; + archives: RuntimePackArtifact[]; + }; + ffi: { + format: number | null; + symbols: string[]; + libraries: string[]; + }; + link: { + input_order: string[]; + driver_flags: string[]; + system_libraries: string[]; + frameworks: string[]; + }; +} + +/** Format the same external-link contract after either host verifies a pack. */ +export function formatNativeLinkInfo(options: { + programObject: string; + target: NativeTargetSpec; + ffi: FfiProfile | null; +}, compilerVersion: string, pack: { + root: string; + manifest: RuntimePackManifest; + selected: RuntimePackArtifacts; + flavor: "release" | "dev"; +}): NativeLinkInfo { + const ffiLibraries = options.ffi?.libraries ?? []; + const plan = executableLinkInputs({ + target: options.target, programObject: options.programObject, + ffiLibraries, ffiSystemLibraries: options.ffi?.systemLibraries ?? [], + ffiFrameworks: options.ffi?.frameworks ?? [], runtimeObjects: pack.selected.runtime.map((artifact) => join(pack.root, artifact.path)), + runtimeArchives: pack.selected.archives.map((artifact) => join(pack.root, artifact.path)), runtimeSystemLibraries: pack.selected.systemLibraries, + optimization: pack.flavor, + }); + return { + schema: "scriptc.native-link-info.v1", + format: 1, + compiler_version: compilerVersion, + object_abi: { + stability: EXTERNAL_OBJECT_ABI_STABILITY, + compatibility: "exact-runtime-version", + }, + target: { + name: options.target.name, + llvm_triple: options.target.llvmTriple, + architecture: options.target.architecture, + object_format: options.target.objectFormat, + minimum_os: options.target.minimumOs, + relocation_model: options.target.relocationModel, + }, + program: { + object: options.programObject, + entry_symbol: "main", + undefined_runtime_symbol_prefix: "scr_", + }, + runtime_abi: { + version: RUNTIME_ABI_VERSION, + marker: RUNTIME_ABI_MARKER, + }, + runtime_pack: { + kind: "precompiled", + package: pack.manifest.package, + version: pack.manifest.version, + root: pack.root, + path_base: "runtime_pack.root", + flavor: pack.flavor, + objects: pack.selected.runtime, + archives: pack.selected.archives, + }, + ffi: { + format: options.ffi?.ffiFormat ?? null, + symbols: options.ffi?.functions.filter((fn) => !fn.callbackOperation).map((fn) => fn.symbol) ?? [], + libraries: [...ffiLibraries], + }, + link: { + input_order: plan.inputs, + driver_flags: plan.driverFlags, + system_libraries: plan.systemLibraries, + frameworks: [...(options.ffi?.frameworks ?? [])], + }, + }; +} diff --git a/packages/compiler/src/backend/native-link-info.ts b/packages/compiler/src/backend/native-link-info.ts index 9bb52201..45f28877 100644 --- a/packages/compiler/src/backend/native-link-info.ts +++ b/packages/compiler/src/backend/native-link-info.ts @@ -1,92 +1,10 @@ +import { formatNativeLinkInfo, type NativeLinkInfo, type NativeLinkFeatures } from "./native-link-info-core.js"; +export type { NativeLinkInfo, NativeLinkFeatures } from "./native-link-info-core.js"; import type { FfiProfile } from "../ffi/ffi-manifest.js"; import { compilerReleaseVersion } from "../library/sidecar.js"; -import { EXTERNAL_OBJECT_ABI_STABILITY, RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "./runtime-abi.js"; -import { executableLinkInputs } from "./link-plan-core.js"; import { loadRuntimePack } from "./runtime-pack.js"; -import type { RuntimePackArtifact } from "./runtime-pack-core.js"; import type { NativeTargetSpec } from "./targets.js"; -export interface NativeLinkFeatures { - dynamic: boolean; - regex: boolean; - copying: boolean; - textDecoderLegacy: boolean; - fileHandle: boolean; - fetch: boolean; - netIsland: boolean; - zlib: boolean; - assert: boolean; - inspect: boolean; - dynInvoke: boolean; - dc: boolean; - dynAsync: boolean; - events: boolean; - emitter: boolean; - symbol: boolean; - bigint: boolean; - searchParams: boolean; - qs: boolean; - parseArgs: boolean; - stream: boolean; - net: boolean; - http: boolean; - http2: boolean; - dgram: boolean; - watch: boolean; - foreignFfi: boolean; - nodeTest: boolean; - tls: boolean; - tlsCa: boolean; -} - -export interface NativeLinkInfo { - schema: "scriptc.native-link-info.v1"; - format: 1; - compiler_version: string; - object_abi: { - stability: "experimental"; - compatibility: "exact-runtime-version"; - }; - target: { - name: NativeTargetSpec["name"]; - llvm_triple: NativeTargetSpec["llvmTriple"]; - architecture: NativeTargetSpec["architecture"]; - object_format: NativeTargetSpec["objectFormat"]; - minimum_os: NativeTargetSpec["minimumOs"]; - relocation_model: NativeTargetSpec["relocationModel"]; - }; - program: { - object: string; - entry_symbol: "main"; - undefined_runtime_symbol_prefix: "scr_"; - }; - runtime_abi: { - version: typeof RUNTIME_ABI_VERSION; - marker: typeof RUNTIME_ABI_MARKER; - }; - runtime_pack: { - kind: "precompiled"; - package: string; - version: string; - root: string; - path_base: "runtime_pack.root"; - flavor: "release" | "dev"; - objects: RuntimePackArtifact[]; - archives: RuntimePackArtifact[]; - }; - ffi: { - format: number | null; - symbols: string[]; - libraries: string[]; - }; - link: { - input_order: string[]; - driver_flags: string[]; - system_libraries: string[]; - frameworks: string[]; - }; -} - export async function createNativeLinkInfo(options: { programObject: string; target: NativeTargetSpec; @@ -100,59 +18,8 @@ export async function createNativeLinkInfo(options: { target: options.target, features: options.features, optimization: options.optimization ?? "release", ...(options.env === undefined ? {} : { env: options.env }), }); - const ffiLibraries = options.ffi?.libraries ?? []; - const plan = executableLinkInputs({ - target: options.target, programObject: options.programObject, - ffiLibraries, ffiSystemLibraries: options.ffi?.systemLibraries ?? [], - ffiFrameworks: options.ffi?.frameworks ?? [], runtimeObjects: pack.runtimeObjects, - runtimeArchives: pack.archives, runtimeSystemLibraries: pack.systemLibraries, - optimization: pack.flavor, + return formatNativeLinkInfo(options, compilerReleaseVersion(), { + root: pack.root, manifest: pack.manifest, flavor: pack.flavor, + selected: { features: pack.features, runtime: pack.selectedRuntimeArtifacts, archives: pack.selectedArchiveArtifacts, systemLibraries: pack.systemLibraries }, }); - return { - schema: "scriptc.native-link-info.v1", - format: 1, - compiler_version: compilerReleaseVersion(), - object_abi: { - stability: EXTERNAL_OBJECT_ABI_STABILITY, - compatibility: "exact-runtime-version", - }, - target: { - name: options.target.name, - llvm_triple: options.target.llvmTriple, - architecture: options.target.architecture, - object_format: options.target.objectFormat, - minimum_os: options.target.minimumOs, - relocation_model: options.target.relocationModel, - }, - program: { - object: options.programObject, - entry_symbol: "main", - undefined_runtime_symbol_prefix: "scr_", - }, - runtime_abi: { - version: RUNTIME_ABI_VERSION, - marker: RUNTIME_ABI_MARKER, - }, - runtime_pack: { - kind: "precompiled", - package: pack.manifest.package, - version: pack.manifest.version, - root: pack.root, - path_base: "runtime_pack.root", - flavor: pack.flavor, - objects: pack.selectedRuntimeArtifacts, - archives: pack.selectedArchiveArtifacts, - }, - ffi: { - format: options.ffi?.ffiFormat ?? null, - symbols: options.ffi?.functions.filter((fn) => !fn.callbackOperation).map((fn) => fn.symbol) ?? [], - libraries: [...ffiLibraries], - }, - link: { - input_order: plan.inputs, - driver_flags: plan.driverFlags, - system_libraries: plan.systemLibraries, - frameworks: [...(options.ffi?.frameworks ?? [])], - }, - }; } diff --git a/packages/compiler/src/backend/native-toolchain.ts b/packages/compiler/src/backend/native-toolchain.ts index 6c3434c1..5cbccb01 100644 --- a/packages/compiler/src/backend/native-toolchain.ts +++ b/packages/compiler/src/backend/native-toolchain.ts @@ -1,3 +1,8 @@ +import { driverTraceCandidates, linkTraceCandidate } from "./link-trace.js"; +import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "./toolchain-environment.js"; +export { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint, type ToolchainEnvironmentCachePolicy } from "./toolchain-environment.js"; +import { IPHONEOS_MIN_VERSION, ANDROID_MIN_API, isIosTarget, isAndroidTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal, configuredTargetPlatform } from "./target-platform.js"; +export { IPHONEOS_MIN_VERSION, ANDROID_MIN_API, isIosTarget, isAndroidTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal, configuredTargetPlatform } from "./target-platform.js"; import { InternalCompilerError } from "../errors.js"; import { execFile, spawnSync } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -123,114 +128,6 @@ export function executableSectionEliminationFlags(platform: string): { } } -/** Environment variables consumed by clang, its linker/subtools, or the - * platform SDK selection. They are implicit command-line inputs: changing one - * must never reuse an artifact produced under the old toolchain posture. */ -const TOOLCHAIN_ENV_KEYS = [ - "COMPILER_PATH", - "GCC_EXEC_PREFIX", - "CPATH", - "C_INCLUDE_PATH", - "CPLUS_INCLUDE_PATH", - "OBJC_INCLUDE_PATH", - "OBJCPLUS_INCLUDE_PATH", - "LIBRARY_PATH", - "LD_LIBRARY_PATH", - "LD_RUN_PATH", - "DYLD_LIBRARY_PATH", - "DYLD_FRAMEWORK_PATH", - "DYLD_FALLBACK_LIBRARY_PATH", - "DYLD_FALLBACK_FRAMEWORK_PATH", - "SDKROOT", - "DEVELOPER_DIR", - "MACOSX_DEPLOYMENT_TARGET", - "IPHONEOS_DEPLOYMENT_TARGET", - "TVOS_DEPLOYMENT_TARGET", - "WATCHOS_DEPLOYMENT_TARGET", - "DRIVERKIT_DEPLOYMENT_TARGET", - "XROS_DEPLOYMENT_TARGET", - "CCC_OVERRIDE_OPTIONS", - "CCC_ADD_ARGS", - "CLANG_CONFIG_FILE_SYSTEM_DIR", - "CLANG_CONFIG_FILE_USER_DIR", - "CC", - "CFLAGS", - "CPPFLAGS", - "LDFLAGS", - "AR", - "RANLIB", - "CMAKE_GENERATOR", - "CMAKE_TOOLCHAIN_FILE", - "ZIG_LIB_DIR", - "ZIG_LIBC", - "SOURCE_DATE_EPOCH", - "ZERO_AR_DATE", - "LANG", - "LC_ALL", - "LC_CTYPE", -] as const; - -/** Toolchain variables whose values name mutable files/directories consumed - * while compiling a TU (or can inject arbitrary compiler options). Hashing the - * value is insufficient: a header, SDK, config, compiler helper, or loaded - * dylib can change in place while the spelling remains stable. In that posture - * neither complete artifacts nor per-TU runtime objects are safe to reuse. */ -const MUTABLE_COMPILE_ENV_KEYS = [ - "COMPILER_PATH", - "GCC_EXEC_PREFIX", - "CPATH", - "C_INCLUDE_PATH", - "CPLUS_INCLUDE_PATH", - "OBJC_INCLUDE_PATH", - "OBJCPLUS_INCLUDE_PATH", - "LD_LIBRARY_PATH", - "DYLD_LIBRARY_PATH", - "DYLD_FRAMEWORK_PATH", - "DYLD_FALLBACK_LIBRARY_PATH", - "DYLD_FALLBACK_FRAMEWORK_PATH", - "SDKROOT", - "DEVELOPER_DIR", - "CCC_OVERRIDE_OPTIONS", - "CCC_ADD_ARGS", - "CLANG_CONFIG_FILE_SYSTEM_DIR", - "CLANG_CONFIG_FILE_USER_DIR", - // `zig cc` resolves its bundled headers/runtime through ZIG_LIB_DIR and a - // caller-selected native libc description through ZIG_LIBC. Both values name - // mutable compiler inputs whose contents can change behind a stable path. - "ZIG_LIB_DIR", - "ZIG_LIBC", -] as const; - -/** These variables only redirect link-time inputs. Runtime objects remain - * reusable, but a complete executable could otherwise retain a library that - * was rebuilt in place behind the same search-path spelling. */ -const MUTABLE_LINK_ENV_KEYS = ["LIBRARY_PATH", "LD_RUN_PATH"] as const; - -export interface ToolchainEnvironmentCachePolicy { - completeArtifacts: boolean; - runtimeObjects: boolean; -} - -export function toolchainEnvironmentCachePolicy( - env: NodeJS.ProcessEnv = process.env, -): ToolchainEnvironmentCachePolicy { - const mutableCompileInput = MUTABLE_COMPILE_ENV_KEYS.some((name) => env[name] !== undefined); - const mutableLinkInput = MUTABLE_LINK_ENV_KEYS.some((name) => env[name] !== undefined); - return { - completeArtifacts: !mutableCompileInput && !mutableLinkInput, - runtimeObjects: !mutableCompileInput, - }; -} - -export function toolchainEnvironmentFingerprint(env: NodeJS.ProcessEnv = process.env): string { - const hash = createHash("sha256").update("toolchain-env-v1\0"); - for (const name of TOOLCHAIN_ENV_KEYS) { - const value = env[name]; - hash.update(name).update(value === undefined ? "\0unset\0" : "\0set\0").update(value ?? "").update("\0"); - } - return hash.digest("hex"); -} - /** Inputs that can change which native tool/runtime implementation an * executable build selects before compileC has a chance to rediscover it. * The early whole-program cache keys this exact posture before restoring a @@ -645,59 +542,6 @@ export function isZigDriver(driver: Pick): boolean { * embedder's side of the contract: Xcode links iOS archives against the * selected SDK, and Gradle/NDK builds link Android archives against the * API-26+ bionic stubs. */ -export const IPHONEOS_MIN_VERSION = "15.0"; -export const ANDROID_MIN_API = 26; - -const MOBILE_LIBRARY_TARGETS = [ - "aarch64-apple-ios", - "aarch64-apple-ios-simulator", - "aarch64-linux-android", -] as const; - -export function isIosTarget(target: string | null): boolean { - return target === "aarch64-apple-ios" || target === "aarch64-apple-ios-simulator"; -} - -export function isAndroidTarget(target: string | null): boolean { - return target === "aarch64-linux-android"; -} - -export function isMobileTarget(target: string | null): boolean { - return isIosTarget(target) || isAndroidTarget(target); -} - -/** The canonical mobile triple SCRIPTC_TARGET selects, or null when the - * environment names none. Pure string inspection — safe to consult before - * any toolchain discovery runs. */ -export function mobileLibraryTarget(env: NodeJS.ProcessEnv = process.env): string | null { - const target = env["SCRIPTC_TARGET"] ?? ""; - return isMobileTarget(target) ? target : null; -} - -/** The admission verdict for a mobile-family triple: null when the spelling - * and host pairing are supported, otherwise the refusal text (the same text - * resolveCc throws and compileLibrary reports as SC3002). Pure string/host - * inspection — no discovery, no subprocess. */ -export function mobileTargetRefusal( - target: string, - hostPlatform: NodeJS.Platform = process.platform, -): string | null { - if (isIosTarget(target)) { - return hostPlatform === "darwin" - ? null - : `${target} library archives build on macOS hosts only (the Apple iOS SDK sysroot and Mach-O symbol localization live there); this host is ${hostPlatform}`; - } - if (isAndroidTarget(target)) return null; - // A near-miss mobile spelling must refuse with the supported set named, - // never reach zig with no sysroot wired (the compile would fail on the - // first libc header) or produce an artifact for an unverified device - // class. - if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { - return `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`; - } - return null; -} - /** The Apple SDK root for one mobile platform, discovered through xcrun the * way Xcode's own build system selects it. Memoized per SDK name and * selection environment: production rediscovers per process, and the two @@ -906,34 +750,6 @@ function isMuslTarget(driver: Pick): boolean { * analyze(): the FRONTEND consults it too (path.sep / os.EOL literals and * the path-module binding follow the target — a win32 triple compiles * Node-on-Windows semantics, path.win32 backing the bare module). */ -export function configuredTargetPlatform( - env: NodeJS.ProcessEnv = process.env, - hostPlatform: NodeJS.Platform = process.platform, -): string { - const target = env["SCRIPTC_TARGET"] ?? ""; - if (target === "") return hostPlatform; - if (target === "wasm32-wasi") return "wasi"; - if (target.includes("wasi")) { - throw new Error(`unsupported WASI target '${target}' (supported: wasm32-wasi)`); - } - // iOS is a darwin-family target: Mach-O objects, ld64 localization, - // POSIX path/EOL semantics. Android falls to the linux arm below — - // bionic is a linux libc and its archives are ordinary ELF. - if (isIosTarget(target)) return "darwin"; - if (isAndroidTarget(target)) return "linux"; - if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { - throw new Error( - `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`, - ); - } - if (target.includes("linux")) return "linux"; - if (target.includes("windows")) return "win32"; - if (target.includes("macos") || target.includes("darwin")) return "darwin"; - throw new Error( - `unsupported target '${target}' (supported OS families: linux, windows, macos/darwin, wasm32-wasi)`, - ); -} - export function targetPlatform(driver: CcDriver): string { if (driver.target === null) return process.platform; return configuredTargetPlatform({ SCRIPTC_TARGET: driver.target }); @@ -1039,6 +855,7 @@ export interface LibArchiveOptions { * archive, byte-for-byte. */ threadInstances?: boolean; /** IR-detected link gates (the compileC precedent, refusal-narrowed). */ + dynInvoke?: boolean; regex?: boolean; assert?: boolean; inspect?: boolean; @@ -1177,6 +994,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ...(opts.assert && opts.bigint ? ["scr_bigint_assert.c"] : []), ...(opts.searchParams ? ["scr_url_params.c"] : []), ...(opts.emitter ? ["scr_events_emitter.c", "scr_dyn_handle.c"] : []), + ...(opts.dynInvoke ? ["scr_dyn_invoke.c"] : []), ...(opts.zlib ? ["scr_zlib.c"] : []), ...(opts.copying ? ["scr_copying.c"] : []), ]; @@ -2978,29 +2796,6 @@ function implicitToolchainFingerprint( ); } -function linkTraceCandidate(line: string): string[] { - const trimmed = line.trim().replace(/^(?:LOAD|load)\s+/, ""); - if (trimmed === "") return []; - const unquoted = - (trimmed.startsWith('"') && trimmed.endsWith('"')) || - (trimmed.startsWith("'") && trimmed.endsWith("'")) - ? trimmed.slice(1, -1) - : trimmed; - const candidates = [unquoted]; - const member = unquoted.lastIndexOf("("); - if (member > 0 && unquoted.endsWith(")")) candidates.push(unquoted.slice(0, member)); - return candidates; -} - -function driverTraceCandidates(line: string): string[] { - const candidates: string[] = []; - for (const match of line.matchAll(/"((?:\\.|[^"\\])*)"|'([^']*)'|(\S+)/g)) { - const token = (match[1] ?? match[2] ?? match[3] ?? "").replace(/\\(["\\])/g, "$1"); - if (token !== "") candidates.push(token); - } - return candidates; -} - async function existingDriverTracePaths( output: string, cwd: string, diff --git a/packages/compiler/src/backend/native-tools.ts b/packages/compiler/src/backend/native-tools.ts index ff30e97b..ab9e693a 100644 --- a/packages/compiler/src/backend/native-tools.ts +++ b/packages/compiler/src/backend/native-tools.ts @@ -1,16 +1,12 @@ /** Native tool invocation without a JavaScript host or shell command construction. */ -import { spawnSync } from "node:child_process"; +import { execFileSync } from "node:child_process"; import { readFileSync, statSync } from "node:fs"; import type { NativeHelperSpec, NativeTargetSpec } from "./targets.js"; -import { validateNativeCodegenVersion } from "./native-codegen-core.js"; +import { validateNativeCodegenVersion, type NativeCodegenOutputKind } from "./native-codegen-core.js"; -export function runNativeTool(executable: string, args: string[]): string { - const result = spawnSync(executable, args, { encoding: "utf8" }); - if (result.error) throw new Error(`${executable}: ${result.error.message}`); - if (result.status !== 0) { - throw new Error(`${executable} failed (${result.signal ?? String(result.status)}): ${result.stderr.trim()}`); - } - return result.stdout; +export function runNativeTool(executable: string, args: string[], cwd?: string, env?: NodeJS.ProcessEnv): string { + return execFileSync(executable, args, { encoding: "utf8", stdio: "pipe", maxBuffer: 64 * 1024 * 1024, + cwd: cwd ?? process.cwd(), env: env ?? process.env }); } export function requireNativeArtifact(path: string): void { @@ -28,6 +24,22 @@ export function emitNativeObject(options: { outputPath: string; sourcePath: string; optimization: "release" | "dev"; + outputKind?: NativeCodegenOutputKind; +}): void { + verifyNativeHelper(options); + runNativeTool(options.executable, [ + "emit", "--input", options.inputPath, "--output", options.outputPath, "--filetype", options.outputKind ?? "obj", + "--target", options.target.llvmTriple, "--opt-level", options.optimization === "dev" ? "0" : "2", + "--relocation-model", options.target.relocationModel, "--diagnostic-format", "json", + "--source-path", options.sourcePath, + ]); + requireNativeArtifact(options.outputPath); +} + +/** Verify installed identity even when a caller can reuse a cached object. */ +export function verifyNativeHelper(options: { + executable: string; packageRoot: string; compilerVersion: string; + target: NativeTargetSpec; helper: NativeHelperSpec; }): void { const identity = JSON.parse(readFileSync(options.packageRoot + "/package.json", "utf8")) as { name: string; version: string }; if (identity.name !== options.helper.packageName || identity.version !== options.compilerVersion) { @@ -35,11 +47,4 @@ export function emitNativeObject(options: { } const version = JSON.parse(runNativeTool(options.executable, ["version", "--format=json"])) as Record; validateNativeCodegenVersion(version, options.target, options.helper, options.compilerVersion); - runNativeTool(options.executable, [ - "emit", "--input", options.inputPath, "--output", options.outputPath, "--filetype", "obj", - "--target", options.target.llvmTriple, "--opt-level", options.optimization === "dev" ? "0" : "2", - "--relocation-model", options.target.relocationModel, "--diagnostic-format", "json", - "--source-path", options.sourcePath, - ]); - requireNativeArtifact(options.outputPath); } diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 5b79448d..e1f50a69 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -199,7 +199,10 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) for (let i = 0; i < sections.length; i++) sectionMap.push(sectionKept[i] === true ? next++ : -1); } for (let i = 0; i < sections.length; i++) { - if (sectionKept[i] === true) sections[i]!.flags &= ~SHF_GROUP; + if (sectionKept[i] === true) { + const section = sections[i]!; + section.flags = section.flags & ~SHF_GROUP; + } } // Demote and drop decisions per symbol. A symbol anchored to a dropped @@ -228,7 +231,9 @@ export function localizeElfObject(object: Uint8Array, keep: ReadonlySet) } const symbolOrder = [...localOrder, ...globalOrder]; const symbolMap: number[] = new Array(symCount).fill(-1); - symbolOrder.forEach((oldIndex, newIndex) => (symbolMap[oldIndex] = newIndex)); + symbolOrder.forEach((oldIndex, newIndex) => { + symbolMap[oldIndex] = newIndex; + }); const newSymtabData = new Uint8Array(symbolOrder.length * 24); const newSymtabView = new DataView(newSymtabData.buffer); @@ -412,7 +417,8 @@ function parseCoff(object: Uint8Array, label: string): CoffObject { const sectionName = (raw: Uint8Array): string => { if (raw[0] === 0x2f /* '/' */) { const spelled = textDecoder.decode(raw.subarray(1)).replace(/\0+$/, "").trim(); - const offset = Number.parseInt(spelled, 10); + const decimal = /^\d+/.exec(spelled); + const offset = decimal === null ? NaN : Number(decimal[0]); if (!Number.isFinite(offset)) fail(`${label}: malformed long section name`); return readCString(strtab, offset); } @@ -563,7 +569,7 @@ export function mergeAndLocalizeCoffObjects( ]; for (const object of objects) { if (object.machine !== IMAGE_FILE_MACHINE_AMD64) { - fail(`${object.label}: unsupported COFF machine 0x${object.machine.toString(16)}`); + fail(`${object.label}: unsupported COFF machine ${object.machine}`); } } @@ -601,7 +607,7 @@ export function mergeAndLocalizeCoffObjects( for (const sym of object.symbols) { if (!coffIsUndefined(sym)) continue; if (selectedDefinitions.has(sym.name)) continue; - for (const candidate of definers.get(sym.name) ?? []) { + for (const candidate of definers.get(sym.name) ?? new Array()) { if (included[candidate] !== true) { included[candidate] = true; addDefinitions(objects[candidate]!); diff --git a/packages/compiler/src/backend/runtime-pack-native.ts b/packages/compiler/src/backend/runtime-pack-native.ts index c6b223e7..2fc94730 100644 --- a/packages/compiler/src/backend/runtime-pack-native.ts +++ b/packages/compiler/src/backend/runtime-pack-native.ts @@ -8,7 +8,7 @@ import type { NativeLinkFeatures } from "./native-link-info.js"; import type { NativeTargetSpec } from "./targets.js"; import { RuntimePackError, parseRuntimePackManifest, selectRuntimePackArtifacts, validateRuntimePackIdentity, - type RuntimePackArtifact, + type RuntimePackArtifact, type RuntimePackArtifacts, type RuntimePackManifest, type RuntimePackMode, } from "./runtime-pack-core.js"; export interface NativeRuntimePack { @@ -17,6 +17,28 @@ export interface NativeRuntimePack { systemLibraries: string[]; } +export interface NativeRuntimeSelection { + root: string; + manifest: RuntimePackManifest; + packageText: string; + manifestText: string; + selected: RuntimePackArtifacts; + flavor: "release" | "dev"; +} + +export function selectNativeRuntimePack( + root: string, target: NativeTargetSpec, compilerVersion: string, + features: NativeLinkFeatures, flavor: "release" | "dev", mode: RuntimePackMode = "executable", +): NativeRuntimeSelection { + const packageText = readFileSync(join(root, "package.json"), "utf8"); + const manifestText = readFileSync(join(root, "runtime-pack.json"), "utf8"); + const identity = JSON.parse(packageText) as { name?: string; version?: string }; + const manifest = parseRuntimePackManifest(JSON.parse(manifestText)); + validateRuntimePackIdentity(manifest, identity.name, identity.version, target, compilerVersion); + const selected = selectRuntimePackArtifacts(manifest, features, flavor, process.env, mode); + return { root, manifest, packageText, manifestText, selected, flavor }; +} + function stageArtifact(root: string, stage: string, artifact: RuntimePackArtifact): string { const source = join(root, artifact.path); const destination = join(stage, artifact.path); @@ -48,15 +70,15 @@ export function stageNativeRuntimePack( compilerVersion: string, features: NativeLinkFeatures, flavor: "release" | "dev", + mode: RuntimePackMode = "executable", ): NativeRuntimePack { + return stageNativeRuntimeSelection(selectNativeRuntimePack(root, target, compilerVersion, features, flavor, mode), stageRoot); +} + +export function stageNativeRuntimeSelection(selection: NativeRuntimeSelection, stageRoot: string): NativeRuntimePack { + const { root, manifest, selected, packageText, manifestText } = selection; const packagePath = join(root, "package.json"); const manifestPath = join(root, "runtime-pack.json"); - const packageText = readFileSync(packagePath, "utf8"); - const manifestText = readFileSync(manifestPath, "utf8"); - const identity = JSON.parse(packageText) as { name?: string; version?: string }; - const manifest = parseRuntimePackManifest(JSON.parse(manifestText)); - validateRuntimePackIdentity(manifest, identity.name, identity.version, target, compilerVersion); - const selected = selectRuntimePackArtifacts(manifest, features, flavor); const runtimeObjects = selected.runtime.map((artifact) => stageArtifact(root, stageRoot, artifact)); const archives = selected.archives.map((artifact) => stageArtifact(root, stageRoot, artifact)); for (const license of manifest.licenses) { diff --git a/packages/compiler/src/backend/target-diagnostics.ts b/packages/compiler/src/backend/target-diagnostics.ts new file mode 100644 index 00000000..6514fab2 --- /dev/null +++ b/packages/compiler/src/backend/target-diagnostics.ts @@ -0,0 +1,110 @@ +import { everyModuleNode } from "../ir/traverse.js"; +import type { LlvmUnsupportedError } from "./llvm/emitter.js"; +import type { ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import { moduleUsesFetch, moduleEmbedsBuiltin, type IrModule, type SrcLoc } from "../ir/ir.js"; + +/** The LLVM backend's tier refusal as a diagnostic. SC3xxx = backend + * coverage (the program is fine — this backend doesn't compile it yet); + * the parenthesized kind tag is machine-readable for the differential + * harness's histogram. */ +export function llvmRefusalDiag(err: LlvmUnsupportedError, entryPath: string): ScrDiagnostic { + return { + code: "SC3001", + message: err.message, + loc: err.loc ?? { file: entryPath, start: 0, end: 0 }, + }; +} + +/** A valid program surface that the selected execution target cannot host. + * SC3xxx stays the backend/target-coverage family: source semantics are + * valid, but this target deliberately refuses them instead of emitting a + * binary that traps later. */ +export function targetRefusalDiag(target: string, surface: string, loc: SrcLoc): ScrDiagnostic { + return { + code: "SC3002", + message: `${target} target does not support ${surface}`, + loc, + }; +} + +/** APIs that require host capabilities absent from portable WASI Preview 1. + * These are target diagnostics, not backend-tier gaps: the same language IR + * (including async, generators, and the dynamic island) is otherwise valid. + * Keep the fine-grained walk first so diagnostics point at the API use; the + * embedded-module checks are the entry-anchored safety net for island code. */ +export function moduleWasiUnavailableSurface(mod: IrModule): { surface: string; loc: SrcLoc } | null { + const entryLoc: SrcLoc = { file: mod.sourceFile, start: 0, end: 0 }; + const prefixes: readonly (readonly [string, string])[] = [ + ["cp.", "child processes (WASI Preview 1 has no process-spawning API)"], + ["child.", "child processes (WASI Preview 1 has no process-spawning API)"], + ["spawnRes.", "child processes (WASI Preview 1 has no process-spawning API)"], + ["net.", "network sockets (WASI Preview 1 has no socket API)"], + ["http.", "network sockets (WASI Preview 1 has no socket API)"], + ["https.", "network sockets (WASI Preview 1 has no socket API)"], + ["http2.", "network sockets (WASI Preview 1 has no socket API)"], + ["h2.", "network sockets (WASI Preview 1 has no socket API)"], + ["dgram.", "network sockets (WASI Preview 1 has no socket API)"], + ["dns.", "network sockets (WASI Preview 1 has no socket API)"], + ["tls.", "network sockets (WASI Preview 1 has no socket API)"], + ["fetch.", "network-backed fetch (WASI Preview 1 has no socket API)"], + ["fs.watch", "filesystem watching (WASI Preview 1 has no notification API)"], + ["watcher.", "filesystem watching (WASI Preview 1 has no notification API)"], + ]; + const kinds: ReadonlyMap = new Map([ + ["child", "child processes (WASI Preview 1 has no process-spawning API)"], + ["spawnRes", "child processes (WASI Preview 1 has no process-spawning API)"], + ["childStream", "child processes (WASI Preview 1 has no process-spawning API)"], + ["childWriter", "child processes (WASI Preview 1 has no process-spawning API)"], + ["netServer", "network sockets (WASI Preview 1 has no socket API)"], + ["netSocket", "network sockets (WASI Preview 1 has no socket API)"], + ["http2Session", "network sockets (WASI Preview 1 has no socket API)"], + ["http2Stream", "network sockets (WASI Preview 1 has no socket API)"], + ["dgramSocket", "network sockets (WASI Preview 1 has no socket API)"], + ["fsWatcher", "filesystem watching (WASI Preview 1 has no notification API)"], + ["httpReq", "network sockets (WASI Preview 1 has no socket API)"], + ["httpRes", "network sockets (WASI Preview 1 has no socket API)"], + ["httpClientReq", "network sockets (WASI Preview 1 has no socket API)"], + ["secureCtx", "network sockets (WASI Preview 1 has no socket API)"], + ]); + let found: { surface: string; loc: SrcLoc } | null = null; + everyModuleNode(mod, { + type: (node, loc) => { + const surface = kinds.get(node.kind); + if (surface === undefined) return true; + found = { surface, loc }; + return false; + }, + stmt: () => true, + expr: (node) => { + if (node.kind !== "libCall") return true; + const loc = node.loc; + if (node.fn === "process.kill" || node.fn === "process.killNum" || + node.fn === "process.onSignal" || node.fn === "process.offSignal") { + found = { surface: "OS signals (WASI Preview 1 has no signal API)", loc }; + return false; + } + if (node.fn === "os.networkInterfaces") { + found = { surface: "network-interface enumeration (WASI Preview 1 has no interface API)", loc }; + return false; + } + for (const [prefix, surface] of prefixes) { + if (node.fn.startsWith(prefix)) { + found = { surface, loc }; + return false; + } + } + return true; + }, + }); + if (found !== null) return found; + + if (moduleUsesFetch(mod)) { + return { surface: "network-backed fetch (WASI Preview 1 has no socket API)", loc: entryLoc }; + } + for (const builtin of ["node:http", "node:https", "node:net", "node:tls"]) { + if (moduleEmbedsBuiltin(mod, builtin)) { + return { surface: `${builtin} networking (WASI Preview 1 has no socket API)`, loc: entryLoc }; + } + } + return null; +} diff --git a/packages/compiler/src/backend/target-platform.ts b/packages/compiler/src/backend/target-platform.ts new file mode 100644 index 00000000..84f49eec --- /dev/null +++ b/packages/compiler/src/backend/target-platform.ts @@ -0,0 +1,80 @@ +export const IPHONEOS_MIN_VERSION = "15.0"; +export const ANDROID_MIN_API = 26; + +const MOBILE_LIBRARY_TARGETS = [ + "aarch64-apple-ios", + "aarch64-apple-ios-simulator", + "aarch64-linux-android", +] as const; + +export function isIosTarget(target: string | null): boolean { + return target === "aarch64-apple-ios" || target === "aarch64-apple-ios-simulator"; +} + +export function isAndroidTarget(target: string | null): boolean { + return target === "aarch64-linux-android"; +} + +export function isMobileTarget(target: string | null): boolean { + return isIosTarget(target) || isAndroidTarget(target); +} + +/** The canonical mobile triple SCRIPTC_TARGET selects, or null when the + * environment names none. Pure string inspection — safe to consult before + * any toolchain discovery runs. */ +export function mobileLibraryTarget(env: NodeJS.ProcessEnv = process.env): string | null { + const target = env["SCRIPTC_TARGET"] ?? ""; + return isMobileTarget(target) ? target : null; +} + +/** The admission verdict for a mobile-family triple: null when the spelling + * and host pairing are supported, otherwise the refusal text (the same text + * resolveCc throws and compileLibrary reports as SC3002). Pure string/host + * inspection — no discovery, no subprocess. */ +export function mobileTargetRefusal( + target: string, + hostPlatform: NodeJS.Platform = process.platform, +): string | null { + if (isIosTarget(target)) { + return hostPlatform === "darwin" + ? null + : `${target} library archives build on macOS hosts only (the Apple iOS SDK sysroot and Mach-O symbol localization live there); this host is ${hostPlatform}`; + } + if (isAndroidTarget(target)) return null; + // A near-miss mobile spelling must refuse with the supported set named, + // never reach zig with no sysroot wired (the compile would fail on the + // first libc header) or produce an artifact for an unverified device + // class. + if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { + return `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`; + } + return null; +} + +export function configuredTargetPlatform( + env: NodeJS.ProcessEnv = process.env, + hostPlatform: NodeJS.Platform = process.platform, +): string { + const target = env["SCRIPTC_TARGET"] ?? ""; + if (target === "") return hostPlatform; + if (target === "wasm32-wasi") return "wasi"; + if (target.includes("wasi")) { + throw new Error(`unsupported WASI target '${target}' (supported: wasm32-wasi)`); + } + // iOS is a darwin-family target: Mach-O objects, ld64 localization, + // POSIX path/EOL semantics. Android falls to the linux arm below — + // bionic is a linux libc and its archives are ordinary ELF. + if (isIosTarget(target)) return "darwin"; + if (isAndroidTarget(target)) return "linux"; + if (/(?:^|-)(?:ios|tvos|watchos|visionos|android)/.test(target)) { + throw new Error( + `unsupported mobile target '${target}' (supported: ${MOBILE_LIBRARY_TARGETS.join(", ")})`, + ); + } + if (target.includes("linux")) return "linux"; + if (target.includes("windows")) return "win32"; + if (target.includes("macos") || target.includes("darwin")) return "darwin"; + throw new Error( + `unsupported target '${target}' (supported OS families: linux, windows, macos/darwin, wasm32-wasi)`, + ); +} diff --git a/packages/compiler/src/backend/targets.test.ts b/packages/compiler/src/backend/targets.test.ts index 1fe162e7..6eb585cb 100644 --- a/packages/compiler/src/backend/targets.test.ts +++ b/packages/compiler/src/backend/targets.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { IOS_ARM64_TARGET, IOS_SIMULATOR_ARM64_TARGET, @@ -19,7 +19,46 @@ import { windowsSubsystemLinkerArgs, } from "./targets.js"; +afterEach(() => vi.restoreAllMocks()); + describe("native code-generation targets", () => { + test("non-Linux and unsupported hosts never collect a process report", async () => { + vi.resetModules(); + const targets = await import("./targets.js"); + const report = vi.spyOn(process.report, "getReport").mockImplementation(() => { + throw new Error("unexpected host libc probe"); + }); + for (const [platform, arch, release] of [ + ["darwin", "arm64", "24.0.0"], ["darwin", "x64", "24.0.0"], ["win32", "x64", "10.0.0"], + ] as const) { + expect(targets.nativeCodegenTarget({}, platform, arch, release)).not.toBeNull(); + expect(targets.nativeCodegenTargetRefusal({}, platform, arch, release)).toBeNull(); + expect(targets.nativeHelperForTarget(LINUX_X64_GNU_TARGET, platform, arch)).not.toBeNull(); + } + expect(targets.nativeCodegenTarget({}, "linux", "ia32", "6.8.0")).toBeNull(); + expect(targets.nativeHelperForTarget(LINUX_X64_GNU_TARGET, "linux", "ia32")).toBeNull(); + expect(report).not.toHaveBeenCalled(); + }); + + test.each(["gnu", "musl"] as const)("probes the %s host libc once across target and helper selection", async (libc) => { + vi.resetModules(); + const targets = await import("./targets.js"); + const report = vi.spyOn(process.report, "getReport").mockReturnValue({ + header: libc === "gnu" ? { glibcVersionRuntime: "2.36" } : {}, + } as ReturnType); + const target = libc === "gnu" ? LINUX_X64_GNU_TARGET : LINUX_X64_MUSL_TARGET; + expect(targets.nativeCodegenTarget({}, "linux", "x64", "6.8.0")).toEqual(target); + expect(targets.nativeCodegenTargetRefusal({}, "linux", "x64", "6.8.0")).toBeNull(); + expect(targets.nativeHelperForTarget(target, "linux", "x64")?.packageName).toBe(`@scriptc/llvm-linux-x64-${libc}`); + expect(targets.nativeCodegenTarget({ SCRIPTC_TARGET: "wasm32-wasi" }, "linux", "x64", "6.8.0")).toEqual(WASM32_WASI_TARGET); + expect(report).toHaveBeenCalledTimes(1); + + // Explicit host descriptions remain independent of the process memo. + expect(targets.nativeCodegenTarget({}, "linux", "x64", "6.8.0", "gnu")).toEqual(LINUX_X64_GNU_TARGET); + expect(targets.nativeCodegenTarget({}, "linux", "x64", "6.8.0", "musl")).toEqual(LINUX_X64_MUSL_TARGET); + expect(report).toHaveBeenCalledTimes(1); + }); + test("matches Clang's narrow integer ABI independently of the build host", () => { for (const target of [MACOS_ARM64_TARGET, MACOS_X64_TARGET, LINUX_X64_GNU_TARGET, LINUX_X64_MUSL_TARGET, WASM32_WASI_TARGET]) { expect(ffiExtendsNarrowIntegers(target.llvmTriple, "win32", "x64")).toBe(true); diff --git a/packages/compiler/src/backend/targets.ts b/packages/compiler/src/backend/targets.ts index 1824f986..cdd96a5d 100644 --- a/packages/compiler/src/backend/targets.ts +++ b/packages/compiler/src/backend/targets.ts @@ -294,7 +294,7 @@ export const WASM32_WASI_TARGET: NativeTargetSpec = { linkerTargetTriple: "wasm32-wasi", runtimeSystemLibraries: ["wasi-emulated-signal", "wasi-emulated-process-clocks"], runtimeCompileDefines: ["_GNU_SOURCE", "_WASI_EMULATED_SIGNAL", "_WASI_EMULATED_PROCESS_CLOCKS"], - supports: { asm: true, obj: true, exe: true, library: false }, + supports: { asm: true, obj: true, exe: true, library: true }, helperPackage: DARWIN_ARM64_HELPER.packageName, helper: DARWIN_ARM64_HELPER, llvmBackend: "WebAssembly", hostHelpers: { @@ -340,24 +340,29 @@ export const NATIVE_TARGETS = [ IOS_ARM64_TARGET, IOS_SIMULATOR_ARM64_TARGET, ANDROID_ARM64_TARGET, ] as const; +let linuxLibcMemo: LinuxLibc | undefined; + function detectedLinuxLibc(): LinuxLibc { + if (linuxLibcMemo !== undefined) return linuxLibcMemo; // Node exposes glibc's runtime version without any external command or // filesystem probe. Its absence on Linux is the portable musl signal used - // by npm's own optional-dependency selection conventions. + // by npm's own optional-dependency selection conventions. The host libc + // cannot change within this process, and generating a full report is costly. const report = process.report?.getReport() as { header?: { glibcVersionRuntime?: unknown } } | undefined; const header = report?.header; - return typeof header?.glibcVersionRuntime === "string" ? "gnu" : "musl"; + linuxLibcMemo = typeof header?.glibcVersionRuntime === "string" ? "gnu" : "musl"; + return linuxLibcMemo; } function helperHost( platform: NodeJS.Platform, arch: string, - linuxLibc: LinuxLibc = detectedLinuxLibc(), + linuxLibc?: LinuxLibc, ): NativeHelperHost | null { if (platform === "darwin" && arch === "arm64") return "darwin-arm64"; if (platform === "darwin" && arch === "x64") return "darwin-x64"; - if (platform === "linux" && arch === "x64") return `linux-x64-${linuxLibc}`; - if (platform === "linux" && arch === "arm64") return `linux-arm64-${linuxLibc}`; + if (platform === "linux" && arch === "x64") return `linux-x64-${linuxLibc ?? detectedLinuxLibc()}`; + if (platform === "linux" && arch === "arm64") return `linux-arm64-${linuxLibc ?? detectedLinuxLibc()}`; if (platform === "win32" && arch === "x64") return "win32-x64"; return null; } @@ -385,19 +390,18 @@ function nativeHostTarget( hostPlatform: NodeJS.Platform, hostArch: string, hostRelease: string, - linuxLibc: LinuxLibc, + linuxLibc?: LinuxLibc, ): NativeTargetSpec | null { if (hostPlatform === "darwin") { const major = Number.parseInt(hostRelease.split(".", 1)[0] ?? "", 10); if (!Number.isFinite(major) || major < 24) return null; return hostArch === "arm64" ? MACOS_ARM64_TARGET : hostArch === "x64" ? MACOS_X64_TARGET : null; } - if (hostPlatform === "linux") { + if (hostPlatform === "linux" && (hostArch === "x64" || hostArch === "arm64")) { + const libc = linuxLibc ?? detectedLinuxLibc(); return hostArch === "x64" - ? linuxLibc === "musl" ? LINUX_X64_MUSL_TARGET : LINUX_X64_GNU_TARGET - : hostArch === "arm64" - ? linuxLibc === "musl" ? LINUX_ARM64_MUSL_TARGET : LINUX_ARM64_GNU_TARGET - : null; + ? libc === "musl" ? LINUX_X64_MUSL_TARGET : LINUX_X64_GNU_TARGET + : libc === "musl" ? LINUX_ARM64_MUSL_TARGET : LINUX_ARM64_GNU_TARGET; } return hostPlatform === "win32" && hostArch === "x64" ? WINDOWS_X64_MSVC_TARGET : null; } @@ -435,20 +439,12 @@ function requestedTarget( } } -/** Select only a fully described scriptc target. LLVM accepting an arbitrary - * triple is never evidence of its object ABI, runtime pack, link, or run. */ -export function nativeCodegenTarget( - env: NodeJS.ProcessEnv = process.env, - hostPlatform: NodeJS.Platform = process.platform, - hostArch: string = process.arch, - hostRelease: string = release(), - linuxLibc: LinuxLibc = detectedLinuxLibc(), +/** Native distributions already know their host ABI, including Linux libc. */ +export function selectNativeTarget( + raw: string, host: NativeTargetSpec, hostPlatform: string = process.platform, ): NativeTargetSpec | null { - const host = nativeHostTarget(hostPlatform, hostArch, hostRelease, linuxLibc); - const target = requestedTarget(env["SCRIPTC_TARGET"] ?? "", host, hostPlatform); - if (host === null || target === null || nativeHelperForTarget(target, hostPlatform, hostArch, linuxLibc) === null) return null; - // Cross ELF links use Zig's target libc. A native clang driver cannot - // infer or provide another architecture's CRT and sysroot. + const target = requestedTarget(raw, host, hostPlatform as NodeJS.Platform); + if (target === null) return null; if (target.platform === "linux" && target.name !== host.name && target.name !== "android-arm64") { return { ...target, defaultLinker: "zig", defaultLinkerArgs: ["cc"], @@ -461,12 +457,27 @@ export function nativeCodegenTarget( return target; } +/** Select only a fully described scriptc target. LLVM accepting an arbitrary + * triple is never evidence of its object ABI, runtime pack, link, or run. */ +export function nativeCodegenTarget( + env: NodeJS.ProcessEnv = process.env, + hostPlatform: NodeJS.Platform = process.platform, + hostArch: string = process.arch, + hostRelease: string = release(), + linuxLibc?: LinuxLibc, +): NativeTargetSpec | null { + const host = nativeHostTarget(hostPlatform, hostArch, hostRelease, linuxLibc); + const target = host === null ? null : selectNativeTarget(env["SCRIPTC_TARGET"] ?? "", host, hostPlatform); + if (host === null || target === null || nativeHelperForTarget(target, hostPlatform, hostArch, linuxLibc) === null) return null; + return target; +} + export function nativeCodegenTargetRefusal( env: NodeJS.ProcessEnv = process.env, hostPlatform: NodeJS.Platform = process.platform, hostArch: string = process.arch, hostRelease: string = release(), - linuxLibc: LinuxLibc = detectedLinuxLibc(), + linuxLibc?: LinuxLibc, ): string | null { if (nativeCodegenTarget(env, hostPlatform, hostArch, hostRelease, linuxLibc) !== null) return null; const requested = env["SCRIPTC_TARGET"] ?? ""; diff --git a/packages/compiler/src/backend/toolchain-environment.ts b/packages/compiler/src/backend/toolchain-environment.ts new file mode 100644 index 00000000..7c8a749d --- /dev/null +++ b/packages/compiler/src/backend/toolchain-environment.ts @@ -0,0 +1,110 @@ +import { createHash } from "node:crypto"; + +/** Environment variables consumed by clang, its linker/subtools, or the + * platform SDK selection. They are implicit command-line inputs: changing one + * must never reuse an artifact produced under the old toolchain posture. */ +const TOOLCHAIN_ENV_KEYS: readonly string[] = [ + "COMPILER_PATH", + "GCC_EXEC_PREFIX", + "CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "OBJC_INCLUDE_PATH", + "OBJCPLUS_INCLUDE_PATH", + "LIBRARY_PATH", + "LD_LIBRARY_PATH", + "LD_RUN_PATH", + "DYLD_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FALLBACK_FRAMEWORK_PATH", + "SDKROOT", + "DEVELOPER_DIR", + "MACOSX_DEPLOYMENT_TARGET", + "IPHONEOS_DEPLOYMENT_TARGET", + "TVOS_DEPLOYMENT_TARGET", + "WATCHOS_DEPLOYMENT_TARGET", + "DRIVERKIT_DEPLOYMENT_TARGET", + "XROS_DEPLOYMENT_TARGET", + "CCC_OVERRIDE_OPTIONS", + "CCC_ADD_ARGS", + "CLANG_CONFIG_FILE_SYSTEM_DIR", + "CLANG_CONFIG_FILE_USER_DIR", + "CC", + "CFLAGS", + "CPPFLAGS", + "LDFLAGS", + "AR", + "RANLIB", + "CMAKE_GENERATOR", + "CMAKE_TOOLCHAIN_FILE", + "ZIG_LIB_DIR", + "ZIG_LIBC", + "SOURCE_DATE_EPOCH", + "ZERO_AR_DATE", + "LANG", + "LC_ALL", + "LC_CTYPE", +]; + +/** Toolchain variables whose values name mutable files/directories consumed + * while compiling a TU (or can inject arbitrary compiler options). Hashing the + * value is insufficient: a header, SDK, config, compiler helper, or loaded + * dylib can change in place while the spelling remains stable. In that posture + * neither complete artifacts nor per-TU runtime objects are safe to reuse. */ +const MUTABLE_COMPILE_ENV_KEYS: readonly string[] = [ + "COMPILER_PATH", + "GCC_EXEC_PREFIX", + "CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "OBJC_INCLUDE_PATH", + "OBJCPLUS_INCLUDE_PATH", + "LD_LIBRARY_PATH", + "DYLD_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FALLBACK_FRAMEWORK_PATH", + "SDKROOT", + "DEVELOPER_DIR", + "CCC_OVERRIDE_OPTIONS", + "CCC_ADD_ARGS", + "CLANG_CONFIG_FILE_SYSTEM_DIR", + "CLANG_CONFIG_FILE_USER_DIR", + // `zig cc` resolves its bundled headers/runtime through ZIG_LIB_DIR and a + // caller-selected native libc description through ZIG_LIBC. Both values name + // mutable compiler inputs whose contents can change behind a stable path. + "ZIG_LIB_DIR", + "ZIG_LIBC", +]; + +/** These variables only redirect link-time inputs. Runtime objects remain + * reusable, but a complete executable could otherwise retain a library that + * was rebuilt in place behind the same search-path spelling. */ +const MUTABLE_LINK_ENV_KEYS: readonly string[] = ["LIBRARY_PATH", "LD_RUN_PATH"]; + +export interface ToolchainEnvironmentCachePolicy { + completeArtifacts: boolean; + runtimeObjects: boolean; +} + +export function toolchainEnvironmentCachePolicy( + env: NodeJS.ProcessEnv = process.env, +): ToolchainEnvironmentCachePolicy { + const mutableCompileInput = MUTABLE_COMPILE_ENV_KEYS.some((name) => env[name] !== undefined); + const mutableLinkInput = MUTABLE_LINK_ENV_KEYS.some((name) => env[name] !== undefined); + return { + completeArtifacts: !mutableCompileInput && !mutableLinkInput, + runtimeObjects: !mutableCompileInput, + }; +} + +export function toolchainEnvironmentFingerprint(env: NodeJS.ProcessEnv = process.env): string { + const hash = createHash("sha256").update("toolchain-env-v1\0"); + for (const name of TOOLCHAIN_ENV_KEYS) { + const value = env[name]; + const text: string = value ?? ""; + hash.update(name).update(value === undefined ? "\0unset\0" : "\0set\0").update(text).update("\0"); + } + return hash.digest("hex"); +} diff --git a/packages/compiler/src/backend/vendor-archives.ts b/packages/compiler/src/backend/vendor-archives.ts index 608de233..cfc4cf18 100644 --- a/packages/compiler/src/backend/vendor-archives.ts +++ b/packages/compiler/src/backend/vendor-archives.ts @@ -9,15 +9,8 @@ import type { CcDriver } from "./native-toolchain.js"; const execFileAsync = promisify(execFile); -/** The pinned QuickJS, mbedTLS, and zlib inputs used by native recipes. */ -export const QJS_COMMIT = "3c8f3d68953955950074c41c6e4d999562ae82a7"; -export const MBEDTLS_VERSION = "3.6.7"; -export const ZLIB_VERSION = "1.3.1"; -/** Exact translation-unit membership shared by cache builds and external - * source-pack recipes. */ -export const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"] as const; -export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const; -export const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c"] as const; +import { QJS_COMMIT, MBEDTLS_VERSION, ZLIB_VERSION, QJS_ENGINE_SOURCES, LRE_SOURCES, ZLIB_SOURCES } from "./vendor-inputs.js"; +export { QJS_COMMIT, MBEDTLS_VERSION, ZLIB_VERSION, QJS_ENGINE_SOURCES, LRE_SOURCES, ZLIB_SOURCES } from "./vendor-inputs.js"; export interface VendorArchiveContext { runtimeSrcDir(): string; diff --git a/packages/compiler/src/backend/vendor-inputs.ts b/packages/compiler/src/backend/vendor-inputs.ts new file mode 100644 index 00000000..a9b4b4ad --- /dev/null +++ b/packages/compiler/src/backend/vendor-inputs.ts @@ -0,0 +1,9 @@ +/** The pinned QuickJS, mbedTLS, and zlib inputs used by native recipes. */ +export const QJS_COMMIT = "3c8f3d68953955950074c41c6e4d999562ae82a7"; +export const MBEDTLS_VERSION = "3.6.7"; +export const ZLIB_VERSION = "1.3.1"; +/** Exact translation-unit membership shared by cache builds and external + * source-pack recipes. */ +export const QJS_ENGINE_SOURCES = ["dtoa.c", "libregexp.c", "libunicode.c", "quickjs.c"] as const; +export const LRE_SOURCES = ["libregexp.c", "libunicode.c"] as const; +export const ZLIB_SOURCES = ["adler32.c", "compress.c", "crc32.c", "deflate.c", "infback.c", "inffast.c", "inflate.c", "inftrees.c", "trees.c", "uncompr.c", "zutil.c"] as const; diff --git a/packages/compiler/src/cli/command.ts b/packages/compiler/src/cli/command.ts new file mode 100644 index 00000000..12d9fdce --- /dev/null +++ b/packages/compiler/src/cli/command.ts @@ -0,0 +1,325 @@ +import { rmSync, statSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { isExactExternalTypeSpecifier } from "../frontend/program.js"; +import { renderDiagnostics } from "../diagnostics/render.js"; +import { renderCoverage } from "../coverage/report.js"; +import { setProvenanceSources } from "../frontend/provenance-registry.js"; +import type { CliHost, NativeCacheWarmProfile } from "./host.js"; +import { resolveOutputOptions } from "./output-options.js"; +import { selectOutputPaths } from "./paths.js"; +import { CLI_OPTIONS, USAGE } from "./usage.js"; + +/* The exit discipline: NEVER process.exit() after writing output. stdout/ + * stderr to a PIPE are async streams — process.exit() drops whatever libuv + * hasn't flushed yet, which truncates large diagnostic renders at the pipe + * buffer (observed: 64KB cut mid-code-frame). Every path sets + * process.exitCode and returns instead; Node exits naturally once the + * streams drain. */ +class CliExit extends Error { + constructor(readonly code: number) { + super(`exit ${code}`); + this.name = "CliExit"; + } +} + +function fail(msg: string): never { + process.stderr.write(msg + "\n"); + throw new CliExit(1); +} + +/** parseArgs, with its throw turned into the CLI's own one-line error. + * Unparseable arguments are a USER error — an unknown flag or a missing + * value used to reach the top level as an uncaught ERR_PARSE_ARGS_* and + * print a Node stack trace over the user's terminal. */ +function parseCli(args: string[]): ReturnType> { + try { + return parseArgs({ args, options: CLI_OPTIONS, allowPositionals: true, allowNegative: true }); + } catch (err) { + // parseArgs appends a paragraph about `--` and positionals to the + // unknown-option message; the first sentence is the part that names + // what was wrong, and USAGE below already covers what was meant. + const raw = err instanceof Error ? err.message : String(err); + const msg = raw.split("\n")[0]!.split(". ")[0]!; + fail(`scriptc: ${msg}\n\n${USAGE}`); + } +} + +async function main(args: string[], host: CliHost): Promise { + const { values, positionals } = parseCli(args); + const externalTypeArgs = values["external-types"] ?? []; + + if (values.version) { + process.stdout.write(`${host.version()}\n`); + return 0; + } + + if (values.help || positionals.length === 0) { + process.stdout.write(USAGE); + return values.help ? 0 : 1; + } + + const [command, inputArg] = positionals; + if (command === "cache") { + if (inputArg !== "warm") fail(`unknown cache command "${inputArg ?? ""}" (supported: warm)\n\n${USAGE}`); + if (values.lib || values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.ffi !== undefined || values.profile !== undefined || values.strip || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || values["provenance-sources"] || externalTypeArgs.length > 0 || values.out !== undefined || values["emit-ir"] || !values["keep-llvm"]) { + fail(`scriptc cache warm takes only native optimization/sanitizer options and profile names\n\n${USAGE}`); + } + const optimization = values.optimization; + if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { + fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); + } + const profileArgs = positionals.slice(2); + const knownProfiles = new Set(["runtime", "tls", "dynamic"]); + for (const profile of profileArgs) { + if (!knownProfiles.has(profile as NativeCacheWarmProfile)) { + fail(`unknown cache warm profile "${profile}" (supported: runtime, tls, dynamic)`); + } + } + let result; + try { + result = await host.warmNativeCaches({ + ...(optimization === undefined ? {} : { optimization }), + sanitize: values.sanitize, + ...(profileArgs.length === 0 + ? {} + : { profiles: profileArgs as NativeCacheWarmProfile[] }), + }); + } catch (error) { + fail(`scriptc: ${error instanceof Error ? error.message : String(error)}`); + } + process.stdout.write(`${result.cacheRoot}\n`); + for (const profile of result.profiles) { + process.stdout.write(`${profile.profile}\t${Math.round(profile.elapsedMs)}ms\n`); + } + return 0; + } + if (command !== "build" && command !== "run" && command !== "coverage") { + fail(`unknown command "${command}"\n\n${USAGE}`); + } + if (values.lib) { + // LIBRARY mode: the profile names the entry module and pins the + // emission; the executable lane's mode flags have no meaning here + // (library artifacts are static-tier only, and there is no fallback + // concept — bare npm specifiers are static-or-refuse: the npm-static + // eligibility bar runs automatically, eligible packages compile into + // the graph, ineligible ones refuse with SC4013). + if (command !== "build") fail(`--lib is a build mode (scriptc build --lib --profile )\n\n${USAGE}`); + const profileArg = values.profile; + if (!profileArg) fail(`scriptc build --lib needs --profile \n\n${USAGE}`); + if (inputArg) { + fail("scriptc build --lib takes no input positional: the profile names the entry module"); + } + if (values.dynamic || values.backend !== undefined || values.emit !== undefined || values.print !== undefined || values.optimization !== undefined || values.strip || values.ffi !== undefined || values["windows-subsystem"] !== undefined || (values["npm-static"] ?? []).length > 0 || externalTypeArgs.length > 0) { + fail( + "scriptc build --lib takes no --dynamic/--backend/--emit/--print/--optimization/--strip/--windows-subsystem/--npm-static/--ffi/--external-types: the profile pins the emission and optimization, npm imports are judged automatically, outbound FFI belongs to executable builds, and external type mappings belong to coverage", + ); + } + const profilePath = resolve(profileArg); + const libOutDir = values.out ? dirname(resolve(values.out)) : join(dirname(profilePath), ".scriptc"); + const result = await host.compileLibrary({ + profilePath, + outDir: libOutDir, + ...(values.out ? { outPath: resolve(values.out) } : {}), + emitIr: values["emit-ir"], + sanitize: values.sanitize, + }); + if (!result.ok) { + const color = process.stderr.isTTY ?? false; + process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); + const n = result.diagnostics.length; + process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); + return 1; + } + if (!values["keep-llvm"]) rmSync(result.llvmPath, { force: true }); + process.stdout.write(`${result.archivePath}\n`); + // The contract sidecar rides the same invocation when the profile + // declares one — name it so the embedder's tooling knows where to look. + if (result.sidecarPath !== undefined) process.stdout.write(`${result.sidecarPath}\n`); + return 0; + } + if (values["emit-ir"] && (command === "build" || command === "run")) { + process.stderr.write("scriptc: warning: --emit-ir is deprecated; use --emit=ir for IR as the primary output\n"); + } + if (!inputArg) fail(`missing input file\n\n${USAGE}`); + const input = resolve(inputArg); + if (command === "coverage" && values.emit !== undefined) { + fail(`--emit is a build/run option\n\n${USAGE}`); + } + if (command === "coverage" && values.strip) { + fail(`--strip is a build/run option\n\n${USAGE}`); + } + if (values.print !== undefined && values.print !== "native-link-info") { + fail(`unknown print kind "${values.print}" (supported: native-link-info)\n\n${USAGE}`); + } + const printNativeLinkInfo = values.print === "native-link-info"; + if (printNativeLinkInfo && command !== "build") { + fail(`--print=native-link-info is a build option\n\n${USAGE}`); + } + if (printNativeLinkInfo && values.emit !== undefined && values.emit !== "obj") { + fail(`--print=native-link-info requires --emit=obj\n\n${USAGE}`); + } + if (externalTypeArgs.length > 0 && command !== "coverage") { + fail(`--external-types is a coverage-only option\n\n${USAGE}`); + } + const externalTypes: Record = Object.create(null) as Record; + for (const mapping of externalTypeArgs) { + const equals = mapping.indexOf("="); + if (equals <= 0 || equals === mapping.length - 1) { + fail(`invalid --external-types mapping ${JSON.stringify(mapping)} (expected )`); + } + const specifier = mapping.slice(0, equals).trim(); + const declarationArg = mapping.slice(equals + 1).trim(); + if (!isExactExternalTypeSpecifier(specifier)) { + fail(`invalid --external-types specifier ${JSON.stringify(specifier)} (expected an exact bare package specifier)`); + } + if (!/\.d\.(?:ts|mts|cts)$/.test(declarationArg)) { + fail(`invalid --external-types declaration ${JSON.stringify(declarationArg)} (expected a .d.ts, .d.mts, or .d.cts file)`); + } + if (externalTypes[specifier] !== undefined) { + fail(`duplicate --external-types mapping for ${JSON.stringify(specifier)}`); + } + const declarationPath = resolve(declarationArg); + try { + if (!statSync(declarationPath).isFile()) throw new Error("not a file"); + } catch { + fail(`--external-types declaration does not name a readable file: ${declarationPath}`); + } + externalTypes[specifier] = declarationPath; + } + const ffiProfilePath = values.ffi !== undefined ? resolve(values.ffi) : undefined; + if (values.backend !== undefined && values.backend !== "llvm") { + fail(`unknown backend "${values.backend}" (supported: llvm)\n\n${USAGE}`); + } + const optimization = values.optimization; + if (optimization !== undefined && optimization !== "release" && optimization !== "dev") { + fail(`unknown optimization "${optimization}" (supported: release, dev)\n\n${USAGE}`); + } + const windowsSubsystem = values["windows-subsystem"]; + if (windowsSubsystem !== undefined && windowsSubsystem !== "console" && windowsSubsystem !== "gui") { + fail(`unknown Windows subsystem "${windowsSubsystem}" (supported: console, gui)\n\n${USAGE}`); + } + if (windowsSubsystem !== undefined && command === "coverage") { + fail(`--windows-subsystem is only supported for executable builds\n\n${USAGE}`); + } + const output = command === "coverage" + ? null + : resolveOutputOptions(command, { + ...(values.emit === undefined && !printNativeLinkInfo + ? {} + : { emit: values.emit ?? "obj" }), + emitIr: values["emit-ir"], + ...(values.backend === undefined ? {} : { backend: values.backend }), + keepLlvm: values["keep-llvm"], + sanitize: values.sanitize, + ...(values.optimization === undefined ? {} : { optimization: values.optimization }), + strip: values.strip, + ...(windowsSubsystem === undefined ? {} : { windowsSubsystem }), + ...(values.ffi === undefined ? {} : { ffi: values.ffi }), + }); + if (output !== null && !output.ok) fail(`${output.message}\n\n${USAGE}`); + const backend = output === null ? undefined : output.backend; + + // --npm-static: repeatable and comma-splittable; the literal "auto" + // switches to eligibility-based detection (mixing "auto" with names + // is rejected — the shapes answer different questions). + const npmStaticRaw = (values["npm-static"] ?? []).flatMap((v) => v.split(",")).map((v) => v.trim()).filter((v) => v !== ""); + let npmStatic: string[] | "auto" | undefined; + if (npmStaticRaw.includes("auto")) { + if (npmStaticRaw.length > 1) fail(`--npm-static auto cannot be combined with package names\n\n${USAGE}`); + npmStatic = "auto"; + } else if (npmStaticRaw.length > 0) { + npmStatic = npmStaticRaw; + } + + // --provenance-sources resolves BEFORE the program loads (tsgo needs the + // source "paths" at creation): attestations and source trees fetch (or + // ride the content-addressed cache / the offline manifest), the registry + // installs, and every fallback prints as a note — never a failure. + const provenance = values["provenance-sources"] ? await host.resolveProvenanceSources(input) : null; + if (provenance !== null) { + setProvenanceSources(provenance); + for (const pkg of provenance.packages) { + process.stderr.write( + `provenance: ${pkg.name}@${pkg.version} ← ${pkg.repo.replace(/^git\+/, "")} @ ${pkg.commit.slice(0, 12)} (source compiles statically)\n`, + ); + } + for (const note of provenance.notes) process.stderr.write(`provenance: ${note}\n`); + } + + if (command === "coverage") { + const { coverage, sourceTexts } = await host.analyze(input, { + dynamic: values.dynamic, + ...(npmStatic !== undefined ? { npmStatic } : {}), + ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), + ...(Object.keys(externalTypes).length > 0 ? { externalTypes } : {}), + }); + const color = process.stdout.isTTY ?? false; + process.stdout.write(renderCoverage(coverage, { color, sourceTexts }) + "\n"); + return coverage.preflightFailed ? 1 : 0; + } + + if (output === null || !output.ok) throw new Error("internal output-option state"); + if (windowsSubsystem !== undefined && host.sourceTargetPlatform() !== "win32") { + fail(`--windows-subsystem requires a Windows executable target\n\n${USAGE}`); + } + const { outDir, outPath } = selectOutputPaths(input, output.cliOutputKind, values.out, + output.cliOutputKind === "exe" ? host.sourceTargetPlatform() : undefined); + + let nativeLinkInfo: object | undefined; + const build = async (): Promise => { + const result = await host.compile(input, { + outPath, + outDir, + outputKind: output.outputKind, + emitIr: output.emitIr, + sanitize: values.sanitize, + dynamic: values.dynamic, + ...(backend !== undefined ? { backend } : {}), + ...(optimization !== undefined ? { optimization } : {}), + ...(values.strip ? { strip: true } : {}), + ...(windowsSubsystem !== undefined ? { windowsSubsystem } : {}), + ...(npmStatic !== undefined ? { npmStatic } : {}), + ...(ffiProfilePath !== undefined ? { ffiProfilePath } : {}), + ...(printNativeLinkInfo ? { nativeLinkInfo: true } : {}), + }); + if (!result.ok) { + const color = process.stderr.isTTY ?? false; + process.stderr.write(renderDiagnostics(result.diagnostics, result.sourceTexts, { color }) + "\n"); + const n = result.diagnostics.length; + process.stderr.write(`\n${n} error${n === 1 ? "" : "s"}.\n`); + throw new CliExit(1); + } + if (result.artifact.kind === "exe") { + if (!values["keep-llvm"]) rmSync(result.artifact.translationUnitPath, { force: true }); + } else if (result.artifact.kind === "obj") { + nativeLinkInfo = result.artifact.nativeLinkInfo; + } + return result.artifact.path; + }; + + const binary = await build(); + + if (command === "run") { + return host.run(binary); + } + + if (printNativeLinkInfo) { + if (nativeLinkInfo === undefined) throw new Error("internal native-link-info state"); + // Keep stdout pure JSON for tooling; the ordinary artifact path is in + // program.object inside the document. + process.stdout.write(`${JSON.stringify(nativeLinkInfo, null, 2)}\n`); + } else { + process.stdout.write(`${binary}\n`); + } + return 0; +} + +/** Both installed and seed commands use this argument and diagnostic contract. */ +export async function runCli(args: string[], host: CliHost): Promise { + try { return await main(args, host); } + catch (err) { + if (err instanceof Error && err.name === "CliExit") return Number(err.message.slice(5)); + throw err; + } +} diff --git a/packages/compiler/src/cli/host.ts b/packages/compiler/src/cli/host.ts new file mode 100644 index 00000000..f2cd6c8b --- /dev/null +++ b/packages/compiler/src/cli/host.ts @@ -0,0 +1,20 @@ +import type { AnalyzeOptions, AnalyzeResult, CompileLibraryOptions, CompileLibraryResult, CompileRequestOptions, CompileRequestResult } from "../compile-types.js"; +import type { ProvenanceSources } from "../frontend/provenance-registry.js"; + +export type NativeCacheWarmProfile = "runtime" | "tls" | "dynamic"; + +/** Only host operations vary between the seed and installed compiler. */ +export interface CliHost { + version: () => string; + sourceTargetPlatform: () => string; + analyze: (entry: string, options: AnalyzeOptions) => Promise; + compile: (entry: string, options: CompileRequestOptions) => Promise; + compileLibrary: (options: CompileLibraryOptions) => Promise; + resolveProvenanceSources: (entry: string) => Promise; + warmNativeCaches: (options: { + optimization?: "release" | "dev"; + sanitize: boolean; + profiles?: NativeCacheWarmProfile[]; + }) => Promise<{ cacheRoot: string; profiles: { profile: NativeCacheWarmProfile; elapsedMs: number }[] }>; + run: (binary: string) => Promise; +} diff --git a/packages/compiler/src/cli/output-options.ts b/packages/compiler/src/cli/output-options.ts new file mode 100644 index 00000000..9dec4c91 --- /dev/null +++ b/packages/compiler/src/cli/output-options.ts @@ -0,0 +1,91 @@ +import type { CompileOutputKind } from "../compile-types.js"; +import type { CliOutputKind } from "./paths.js"; + +export interface OutputOptionValues { + emit?: string; + emitIr: boolean; + backend?: string; + keepLlvm: boolean; + sanitize: boolean; + optimization?: string; + strip?: boolean; + windowsSubsystem?: string; + ffi?: string; +} + +export type OutputOptionResolution = + | { + ok: true; + outputKind: CompileOutputKind; + cliOutputKind: CliOutputKind; + backend?: "llvm"; + emitIr: boolean; + deprecateEmitIr: boolean; + } + | { ok: false; message: string }; + +const SOURCE_KINDS = new Set(["ir", "llvm"]); +const NATIVE_ARTIFACT_KINDS = new Set(["asm", "obj"]); + +/** Pure compatibility/validation matrix for build/run output selection. */ +export function resolveOutputOptions( + command: "build" | "run", + values: OutputOptionValues, +): OutputOptionResolution { + if (values.backend !== undefined && values.backend !== "llvm") { + return { ok: false, message: `unknown backend "${values.backend}" (supported: llvm)` }; + } + const backend = values.backend as "llvm" | undefined; + const rawEmit = values.emit; + if ( + rawEmit !== undefined && rawEmit !== "ir" && rawEmit !== "llvm" && + rawEmit !== "asm" && rawEmit !== "obj" && rawEmit !== "exe" + ) { + return { + ok: false, + message: `unknown emit kind "${rawEmit}" (supported: ir, llvm, asm, obj, exe)`, + }; + } + const emit = (rawEmit ?? "exe") as CliOutputKind; + if (command === "run" && emit !== "exe") { + return { ok: false, message: `scriptc run requires --emit=exe` }; + } + if (values.emitIr && rawEmit !== undefined && emit !== "ir" && emit !== "exe") { + return { ok: false, message: `--emit-ir cannot be combined with --emit=${emit}; use --emit=ir` }; + } + if (values.emitIr && emit === "ir") { + return { ok: false, message: `--emit-ir and --emit=ir select the same output; use --emit=ir` }; + } + if (values.windowsSubsystem !== undefined && emit !== "exe") { + return { ok: false, message: `--windows-subsystem is only supported with --emit=exe` }; + } + if (values.strip && emit !== "exe") { + return { ok: false, message: `--strip is only supported with --emit=exe` }; + } + if (emit === "ir" && backend !== undefined) { + return { ok: false, message: `--emit=ir cannot be combined with --backend; IR is emitted before backend selection` }; + } + if (SOURCE_KINDS.has(emit)) { + if (!values.keepLlvm) { + return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; + } + if (values.sanitize) { + return { ok: false, message: `--sanitize is only meaningful with --emit=exe` }; + } + if (values.optimization !== undefined) { + return { ok: false, message: `--optimization is only meaningful with --emit=exe` }; + } + } + if (NATIVE_ARTIFACT_KINDS.has(emit) && !values.keepLlvm) { + return { ok: false, message: `--no-keep-llvm is only meaningful with --emit=exe` }; + } + const outputKind = emit as CompileOutputKind; + return { + ok: true, + outputKind, + cliOutputKind: emit, + ...(emit === "ir" && backend === undefined ? {} : { backend: "llvm" as const }), + emitIr: values.emitIr && emit === "exe", + deprecateEmitIr: values.emitIr, + }; +} diff --git a/packages/compiler/src/cli/paths.ts b/packages/compiler/src/cli/paths.ts new file mode 100644 index 00000000..3ea4c664 --- /dev/null +++ b/packages/compiler/src/cli/paths.ts @@ -0,0 +1,61 @@ +import { basename, dirname, join, resolve } from "node:path"; +import { configuredTargetPlatform as sourceTargetPlatform } from "../backend/target-platform.js"; +export { wasiEnvironment, wasiPreopens } from "./wasi-paths.js"; + +export type CliOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; + +const POSIX_SUFFIXES: Record = { + ir: ".ir.json", + llvm: ".ll", + asm: ".s", + obj: ".o", + exe: "", +}; + +const WINDOWS_SUFFIXES: Record = { + ...POSIX_SUFFIXES, + asm: ".asm", + obj: ".obj", + exe: ".exe", +}; + +export function defaultOutputName( + stem: string, + kind: CliOutputKind, + platform?: string, +): string { + const selectedPlatform = platform ?? (kind === "exe" ? sourceTargetPlatform() : process.platform); + if (kind === "exe" && selectedPlatform === "wasi") return `${stem}.wasm`; + return `${stem}${(selectedPlatform === "win32" ? WINDOWS_SUFFIXES : POSIX_SUFFIXES)[kind]}`; +} + +export interface OutputPaths { + outDir: string; + outPath: string; +} + +/** One authority for explicit and default primary artifact paths. */ +export function selectOutputPaths( + input: string, + kind: CliOutputKind, + explicitOut?: string, + platform?: string, +): OutputPaths { + const absoluteInput = resolve(input); + const outDir = explicitOut === undefined + ? join(dirname(absoluteInput), ".scriptc") + : dirname(resolve(explicitOut)); + const stem = basename(absoluteInput).replace(/\.(ts|mts|cts|js|mjs|cjs|c|ll)$/, ""); + return { + outDir, + outPath: explicitOut === undefined + ? join(outDir, defaultOutputName(stem, kind, platform)) + : resolve(explicitOut), + }; +} + +/** Default executable filename for the build target. Explicit --out paths + * stay exact; only scriptc's generated default needs the Windows PE suffix. */ +export function defaultExecutableName(stem: string, platform: string = sourceTargetPlatform()): string { + return defaultOutputName(stem, "exe", platform); +} diff --git a/packages/compiler/src/cli/usage.ts b/packages/compiler/src/cli/usage.ts new file mode 100644 index 00000000..062e2653 --- /dev/null +++ b/packages/compiler/src/cli/usage.ts @@ -0,0 +1,86 @@ +export const USAGE = `scriptc — TypeScript/JavaScript to native and WebAssembly executables (experimental) + +Usage: + scriptc build [options] compile to an executable or source artifact + scriptc run [options] compile and run + scriptc coverage how much compiles statically, and why not + scriptc coverage --dynamic what a --dynamic build compiles, and what still blocks it + scriptc coverage --external-types + type-resolve an embedder-provided module for analysis + scriptc build --lib --profile library mode: compile the profile's entry + module to a linkable static archive + (.lib.a), or a Wasm reactor + (.wasm) on wasm32-wasi, + exporting the profile symbols; a profile + with a sidecar section also gets the + contract sidecar JSON beside the archive + scriptc cache warm [runtime|tls|dynamic…] prebuild expensive native cache families + for the current compiler/SDK/target + +Options: + -o, --out primary output path (default: .scriptc/) + --emit primary output: ir, llvm, asm, obj, or exe + (default: exe). asm/obj use the matching platform helper + --print print machine-readable metadata instead of the output path + (native-link-info implies --emit=obj and never links) + --backend code generator (llvm) + --optimization + native optimization posture (default: release/-O2). dev + uses -O0, source breakpoints, and cached LLVM object shards; + macOS executable builds also produce an adjacent .dSYM + --strip remove symbol/debug payload from the linked executable + for smaller builds (opt in; --emit=exe only) + --windows-subsystem + Windows executable subsystem (default: console). gui + prevents Windows from opening a console window + --keep-llvm keep generated LLVM IR beside the executable (default) + --no-keep-llvm delete generated LLVM IR after compiling + --emit-ir also write IR beside an executable or library archive; + deprecated for executables: use --emit=ir for primary IR + --sanitize build with ASan + runtime RC audit + --dynamic embed the dynamic engine (adds ~620KB; static stays the default) + --ffi bind signature-only TypeScript declarations to native + C symbols and link the manifest's archives/libraries + --npm-static + compile the named npm packages' shipped JS statically as + program modules (repeatable; "auto" opts in every eligible + direct import: own .d.ts, unminified JS, no build-transform + markers). A package preflight refuses falls back to the + island (--dynamic) with a coverage-report note — opt-in, + experimental + --provenance-sources + EXPERIMENTAL: compile npm dependencies from their + provenance-attested SOURCE (fetched at the attested + commit) as static program modules; packages without a + usable attestation keep the island path (a note, never + a failure) + --external-types + coverage only: map an exact bare module specifier to a + local declaration file. The declaration supplies types + for analysis; the host module remains an explicit + external-boundary blocker (repeatable) + -h, --help show this help + -v, --version print the version +`; + +export const CLI_OPTIONS = { + out: { type: "string", short: "o" }, + emit: { type: "string" }, + print: { type: "string" }, + backend: { type: "string" }, + optimization: { type: "string" }, + strip: { type: "boolean", default: false }, + "windows-subsystem": { type: "string" }, + "keep-llvm": { type: "boolean", default: true }, + "emit-ir": { type: "boolean", default: false }, + sanitize: { type: "boolean", default: false }, + dynamic: { type: "boolean", default: false }, + ffi: { type: "string" }, + "npm-static": { type: "string", multiple: true }, + "provenance-sources": { type: "boolean", default: false }, + "external-types": { type: "string", multiple: true }, + lib: { type: "boolean", default: false }, + profile: { type: "string" }, + help: { type: "boolean", short: "h", default: false }, + version: { type: "boolean", short: "v", default: false }, +} as const; diff --git a/packages/compiler/src/cli/wasi-paths.ts b/packages/compiler/src/cli/wasi-paths.ts new file mode 100644 index 00000000..07108936 --- /dev/null +++ b/packages/compiler/src/cli/wasi-paths.ts @@ -0,0 +1,45 @@ +import { tmpdir } from "node:os"; +import { wasiGuestPath } from "../wasi-paths.js"; + +/** Host paths exposed by `scriptc run` to a WASI Preview 1 module. Guest + * `/tmp` maps to the host's real platform temp directory instead of assuming + * the POSIX spelling exists (notably false on Windows). */ +export function wasiPreopens( + cwd: string = process.cwd(), + hostTmp: string = tmpdir(), +): Record { + return { "/": cwd, "/tmp": hostTmp }; +} + +/** Environment inherited by a WASI module. Host-absolute directory values + * must not claim paths outside the guest namespace: `/` is the module's + * capability root/home/cwd and `/tmp` is its writable temporary directory. */ +export function wasiEnvironment( + env: NodeJS.ProcessEnv = process.env, + cwd: string = process.cwd(), + hostTmp: string = tmpdir(), +): Record { + const guest: Record = {}; + for (const [name, value] of Object.entries(env)) { + if (value !== undefined) guest[name] = value; + } + + guest["PWD"] = "/"; + guest["HOME"] = "/"; + guest["TMPDIR"] = "/tmp"; + if (guest["USERPROFILE"] !== undefined) guest["USERPROFILE"] = "/"; + if (guest["TMP"] !== undefined) guest["TMP"] = "/tmp"; + if (guest["TEMP"] !== undefined) guest["TEMP"] = "/tmp"; + + // These optional shell/package-manager paths retain their meaning only + // when they fall under a capability the runner actually exposes. + for (const key of ["OLDPWD", "INIT_CWD"]) { + const value = guest[key]; + if (value === undefined) continue; + const mapped = wasiGuestPath(value, cwd, hostTmp); + if (mapped === null) delete guest[key]; + else guest[key] = mapped; + } + + return guest; +} diff --git a/packages/compiler/src/cli/wasi-runner.ts b/packages/compiler/src/cli/wasi-runner.ts new file mode 100644 index 00000000..6fb88bc8 --- /dev/null +++ b/packages/compiler/src/cli/wasi-runner.ts @@ -0,0 +1,39 @@ +#!/usr/bin/env node +/* The subprocess host for `scriptc run` on wasm32-wasi. Keeping the WASI + * instance outside the CLI process preserves native run's exit isolation: + * process.exit(), traps, and signals cannot take the compiler process down. + */ +import { readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { WASI } from "node:wasi"; +import { wasiGuestPath } from "../wasi-paths.js"; +import { wasiEnvironment, wasiPreopens } from "./wasi-paths.js"; + +type WasiInstance = Parameters[0]; +declare const WebAssembly: { + instantiate( + bytes: Uint8Array, + imports: ReturnType, + ): Promise<{ instance: WasiInstance }>; +}; + +const binary = process.argv[2]; +if (binary === undefined) throw new Error("scriptc WASI runner needs a module path"); +const cwd = process.cwd(); +const hostTmp = tmpdir(); + +const wasi = new WASI({ + version: "preview1", + args: [wasiGuestPath(binary, cwd, hostTmp) ?? binary], + env: wasiEnvironment(process.env, cwd, hostTmp), + // A native scriptc executable inherits access to the caller's filesystem. + // WASI is capability-based, so expose the caller's working tree as `/` + // and the platform's actual temporary directory as guest `/tmp`. + preopens: wasiPreopens(cwd, hostTmp), + returnOnExit: true, +}); +const instantiated = await WebAssembly.instantiate( + await readFile(binary), + wasi.getImportObject(), +); +process.exitCode = wasi.start(instantiated.instance); diff --git a/packages/compiler/src/compile-types.ts b/packages/compiler/src/compile-types.ts new file mode 100644 index 00000000..d39e3a11 --- /dev/null +++ b/packages/compiler/src/compile-types.ts @@ -0,0 +1,157 @@ +import type { WindowsSubsystem } from "./backend/targets.js"; +import type { NativeLinkInfo } from "./backend/native-link-info.js"; +import type { ScrDiagnostic } from "./diagnostics/diagnostic.js"; +import type { CoverageInput } from "./coverage/report.js"; + +export type CompileOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; + +export interface CompileBaseOptions { + /** Primary artifact path. The CLI supplies the output-kind default. */ + outPath: string; + /** Where generated intermediates and compatibility side artifacts land. */ + outDir: string; + /** @deprecated This option no longer controls output cleanup; sibling artifacts are retained. */ + defaultOutputPath?: boolean; + /** Compatibility-only additive IR side artifact for executable builds. + * The CLI's deprecated --emit-ir flag supplies this option. */ + emitIr?: boolean; + sanitize?: boolean; + /** Embed the dynamic-island engine (--dynamic). Off = the static default: + * island constructs are diagnostics and nothing about codegen or linking + * changes. */ + dynamic?: boolean; + /** LLVM is the production code generator. */ + backend?: "llvm"; + /** Native optimization posture. Release is the shipped -O2 default; dev + * uses -O0, source line tables, and stable multi-TU object caching for + * large LLVM programs. Darwin executables include an adjacent .dSYM. */ + optimization?: "release" | "dev"; + /** Remove symbol/debug payload from an executable at link time. */ + strip?: boolean; + /** Windows PE executable subsystem. Console is the default; GUI suppresses + * automatic console-window creation. Only valid for Windows executables. */ + windowsSubsystem?: WindowsSubsystem; + /** --npm-static: package names whose shipped, unminified JS compiles + * STATICALLY as program modules (inference types the bodies; statements + * the lowering cannot prove become runtime fences). "auto" opts in every + * directly-imported package passing the eligibility heuristics (own + * .d.ts, unminified JS, no build-transform markers). A package whose + * preflight refuses marks itself an offender and falls back to the + * island (--dynamic) or the requires-dynamic diagnostic (static builds) + * — never a silent misbuild. Off by default: nothing changes without + * the flag. */ + npmStatic?: readonly string[] | "auto"; + /** Outbound native FFI manifest. Its signature-only TypeScript bindings + * lower to direct C ABI calls. Source outputs retain those declarations; + * archive/system-library inputs join only an executable link. */ + ffiProfilePath?: string; + /** Attach the machine-readable external link recipe to an object result. + * Valid only with outputKind "obj"; it never invokes a linker. */ + nativeLinkInfo?: boolean; +} + +/** Executable compile options. This remains the compatibility type for the + * historical compile() API, whose omitted output kind means executable. */ +export interface CompileOptions extends CompileBaseOptions { + outputKind?: "exe"; + /** Internal validation lane retained for helper-object artifact tests. + * Supported ordinary LLVM executable builds select this path automatically. */ + nativeProgramObject?: boolean; +} + +/** Source-artifact compile options, discriminated by the required kind. */ +export interface CompileSourceOptions extends CompileBaseOptions { + outputKind: Exclude; +} + +/** Internal/dynamic request shape for callers that select the kind at runtime. + * Statically executable/source callers should prefer the narrower interfaces. */ +export interface CompileRequestOptions extends CompileBaseOptions { + outputKind?: CompileOutputKind; + /** Internal validation lane for executable requests. */ + nativeProgramObject?: boolean; +} + +export type CompileArtifact = + | { kind: "ir"; path: string } + | { kind: "llvm"; path: string } + | { kind: "asm"; path: string } + | { kind: "obj"; path: string; nativeLinkInfo?: NativeLinkInfo } + | { + kind: "exe"; + path: string; + translationUnitPath: string; + backend: "llvm"; + + }; + +export type CompileFailure = { + ok: false; + diagnostics: ScrDiagnostic[]; + sourceTexts: Map; +}; + +export type CompileSourceResult = + | { ok: true; artifact: Extract } + | CompileFailure; + +/** Historical executable result shape retained for source compatibility. */ +export type CompileResult = + | { + ok: true; + binaryPath: string; + llvmPath: string; + irPath?: string; + backend: "llvm"; + + } + | CompileFailure; + +export type CompileExecutableResult = + /** The generated LLVM source is retained beside the executable. */ + | (Extract & { + artifact: Extract; + }) + | CompileFailure; + +/** Result union for callers that choose outputKind dynamically. */ +export type CompileRequestResult = CompileSourceResult | CompileExecutableResult; + +export interface CompileLibraryOptions { + profilePath: string; + /** Where the archive and the kept program TU land. */ + outDir: string; + /** Artifact path. Default: .lib.a, or .wasm for wasm32-wasi. */ + outPath?: string; + emitIr?: boolean; + sanitize?: boolean; +} + +export type CompileLibraryResult = + /** `sidecarPath` is present exactly when the profile declares a + * `sidecar` section: the contract JSON written beside the archive by + * the same invocation (ask 2). */ + | { ok: true; archivePath: string; llvmPath: string; backend: "llvm"; irPath?: string; sidecarPath?: string } + | { ok: false; diagnostics: ScrDiagnostic[]; sourceTexts: Map }; + +export interface AnalyzeOptions { + /** Analyze as a --dynamic build (island constructs lower instead of + * producing requires-dynamic diagnostics). */ + dynamic?: boolean; + /** --npm-static (see CompileOptions.npmStatic): the analysis compiles + * opted-in packages' JS as program modules and the coverage report + * carries each package's static/fallback status. */ + npmStatic?: readonly string[] | "auto"; + /** Analyze with the outbound native bindings from this FFI manifest. */ + ffiProfilePath?: string; + /** Coverage-only external host type surfaces: exact bare module + * specifier → local declaration file. The checker uses the declarations + * to analyze project code, but imported runtime values remain explicit + * SC1010 blockers rather than being counted as executable. */ + externalTypes?: Readonly>; +} + +export interface AnalyzeResult { + coverage: CoverageInput; + sourceTexts: Map; +} diff --git a/packages/compiler/src/coverage/report.ts b/packages/compiler/src/coverage/report.ts index 3af5092d..778b8024 100644 --- a/packages/compiler/src/coverage/report.ts +++ b/packages/compiler/src/coverage/report.ts @@ -190,21 +190,21 @@ export function renderCoverage(input: CoverageInput, opts: { color?: boolean; so // surface as SC2030 blockers below; "lazy trap" rows DON'T fail the // build — the binary embeds Node's call-time error and the call throws // at runtime, exactly where Node would have failed. - const builtins = (input.dynamic && input.npmBuiltins) || []; - const traps = (input.dynamic && input.npmLazyTraps) || []; + const builtins = input.dynamic ? input.npmBuiltins ?? [] : []; + const traps = input.dynamic ? input.npmLazyTraps ?? [] : []; if (builtins.length > 0 || traps.length > 0) { out.push( ` ${c(DIM, traps.length > 0 ? "embedded npm code imports Node builtins and unresolved specifiers:" : "embedded npm code imports Node builtins:")}`, ); - const widestB = Math.max( + const widestB = Math.max(...[ ...builtins.map((b) => b.builtin.length), ...traps.map((t) => t.specifier.length), - ); + ]); // Pad the plain words before coloring — escape codes have no width. - const widestS = Math.max( + const widestS = Math.max(...[ ...builtins.map((b) => (b.shimmed ? 7 : b.lazy ? 23 : 11)), ...traps.map(() => 24), - ); + ]); for (const b of builtins) { const status = b.shimmed ? c(GREEN, "shimmed".padEnd(widestS)) @@ -247,7 +247,7 @@ export function renderCoverage(input: CoverageInput, opts: { color?: boolean; so let pFailed = 0; let pIsland = 0; const dir = pkg.dir.endsWith("/") ? pkg.dir : `${pkg.dir}/`; - for (const [file, s] of input.statsByFile ?? []) { + for (const [file, s] of input.statsByFile ?? new Map()) { if (!file.startsWith(dir)) continue; pTotal += s.total; pFailed += s.failed; diff --git a/packages/compiler/src/coverage/surface-manifest.ts b/packages/compiler/src/coverage/surface-manifest.ts index bafa96ba..55cb8e24 100644 --- a/packages/compiler/src/coverage/surface-manifest.ts +++ b/packages/compiler/src/coverage/surface-manifest.ts @@ -187,7 +187,7 @@ export function generateSurfaceManifest(compilerVersion: string): SurfaceManifes ["for-using-of", "for (using ... of ...) over arrays", "each array element is disposed at the end of its iteration, including break and continue paths"], ["finally-abrupt-completions", "abrupt completions through finally", "return, throw, break, continue, and labeled jumps run crossed finally blocks; a finally completion replaces the pending one"], ["spread-arguments", "spread arguments", "non-empty fixed tuples flatten into fixed signatures with evaluate-once ordering; arrays, Sets, and statically represented class iterables spread into typed rest parameters"], - ] as const) { + ] as [string, string, string][]) { add({ id: `syntax.${id}`, kind: "syntax", name, status: "static", note }); } diff --git a/packages/compiler/src/executable/prepare.ts b/packages/compiler/src/executable/prepare.ts new file mode 100644 index 00000000..3c644d97 --- /dev/null +++ b/packages/compiler/src/executable/prepare.ts @@ -0,0 +1,85 @@ +import type { CompileFailure, CompileRequestOptions } from "../compile-types.js"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { checkerPanicDiag, iceDiag, isCheckerPanic, type ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import type { IrModule, SrcLoc } from "../ir/ir.js"; +import { validateModule } from "../ir/validate.js"; +import { moduleWasiUnavailableSurface, targetRefusalDiag } from "../backend/target-diagnostics.js"; +import type { LowerResult, LowerStats } from "../frontend/lowering/lowerer.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import type { CompilationTiming } from "../timing.js"; + +export interface PreparedExecutableModule { + ok: true; + mod: IrModule; + sourceTexts: Map; + stats: LowerStats; +} + +/** Release the parser/checker before native object generation begins. */ +export function prepareExecutableModule( + entryPath: string, + opts: CompileRequestOptions, + ffi: FfiProfile | null, + buildPlatform: string, + createFrontend: FrontendFactory, + timing: CompilationTiming = () => {}, +): PreparedExecutableModule | CompileFailure { + const fe = createFrontend(entryPath, opts.npmStatic); + timing("frontend-load"); + let lowered: LowerResult; + let sourceTexts: Map; + // The frontend (and its tsgo server) is released as soon as lowering + // ends — clang and the link never hold it open. + try { + const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ + ok: false, + diagnostics, + sourceTexts: fe.sourceTexts(), + }); + + if (fe.preflight.length > 0) return fail(fe.preflight); + + try { + lowered = fe.lower({ + dynamic: opts.dynamic ?? false, + targetPlatform: buildPlatform, + ...(ffi !== null ? { ffiImports: ffi.functions } : {}), + }); + timing("lower"); + } catch (e) { + // The last-resort panic fence: an upstream tsgo panic that crossed a + // checker call no statement/collection fence wrapped still becomes a + // clean failed compile (anchored at the entry), never a crashed CLI. + if (!isCheckerPanic(e)) throw e; + return fail([ + checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 }), + ]); + } + if (lowered.module === null) return fail(lowered.diagnostics); + + const validation = validateModule(lowered.module); + timing("ir-validate"); + if (validation.length > 0) { + return fail(validation.map((v) => iceDiag(v.message, v.loc))); + } + if (buildPlatform === "wasi") { + const entryLoc: SrcLoc = { file: entryPath, start: 0, end: 0 }; + if (opts.sanitize) { + return fail([targetRefusalDiag("wasm32-wasi", "--sanitize", entryLoc)]); + } + if (ffi !== null) { + return fail([targetRefusalDiag("wasm32-wasi", "native FFI manifests", entryLoc)]); + } + const unavailable = moduleWasiUnavailableSurface(lowered.module); + if (unavailable !== null) { + return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); + } + } + sourceTexts = fe.sourceTexts(); + } finally { + fe.dispose(); + timing("frontend-dispose"); + } + + return { ok: true, mod: lowered.module!, sourceTexts, stats: lowered.stats }; +} diff --git a/packages/compiler/src/frontend/analysis.ts b/packages/compiler/src/frontend/analysis.ts new file mode 100644 index 00000000..32b92413 --- /dev/null +++ b/packages/compiler/src/frontend/analysis.ts @@ -0,0 +1,90 @@ +import type { AnalyzeOptions, AnalyzeResult } from "../compile-types.js"; +import { loadFfiProfile, type FfiProfile } from "../ffi/ffi-manifest.js"; +import { provenanceSources } from "./provenance-registry.js"; +import type { FrontendFactory } from "./pipeline.js"; + +/** Analysis without codegen: how much of the program compiles statically. + * Unlike compile(), lowering diagnostics are data here, not failure. */ +export function analyzeWithFrontend(entryPath: string, opts: AnalyzeOptions, buildPlatform: string, createFrontend: FrontendFactory): AnalyzeResult { + let ffi: FfiProfile | null = null; + if (opts.ffiProfilePath !== undefined) { + const loaded = loadFfiProfile(opts.ffiProfilePath); + if (!loaded.ok) { + return { + coverage: { + file: entryPath, + dynamic: opts.dynamic ?? false, + stats: { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }, + diagnostics: loaded.diagnostics, + preflightFailed: true, + }, + sourceTexts: new Map(), + }; + } + ffi = loaded.profile; + } + const fe = createFrontend(entryPath, opts.npmStatic, opts.externalTypes); + try { + const emptyStats = { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }; + + const preflight = fe.preflight; + // Import-FORM fences don't stop the analysis: the module graph is still + // computable (a fenced import contributes no edges), the imported + // bindings poison at their use sites, and the fences join the blockers + // list beside statement-level ones — the report shows a statement + // percentage instead of stopping at the import lines. Everything else — + // tsc errors, config incompatibilities, circular imports — still stops + // at preflight (no trustworthy program to lower). Builds are unchanged: + // compile() fails on every preflight diagnostic exactly as before. + const IMPORT_FENCES = new Set(["SC1010", "SC1012", "SC1013", "SC1014", "SC1015"]); + if (preflight.some((d) => !IMPORT_FENCES.has(d.code))) { + return { + coverage: { + file: entryPath, + dynamic: opts.dynamic ?? false, + stats: emptyStats, + diagnostics: preflight, + ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), + preflightFailed: true, + }, + sourceTexts: fe.sourceTexts(), + }; + } + // Coverage is whole-program by design: builds stop at what the entry + // reaches, but the analysis additionally lowers the unreached remainder + // (throwaway) so the report covers everything the source declares — with + // the unreached share in its own group. + const lowered = fe.lower({ + dynamic: opts.dynamic ?? false, + coverage: true, + targetPlatform: buildPlatform, + ...(ffi !== null ? { ffiImports: ffi.functions } : {}), + }); + const provenance = provenanceSources(); + return { + coverage: { + file: entryPath, + dynamic: opts.dynamic ?? false, + stats: lowered.stats, + // The import fences report as blockers alongside the statement-level + // ones (use sites of the fenced bindings emit matching diagnostics, + // which the report groups with these). + diagnostics: [...preflight, ...lowered.diagnostics], + ...(lowered.runtimeFences.length > 0 ? { runtimeFences: lowered.runtimeFences } : {}), + ...(lowered.unreached ? { unreached: lowered.unreached } : {}), + ...(lowered.npmBuiltins ? { npmBuiltins: lowered.npmBuiltins } : {}), + ...(lowered.npmLazyTraps ? { npmLazyTraps: lowered.npmLazyTraps } : {}), + ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), + // --provenance-sources: the per-package attribution inputs (the + // report aggregates statsByFile under each package's source dir). + ...(provenance !== null ? { provenance } : {}), + ...(lowered.statsByFile ? { statsByFile: lowered.statsByFile } : {}), + ...(lowered.provenanceElided ? { provenanceElided: lowered.provenanceElided } : {}), + preflightFailed: false, + }, + sourceTexts: fe.sourceTexts(), + }; + } finally { + fe.dispose(); + } +} diff --git a/packages/compiler/src/frontend/comptime-node.ts b/packages/compiler/src/frontend/comptime-node.ts index 8331f886..ee6bb1c2 100644 --- a/packages/compiler/src/frontend/comptime-node.ts +++ b/packages/compiler/src/frontend/comptime-node.ts @@ -1,10 +1,13 @@ import vm from "node:vm"; -import ts5 from "typescript5"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); /** The Node client's compile-time evaluator. Only source text and the * resulting value cross this boundary; TS5 ASTs never enter the frontend. * Every invocation gets fresh globals and a bounded execution window. */ export function evaluateNodeComptime(source: string, timeoutMs: number): unknown { + const ts5 = require("typescript5") as typeof import("typescript5"); const js = ts5.transpileModule(`(${source})()`, { compilerOptions: { target: ts5.ScriptTarget.ESNext }, }).outputText; diff --git a/packages/compiler/src/frontend/dts-paths.ts b/packages/compiler/src/frontend/dts-paths.ts index 34cbf4b5..79113b56 100644 --- a/packages/compiler/src/frontend/dts-paths.ts +++ b/packages/compiler/src/frontend/dts-paths.ts @@ -1,16 +1,29 @@ import { createRequire } from "node:module"; +import { join } from "node:path"; import { npmPackageNameOf } from "./workspace-registry.js"; import { tsgoPath } from "./ts7/session-path.js"; export { tsgoPath } from "./ts7/session-path.js"; const require = createRequire(import.meta.url); +let installedDeclarationRoot: string | null = null; + +/** A native distribution supplies its relocatable declaration directory. */ +export function setDeclarationRoot(root: string): void { installedDeclarationRoot = root; } + +function declarationPath(name: string): string { + return tsgoPath(installedDeclarationRoot === null + ? name === "scriptc.d.ts" ? require.resolve("@scriptc/compiler/scriptc.d.ts") + : name === "scriptc-overrides.d.ts" ? require.resolve("@scriptc/compiler/scriptc-overrides.d.ts") + : require.resolve("@scriptc/compiler/scriptc-node-fallback.d.ts") + : join(installedDeclarationRoot, name)); +} /** Path of the shipped ambient declarations — the always-shipped CORE * (comptime/__island_eval, setTimeout). Part of EVERY program scriptc * builds, the project-world preflight program included. */ export function ambientDtsPath(): string { - return tsgoPath(require.resolve("@scriptc/compiler/scriptc.d.ts")); + return declarationPath("scriptc.d.ts"); } /** Path of the shipped divergence/precision OVERRIDES (JSON.parse(): @@ -19,7 +32,7 @@ export function ambientDtsPath(): string { * so a project that typechecks under its own tsc never fails preflight over * an override-manufactured error (checkPreflight). */ export function overridesDtsPath(): string { - return tsgoPath(require.resolve("@scriptc/compiler/scriptc-overrides.d.ts")); + return declarationPath("scriptc-overrides.d.ts"); } /** Path of the shipped FALLBACK declarations (console, process, node:fs) — @@ -27,7 +40,7 @@ export function overridesDtsPath(): string { * With @types/node, the project's real Node types stand in and this file * stands down (its declaration forms would collide). */ export function fallbackDtsPath(): string { - return tsgoPath(require.resolve("@scriptc/compiler/scriptc-node-fallback.d.ts")); + return declarationPath("scriptc-node-fallback.d.ts"); } /** True for files belonging to the adopted Node type surface: the diff --git a/packages/compiler/src/frontend/input-tracker.test.ts b/packages/compiler/src/frontend/input-tracker.test.ts index 59f5ef98..085f443d 100644 --- a/packages/compiler/src/frontend/input-tracker.test.ts +++ b/packages/compiler/src/frontend/input-tracker.test.ts @@ -176,3 +176,27 @@ test("failed directory enumeration invalidates when access is restored", async ( await chmod(packages, 0o700); } }); + +test("synchronous tracking restores parents after throws and propagates unstable inputs", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-inputs-")); + scratch.push(dir); + const first = join(dir, "first.ts"); + const second = join(dir, "second.ts"); + await writeFile(first, "export const first = 1;"); + await writeFile(second, "export const second = 2;"); + const parent = new FrontendInputTracker(); + const child = new FrontendInputTracker(); + parent.runSynchronous(() => { + expect(() => child.runSynchronous(() => { + trackedReadFile(first); + markFrontendInputsUnstable(); + throw new Error("interrupted"); + })).toThrow("interrupted"); + trackedReadFile(second); + }); + expect(parent.snapshot().probes.filter((probe) => probe.op === "file").map((probe) => probe.path)).toEqual([first, second]); + expect(parent.snapshot().stable).toBe(false); + const before = parent.snapshot(); + trackedFileExists(join(dir, "outside.ts")); + expect(parent.snapshot()).toEqual(before); +}); diff --git a/packages/compiler/src/frontend/input-tracker.ts b/packages/compiler/src/frontend/input-tracker.ts index 685e6768..ab18b373 100644 --- a/packages/compiler/src/frontend/input-tracker.ts +++ b/packages/compiler/src/frontend/input-tracker.ts @@ -41,8 +41,8 @@ export interface FrontendSemanticMatch { * named artifacts and a generated directory's formerly-missing observation * are excluded. */ export interface FrontendInputExclusions { - outputPaths?: Iterable; - outputDirectories?: Iterable; + outputPaths?: readonly string[]; + outputDirectories?: readonly string[]; } function frontendSourceDigest(text: string): string { @@ -73,6 +73,7 @@ function systemRealpath(path: string): string { } const activeTracker = new AsyncLocalStorage(); +let synchronousTracker: FrontendInputTracker | undefined; export class FrontendInputTracker { private readonly probes = new Map(); @@ -88,6 +89,23 @@ export class FrontendInputTracker { }); } + /** The native frontend is synchronous; its typed result need not cross an + * async-context callback boundary. Nested trackers still propagate probes. */ + runSynchronous(fn: () => T): T { + const parent = synchronousTracker; + // Install the active tracker so filesystem callbacks can record probes. + // eslint-disable-next-line @typescript-eslint/no-this-alias + synchronousTracker = this; + try { return fn(); } + finally { + synchronousTracker = parent; + if (parent !== undefined && parent !== this) { + for (const probe of this.probes.values()) parent.record(probe); + if (!this.stable) parent.markUnstable(); + } + } + } + record(probe: FrontendInputProbe): void { const key = `${probe.op}\0${probe.path}`; const previous = this.probes.get(key); @@ -140,13 +158,13 @@ export class FrontendInputTracker { } function record(probe: FrontendInputProbe): void { - activeTracker.getStore()?.record(probe); + (synchronousTracker ?? activeTracker.getStore())?.record(probe); } /** Mark the active frontend as unsafe to persist because an exact host query * bypassed the tracked filesystem wrappers. No-op outside a frontend run. */ export function markFrontendInputsUnstable(): void { - activeTracker.getStore()?.markUnstable(); + (synchronousTracker ?? activeTracker.getStore())?.markUnstable(); } export function trackedReadFile(path: string): string | null { @@ -411,11 +429,11 @@ export function frontendInputsSemanticallyMatch( /** Pure validator used by the persistent-cache reader before any path probes. */ export function validFrontendInputSnapshot(snapshot: unknown): snapshot is FrontendInputSnapshot { if (snapshot === null || typeof snapshot !== "object") return false; - const candidate = snapshot as Partial; + const candidate = snapshot as { version?: unknown; stable?: unknown; probes?: unknown }; if (candidate.version !== 1 || candidate.stable !== true || !Array.isArray(candidate.probes)) return false; return candidate.probes.every((probe) => { if (probe === null || typeof probe !== "object") return false; - const value = probe as Partial; + const value = probe as Record; if (typeof value.path !== "string" || typeof value.op !== "string") return false; switch (value.op) { case "file": diff --git a/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts b/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts index e2319323..19bd6ab9 100644 --- a/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts +++ b/packages/compiler/src/frontend/lowering/array-indexed-mutation.ts @@ -1,4 +1,4 @@ -import { BOOL, F64, type IrExpr, type IrStmt, type IrType, type SrcLoc } from "../../ir/ir.js"; +import { BOOL, DYN, F64, NULL_T, type IrExpr, type IrStmt, type IrType, type SrcLoc } from "../../ir/ir.js"; import { numLit, varRef } from "../../ir/build.js"; import { typeKey } from "../type-mapper.js"; import type { Lowerer } from "./lowerer.js"; @@ -40,8 +40,12 @@ export function lowerArrayFill( arrType: IrType & { kind: "array" }, loc: SrcLoc, ): IrExpr { - const valueType = writeUndefined ? F64 : value!.type; - const key = "indexed:fill:" + typeKey(arrType.elem) + ":" + typeKey(valueType) + ":" + writeUndefined; + const writeNull = value?.type.kind === "nullT"; + // Unit literals have no parameter ABI. Pass a numeric placeholder and + // synthesize the unit inside the helper's correctly tagged store. + const discardValue = writeUndefined || writeNull; + const valueType = discardValue ? F64 : value!.type; + const key = "indexed:fill:" + typeKey(arrType.elem) + ":" + typeKey(valueType) + ":" + writeUndefined + ":" + writeNull; let name = lowerer.arrHofHelpers.get(key); if (!name) { name = "%arr.fill." + lowerer.arrHofHelpers.size; @@ -51,7 +55,7 @@ export function lowerArrayFill( const i = varRef("i.0", F64, loc); const body: IrStmt[] = writeUndefined ? [{ kind: "arraySetUndefined", arr: a, index: i, loc }] - : [arrayValueStore(lowerer, a, i, varRef("v.0", valueType, loc), arrType.elem, loc)]; + : [arrayValueStore(lowerer, a, i, writeNull ? { kind: "unitLit", unit: "null", type: NULL_T, loc } : varRef("v.0", valueType, loc), arrType.elem, loc)]; lowerer.liftedFns.push({ name, params: [ @@ -88,8 +92,9 @@ export function lowerArrayFill( loc, }); } - const valueArg: IrExpr = value && writeUndefined - ? { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: value, loc }], result: numLit(0, loc), type: F64, loc } + const valueArg: IrExpr = discardValue + ? value === null || value.kind === "unitLit" ? numLit(0, loc) + : { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: lowerer.coerceToExpected(value, DYN), loc }], result: numLit(0, loc), type: F64, loc } : value ?? numLit(0, loc); return { kind: "call", callee: name, args: [receiver, valueArg, start, end], type: arrType, loc }; } diff --git a/packages/compiler/src/frontend/lowering/class-callbacks.ts b/packages/compiler/src/frontend/lowering/class-callbacks.ts new file mode 100644 index 00000000..a7bc525b --- /dev/null +++ b/packages/compiler/src/frontend/lowering/class-callbacks.ts @@ -0,0 +1,62 @@ +import * as ts from "../ts7/adapter.js"; +import { BOOL, DYN, STRING, VOID, isClassOwnEnumerableFieldName, type IrExpr, type IrStmt, type SrcLoc } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import { locOf } from "../program.js"; +import { dynUndefinedExpr, type Lowerer } from "./lowerer.js"; +import { findMethodOn, type ClassInfo } from "./lower-classes.js"; +import { classPropertiesHelper } from "./class-dynamic-dispatch.js"; +import { tryLowerExpression } from "./expressions/try-lower-expression.js"; + +/** Zero-argument notification methods can be replaced by own callbacks. + * Other compiled signatures retain their method-replacement fence. */ +export function isClassCallback(lowerer: Lowerer, info: ClassInfo, name: string): boolean { + if (!isClassOwnEnumerableFieldName(name) || name.startsWith("get:") || name.startsWith("set:")) return false; + if (info.def.runtime || info.builtinError || info.builtinEmitter || info.builtinStream || info.fields.has(name)) return false; + const found = findMethodOn(lowerer, info, name); + return !!found && found.sig.params.length === 0 && found.sig.ret.kind === "void" && !found.sig.abstract; +} + +function callbackBag(lowerer: Lowerer, receiver: IrExpr, loc: SrcLoc): IrExpr { + return { kind: "call", callee: classPropertiesHelper(lowerer, loc).name, + args: [lowerer.coerceToExpected(receiver, DYN)], type: DYN, loc }; +} + +export function lowerClassCallbackAssign(lowerer: Lowerer, expr: ts.BinaryExpression): IrStmt | null { + const access = expr.left; + if (!ts.isPropertyAccessExpression(access) || access.questionDotToken) return null; + const receiver = tryLowerExpression(lowerer, access.expression); + if (!receiver) return null; + const info = receiver.type.kind === "object" ? lowerer.classes.get(receiver.type.className) : undefined; + if (!info || !isClassCallback(lowerer, info, access.name.text)) return null; + const loc = locOf(expr); + return { kind: "exprStmt", expr: { kind: "libCall", fn: "dyn.keySet", args: [callbackBag(lowerer, receiver, loc), + { kind: "strLit", value: access.name.text, type: STRING, loc }, lowerer.lowerExprExpecting(expr.right, DYN)], type: VOID, loc }, loc }; +} + +/** Read the own property before arguments run, retaining callback identity + * and the original native receiver for JavaScript's `this` binding. */ +export function classCallbackValue(lowerer: Lowerer, receiver: IrExpr, name: string, fallback: IrExpr, loc: SrcLoc): IrExpr { + const bag = lowerer.declareHiddenLocal("%callbackBag", DYN); + const value = varRef(bag.id, DYN, loc); + const key: IrExpr = { kind: "strLit", value: name, type: STRING, loc }; + const result: IrExpr = { kind: "ternary", cond: { kind: "libCall", fn: "dyn.hasOwn", args: [value, key], type: BOOL, loc }, + then: { kind: "dynKeyGet", value, key, type: DYN, loc }, else_: lowerer.coerceToExpected(fallback, DYN), type: DYN, loc }; + return { kind: "seqExpr", stmts: [{ kind: "varDecl", localId: bag.id, init: callbackBag(lowerer, receiver, loc), loc }], result, type: DYN, loc }; +} + +export function classCallbackCall(lowerer: Lowerer, receiver: IrExpr, name: string, args: IrExpr[], fallback: IrExpr, loc: SrcLoc): IrExpr { + const bag = lowerer.declareHiddenLocal("%callbackBag", DYN); + const own = lowerer.declareHiddenLocal("%callbackOwn", BOOL); + const callback = lowerer.declareHiddenLocal("%callback", DYN); + const value = varRef(bag.id, DYN, loc); + const key: IrExpr = { kind: "strLit", value: name, type: STRING, loc }; + const call: IrExpr = { kind: "dynCall", callee: varRef(callback.id, DYN, loc), receiver: lowerer.coerceToExpected(receiver, DYN), + calleeName: name, args, type: DYN, loc }; + const result: IrExpr = { kind: "ternary", cond: varRef(own.id, BOOL, loc), then: call, + else_: { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: fallback, loc }], result: dynUndefinedExpr(loc), type: DYN, loc }, type: DYN, loc }; + return { kind: "seqExpr", stmts: [ + { kind: "varDecl", localId: bag.id, init: callbackBag(lowerer, receiver, loc), loc }, + { kind: "varDecl", localId: own.id, init: { kind: "libCall", fn: "dyn.hasOwn", args: [value, key], type: BOOL, loc }, loc }, + { kind: "varDecl", localId: callback.id, init: { kind: "dynKeyGet", value, key, type: DYN, loc }, loc }, + ], result, type: DYN, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/class-descriptors.ts b/packages/compiler/src/frontend/lowering/class-descriptors.ts new file mode 100644 index 00000000..4ed75c85 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/class-descriptors.ts @@ -0,0 +1,70 @@ +import * as ts from "../ts7/adapter.js"; +import { DYN, type IrExpr, type IrStmt, isDynTypedRefType } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import { locOf } from "../program.js"; +import type { Lowerer } from "./lowerer.js"; +import type { ClassInfo } from "./lower-classes.js"; +import { isCompiledPrototypeMember } from "./class-prototypes.js"; +import { classPropertiesHelper } from "./class-dynamic-dispatch.js"; + +function literalName(name: ts.PropertyName): string | null { + return ts.isIdentifier(name) || ts.isStringLiteral(name) ? name.text : null; +} + +function bagOnlyProperty(lowerer: Lowerer, owner: ClassInfo, name: string): boolean { + return !owner.fields.has(name) && !isCompiledPrototypeMember(lowerer, owner, name) && + owner.subclasses.every((child) => bagOnlyProperty(lowerer, child, name)); +} + +/** A named bag property needs no snapshot of unrelated native fields, + * which may contain recursive or otherwise opaque values. */ +export function lowerClassDescriptorRead(lowerer: Lowerer, call: ts.CallExpression, target: IrExpr): IrExpr | null { + if (!isDynTypedRefType(target.type)) return null; + const info = lowerer.classes.get(target.type.className); + const key = call.arguments[1]!; + if (!info || info.def.runtime || info.builtinError || info.builtinEmitter || info.builtinStream || + !ts.isStringLiteral(key) || !bagOnlyProperty(lowerer, info, key.text)) return null; + const loc = locOf(call); + const bag: IrExpr = { kind: "call", callee: classPropertiesHelper(lowerer, loc).name, + args: [lowerer.coerceToExpected(target, DYN)], type: DYN, loc }; + return { kind: "libCall", fn: "dyn.getOwnPropertyDescriptor", args: [bag, lowerer.lowerExprExpecting(key, DYN)], type: DYN, loc }; +} + +/** Native layout fields cannot change descriptors. New named data properties + * live in the instance's shared bag, preserving attributes and identity. */ +export function lowerClassDataDescriptor(lowerer: Lowerer, call: ts.CallExpression, member: string, target: IrExpr): IrExpr | null { + if (!isDynTypedRefType(target.type)) return null; + const info = lowerer.classes.get(target.type.className); + if (!info || info.def.runtime || info.builtinError || info.builtinEmitter || info.builtinStream) return null; + const safeName = (owner: ClassInfo, name: string): boolean => bagOnlyProperty(lowerer, owner, name); + const descriptor = (node: ts.Expression): boolean => ts.isObjectLiteralExpression(node) && + node.properties.every((p) => (ts.isPropertyAssignment(p) || ts.isShorthandPropertyAssignment(p)) && + ["value", "writable", "enumerable", "configurable"].includes(literalName(p.name) ?? "")); + const descriptors = call.arguments[member === "defineProperty" ? 2 : 1]!; + if (member === "defineProperty") { + const key = call.arguments[1]!; + if (!ts.isStringLiteral(key) || !safeName(info, key.text) || !descriptor(descriptors)) return null; + } else { + if (!ts.isObjectLiteralExpression(descriptors) || !descriptors.properties.every((p) => { + if (!ts.isPropertyAssignment(p)) return false; + const name = literalName(p.name); + return name !== null && name !== "__proto__" && safeName(info, name) && descriptor(p.initializer); + })) return null; + } + const loc = locOf(call); + const receiver = lowerer.declareHiddenLocal("%descriptorReceiver", target.type); + const desc = lowerer.declareHiddenLocal("%descriptors", DYN); + const value = varRef(receiver.id, target.type, loc); + const boxed = lowerer.coerceToExpected(value, DYN); + const helper = classPropertiesHelper(lowerer, loc); + const bag: IrExpr = { kind: "call", callee: helper.name, args: [boxed], type: DYN, loc }; + const args = member === "defineProperty" + ? [bag, lowerer.lowerExprExpecting(call.arguments[1]!, DYN), varRef(desc.id, DYN, loc)] + : [bag, varRef(desc.id, DYN, loc)]; + const stmts: IrStmt[] = [ + { kind: "varDecl", localId: receiver.id, init: target, loc }, + { kind: "varDecl", localId: desc.id, init: lowerer.lowerExprExpecting(descriptors, DYN), loc }, + { kind: "exprStmt", expr: { kind: "libCall", fn: member === "defineProperty" ? "dyn.defineProperty" : "dyn.defineProps", args, type: DYN, loc }, loc }, + ]; + return { kind: "seqExpr", stmts, result: value, type: value.type, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts index 0bc9ac3c..1b173cb3 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "vitest"; import { DYN, F64, STRING, VOID, type IrExpr, type IrFunction, type IrStmt } from "../../ir/ir.js"; import { ClassDynamicDispatch } from "./class-dynamic-dispatch.js"; import type { Lowerer } from "./lowerer.js"; +import type { ClassInfo } from "./lower-classes.js"; const loc = { file: "dispatch.ts", start: 0, end: 1 }; const variable = (localId: string): IrExpr => ({ kind: "varRef", localId, type: DYN, loc }); @@ -58,3 +59,40 @@ test("discovers dispatch sites added during a later reachability pass", () => { expect(dispatch.process(lowerer, [boxed, target])).toBe(true); expect(target.body[0]).toMatchObject({ kind: "exprStmt", expr: { kind: "call" } }); }); + +test("known class property bags preserve one owned receiver without boxing", () => { + const { lowerer, liftedFns } = context(); + const type = { kind: "object", className: "Widget" } as const; + const info = { def: { name: "Widget", fields: [] }, fields: new Map(), base: null, subclasses: [] } as unknown as ClassInfo; + lowerer.classes.set("Widget", info); + const receiver: IrExpr = { kind: "call", callee: "makeWidget", args: [], type, loc }; + const target = fn("target", [statement({ kind: "call", callee: "%dyn.class.properties", + args: [{ kind: "dynFrom", value: receiver, type: DYN, loc }], type: DYN, loc })]); + const dispatch = new ClassDynamicDispatch(); + expect(dispatch.process(lowerer, [target])).toBe(true); + expect(target.body).toEqual([statement({ kind: "call", callee: "%class.properties:Widget", args: [receiver], type: DYN, loc })]); + const helper = liftedFns.find((item) => item.name === "%class.properties:Widget")!; + expect(helper.params.map((param) => param.type)).toEqual([type]); + expect(helper.body).toMatchObject([ + { kind: "if", cond: { kind: "dynTest", test: "undefined", value: { kind: "fieldGet", className: "Widget" } }, + then: [{ kind: "fieldSet", obj: { kind: "varRef", type }, value: { kind: "dynObjLit" } }] }, + { kind: "return", value: { kind: "fieldGet", className: "Widget" } }, + ]); + expect(dispatch.process(lowerer, [target, ...liftedFns])).toBe(false); + // Prototype assignments can appear in a later reachable body. Both + // lookup paths must then initialize the same inherited property bag. + info.def.prototypeDataHelper = "%prototype.Widget"; + expect(dispatch.process(lowerer, [target, ...liftedFns])).toBe(true); + expect(helper.body[0]).toMatchObject({ then: [{ value: { kind: "libCall", fn: "dyn.objCreate", + args: [{ kind: "call", callee: "%prototype.Widget" }] } }] }); + expect(dispatch.process(lowerer, [target, ...liftedFns])).toBe(false); +}); + +test("unknown receivers keep checked class property dispatch", () => { + const { lowerer, boxed } = context(); + const expr: IrExpr = { kind: "call", callee: "%dyn.class.properties", args: [variable("value")], type: DYN, loc }; + const target = fn("target", [statement(expr)]); + const body = target.body; + new ClassDynamicDispatch().process(lowerer, [boxed, target]); + expect(target.body).toBe(body); +}); diff --git a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts index a0c333c2..c8586b20 100644 --- a/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts +++ b/packages/compiler/src/frontend/lowering/class-dynamic-dispatch.ts @@ -1,4 +1,4 @@ -import { BOOL, DYN, DYN_CLASS_PROPERTIES as PROPERTY_BAG, STRING, VOID, canConvertToDyn, canDynCheckTo, isClassOwnEnumerableFieldName, isDynTypedRefType, isUnitType, typeEquals, typeKey, type IrExpr, type IrFunction, type IrStmt, type IrType } from "../../ir/ir.js"; +import { BOOL, DYN, DYN_CLASS_PROPERTIES as PROPERTY_BAG, STRING, VOID, canConvertToDyn, canDynCheckTo, isClassOwnEnumerableFieldName, isDynTypedRefType, isUnitType, typeEquals, typeKey, type IrExpr, type IrFunction, type IrStmt, type IrType, type SrcLoc } from "../../ir/ir.js"; import { streamTypedRefEligible } from "../../ir/analysis.js"; import { varRef } from "../../ir/build.js"; import { everyStmtList, transformStmtList } from "../../ir/traverse.js"; @@ -11,14 +11,22 @@ import { GeneratorDynamicDispatch } from "./generator-dynamic-dispatch.js"; import { classMethodValue } from "./class-method-values.js"; import * as ts from "../ts7/adapter.js"; import { SYMBOL_T } from "../../ir/ir.js"; +import { isClassCallback } from "./class-callbacks.js"; type Invoke = Extract; interface Dispatch { source: Invoke; fn: IrFunction; + callback: IrFunction; classes: Set; } +interface PropertyReceiver { + info: ClassInfo; + capsule: ClassInfo; + key: string; +} + interface PropertyDispatch { name: string; write: boolean; @@ -26,11 +34,36 @@ interface PropertyDispatch { classes: Set; } +export function classInstanceOf(lowerer: Lowerer, value: IrExpr, info: ClassInfo, loc: SrcLoc): IrExpr { + const name = `%dyn.class.instanceof:${info.def.name}`; + if (!lowerer.liftedFns.some((fn) => fn.name === name)) lowerer.liftedFns.push({ + name, params: [{ localId: "value", name: "value", type: DYN }], returnType: BOOL, + locals: [{ id: "value", name: "value", type: DYN, mutable: false }], + body: [{ kind: "return", value: { kind: "boolLit", value: false, type: BOOL, loc }, loc }], loc, + }); + return { kind: "call", callee: name, args: [value], type: BOOL, loc }; +} + +export function classPropertiesHelper(lowerer: Lowerer, loc: SrcLoc): IrFunction { + const name = "%dyn.class.properties"; + const existing = lowerer.liftedFns.find((fn) => fn.name === name); + if (existing) return existing; + const helper: IrFunction = { + name, params: [{ localId: "p.0", name: "value", type: DYN }], returnType: DYN, + locals: [{ id: "p.0", name: "value", type: DYN, mutable: false }], + body: [{ kind: "return", value: varRef("p.0", DYN, loc), loc }], loc, + }; + lowerer.liftedFns.push(helper); + return helper; +} + /** Calls on native class capsules keep the instance's compiled methods. The * reachable-body fixed point discovers both the boxed classes and method * names before generating checked native dispatch; ordinary dyn receivers * continue through their existing runtime implementation. */ export class ClassDynamicDispatch { + constructor(private readonly asyncFree = false) {} + private readonly generatorDispatch = new GeneratorDynamicDispatch(); private readonly boxed = new Set(); private readonly dispatches = new Map(); @@ -39,8 +72,10 @@ export class ClassDynamicDispatch { private propertyBag: IrFunction | null = null; private readonly errorStrings = new Map(); private readonly bagClasses = new Set(); - private readonly bagInitializers = new Map>(); + private readonly bagInitializers = new Map[]>(); + private readonly typedPropertyBags = new Map(); private readonly generated = new Set(); + private readonly instanceTests = new Map>(); process(lowerer: Lowerer, functions: readonly IrFunction[]): boolean { let changed = refreshErrorMethodDispatch(lowerer, functions); @@ -68,6 +103,9 @@ export class ClassDynamicDispatch { case "dynFrom": discover(expr.value.type); break; + case "call": + if (expr.callee === "%dyn.class.properties" && expr.args[0]?.kind === "dynFrom" && isDynTypedRefType(expr.args[0].value.type)) rewrite.add(fn); + break; case "dynKeyGet": case "dynInvoke": rewrite.add(fn); @@ -81,26 +119,41 @@ export class ClassDynamicDispatch { }, }); if (this.boxed.size === 0 && rewrite.size === 0) return changed; + for (const fn of functions) { + if (!fn.name.startsWith("%dyn.class.instanceof:")) continue; + const target = fn.name.slice("%dyn.class.instanceof:".length); + let checked = this.instanceTests.get(fn.name); + if (!checked) { checked = new Set(); this.instanceTests.set(fn.name, checked); } + for (const className of this.boxed) { + if (checked.has(className)) continue; + checked.add(className); + const subtype = className === target || lowerer.isSubclassOf(className, target); + if (!subtype && !lowerer.isSubclassOf(target, className)) continue; + const loc = fn.loc; + const value = varRef("value", DYN, loc); + const type: IrType = { kind: "object", className }; + const result: IrExpr = subtype ? { kind: "boolLit", value: true, type: BOOL, loc } + : { kind: "instanceOf", value: { kind: "dynCheck", value, type, loc }, className: target, type: BOOL, loc }; + fn.body.unshift({ kind: "if", cond: { kind: "libCall", fn: "dyn.typedRefIs", args: [value, + { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc }, then: [{ kind: "return", value: result, loc }], else_: null, loc }); + changed = true; + } + } if (!this.propertyBag) { - const loc = functions[0]!.loc; - this.propertyBag = { - name: "%dyn.class.properties", params: [{ localId: "p.0", name: "value", type: DYN }], returnType: DYN, - locals: [{ id: "p.0", name: "value", type: DYN, mutable: false }], - body: [{ kind: "return", value: varRef("p.0", DYN, loc), loc }], loc, - }; + this.propertyBag = classPropertiesHelper(lowerer, functions[0]!.loc); this.generated.add(this.propertyBag); - lowerer.liftedFns.push(this.propertyBag); changed = true; } for (const className of this.boxed) { const info = lowerer.classes.get(className); if (!info || info.builtinEmitter || info.builtinStream || info.builtinError) continue; - const existingInit = this.bagInitializers.get(className); - if (existingInit && existingInit.value.kind === "dynObjLit" && hasClassPrototypeData(info)) { - const prototype = classPrototypeData(lowerer, info, existingInit.loc, existingInit.obj); - if (prototype) { - existingInit.value = { kind: "libCall", fn: "dyn.objCreate", args: [prototype], type: DYN, loc: existingInit.loc }; - changed = true; + for (const existingInit of this.bagInitializers.get(className) ?? []) { + if (existingInit.value.kind === "dynObjLit" && hasClassPrototypeData(info)) { + const prototype = classPrototypeData(lowerer, info, existingInit.loc, existingInit.obj); + if (prototype) { + existingInit.value = { kind: "libCall", fn: "dyn.objCreate", args: [prototype], type: DYN, loc: existingInit.loc }; + changed = true; + } } } if (this.bagClasses.has(className)) continue; @@ -117,7 +170,7 @@ export class ClassDynamicDispatch { ? { kind: "libCall", fn: "dyn.objCreate", args: [prototype], type: DYN, loc } : { kind: "dynObjLit", fields: [], type: DYN, loc }, loc, }; - this.bagInitializers.set(className, initialize); + this.bagInitializers.set(className, [initialize]); this.propertyBag.body.unshift({ kind: "if", cond: { kind: "libCall", fn: "dyn.typedRefIs", args: [value, { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc }, then: [ @@ -174,6 +227,16 @@ export class ClassDynamicDispatch { } return { kind: "call", callee: errorString.name, args: expr.args, type: STRING, loc: expr.loc }; } + if (expr.kind === "call" && expr.callee === this.propertyBag!.name) { + const boxed = expr.args[0]; + if (boxed?.kind === "dynFrom" && isDynTypedRefType(boxed.value.type)) { + const helper = this.typedPropertyBag(lowerer, boxed.value.type, expr.loc); + if (helper) { + changed = true; + return { ...expr, callee: helper.name, args: [boxed.value] }; + } + } + } const computedRead = expr.kind === "dynKeyGet" && expr.key.kind !== "strLit" ? expr : null; const computedWrite = expr.kind === "libCall" && (expr.fn === "dyn.keySetComputed" || expr.fn === "dyn.keySet" && expr.args[1]?.kind !== "strLit") ? expr : null; const computedProbe = expr.kind === "libCall" && ["dyn.hasKeyComputed", "dyn.hasOwnComputed", "dyn.propertyIsEnumerableComputed"].includes(expr.fn) ? expr : null; @@ -271,6 +334,12 @@ export class ClassDynamicDispatch { let dispatch = this.dispatches.get(key); if (!dispatch) { const params = [invoke.recv, ...invoke.args].map((_, i) => ({ localId: `p.${i}`, name: `p${i}`, type: DYN })); + params.splice(1, 0, { localId: "p.callback", name: "callback", type: DYN }); + const callback: IrFunction = { + name: `%dyn.class.callback.${this.dispatches.size}`, params: [params[0]!], returnType: DYN, + locals: [{ id: "p.0", name: "receiver", type: DYN, mutable: false }], + body: [{ kind: "return", value: dynUndefinedExpr(expr.loc), loc: expr.loc }], loc: expr.loc, + }; const helper: IrFunction = { name: `%dyn.class.call.${this.dispatches.size}`, params, returnType: DYN, locals: params.map((p) => ({ id: p.localId, name: p.name, type: DYN, mutable: false })), @@ -289,13 +358,21 @@ export class ClassDynamicDispatch { }, loc }, ], else_: null, loc }); } - dispatch = { source: invoke, fn: helper, classes: new Set() }; + dispatch = { source: invoke, fn: helper, callback, classes: new Set() }; this.dispatches.set(key, dispatch); this.generated.add(helper); - lowerer.liftedFns.push(helper); + this.generated.add(callback); + lowerer.liftedFns.push(helper, callback); changed = true; } - return { kind: "call", callee: dispatch.fn.name, args: [invoke.recv, ...invoke.args], type: DYN, loc: expr.loc }; + // Resolve an own callback before argument effects can replace it. + const local = { id: `%dispatch.receiver.${fn.locals.length}`, name: "receiver", type: DYN, mutable: false }; + fn.locals.push(local); + const receiver = varRef(local.id, DYN, expr.loc); + return { kind: "seqExpr", stmts: [{ kind: "varDecl", localId: local.id, init: invoke.recv, loc: expr.loc }], result: { + kind: "call", callee: dispatch.fn.name, args: [receiver, + { kind: "call", callee: dispatch.callback.name, args: [receiver], type: DYN, loc: expr.loc }, ...invoke.args], type: DYN, loc: expr.loc, + }, type: DYN, loc: expr.loc }; }, }); } @@ -306,6 +383,13 @@ export class ClassDynamicDispatch { const loc = dispatch.source.loc; const type: IrType = { kind: "object", className: info.def.name }; const receiver = varRef("p.0", DYN, loc); + if (isClassCallback(lowerer, info, dispatch.source.method)) { + const bag: IrExpr = { kind: "call", callee: this.propertyBag!.name, args: [receiver], type: DYN, loc }; + dispatch.callback.body.unshift({ kind: "if", cond: { + kind: "libCall", fn: "dyn.typedRefIs", args: [receiver, { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc, + }, then: [{ kind: "return", value: { kind: "libCall", fn: "dyn.getOwnPropertyDescriptor", args: [bag, + lowerer.coerceToExpected({ kind: "strLit", value: dispatch.source.method, type: STRING, loc }, DYN)], type: DYN, loc }, loc }], else_: null, loc }); + } const before = lowerer.diags.length; let branch: IrStmt[]; try { @@ -325,16 +409,19 @@ export class ClassDynamicDispatch { } } for (const dispatch of this.properties.values()) { - for (const info of this.propertyCandidates(lowerer, dispatch.name)) { - if (dispatch.classes.has(info.def.name)) continue; - dispatch.classes.add(info.def.name); + for (const plan of this.propertyReceivers(lowerer)) { + const { info } = plan; + if (!isClassOwnEnumerableFieldName(dispatch.name) || + !info.fields.has(dispatch.name) && !findMethodOn(lowerer, info, `get:${dispatch.name}`) && !findMethodOn(lowerer, info, `set:${dispatch.name}`) && + !findMethodOn(lowerer, info, dispatch.name) && !findGenericMethodOn(lowerer, info, dispatch.name)) continue; + if (dispatch.classes.has(plan.key)) continue; + dispatch.classes.add(plan.key); const loc = dispatch.fn.loc; - const type: IrType = { kind: "object", className: info.def.name }; - const receiver = varRef("p.0", DYN, loc); + const { receiver, condition } = this.propertyReceiver(plan, loc); const before = lowerer.diags.length; let branch: IrStmt[]; try { - branch = this.propertyBody(lowerer, dispatch, info, { kind: "dynCheck", value: receiver, type, loc }); + branch = this.propertyBody(lowerer, dispatch, info, receiver); } catch (error) { if (!(error instanceof PoisonError) || !info.decl) throw error; const fence = lowerer.deferToRuntimeFence(before, info.decl, { kind: "statement" }); @@ -342,23 +429,18 @@ export class ClassDynamicDispatch { branch = [fence]; } dispatch.fn.body.unshift({ - kind: "if", cond: { - kind: "libCall", fn: "dyn.typedRefIs", args: [receiver, { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc, - }, then: branch, else_: null, loc, + kind: "if", cond: condition, then: branch, else_: null, loc, }); changed = true; } } for (const dispatch of this.computed.values()) { - for (const className of this.boxed) { - if (dispatch.classes.has(className)) continue; - const info = lowerer.classes.get(className); - if (!info || info.builtinEmitter || info.builtinStream || info.builtinError) continue; - dispatch.classes.add(className); + for (const plan of this.propertyReceivers(lowerer)) { + if (dispatch.classes.has(plan.key)) continue; + const { info } = plan; + dispatch.classes.add(plan.key); const loc = dispatch.fn.loc; - const type: IrType = { kind: "object", className }; - const value = varRef("p.0", DYN, loc); - const receiver: IrExpr = { kind: "dynCheck", value, type, loc }; + const { receiver, condition } = this.propertyReceiver(plan, loc); const memberBody = (name: string): IrStmt[] => dispatch.probe ? [{ kind: "return", value: { kind: "boolLit", value: info.fields.has(name) || dispatch.probe === "in", type: BOOL, loc }, loc }] : this.propertyBody(lowerer, { ...dispatch, name }, info, receiver); @@ -395,7 +477,7 @@ export class ClassDynamicDispatch { branches.push({ kind: "if", cond: { kind: "dynScalarEq", left: varRef("p.key", DYN, loc), right: symbol, type: BOOL, loc }, then, else_: null, loc }); } if (branches.length) { - dispatch.fn.body.unshift({ kind: "if", cond: { kind: "libCall", fn: "dyn.typedRefIs", args: [value, { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc }, then: [ + dispatch.fn.body.unshift({ kind: "if", cond: condition, then: [ { kind: "if", cond: { kind: "dynTest", test: "symbol", value: varRef("p.key", DYN, loc), type: BOOL, loc }, then: branches, else_: null, loc }, ], else_: null, loc }); dispatch.branchIndex++; @@ -426,15 +508,43 @@ export class ClassDynamicDispatch { kind: "strEq", left: varRef(dispatch.keyLocal, STRING, loc), right: { kind: "strLit", value: name, type: STRING, loc }, negated: false, type: BOOL, loc, }, then: body, else_: null, loc }); } - dispatch.fn.body.splice(dispatch.branchIndex, 0, { kind: "if", cond: { - kind: "libCall", fn: "dyn.typedRefIs", args: [value, { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc, - }, then: branch, else_: null, loc }); + dispatch.fn.body.splice(dispatch.branchIndex, 0, { kind: "if", cond: condition, then: branch, else_: null, loc }); changed = true; } } return changed; } + /** A known native class can access its shared bag directly. Keep the + * receiver as an owned parameter so calls and temporary instances retain + * their ordinary evaluation and lifetime rules without a boxed capsule. */ + private typedPropertyBag(lowerer: Lowerer, type: Extract, loc: SrcLoc): IrFunction | null { + const existing = this.typedPropertyBags.get(type.className); + if (existing) return existing; + const initializers = this.bagInitializers.get(type.className); + if (!initializers) return null; + const receiver = varRef("p.0", type, loc); + const bag: IrExpr = { kind: "fieldGet", obj: receiver, className: type.className, field: PROPERTY_BAG, type: DYN, loc }; + const info = lowerer.classes.get(type.className)!; + const prototype = hasClassPrototypeData(info) ? classPrototypeData(lowerer, info, loc, receiver) : null; + const initialize: Extract = { ...initializers[0]!, obj: receiver, value: prototype + ? { kind: "libCall", fn: "dyn.objCreate", args: [prototype], type: DYN, loc } + : { kind: "dynObjLit", fields: [], type: DYN, loc }, loc }; + const helper: IrFunction = { + name: `%class.properties:${type.className}`, params: [{ localId: "p.0", name: "value", type }], returnType: DYN, + locals: [{ id: "p.0", name: "value", type, mutable: false }], loc, + body: [ + { kind: "if", cond: { kind: "dynTest", test: "undefined", value: bag, type: BOOL, loc }, then: [initialize], else_: null, loc }, + { kind: "return", value: bag, loc }, + ], + }; + initializers.push(initialize); + this.typedPropertyBags.set(type.className, helper); + this.generated.add(helper); + lowerer.liftedFns.push(helper); + return helper; + } + /** The hidden bag is part of the native object layout, so every capsule * shares it and normal class tracing/disposal owns its values. Insert it * at the same prefix offset throughout a hierarchy, including classes @@ -454,14 +564,34 @@ export class ClassDynamicDispatch { add(root); } - private propertyCandidates(lowerer: Lowerer, name: string): ClassInfo[] { - if (!isClassOwnEnumerableFieldName(name)) return []; - return [...this.boxed].flatMap((className) => { - const info = lowerer.classes.get(className); - if (!info || info.builtinEmitter || info.builtinStream || info.builtinError) return []; - return info.fields.has(name) || findMethodOn(lowerer, info, `get:${name}`) || findMethodOn(lowerer, info, `set:${name}`) || - findMethodOn(lowerer, info, name) || findGenericMethodOn(lowerer, info, name) ? [info] : []; - }); + /** A capsule retains its static type, while its object may be a subclass. + * Preorder insertion lets later branches test the most derived layout first. */ + private propertyReceivers(lowerer: Lowerer): PropertyReceiver[] { + const plans: PropertyReceiver[] = []; + for (const name of this.boxed) { + const capsule = lowerer.classes.get(name); + if (!capsule || capsule.builtinEmitter || capsule.builtinStream || capsule.builtinError) continue; + const visit = (info: ClassInfo): void => { + plans.push({ info, capsule, key: JSON.stringify([name, info.def.name]) }); + for (const child of info.subclasses) visit(child); + }; + visit(capsule); + } + return plans; + } + + private propertyReceiver(plan: PropertyReceiver, loc: SrcLoc): { receiver: IrExpr; condition: IrExpr } { + const value = varRef("p.0", DYN, loc); + const type: IrType = { kind: "object", className: plan.capsule.def.name }; + const checked: IrExpr = { kind: "dynCheck", value, type, loc }; + const matches: IrExpr = { kind: "libCall", fn: "dyn.typedRefIs", args: [value, + { kind: "strLit", value: typeKey(type), type: STRING, loc }], type: BOOL, loc }; + if (plan.info === plan.capsule) return { receiver: checked, condition: matches }; + return { + receiver: { kind: "downcast", value: checked, type: { kind: "object", className: plan.info.def.name }, loc }, + condition: { kind: "ternary", cond: matches, then: { kind: "instanceOf", value: checked, + className: plan.info.def.name, type: BOOL, loc }, else_: { kind: "boolLit", value: false, type: BOOL, loc }, type: BOOL, loc }, + }; } private propertyBody(lowerer: Lowerer, dispatch: PropertyDispatch, info: ClassInfo, receiver: IrExpr): IrStmt[] { @@ -478,6 +608,14 @@ export class ClassDynamicDispatch { return [{ kind: "return", value: errorToStringMethod(lowerer, receiver), loc }]; } if (!field && !accessor && info.decl) { + const declared = findMethodOn(lowerer, info, name); + // Untyped dispatch considers native classes that may never reach + // this lookup. Libraries cannot expose fibers or promises through + // those speculative branches. Direct typed references still reach + // the ordinary graph-wide async_free diagnostic. + if (this.asyncFree && (declared?.sig.gen || declared?.sig.async)) { + return [{ kind: "runtimeFence", code: "SC4005", message: `reading async or generator method '${name}' through an untyped library lookup is not supported`, loc }]; + } const generic = findGenericMethodOn(lowerer, info, name); // A computed lookup has branches for every visible name. Generic // methods need a concrete specialization before they can escape; @@ -538,6 +676,9 @@ export class ClassDynamicDispatch { const { method, loc } = dispatch.source; const fence = (): IrStmt[] => [{ kind: "runtimeFence", code: "SC2020", message: `calling '${method}' on this native class through an untyped value is not supported yet`, loc }]; const methodInfo = findMethodOn(lowerer, info, method); + if (this.asyncFree && (methodInfo?.sig.gen || methodInfo?.sig.async)) { + return [{ kind: "runtimeFence", code: "SC4005", message: `calling async or generator method '${method}' through an untyped library value is not supported`, loc }]; + } if (methodInfo?.declarer.builtinError && method === "toString") { return [{ kind: "return", value: { kind: "dynFrom", value: errorToStringCall(lowerer, receiver), type: DYN, loc }, loc }]; } @@ -605,6 +746,14 @@ export class ClassDynamicDispatch { if (virtual) lowerer.noteVirtualEdge(owner, method); else lowerer.noteEdge(callee); } + if (isClassCallback(lowerer, info, method)) { + const descriptor = varRef("p.callback", DYN, loc); + body.unshift({ kind: "if", cond: { kind: "dynTest", test: "undefined", negated: true, value: descriptor, type: BOOL, loc }, then: [ + { kind: "return", value: { kind: "dynCall", callee: { kind: "dynKeyGet", value: descriptor, + key: { kind: "strLit", value: "value", type: STRING, loc }, type: DYN, loc }, receiver: varRef("p.0", DYN, loc), + args: incoming, calleeName: dispatch.source.calleeName, type: DYN, loc }, loc }, + ], else_: null, loc }); + } return body; } } diff --git a/packages/compiler/src/frontend/lowering/class-instance-constructor.ts b/packages/compiler/src/frontend/lowering/class-instance-constructor.ts new file mode 100644 index 00000000..84ea137e --- /dev/null +++ b/packages/compiler/src/frontend/lowering/class-instance-constructor.ts @@ -0,0 +1,59 @@ +import * as ts from "../ts7/adapter.js"; +import { BOOL, DYN, STRING, VOID, type IrExpr, type IrStmt } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import { locOf } from "../program.js"; +import type { Lowerer } from "./lowerer.js"; +import { classValueRef, upcastTo, type ClassInfo } from "./lower-classes.js"; +import { classPropertiesHelper } from "./class-dynamic-dispatch.js"; + +/** Select the instance's concrete class before evaluating constructor + * arguments. Each branch completes arguments against its own constructor, + * including subclasses with different defaults or parameter counts. */ +export function lowerInstanceConstructorNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null { + let access = expr.expression; + while (ts.isParenthesizedExpression(access)) access = access.expression; + if (!ts.isPropertyAccessExpression(access) || access.name.text !== "constructor" || access.questionDotToken) return null; + const receiver = lowerer.lowerExpr(access.expression); + if (receiver.type.kind !== "object") return null; + const info = lowerer.classes.get(receiver.type.className); + if (!info || info.def.runtime) return null; + const loc = locOf(expr); + const local = lowerer.declareHiddenLocal("%constructorReceiver", receiver.type); + const value = varRef(local.id, receiver.type, loc); + const construct = (current: ClassInfo): IrExpr => { + if (current.fields.has("constructor") || current.generic || current.localClass || current.classDecorators) { + lowerer.unsupported("SC1090", expr, "instance constructors on generic, local, decorated, or constructor-shadowing classes"); + } + classValueRef(lowerer, current, expr); + const result: IrExpr = { + kind: "new", className: current.def.name, + args: lowerer.completeArgs(expr.arguments ?? [], current.ctorParams, loc, expr), + type: { kind: "object", className: current.def.name }, loc, + }; + return upcastTo(lowerer, result, info.def.name); + }; + const visit = (current: ClassInfo): IrExpr => { + let result = construct(current); + for (const child of current.subclasses) result = { + kind: "ternary", cond: { kind: "instanceOf", value, className: child.def.name, type: BOOL, loc }, + then: visit(child), else_: result, type: receiver.type, loc, + }; + return result; + }; + const bag: IrExpr = { kind: "call", callee: classPropertiesHelper(lowerer, loc).name, + args: [lowerer.coerceToExpected(value, DYN)], type: DYN, loc }; + const check = "%class.constructor.check"; + if (!lowerer.liftedFns.some((fn) => fn.name === check)) lowerer.liftedFns.push({ + name: check, params: [{ localId: "bag", name: "bag", type: DYN }], returnType: VOID, + locals: [{ id: "bag", name: "bag", type: DYN, mutable: false }], loc, + body: [{ kind: "if", cond: { kind: "libCall", fn: "dyn.hasOwn", args: [varRef("bag", DYN, loc), + { kind: "strLit", value: "constructor", type: STRING, loc }], type: BOOL, loc }, then: [ + { kind: "runtimeFence", code: "SC1090", message: "construction through a replaced instance constructor is not supported yet", loc }, + ], else_: null, loc }, { kind: "return", value: null, loc }], + }); + const stmts: IrStmt[] = [ + { kind: "varDecl", localId: local.id, init: receiver, loc }, + { kind: "exprStmt", expr: { kind: "call", callee: check, args: [bag], type: VOID, loc }, loc }, + ]; + return { kind: "seqExpr", stmts, result: visit(info), type: receiver.type, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/class-method-values.ts b/packages/compiler/src/frontend/lowering/class-method-values.ts index d8ae9e57..b94e9597 100644 --- a/packages/compiler/src/frontend/lowering/class-method-values.ts +++ b/packages/compiler/src/frontend/lowering/class-method-values.ts @@ -7,6 +7,7 @@ import type { Lowerer } from "./lowerer.js"; import { findGenericMethodOn, findMethodOn, genericOverrideBelow, type ClassInfo } from "./lower-classes.js"; import { funcTypeFromParamShapes, implicitDefaultInstance, type ParamShape } from "./lower-calls.js"; import { errorToStringMethod } from "./error-methods.js"; +import { classCallbackValue, isClassCallback } from "./class-callbacks.js"; /** A method value retains its declaration's identity, not the receiver from * extraction. Its native thunk validates the receiver supplied at call time. */ @@ -18,10 +19,19 @@ export function lowerClassMethodValue(lowerer: Lowerer, expr: ts.PropertyAccessE const value = methodValue(lowerer, expr, info); if (!value) return null; const loc = locOf(expr); + const callback = isClassCallback(lowerer, info, method); + const receiver = lowerer.lowerExpr(expr.expression); + const local = callback ? lowerer.declareHiddenLocal("%callbackReceiver", receiver.type) : null; + const reference = local ? varRef(local.id, receiver.type, loc) : receiver; + const finish = (result: IrExpr): IrExpr => { + if (!local) return result; + const selected = classCallbackValue(lowerer, reference, method, result, loc); + return { kind: "seqExpr", stmts: [{ kind: "varDecl", localId: local.id, init: receiver, loc }], result: selected, type: selected.type, loc }; + }; const overrides = [...lowerer.classes.values()].filter((candidate) => candidate !== info && lowerer.isSubclassOf(candidate.def.name, info.def.name) && candidate.methods.has(method)); if (overrides.length === 0) { - return { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: lowerer.lowerExpr(expr.expression), loc }], result: value, type: value.type, loc }; + return finish({ kind: "seqExpr", stmts: [{ kind: "exprStmt", expr: reference, loc }], result: value, type: value.type, loc }); } // Select the declaration when extracting the value. Calling the value // later must not redispatch the method name on a different receiver. @@ -42,7 +52,7 @@ export function lowerClassMethodValue(lowerer: Lowerer, expr: ts.PropertyAccessE lowerer.liftedFns.push({ name, params: [{ localId: "this.0", name: "this", type: receiverType }], returnType: value.type, locals: [{ id: "this.0", name: "this", type: receiverType, mutable: false }], body, loc }); } - return { kind: "call", callee: name, args: [lowerer.lowerExpr(expr.expression)], type: value.type, loc }; + return finish({ kind: "call", callee: name, args: [reference], type: value.type, loc }); } function methodValue(lowerer: Lowerer, expr: ts.PropertyAccessExpression, info: ClassInfo): IrExpr | null { diff --git a/packages/compiler/src/frontend/lowering/class-static-data.ts b/packages/compiler/src/frontend/lowering/class-static-data.ts new file mode 100644 index 00000000..08c88575 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/class-static-data.ts @@ -0,0 +1,54 @@ +import * as ts from "../ts7/adapter.js"; +import { BOOL, DYN, type IrExpr, type IrFunction, type SrcLoc } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import { isJsSourceFile } from "../program.js"; +import { findStaticOn, findGenericStaticOn, type ClassInfo } from "./lower-classes.js"; +import type { Lowerer } from "./lowerer.js"; + +const constructorMembers = new Set([ + "name", "length", "prototype", "__proto__", "constructor", "call", "apply", "bind", + "arguments", "caller", "toString", "toLocaleString", "valueOf", "hasOwnProperty", + "isPrototypeOf", "propertyIsEnumerable", "__defineGetter__", "__defineSetter__", + "__lookupGetter__", "__lookupSetter__", +]); + +/** Expando data on an exact JS constructor. Declared statics and Function + * members retain their existing lowering; this bag cannot replace them. */ +export function classStaticDataFor(lowerer: Lowerer, info: ClassInfo, name: string, loc: SrcLoc): IrExpr | null { + if (!info.decl || !isJsSourceFile(info.decl.getSourceFile()) || name.startsWith("#") || constructorMembers.has(name)) return null; + if (findStaticOn(lowerer, info, name) || findGenericStaticOn(lowerer, info, name)) return null; + for (let c: ClassInfo | null = info; c; c = c.base) { + if (c.decl?.members.some((m) => m.name && + ts.canHaveModifiers(m) && ts.getModifiers(m)?.some((mod) => mod.kind === ts.SyntaxKind.StaticKeyword) && + (ts.isComputedPropertyName(m.name) || ((ts.isIdentifier(m.name) || ts.isStringLiteral(m.name)) && m.name.text === name)))) return null; + } + return classStaticData(lowerer, info, loc); +} + +function classStaticData(lowerer: Lowerer, info: ClassInfo, loc: SrcLoc): IrExpr | null { + if (info.localClass || info.mixinInstance || info.generic || info.genericInstance || info.classDecorators || info.def.runtime || info.builtinEmitter || info.builtinStream || info.builtinError) return null; + if (info.staticDataHelper === undefined) { + const base = info.base ? classStaticData(lowerer, info.base, loc) : null; + if (info.base && base === null) return null; + const name = `%static.data.${info.def.name}`; + const globalId = `%g.${name}`; + const readyId = `${globalId}.ready`; + info.staticDataHelper = name; + lowerer.globalsList.push({ id: globalId, name: `${info.def.name}.staticData`, type: DYN, mutable: true }); + lowerer.globalsList.push({ id: readyId, name: `${info.def.name}.staticDataReady`, type: BOOL, mutable: true }); + const helper: IrFunction = { + name, params: [], returnType: DYN, locals: [], loc, + body: [ + { kind: "if", cond: { kind: "unary", op: "!", operand: varRef(readyId, BOOL, loc), type: BOOL, loc }, then: [ + { kind: "assign", localId: globalId, value: base + ? { kind: "libCall", fn: "dyn.objCreate", args: [base], type: DYN, loc } + : { kind: "dynObjLit", fields: [], type: DYN, loc }, loc }, + { kind: "assign", localId: readyId, value: { kind: "boolLit", value: true, type: BOOL, loc }, loc }, + ], else_: null, loc }, + { kind: "return", value: varRef(globalId, DYN, loc), loc }, + ], + }; + lowerer.liftedFns.push(helper); + } + return { kind: "call", callee: info.staticDataHelper, args: [], type: DYN, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/containers/bytes.ts b/packages/compiler/src/frontend/lowering/containers/bytes.ts index 169a2f8d..4459d9fa 100644 --- a/packages/compiler/src/frontend/lowering/containers/bytes.ts +++ b/packages/compiler/src/frontend/lowering/containers/bytes.ts @@ -193,23 +193,14 @@ export function lowerBytesNew(lowerer: Lowerer, expr: ts.NewExpression, symbol: return { kind: "bytesNew", source: count, type, loc }; } const src = lowerer.lowerExpr(argNode); - if (src.type.kind === "union" && lowerer.armTag(src.type.unionId, UNDEFINED_T) >= 0) { - const present = lowerer.stripUndefinedArm(src.type); - if (present.kind === "array" && present.elem.kind === "f64") { - const undefTag = lowerer.armTag(src.type.unionId, UNDEFINED_T); - const presentTag = lowerer.armTag(src.type.unionId, present); - if (undefTag >= 0 && presentTag >= 0) { - const empty: IrExpr = { kind: "arrayLit", elems: [], type: present, loc }; - const source: IrExpr = { - kind: "ternary", - cond: { kind: "unionIsTag", unionId: src.type.unionId, tag: undefTag, negated: false, value: src, type: BOOL, loc }, - then: empty, - else_: { kind: "unionNarrow", unionId: src.type.unionId, tag: presentTag, value: src, type: present, loc }, - type: present, - loc, - }; - return { kind: "bytesNew", source, type, loc }; - } + if (src.type.kind === "union") { + const arms = lowerer.unions.get(src.type.unionId)?.arms; + if (arms?.every((arm) => arm.kind === "f64" || arm.kind === "bytes" || + typeEquals(arm, UNDEFINED_T) || (arm.kind === "array" && arm.elem.kind === "f64"))) { + // The checked constructor dispatch preserves each arm's copy or + // length semantics, including undefined -> empty, and evaluates + // an effectful source expression exactly once. + return { kind: "bytesNew", source: lowerer.coerceInto(argNode, src, DYN), type, loc }; } } if ( diff --git a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts index c841c4d8..47c4f576 100644 --- a/packages/compiler/src/frontend/lowering/expressions/object-literals.ts +++ b/packages/compiler/src/frontend/lowering/expressions/object-literals.ts @@ -302,7 +302,9 @@ export function lowerDynObjectLiteral( const lowerValue = (): IrExpr => ts.isMethodDeclaration(prop) ? (isJsSourceFile(expr.getSourceFile()) ? lowerer.lowerLambda(prop) : (lowerer.rejectThisInObjectMethod(prop.body ?? prop), lowerer.lowerLambda(prop))) - : lowerer.lowerExpr(valueExpr as ts.Expression); + : !boxValue && ts.isObjectLiteralExpression(valueExpr) + ? lowerer.lowerExprExpecting(valueExpr, DYN) + : lowerer.lowerExpr(valueExpr as ts.Expression); raw = fenceClosureProbe(lowerer, valueExpr, undefined, lowerValue) ?? lowerValue(); @@ -325,6 +327,11 @@ export function lowerDynObjectLiteral( let v = boxValue ? boxValue(valueExpr as ts.Expression, raw) : lowerer.coerceToExpected(raw, DYN); + // A JS object's array-valued properties retain the same source array. + // Repeated references must compare equal and mutations remain shared. + if (!boxValue && isJsSourceFile(prop.getSourceFile()) && v.kind === "dynFrom" && v.value.type.kind === "array") { + v = { ...v, liveRef: true }; + } if (v.type.kind !== "dyn") { const convDiagsBefore = lowerer.diags.length; try { diff --git a/packages/compiler/src/frontend/lowering/lower-builtins.ts b/packages/compiler/src/frontend/lowering/lower-builtins.ts index 168e8415..2e7f4754 100644 --- a/packages/compiler/src/frontend/lowering/lower-builtins.ts +++ b/packages/compiler/src/frontend/lowering/lower-builtins.ts @@ -2303,6 +2303,12 @@ function lowerFsSyncBufferWindow( } if (bi.module === "fs" && (bi.member === "writeFileSync" || bi.member === "appendFileSync") && expr.arguments.length === 2) { const dataIr = lowerer.mapTypeOf(lowerer.typeOf(expr.arguments[1]!)); + if (dataIr?.kind === "union") { + const arms = lowerer.unions.get(dataIr.unionId)?.arms; + if (arms && arms.every((arm) => arm.kind === "string" || (arm.kind === "bytes" && arm.elem === "u8"))) { + return lowerStringOrBytesWrite(lowerer, expr, dataIr, arms, bi.member === "appendFileSync"); + } + } if (dataIr?.kind === "bytes") { if (dataIr.elem !== "u8") { lowerer.noLowering( @@ -4995,17 +5001,49 @@ export function lowerForkCall(lowerer: Lowerer, expr: ts.CallExpression, loc: Sr ); } const node: IrExpr = { kind: "jsonStringify", value, type: STRING, loc }; - if (indent !== "") { - // The compile-time-resolved indent rides as an extra property (the - // node shape in ir/ir.ts is unchanged); the backend re-indents - // the compact serializer output with Node's gap algorithm. - (node as { indent?: string }).indent = indent; - } + if (indent !== "") node.indent = indent; return node; } return null; // unknown members are tsc errors before lowering } +/** A valid string/byte overload must select its runtime entry from the live + * union tag. Coercing the whole argument to the string overload loses bytes. */ +function lowerStringOrBytesWrite(lowerer: Lowerer, call: ts.CallExpression, + dataType: Extract, arms: readonly IrType[], append: boolean): IrExpr { + const loc = locOf(call); + const key = `fs.${append ? "append" : "write"}:${dataType.unionId}`; + let helper = lowerer.widthHelpers.get(key); + if (!helper) { + helper = `%fs.writeData.${lowerer.widthHelpers.size}`; + lowerer.widthHelpers.set(key, helper); + const body: IrStmt[] = []; + for (let tag = 0; tag < arms.length; tag++) { + const arm = arms[tag]!; + const fn: IrLibFn = arm.kind === "bytes" + ? append ? "fs.appendFileSyncBytes" : "fs.writeFileSyncBytes" + : append ? "fs.appendFileSync" : "fs.writeFileSync"; + body.push({ kind: "if", loc, + cond: { kind: "unionIsTag", unionId: dataType.unionId, tag, negated: false, + value: varRef("data.0", dataType, loc), type: BOOL, loc }, + then: [ + { kind: "exprStmt", expr: { kind: "libCall", fn, args: [varRef("path.0", STRING, loc), + { kind: "unionNarrow", unionId: dataType.unionId, tag, value: varRef("data.0", dataType, loc), type: arm, loc }], type: VOID, loc }, loc }, + { kind: "return", value: null, loc }, + ], else_: null, + }); + } + body.push({ kind: "return", value: null, loc }); + lowerer.liftedFns.push({ name: helper, returnType: VOID, loc, + params: [{ localId: "path.0", name: "path", type: STRING }, { localId: "data.0", name: "data", type: dataType }], + locals: [{ id: "path.0", name: "path", type: STRING, mutable: false }, { id: "data.0", name: "data", type: dataType, mutable: false }], + body, + }); + } + return { kind: "call", callee: helper, args: [lowerer.lowerExprExpecting(call.arguments[0]!, STRING), + lowerer.lowerExprExpecting(call.arguments[1]!, dataType)], type: VOID, loc }; +} + function lowerOptionalStringifyRoot(lowerer: Lowerer, value: IrExpr, indent: string, loc: SrcLoc): IrExpr | null { const tags = optionalStringTags(lowerer, value.type); if (!tags || value.type.kind !== "union") return null; @@ -5022,7 +5060,7 @@ function lowerOptionalStringifyRoot(lowerer: Lowerer, value: IrExpr, indent: str type: STRING, loc, }; - if (indent !== "") (serialized as { indent?: string }).indent = indent; + if (indent !== "") serialized.indent = indent; const missing = lowerer.wrappedUndefined(resultT, loc); if (!missing) throw new InternalCompilerError("optional JSON.stringify result needs an undefined arm"); lowerer.liftedFns.push({ @@ -6137,7 +6175,11 @@ function lowerOptionalStringSearchParams(lowerer: Lowerer, init: IrExpr, loc: Sr lowerer.noLowering(`${receiverType.kind === "cryptoHash" ? "Hash" : "Hmac"}.update with ${call.arguments.length} arguments`, call, "update(stringOrBuffer[, inputEncoding]) is supported"); } const dataNode = call.arguments[0]!; - const data = lowerer.lowerExpr(dataNode); + const dataType = lowerer.mapTypeOf(lowerer.typeOf(dataNode)); + // Indexed reads can retain optional storage after a nullish fallback + // or narrowing. Use the proven argument type through a checked coercion. + const data = dataType?.kind === "string" || dataType?.kind === "bytes" + ? lowerer.lowerExprExpecting(dataNode, dataType) : lowerer.lowerExpr(dataNode); const prefix = receiverType.kind === "cryptoHash" ? "crypto.hashUpdate" : "crypto.hmacUpdate"; if (data.type.kind === "bytes" && data.type.elem === "u8") { if (call.arguments.length !== 1) { diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 31cd52a9..85042506 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -1,3 +1,4 @@ +import { objectEnumerationReceiver } from "./object-enumeration-receiver.js"; import { InternalCompilerError } from "../../errors.js"; /* Call lowering: the lowerCall dispatch chain, parameter-shape analysis and * argument completion (optional/default/rest, explicit-undefined ≡ omission), @@ -29,7 +30,8 @@ import { lowerAbsenceProbe, lowerPromiseAllTupleCall, lowerPromiseRejectCall, st import { isSafeToDiscard } from "./expressions/evaluation-safety.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { httpClientFnBindingOf, isStreamUndefCallExpr, lowerCompatReqStreamOptionalCall, lowerHttpClientFnCall } from "./lower-server.js"; -import { EMITTER_API_MEMBERS, exactInstanceClassOf, findGenericMethodOn, lowerClassGenericMethodCall, lowerStaticMethodCall, type ClassInfo } from "./lower-classes.js"; +import { EMITTER_API_MEMBERS, exactInstanceClassOf, findGenericMethodOn, lowerClassGenericMethodCall, lowerStaticMethodCall, returnsOnlyThis, type ClassInfo } from "./lower-classes.js"; +import { classCallbackCall, isClassCallback } from "./class-callbacks.js"; import { emitterRooted, lowerEmitterMethodCall } from "./lower-event-emitter.js"; import { lowerConsoleInspectArg, lowerFormatCall } from "./lower-inspect.js"; import { STREAM_API_MEMBERS, lowerStreamMethodCall, lowerStreamModuleCall, lowerStreamStaticCall, streamSidesOf } from "./lower-stream.js"; @@ -42,6 +44,8 @@ import { rejectStaticThis } from "./static-this.js"; import { fenceNodeModuleMutationCall, lowerRequireCacheKeys } from "./lower-node-module.js"; import { defaultAfterUndefined, lowerOptionalArgument, lowerStaticallyUndefinedArgument, positionNumber } from "./optional-arguments.js"; import { fenceSymbolFieldCopy } from "./symbol-fields.js"; +import { lowerClassDataDescriptor, lowerClassDescriptorRead } from "./class-descriptors.js"; +import { classStaticDataFor } from "./class-static-data.js"; export { bodyReadsArguments }; @@ -183,9 +187,12 @@ export interface GenericFnInfo { * Instance methods take `this` (object:) as param 0 and lower * under the declarer's instantiation bindings (generic-class receivers) * MERGED with the method instantiation's own; statics lower as plain - * module functions with the static-method this/super fence. Absent for + * module functions with an exact receiver or the this/super fence. Absent for * top-level functions and object-literal methods. */ member?: { cls: ClassInfo; kind: "method" | "static" }; + /** Direct static calls specialize lexical this for the proven receiver. */ + staticReceiver?: ClassInfo; + receiverSpecializations?: Map; /** Object-literal generic methods (`{ m(x: T) {...} }` and generic * arrow/function-expression properties): lowered as plain module * functions — `this` inside is fenced (rejectThisInObjectMethod) and the @@ -1749,19 +1756,24 @@ export function genericFnOf(lowerer: Lowerer, ident: ts.Identifier): GenericFnIn lowerer.fnStack.push(fnCtx); lowerer.localClassInstantiations.push({ owner: decl, name: inst.name }); try { - // STATIC generic methods: `this`/`super` name the RECEIVER class (a - // dynamic value) — the lowerStaticMethod fence, applied here because - // generic statics have no non-generic lowering pass. Arrow functions - // are transparent (they inherit the method's `this`); this-binding - // function forms are opaque. + // Static specializations bind lexical this only when the caller + // proved its exact receiver. Other instances retain the fence; + // super remains unsupported. Arrows inherit the lexical receiver. if (info.member?.kind === "static" && decl.body) { rejectStaticThis( lowerer, decl.body, (keyword) => `'${keyword}' in static methods (it names the RECEIVER class — a dynamic value; reference the class by name instead)`, + false, info.staticReceiver !== undefined, ); } const params: IrParam[] = []; + const receiverInit: IrStmt[] = []; + if (info.staticReceiver) { + const value = lowerer.classValueRef(info.staticReceiver, decl); + const local = lowerer.declareThis(value.type); + receiverInit.push({ kind: "varDecl", localId: local.id, init: value, loc: locOf(decl) }); + } if (cls && info.member!.kind === "method") { // Instance methods take `this` as param 0, exactly like plain // `%C.method` functions (lowerClassMethodMemberInner). @@ -1773,7 +1785,7 @@ export function genericFnOf(lowerer: Lowerer, ident: ts.Identifier): GenericFnIn // threaded — a default mentioning T resolves like any body expression. const declared = lowerer.declareParams(decl.parameters, inst.params); params.push(...declared.params); - const body = [...declared.prologue]; + const body = [...receiverInit, ...declared.prologue]; const bodyBlock = blockBodyOf(decl); if (bodyBlock) { body.push(...lowerer.lowerStmts(bodyBlock.statements)); @@ -1882,21 +1894,22 @@ export function genericFnOf(lowerer: Lowerer, ident: ts.Identifier): GenericFnIn // The wrap pass: settle every recorded return onto `final`, in place. for (const e of infer.entries) { if (e.stmt.kind !== "return") continue; - const st = e.stmt as IrStmt & { kind: "return"; value: IrExpr | null }; + // Write through the recorded statement. A narrowed local can acquire a + // different record layout and copy it in the native compiler. const diagsBefore = lowerer.diags.length; try { - if (st.value === null || st.value === undefined) { - if (final.kind === "dyn") st.value = dynUndefinedExpr(st.loc); + if (e.stmt.value === null || e.stmt.value === undefined) { + if (final.kind === "dyn") e.stmt.value = dynUndefinedExpr(e.stmt.loc); else if (final.kind === "union") { - const wrapped = lowerer.wrappedUndefined(final, st.loc); + const wrapped = lowerer.wrappedUndefined(final, e.stmt.loc); if (!wrapped) { lowerer.unsupported("SC1090", e.node ?? decl, `bare 'return' in a function whose inferred return type is '${lowerer.fmt(final)}'`); } - st.value = wrapped; + e.stmt.value = wrapped; } // void final: bare return stands as-is - } else if (!typeEquals(st.value.type, final)) { - st.value = lowerer.coerceInto(e.node ?? decl, st.value, final); + } else if (!typeEquals(e.stmt.value.type, final)) { + e.stmt.value = lowerer.coerceInto(e.node ?? decl, e.stmt.value, final); } } catch (err) { if (!(err instanceof PoisonError)) throw err; @@ -1908,7 +1921,7 @@ export function genericFnOf(lowerer: Lowerer, ident: ts.Identifier): GenericFnIn if (ice.length > 0) lowerer.diags.push(...ice); lowerer.runtimeFences.push(...captured.filter((d) => d.code !== "SC9001")); const first = captured.find((d) => d.code !== "SC9001"); - const mutable = st as unknown as Record; + const mutable = e.stmt as unknown as Record; delete mutable["value"]; mutable["kind"] = "runtimeFence"; mutable["code"] = first?.code ?? "SC1090"; @@ -2172,11 +2185,14 @@ function runtimeOptionalHofGenericBinding( /** The implicit-type-parameter slots of a JS function-like: parallel to * decl.parameters, the param SYMBOL where the slot is a bindable - * implicit-any param (identifier-named, no annotation/JSDoc type, not - * rest/optional/defaulted, never written), null elsewhere. Null overall - * when nothing qualifies — the declaration keeps today's path. */ + * implicit param (identifier-named, broad inferred/JSDoc type, not + * rest/optional/defaulted, never written), null elsewhere. Package JS + * methods may also specialize class parameters: JSDoc names a nominal + * class even when the body accepts other objects with the same members. + * Null overall when nothing qualifies — the declaration keeps its ABI. */ export function implicitAnyParamSymbolsOf(lowerer: Lowerer, - decl: ts.FunctionDeclaration | ts.MethodDeclaration | ts.FunctionExpression | ts.ArrowFunction,): (ts.Symbol | null)[] | null { + decl: ts.FunctionDeclaration | ts.MethodDeclaration | ts.FunctionExpression | ts.ArrowFunction, + classParams = false,): (ts.Symbol | null)[] | null { if (!decl.body) return null; if (decl.asteriskToken) return null; if (decl.modifiers?.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword)) return null; @@ -2193,7 +2209,9 @@ function runtimeOptionalHofGenericBinding( const broadFunction = lowerer.checker.typeToString(t) === "Function"; const broadArray = lowerer.checkerAnyArrayType(t); const broadPromise = broadPromiseParam(lowerer, t); - if ((t.flags & ts.TypeFlags.Any) === 0 && !broadFunction && !broadArray && !broadPromise) return null; + const declaredKind = classParams ? lowerer.mapTypeOf(t)?.kind : undefined; + const classParam = classParams && (declaredKind === "object" || declaredKind === "array"); + if ((t.flags & ts.TypeFlags.Any) === 0 && !broadFunction && !broadArray && !broadPromise && !classParam) return null; const sym = lowerer.checker.getSymbolAtLocation(param.name); if (!sym) return null; if (paramWrittenInBody(lowerer, decl.body!, sym, param.name.text)) return null; @@ -2210,6 +2228,11 @@ function runtimeOptionalHofGenericBinding( * selects return INFERENCE with DYN as the recursion pin. */ function implicitDeclaredReturn(lowerer: Lowerer, info: GenericFnInfo): IrType | null { try { + // Static JS helpers can accept a structurally compatible class despite + // narrower JSDoc (for example a Vector2 target documented as Vector3). + // Infer their result from the specialized body as well. + if (info.member?.kind === "static" && info.decl.parameters.some((param, index) => + info.implicitParams?.[index] && lowerer.mapTypeOf(lowerer.typeOf(param.name))?.kind === "object")) return null; const broadCallbackSymbols = new Set( info.decl.parameters.flatMap((param, index) => { const symbol = info.implicitParams?.[index]; @@ -2247,6 +2270,8 @@ function runtimeOptionalHofGenericBinding( if (retTs.flags & ts.TypeFlags.Any) return null; if (retTs.flags & (ts.TypeFlags.Null | ts.TypeFlags.Undefined)) return DYN; if (lowerer.checker.isArrayType(retTs)) { + if (info.decl.parameters.some((param, index) => info.implicitParams?.[index] && + lowerer.mapTypeOf(lowerer.typeOf(param.name))?.kind === "array")) return null; const elem = lowerer.checker.getTypeArguments(retTs as ts.TypeReference)[0]; // An implicit-any parameter poisons an inferred array result to // any[] even when this instance's body produces one concrete @@ -2412,7 +2437,17 @@ function storedImplicitArgumentType(lowerer: Lowerer, arg: ts.Expression): IrTyp const shapes: ParamShape[] = info.decl.parameters.map((param, i) => info.implicitParams![i] ? { type: DYN, mode: "required" as const } : lowerer.paramShape(param), ); - return internImplicitInstance(lowerer, blame, info, shapes, new Map()); + const argTypes = new Map(); + // Newly bindable nominal class parameters must not retain their JSDoc + // receiver type in the checked fallback. Keep existing broad JS inference. + for (let i = 0; i < info.decl.parameters.length; i++) { + const symbol = info.implicitParams?.[i]; + const kind = lowerer.mapTypeOf(lowerer.typeOf(info.decl.parameters[i]!.name))?.kind; + if (symbol && (kind === "object" || kind === "array")) { + argTypes.set(symbol, lowerer.checker.getUnknownType()); + } + } + return internImplicitInstance(lowerer, blame, info, shapes, argTypes); } function internImplicitInstance(lowerer: Lowerer, blame: ts.Node, @@ -6664,6 +6699,12 @@ const inliningPredicates = new Set(); if (ts.isIdentifier(m.name) || ts.isStringLiteralLike(m.name)) own.add(m.name.text); else return null; // computed static names — the answer isn't static } + if (!own.has(key)) { + const loc = locOf(call); + const data = classStaticDataFor(lowerer, info, key, loc); + if (data) return { kind: "libCall", fn: "dyn.hasOwn", args: [data, + { kind: "strLit", value: key, type: STRING, loc }], type: BOOL, loc }; + } return { kind: "boolLit", value: own.has(key), type: BOOL, loc: locOf(call) }; } @@ -6886,6 +6927,12 @@ const inliningPredicates = new Set(); function reconcileOverloadReturn(lowerer: Lowerer, expr: ts.CallExpression | ts.TaggedTemplateExpression, call: IrExpr): IrExpr { const rsig = lowerer.checker.getResolvedSignature(expr); const rdecl = rsig ? lowerer.checker.signatureDeclaration(rsig) : undefined; + // Fluent JS overrides share the base's return ABI, but their result + // still has the receiver's subclass layout at the call site. + if (call.type.kind === "object" && rdecl && ts.isMethodDeclaration(rdecl) && + isJsSourceFile(rdecl.getSourceFile()) && returnsOnlyThis(rdecl)) { + call = lowerer.maybeNarrow(call, expr); + } const resolvedOverload = rsig && rdecl && (ts.isFunctionDeclaration(rdecl) || ts.isMethodDeclaration(rdecl)) && @@ -9419,6 +9466,15 @@ export function lowerPromiseMethodCall(lowerer: Lowerer, call: ts.CallExpression (member === "getOwnPropertyDescriptor" && call.arguments.length === 2)) { if (call.arguments.some((a) => ts.isSpreadElement(a))) return null; let target = tryLowerExpression(lowerer, call.arguments[0]!); + if (target && member === "defineProperty") { + const native = lowerClassDataDescriptor(lowerer, call, member, target); + if (native) return native; + } + if (target && member === "getOwnPropertyDescriptor" && isDynTypedRefType(target.type)) { + const descriptor = lowerClassDescriptorRead(lowerer, call, target); + if (descriptor) return descriptor; + target = lowerer.coerceToExpected(target, DYN); + } // Error.cause owns a live data-property slot shared with its checked // view. Other native fields do not yet synchronize descriptor edits. const keyNode = call.arguments[1]!; @@ -9453,6 +9509,10 @@ export function lowerPromiseMethodCall(lowerer: Lowerer, call: ts.CallExpression if (member === "defineProperties" && call.arguments.length === 2 && !call.arguments.some((a) => ts.isSpreadElement(a))) { let target = tryLowerExpression(lowerer, call.arguments[0]!); + if (target) { + const native = lowerClassDataDescriptor(lowerer, call, member, target); + if (native) return native; + } // A FUNCTION-typed target boxes through the dyn boundary: the // property table lives on the CLOSURE (shared by every box of this // function value), so defining through a fresh box sticks — the @@ -9687,7 +9747,7 @@ export function lowerPromiseMethodCall(lowerer: Lowerer, call: ts.CallExpression const loc = locOf(call); const resultT = lowerer.irTypeOf(call); if (resultT.kind !== "array") lowerer.badType(call, lowerer.typeOf(call)); // defensive - const receiver = lowerer.lowerExpr(argNode); + const receiver = objectEnumerationReceiver(lowerer, lowerer.lowerExpr(argNode), argIr, loc); if (member === "keys") { // The keys walk is shared with for-in (which iterates exactly the // keys Object.keys answers — one construction, one intern key). @@ -10026,7 +10086,9 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I export function lowerRecordFieldCall(lowerer: Lowerer, call: ts.CallExpression, access: ts.PropertyAccessExpression,): IrExpr | null { if (lowerer.chainBlocked(call)) return null; - const receiverType = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + const stored = ts.isIdentifier(access.expression) + ? lowerer.peekLocal(access.expression)?.type ?? lowerer.globalOf(access.expression)?.type : undefined; + const receiverType = stored?.kind === "record" ? stored : lowerer.mapTypeOf(lowerer.typeOf(access.expression)); if (receiverType?.kind !== "record" && receiverType?.kind !== "union") return null; // A union of records can share a closure-valued field. Its normal // property read performs the tag dispatch before arguments evaluate. @@ -11004,7 +11066,15 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I export function lowerObjectMethodCall(lowerer: Lowerer, call: ts.CallExpression, access: ts.PropertyAccessExpression,): IrExpr | null { if (lowerer.chainBlocked(access, call)) return null; - const mappedReceiver = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + let mappedReceiver = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + // A specialized JS parameter can expose a native field whose checker + // type still belongs to the unspecialized body (plane.normal.dot()). + // Use the lowered receiver's representation and retain its evaluation. + const probe = mappedReceiver?.kind !== "object" && (lowerer.implicitParamTypes !== null || ts.isNewExpression(access.expression) || + isJsSourceFile(call.getSourceFile()) && ts.isCallExpression(access.expression)) + ? tryLowerExpression(lowerer, access.expression) : null; + const specializedReceiver = probe?.type.kind === "object" ? probe : null; + if (specializedReceiver) mappedReceiver = specializedReceiver.type; if (mappedReceiver?.kind === "union") { const dispatched = lowerUnionObjectMethodCall(lowerer, call, access, mappedReceiver) ?? @@ -11025,7 +11095,7 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I : null; if (receiverIr === null) return null; const lowerReceiver = (): IrExpr => { - const receiver = lowerer.lowerExpr(access.expression); + const receiver = specializedReceiver ?? lowerer.lowerExpr(access.expression); const optional = lowerer.runtimeOptionalPropertyReceiver( access.expression, receiver, @@ -11061,7 +11131,7 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I // (lowerClassGenericMethodCall has the exactness rules). if (info && !found) { const gfound = findGenericMethodOn(lowerer, info, access.name.text); - if (gfound) return lowerClassGenericMethodCall(lowerer, call, access, info, gfound); + if (gfound) return lowerClassGenericMethodCall(lowerer, call, access, info, gfound, lowerReceiver()); } // A FUNC-, nullable-FUNC-, or DYN-typed FIELD in call position: // `this.cb()` — the ctor-assigned callback field (countdown.js's @@ -11123,6 +11193,21 @@ export function lowerFunction(lowerer: Lowerer, decl: ts.FunctionDeclaration): I } if (!info || !found) return null; const method = access.name.text; + if (isClassCallback(lowerer, info, method)) { + const loc = locOf(call); + const receiver = lowerReceiver(); + const local = lowerer.declareHiddenLocal("%callbackReceiver", receiver.type); + const value: IrExpr = { kind: "varRef", localId: local.id, type: receiver.type, loc }; + const args = lowerer.completeArgs(call.arguments, found.sig.params, loc, call); + const virtual = lowerer.overrideBelow(info, method); + if (virtual) lowerer.noteVirtualEdge(info, method); + else lowerer.noteEdge(`%${found.declarer.def.name}.${method}`); + const fallback: IrExpr = virtual + ? { kind: "virtualCall", className: info.def.name, method, args: [lowerer.upcastTo(value, info.def.name), ...args], type: found.sig.ret, loc } + : { kind: "call", callee: `%${found.declarer.def.name}.${method}`, args: [lowerer.upcastTo(value, found.declarer.def.name), ...args], type: found.sig.ret, loc }; + const result = classCallbackCall(lowerer, value, method, call.arguments.map((arg) => lowerer.lowerExprExpecting(arg, DYN)), fallback, loc); + return { kind: "seqExpr", stmts: [{ kind: "varDecl", localId: local.id, init: receiver, loc }], result, type: result.type, loc }; + } if (found.declarer.builtinError) { return errorToStringCall(lowerer, lowerReceiver()); } @@ -11205,7 +11290,10 @@ function lowerUnionObjectMethodCall( function paramAbisEqual(left: readonly ParamShape[], right: readonly ParamShape[]): boolean { return left.length === right.length && left.every((shape, i) => { const other = right[i]; - return other !== undefined && shape.mode === other.mode && typeEquals(shape.type, other.type); + return other !== undefined && typeEquals(shape.type, other.type) && + (shape.mode === other.mode || shape.type.kind === "dyn" && + (shape.mode === "required" || shape.mode === "omittable") && + (other.mode === "required" || other.mode === "omittable")); }); } diff --git a/packages/compiler/src/frontend/lowering/lower-classes.ts b/packages/compiler/src/frontend/lowering/lower-classes.ts index 2e79443c..012e9324 100644 --- a/packages/compiler/src/frontend/lowering/lower-classes.ts +++ b/packages/compiler/src/frontend/lowering/lower-classes.ts @@ -8,7 +8,7 @@ import * as ts from "../ts7/adapter.js"; import type { FnCtx, Lowerer } from "./lowerer.js"; import { BOOL, DATE_T, DYN, F64, bytesOf, type IrClassDef, type IrExpr, type IrFunction, type IrLocal, type IrParam, type IrStmt, type IrType, JSVAL, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, STRING, type SrcLoc, UNDEFINED_T, VOID, arrayOf, isSupportedMapKey, isSupportedMapValue, isSupportedSetElem, isUnitType, typeEquals } from "../../ir/ir.js"; import { MAX_GENERIC_INSTANCES, appendImplicitUndefinedReturn, bodyReadsArguments, generatorMeta, genericCallInstance, implicitAnyParamSymbolsOf, implicitCallInstance, implicitMonoFile, omittedArgFor, type GenericFnInfo, type ParamShape } from "./lower-calls.js"; -import { isGenericCallableMemberType, jsOpenObjectType, typeKey } from "../type-mapper.js"; +import { isGenericCallableMemberType, jsOpenObjectType, typeKey, withUnitArm } from "../type-mapper.js"; import { cjsClassExprWholeExportOf, isCjsJsFile, isJsSourceFile, isModuleExportsAccess, isNodeTypesPath, locOf } from "../program.js"; import { PoisonError, dynFallbackType, dynUndefinedExpr, newFnCtx, own } from "./lowerer.js"; import { lowerArrayConstructor, lowerMapSeedNew, lowerSetSeedNew } from "./lower-containers.js"; @@ -24,10 +24,31 @@ import { builtinFenceHintOf } from "./surfaces.js"; import { lowerHttpAgentNew, lowerHttpServerNew } from "./lower-server.js"; import { ambientNsRootOf, ambientUndefReadType, ambientUndefVarRootOf, ambientUndefinedFnSymbolOf, fenceEarlyAliasUse, fenceEarlyNsMemberRef, nsMemberIdentOf, nsUndefRead } from "./lower-namespaces.js"; import { mixinResultBindingClassOf, type MixinInstanceInfo } from "./lower-mixins.js"; -import { rejectStaticThis } from "./static-this.js"; +import { hasLexicalPrivateReference, hasStaticThis, rejectStaticThis } from "./static-this.js"; import { lowerUrlNew } from "./lower-url.js"; import { isNativeProxyInitializer, lowerNativeProxy } from "./expressions/native-proxy.js"; import { classPrototypeData } from "./class-prototypes.js"; +import { classStaticDataFor } from "./class-static-data.js"; +import { lowerObjectFactoryNew } from "./object-factory-new.js"; +import { lowerInstanceConstructorNew } from "./class-instance-constructor.js"; + +export function returnsOnlyThis(member: ts.MethodDeclaration): boolean { + const body = member.body; + const last = body?.statements.at(-1); + if (!body || !last || !ts.isReturnStatement(last)) return false; + let valid = true; + const visit = (node: ts.Node): void => { + if (ts.isFunctionLike(node)) return; + if (ts.isReturnStatement(node)) { + let value = node.expression; + while (value && ts.isParenthesizedExpression(value)) value = value.expression; + if (!value || value.kind !== ts.SyntaxKind.ThisKeyword) valid = false; + } + node.forEachChild(visit); + }; + body.forEachChild(visit); + return valid; +} function functionLocalClass(decl: ts.ClassLikeDeclaration): boolean { for (let parent: ts.Node | undefined = decl.parent; parent && !ts.isSourceFile(parent); parent = parent.parent) { @@ -41,6 +62,10 @@ export function storedClassValueType(lowerer: Lowerer, expression: ts.Expression const member = nsMemberIdentOf(lowerer, expression); if (member) return storedClassValueType(lowerer, member); } + if (expression.kind === ts.SyntaxKind.ThisKeyword) { + const stored = lowerer.resolveThis(); + if (stored?.type.kind === "classval") return stored.type; + } if (ts.isIdentifier(expression)) { const stored = lowerer.peekLocal(expression) ?? lowerer.globalOf(expression); if (stored?.type.kind === "classval") return stored.type; @@ -95,7 +120,7 @@ export interface ClassInfo { * no-dynamic-dispatch semantics by construction). A `gen` entry has a * generator body whose direct calls enter through its spawn wrapper. */ methods: Map; - /** Own JS overrides whose return cannot use the inherited ABI. The slot + /** Own JS overrides whose signature cannot use the inherited ABI. The slot * retains that ABI, but its implementation throws before executing. */ methodEntryFences?: Map; /** OWN GENERIC instance methods (own type parameters — `m(x: T)`), @@ -183,14 +208,16 @@ export interface ClassInfo { * same dynamic story: both are named fences). Accessors and * initializer-less fields keep the fence. */ staticFields: { name: string; type: IrType; initializer: ts.Expression; globalId: string; readonly: boolean }[]; + /** Shared expando data added after a JS class declaration. */ + staticDataHelper?: string; /** STATIC methods — ordinary module functions named `%C.static:m` (the * accessor-colon trick: no user identifier can spell it, and statics * never join vtables, so IrClassDef doesn't know them). `C.m(args)` is * a direct call; `const f = C.m` a zero-capture closure; calls through * class VALUES devirtualize when no strict descendant redeclares the - * member. `this`/`super` inside fence at lowering (JS binds `this` to - * the RECEIVER class — dynamic). Absent on builtin classes. */ - staticMethods?: Map; + * member. Receiver-dependent methods specialize for exact receivers; + * detached methods and `super` retain their fences. Absent on builtins. */ + staticMethods?: Map; /** `static { ... }` blocks, in declaration order. They are DECLARATION-TIME * CODE, not shape: JS runs each block once when the class statement * evaluates, whether or not anything ever references the class — so their @@ -551,16 +578,48 @@ export interface GenericClassInfo { return declared ? (lowerer.classes.get(irName) ?? null) : null; } +/** A dictionary lookup may be absent even when its checker type is a + * required reference. Keep literal nullish fallbacks in inferred JS storage. */ +function jsFieldInitializerType(lowerer: Lowerer, type: IrType, expression: ts.Expression): IrType { + let node = expression; + while (ts.isParenthesizedExpression(node)) node = node.expression; + if (node.kind === ts.SyntaxKind.NullKeyword) return withUnitArm(type, "nullT", lowerer.unions) ?? type; + const inferred = lowerer.typeOf(node); + for (const arm of inferred.isUnionType() ? ts.constituentTypes(inferred) : [inferred]) { + if ((arm.flags & ts.TypeFlags.Undefined) !== 0) type = lowerer.withUndefinedArmOf(type) ?? type; + if ((arm.flags & ts.TypeFlags.Null) !== 0) type = withUnitArm(type, "nullT", lowerer.unions) ?? type; + } + if (ts.isConditionalExpression(node)) { + return jsFieldInitializerType(lowerer, jsFieldInitializerType(lowerer, type, node.whenTrue), node.whenFalse); + } + if (ts.isBinaryExpression(node)) { + if (node.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken) type = jsFieldInitializerType(lowerer, type, node.left); + if (node.operatorToken.kind === ts.SyntaxKind.AmpersandAmpersandToken || + node.operatorToken.kind === ts.SyntaxKind.BarBarToken || + node.operatorToken.kind === ts.SyntaxKind.QuestionQuestionToken) return jsFieldInitializerType(lowerer, type, node.right); + } + return type; +} + /** The undefined-armed union of a JS class property's inferred type — the * honest slot for a field first assigned outside the constructor's top * level (undefined until the write runs, Node-exact). Null when the - * inference is unmappable, checked-dynamic (dyn stays out of class - * fields — KEEP NARROW), or an arm-less kind that cannot join a union + * inference is unmappable or an arm-less kind that cannot join a union * (genResultRecord's list, including scalar-backed Date values). */ function undefArmedFieldType(lowerer: Lowerer, p: ts.Symbol): IrType | null { const t = lowerer.checker.getTypeOfSymbol(p); - const mapped = lowerer.mapTypeOf(t); - if (!mapped || mapped.kind === "void" || mapped.kind === "dyn") return null; + const site = lowerer.checker.declarationsOf(p)[0]; + let mapped = lowerer.mapTypeOf(t) ?? (site ? dynFallbackType(lowerer, site, t) : null); + if (mapped && site) mapped = jsOpenObjectType(site, mapped, lowerer.shapes, lowerer.unions); + if (!mapped || mapped.kind === "void") return null; + for (const declaration of lowerer.checker.declarationsOf(p)) { + const assignment = ts.isBinaryExpression(declaration) ? declaration : declaration.parent; + if (assignment && ts.isBinaryExpression(assignment) && assignment.operatorToken.kind === ts.SyntaxKind.EqualsToken) { + mapped = jsFieldInitializerType(lowerer, mapped, assignment.right); + } + } + // An implicit JS value already carries undefined in its native box. + if (mapped.kind === "dyn") return mapped; const byKey = new Map(); const arms = mapped.kind === "union" ? (lowerer.unions.get(mapped.unionId)?.arms ?? []) : [mapped]; for (const a of arms) { @@ -1293,7 +1352,7 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla const paramProps: NonNullable = []; const staticFields: ClassInfo["staticFields"] = []; - const staticMethods = new Map(); + const staticMethods: NonNullable = new Map(); const staticBlocks: ts.ClassStaticBlockDeclaration[] = []; // GENERIC methods (own type parameters), instance and static: only // the SYNTAX is checked here — parameter/return types mention the @@ -1419,6 +1478,15 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla lowerer.badType(member.name, lowerer.typeOf(member.name)); } staticMethods.set(member.name.text, { params: shapes, ret: ft.ret, member }); + if (implicitMonoFile(decl.getSourceFile()) && !functionLocalClass(decl) && + inst === undefined && decl.typeParameters === undefined) { + const implicit = implicitAnyParamSymbolsOf(lowerer, member, true); + if (implicit) genericStatics.set(member.name.text, { + decl: member, baseName: member.name.text, + qualifiedName: `%${className}.static:${member.name.text}%implicit`, + typeParams: [], instances: new Map(), implicitParams: implicit, + }); + } } // GENERIC static methods monomorphize like top-level generic // functions (`%C.static:m%n`), async ones included — a generic @@ -1918,7 +1986,7 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla !lowerer.findMethodOn(base, member.name.text) && !findGenericMethodOn(lowerer, base, member.name.text) ) { - const implicit = implicitAnyParamSymbolsOf(lowerer, member); + const implicit = implicitAnyParamSymbolsOf(lowerer, member, true); if (implicit) { genericMethods.set(member.name.text, { decl: member, @@ -1932,6 +2000,14 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla } } const { shapes, funcType: ft } = lowerer.lambdaSignature(member); + // JS subclasses can add parameters to a method. Reserve checked + // argument slots in the base before constructing its vtable; shorter + // bodies ignore them and omitted call arguments become undefined. + const virtualArity = lowerer.virtualJsMethodArity.get(member) ?? shapes.length; + if (virtualArity > shapes.length && !bodyReadsArguments(member) && + shapes.every((p) => p.type.kind === "dyn" && (p.mode === "required" || p.mode === "omittable"))) { + while (shapes.length < virtualArity) shapes.push({ type: DYN, mode: "omittable" }); + } if (fields.has(mName)) { lowerer.unsupported("SC1090", member.name, "methods shadowing inherited fields"); } @@ -1960,6 +2036,25 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla // against the STATIC receiver's shape, so `m(x?: number)` and // `m(x: number | undefined)` interchange soundly in overrides. const overridden = lowerer.findMethodOn(base, mName); + if (overridden && isJsSourceFile(member.getSourceFile())) { + // Fluent overrides return the same native receiver. Keep the + // inherited return ABI and upcast `this` when lowering returns. + if (overridden.sig.ret.kind === "object" && base && + (base.def.name === overridden.sig.ret.className || lowerer.isSubclassOf(base.def.name, overridden.sig.ret.className)) && + returnsOnlyThis(member)) ft.ret = overridden.sig.ret; + // An unannotated JS parameter can use the base's typed ABI. + // Defaults stay local to each declaration, so only inherited + // required/optional slots participate in this refinement. + for (let i = 0; i < shapes.length; i++) { + const parameter = member.parameters[i]; + const inherited = overridden.sig.params[i]; + if (parameter && inherited && !parameter.type && !parameter.initializer && !parameter.dotDotDotToken && + shapes[i]!.type.kind === "dyn" && inherited.type.kind !== "dyn" && + (inherited.mode === "required" || inherited.mode === "omittable")) { + shapes[i] = { ...inherited }; + } + } + } // JS overrides may ignore trailing arguments. Retain the base's // checked-value ABI slots as unused parameters, and let shorter // calls fill them with undefined. Typed tails and rest/arguments @@ -1979,23 +2074,26 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla !overridden.sig.params.every((p, i) => typeEquals(p.type, shapes[i]!.type)) || !typeEquals(overridden.sig.ret, ft.ret)) ) { - const returnOnlyJsOverride = + const fencedJsOverride = isJsSourceFile(member.getSourceFile()) && overridden.declarer.decl !== null && isJsSourceFile(overridden.declarer.decl.getSourceFile()) && - overridden.sig.params.length === shapes.length && - overridden.sig.params.every((p, i) => typeEquals(p.type, shapes[i]!.type)) && member.modifiers?.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword) !== true && overridden.sig.async !== true && member.asteriskToken === undefined && overridden.sig.gen === undefined; - if (returnOnlyJsOverride) { + if (fencedJsOverride) { // A dormant JS method must not poison the whole class. Keep // the inherited call/dispatch ABI and refuse at method entry; - // lowering this body with the base return would change JS. + // lowering this body with the base signature would change JS. + const returnOnly = overridden.sig.params.length === shapes.length && + overridden.sig.params.every((p, i) => typeEquals(p.type, shapes[i]!.type)); methodEntryFences.set(mName, unsupportedDiag( "SC1090", locOf(member.name), - `overriding method '${mName}' with a different return type (the native return type must match the base declaration exactly)`, + returnOnly + ? `overriding method '${mName}' with a different return type (the native return type must match the base declaration exactly)` + : `overriding method '${mName}' with a different signature (the native parameter and return types must match the base declaration exactly)`, )); + shapes.splice(0, shapes.length, ...overridden.sig.params); ft.ret = overridden.sig.ret; } else { lowerer.unsupported( @@ -2299,6 +2397,7 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla type = inferredEmptyCollectionFieldType(lowerer, decl, name, rhs) ?? type; } if (!type || type.kind === "void") lowerer.badType(assign, t ?? lowerer.typeOf(assign)); + type = jsFieldInitializerType(lowerer, type, rhs); // A JSDoc claim the BODY contradicts (`@type {Command}` // assigned `undefined` — the lazy-init idiom): the // representation follows the body — the field widens to the @@ -2419,6 +2518,10 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla if (isJsSourceFile(decl.getSourceFile())) { const armed = undefArmedFieldType(lowerer, p); if (armed !== null) { + // Unknown JS properties use the instance bag: they do not + // exist until assigned, and a subclass may declare the same + // name as an ordinary layout field. + if (armed.kind === "dyn") continue; fields.set(p.name, armed); fieldOrder.push({ name: p.name, type: armed, initializer: undefined }); continue; @@ -3055,7 +3158,7 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla * with its OWN storage, exactly JS). */ export function findStaticOn(lowerer: Lowerer, info: ClassInfo | null, name: string): | { declarer: ClassInfo; field: ClassInfo["staticFields"][number]; method?: undefined } - | { declarer: ClassInfo; method: { params: ParamShape[]; ret: IrType; member: ts.MethodDeclaration }; field?: undefined } + | { declarer: ClassInfo; method: { params: ParamShape[]; ret: IrType; member: ts.MethodDeclaration; receiverInfo?: GenericFnInfo }; field?: undefined } | null { for (let c = info; c; c = c.base) { const field = c.staticFields.find((s) => s.name === name); @@ -3174,7 +3277,7 @@ export function collectClassShapeInner(lowerer: Lowerer, decl: ts.ClassLikeDecla * Inherited writes create a new subclass property in JS, so they must not * update the declaring class's global. */ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAccessExpression): { id: string; type: IrType } | null { - if (access.questionDotToken || !ts.isIdentifier(access.expression)) return null; + if (access.questionDotToken || (!ts.isIdentifier(access.expression) && access.expression.kind !== ts.SyntaxKind.ThisKeyword)) return null; const info = lowerer.exactClassOfReceiver(access.expression); if (!info) return null; const found = lowerer.findStaticOn(info, access.name.text); @@ -3196,8 +3299,8 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce * generic member rejection names the site. */ export function lowerStaticFieldRead(lowerer: Lowerer, expr: ts.PropertyAccessExpression): IrExpr | null { if (expr.questionDotToken) return null; - if (!ts.isIdentifier(expr.expression)) return null; - const symbol = lowerer.resolveValueSymbol(expr.expression); + if (!ts.isIdentifier(expr.expression) && expr.expression.kind !== ts.SyntaxKind.ThisKeyword) return null; + const symbol = ts.isIdentifier(expr.expression) ? lowerer.resolveValueSymbol(expr.expression) : null; const info = (symbol ? lowerer.classBySymbol.get(symbol) : undefined) ?? // A require binding over `module.exports = class {…}` (the alias @@ -3205,6 +3308,7 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce // or `C.name` below would fall through to paths that answer for // stdlib globals instead of this class. propertyAssignedClassInfoOf(lowerer, symbol) ?? + exactClassOfReceiver(lowerer, expr.expression) ?? undefined; if (!info) return null; // A decorated name that can REBIND (a replacing decorator): the @@ -3248,6 +3352,8 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce if (expr.name.text === "name" && info.def.jsName !== undefined) { return { kind: "strLit", value: info.def.jsName, type: STRING, loc }; } + const data = classStaticDataFor(lowerer, info, expr.name.text, loc); + if (data) return { kind: "dynKeyGet", value: data, key: { kind: "strLit", value: expr.name.text, type: STRING, loc }, type: DYN, loc }; return null; } @@ -3443,6 +3549,12 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce * hit the declaring class's storage exactly (the shadowing hazards of * general class values don't arise). Null for everything else. */ export function exactClassOfReceiver(lowerer: Lowerer, expr: ts.Expression): ClassInfo | null { + // A class-valued lexical this is introduced only for an exact static + // receiver specialization; ordinary instance this remains an object. + if (expr.kind === ts.SyntaxKind.ThisKeyword) { + const receiver = lowerer.resolveThis(); + return receiver?.type.kind === "classval" ? lowerer.classes.get(receiver.type.className) ?? null : null; + } if (!ts.isIdentifier(expr)) return null; const symbol = lowerer.resolveValueSymbol(expr); if (!symbol) return null; @@ -3584,6 +3696,11 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce export function lowerStaticMethodCall(lowerer: Lowerer, call: ts.CallExpression, access: ts.PropertyAccessExpression,): IrExpr | null { if (access.questionDotToken) return null; + if (access.expression.kind === ts.SyntaxKind.ThisKeyword) { + const receiver = lowerer.resolveThis(); + const info = receiver?.type.kind === "classval" ? lowerer.classes.get(receiver.type.className) : undefined; + if (info) return staticCallOn(lowerer, call, access, info, false); + } // `module.exports.describe()` in a module whose whole export IS a // class expression: the receiver is exactly that class (the kept // export assignment pins it) — the direct-name rules apply. @@ -3683,7 +3800,7 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce `calling the static member '${access.name.text}' through a class value (a subclass of '${info.def.name.replace(/^%|^%m\d+\./, "")}' redeclares it, so the runtime class decides which declaration answers)`, ); } - const instance = genericCallInstance(lowerer, call, gfound.info); + const instance = genericCallInstance(lowerer, call, staticMethodForReceiver(lowerer, access, gfound.info, info, throughValue)); const args = lowerer.completeArgs(call.arguments, instance.params, loc, call); return { kind: "call", callee: instance.name, args, type: instance.returnType, loc }; } @@ -3694,6 +3811,12 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce `calling the static member '${access.name.text}' through a class value (a subclass of '${info.def.name.replace(/^%|^%m\d+\./, "")}' redeclares it, so the runtime class decides which declaration answers)`, ); } + const specialized = findGenericStaticOn(lowerer, info, access.name.text); + if (specialized?.declarer === found.declarer && specialized.info.implicitParams) { + const instance = implicitCallInstance(lowerer, call, staticMethodForReceiver(lowerer, access, specialized.info, info, throughValue)); + const args = lowerer.completeArgs(call.arguments, instance.params, loc, call); + return { kind: "call", callee: instance.name, args, type: instance.returnType, loc }; + } if (found.field !== undefined) { // A func-typed static field in call position: read the global, // call through the value (the ctor-assigned-callback pattern). @@ -3710,12 +3833,44 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce } return { kind: "callValue", callee, args, type: found.field.type.ret, loc }; } + if (found.method.member.body && hasStaticThis(found.method.member)) { + const generic = found.method.receiverInfo ??= { + decl: found.method.member, baseName: access.name.text, + qualifiedName: `%${found.declarer.def.name}.static:${access.name.text}`, + typeParams: [], instances: new Map(), member: { cls: found.declarer, kind: "static" }, + }; + const instance = genericCallInstance(lowerer, call, staticMethodForReceiver(lowerer, access, generic, info, throughValue)); + return { kind: "call", callee: instance.name, + args: lowerer.completeArgs(call.arguments, instance.params, loc, call), type: instance.returnType, loc }; + } const fnName = `%${found.declarer.def.name}.static:${access.name.text}`; lowerer.noteEdge(fnName); const args = lowerer.completeArgs(call.arguments, found.method.params, loc, call); return { kind: "call", callee: fnName, args, type: found.method.ret, loc }; } +function staticMethodForReceiver(lowerer: Lowerer, access: ts.PropertyAccessExpression, method: GenericFnInfo, + receiver: ClassInfo, throughValue: boolean): GenericFnInfo { + if (!hasStaticThis(method.decl)) return method; + // A private name in the inherited body belongs to the declaring class, + // even when the receiver declares a different private with that spelling. + // Do not resolve it by receiver name until the static ABI tracks the brand. + if (method.member && method.member.cls !== receiver && hasLexicalPrivateReference(method.decl)) { + lowerer.unsupported("SC1090", access, "private member references in an inherited receiver-dependent static method"); + } + if (throughValue && !classValueIsExactlyOwn(lowerer, access.expression, receiver)) { + lowerer.unsupported("SC1090", access, "receiver-dependent static methods through a polymorphic class value"); + } + const instances = method.receiverSpecializations ??= new Map(); + let specialized = instances.get(receiver.def.name); + if (!specialized) { + specialized = { ...method, qualifiedName: `${method.qualifiedName}%on.${receiver.def.name}`, + instances: new Map(), staticReceiver: receiver, receiverSpecializations: new Map() }; + instances.set(receiver.def.name, specialized); + } + return specialized; +} + /** Static member access through a class VALUE (`X.m` where X is * classval-typed): devirtualized — the member resolves against the * static class's chain, exact when no strict descendant redeclares it @@ -3739,6 +3894,7 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce } if ( !ts.isIdentifier(expr.expression) && + expr.expression.kind !== ts.SyntaxKind.ThisKeyword && // `module.exports.label` in a class-replaced CJS module: the // receiver is the exact exported class, and the read is // side-effect-free — as bindable as an identifier. @@ -3883,15 +4039,14 @@ export function staticFieldWriteTarget(lowerer: Lowerer, access: ts.PropertyAcce * collected subclass graph cannot answer this question yet. The checker * hierarchy identifies the original declarations through aliases as well. */ export function collectVirtualJsMethods(lowerer: Lowerer, files: readonly ts.SourceFile[]): void { - if (!files.some(implicitMonoFile)) return; + if (!files.some(isJsSourceFile)) return; const visit = (node: ts.Node): void => { if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { - const names = node.members.flatMap((member) => + const members = node.members.filter((member): member is ts.MethodDeclaration => ts.isMethodDeclaration(member) && ts.isIdentifier(member.name) && - !member.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword) - ? [member.name.text] : [], + !member.modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.StaticKeyword), ); - if (node.heritageClauses?.length && names.length > 0) { + if (node.heritageClauses?.length && members.length > 0) { const symbol = lowerer.typeOf(node).getSymbol(); const instance = symbol ? lowerer.checker.getDeclaredTypeOfSymbol(symbol) : null; const seen = new Set(); @@ -3900,11 +4055,19 @@ export function collectVirtualJsMethods(lowerer: Lowerer, files: readonly ts.Sou if (target === undefined || !target.isClassOrInterface() || seen.has(target)) return; seen.add(target); for (const base of lowerer.checker.getBaseTypes(target)) { - for (const name of names) { - const property = lowerer.checker.getPropertyOfType(base, name); + for (const member of members) { + if (!ts.isIdentifier(member.name)) continue; + const property = lowerer.checker.getPropertyOfType(base, member.name.text); for (const declaration of property ? lowerer.checker.declarationsOf(property) : []) { - if (ts.isMethodDeclaration(declaration) && implicitMonoFile(declaration.getSourceFile())) { + if (ts.isMethodDeclaration(declaration) && isJsSourceFile(declaration.getSourceFile())) { lowerer.virtualJsMethods.add(declaration); + if (isJsSourceFile(member.getSourceFile()) && !member.typeParameters && + !member.asteriskToken && !member.modifiers?.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword) && + !bodyReadsArguments(member) && + member.parameters.every((p) => !p.initializer && !p.dotDotDotToken)) { + lowerer.virtualJsMethodArity.set(declaration, + Math.max(lowerer.virtualJsMethodArity.get(declaration) ?? 0, member.parameters.length)); + } } } } @@ -3914,9 +4077,8 @@ export function collectVirtualJsMethods(lowerer: Lowerer, files: readonly ts.Sou if (instance) markBases(instance); } } - node.forEachChild(visit); }; - for (const file of files) visit(file); + for (const file of files) ts.walkPreorder(file, visit); } /** The receiver's EXACT runtime class, when the expression proves it: a @@ -4776,65 +4938,74 @@ export function lowerClassMembers(lowerer: Lowerer, info: ClassInfo): IrFunction forward?: IrExpr[],): IrStmt[] { const out: IrStmt[] = []; let superSeen = false; - for (const stmt of info.ctor!.body!.statements) { - const superCall = - ts.isExpressionStatement(stmt) && - ts.isCallExpression(stmt.expression) && - stmt.expression.expression.kind === ts.SyntaxKind.SuperKeyword - ? stmt.expression - : null; - if (!superCall) { - out.push(...lowerer.lowerStmts([stmt])); - continue; - } - if (!lowerer.suppressStats) { - lowerer.stats.statementsTotal++; - lowerer.bumpFileStat(locOf(stmt).file, "total"); - } - try { - if (superSeen) lowerer.unsupported("SC1090", stmt, "multiple super() calls"); - superSeen = true; - if (info.callableBase) { - if (superCall.arguments.some(ts.isSpreadElement)) lowerer.unsupported("SC1090", superCall, "spreading explicit super arguments into an ordinary constructor"); - const loc = locOf(stmt); - const pack: IrExpr = { kind: "dynArrLit", elems: superCall.arguments.map((arg) => lowerer.lowerExprExpecting(arg, DYN)), type: DYN, loc }; - out.push(lowerer.superCallStmt(info, thisLocal, [pack], loc)); - out.push(...lowerer.fieldInitStmts(info, thisLocal)); - out.push(...paramPropInitStmts(lowerer, info, thisLocal)); + const statements = info.ctor!.body!.statements; + const entry = { stmts: statements, index: 0, ctx: lowerer.ctx, frame: lowerer.scopes[lowerer.scopes.length - 1]!, out }; + lowerer.activeStmtLists.push(entry); + try { + for (let index = 0; index < statements.length; index++) { + entry.index = index; + const stmt = statements[index]!; + const superCall = + ts.isExpressionStatement(stmt) && + ts.isCallExpression(stmt.expression) && + stmt.expression.expression.kind === ts.SyntaxKind.SuperKeyword + ? stmt.expression + : null; + if (!superCall) { + out.push(...lowerer.lowerStmts([stmt])); continue; } - const base = superBaseOf(info)!; - if (base.builtinEmitter && superCall.arguments.length > 0) { - // @types/node admits super({ captureRejections }) — no lowering. - lowerer.unsupported("SC1090", superCall, "EventEmitter constructor options ('captureRejections')"); - } - if (base.builtinStream) { - // super(options?) into a runtime stream base: the stream spoke - // parses the options and binds overridden underscore methods. - out.push(...lowerStreamSuperCall(lowerer, info, base, superCall.arguments, thisLocal, locOf(stmt), stmt)); - out.push(...lowerer.fieldInitStmts(info, thisLocal)); - out.push(...paramPropInitStmts(lowerer, info, thisLocal)); - continue; - } - const args = forward !== undefined - ? forward - : base.builtinError - ? lowerer.errorConstructorArgs(superCall.arguments, locOf(stmt), stmt) - : base.builtinEmitter - ? [] - : lowerer.completeArgs(superCall.arguments, base.ctorParams, locOf(stmt), stmt); - out.push(lowerer.superCallStmt(info, thisLocal, args, locOf(stmt))); - // super() returns → field initializers → parameter-property - // assignments (Node's order, probed) → the rest of the body. - out.push(...lowerer.fieldInitStmts(info, thisLocal)); - out.push(...paramPropInitStmts(lowerer, info, thisLocal)); - } catch (e) { - if (!(e instanceof PoisonError)) throw e; if (!lowerer.suppressStats) { - lowerer.stats.statementsFailed++; - lowerer.bumpFileStat(locOf(stmt).file, "failed"); + lowerer.stats.statementsTotal++; + lowerer.bumpFileStat(locOf(stmt).file, "total"); + } + try { + if (superSeen) lowerer.unsupported("SC1090", stmt, "multiple super() calls"); + superSeen = true; + if (info.callableBase) { + if (superCall.arguments.some(ts.isSpreadElement)) lowerer.unsupported("SC1090", superCall, "spreading explicit super arguments into an ordinary constructor"); + const loc = locOf(stmt); + const pack: IrExpr = { kind: "dynArrLit", elems: superCall.arguments.map((arg) => lowerer.lowerExprExpecting(arg, DYN)), type: DYN, loc }; + out.push(lowerer.superCallStmt(info, thisLocal, [pack], loc)); + out.push(...lowerer.fieldInitStmts(info, thisLocal)); + out.push(...paramPropInitStmts(lowerer, info, thisLocal)); + continue; + } + const base = superBaseOf(info)!; + if (base.builtinEmitter && superCall.arguments.length > 0) { + // @types/node admits super({ captureRejections }) — no lowering. + lowerer.unsupported("SC1090", superCall, "EventEmitter constructor options ('captureRejections')"); + } + if (base.builtinStream) { + // super(options?) into a runtime stream base: the stream spoke + // parses the options and binds overridden underscore methods. + out.push(...lowerStreamSuperCall(lowerer, info, base, superCall.arguments, thisLocal, locOf(stmt), stmt)); + out.push(...lowerer.fieldInitStmts(info, thisLocal)); + out.push(...paramPropInitStmts(lowerer, info, thisLocal)); + continue; + } + const args = forward !== undefined + ? forward + : base.builtinError + ? lowerer.errorConstructorArgs(superCall.arguments, locOf(stmt), stmt) + : base.builtinEmitter + ? [] + : lowerer.completeArgs(superCall.arguments, base.ctorParams, locOf(stmt), stmt); + out.push(lowerer.superCallStmt(info, thisLocal, args, locOf(stmt))); + // super() returns → field initializers → parameter-property + // assignments (Node's order, probed) → the rest of the body. + out.push(...lowerer.fieldInitStmts(info, thisLocal)); + out.push(...paramPropInitStmts(lowerer, info, thisLocal)); + } catch (e) { + if (!(e instanceof PoisonError)) throw e; + if (!lowerer.suppressStats) { + lowerer.stats.statementsFailed++; + lowerer.bumpFileStat(locOf(stmt).file, "failed"); + } } } + } finally { + lowerer.activeStmtLists.pop(); } if (!superSeen) { // tsc guarantees the call exists somewhere; if it wasn't a top-level @@ -5205,6 +5376,24 @@ function assignedThisFieldType(lowerer: Lowerer, expr: ts.NewExpression): IrType export function lowerNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr { const loc = locOf(expr); + const instanceConstructor = lowerInstanceConstructorNew(lowerer, expr); + if (instanceConstructor) return instanceConstructor; + const objectFactory = lowerObjectFactoryNew(lowerer, expr); + if (objectFactory) return objectFactory; + let selected: ts.Expression = expr.expression; + while (ts.isParenthesizedExpression(selected)) selected = selected.expression; + if (ts.isConditionalExpression(selected)) { + const yes = exactClassOfReceiver(lowerer, selected.whenTrue); + const no = exactClassOfReceiver(lowerer, selected.whenFalse); + if (yes && no && !yes.classDecorators && !no.classDecorators) { + const yesType: IrType = { kind: "object", className: yes.def.name }; + const noType: IrType = { kind: "object", className: no.def.name }; + const type: IrType = yes === no ? yesType : { kind: "union", unionId: lowerer.unions.intern([yesType, noType].sort((a, b) => typeKey(a) < typeKey(b) ? -1 : 1)) }; + return { kind: "ternary", cond: lowerer.lowerCondition(selected.condition), + then: lowerer.coerceInto(expr, lowerProgramClassNew(lowerer, expr, yes, loc), type), + else_: lowerer.coerceInto(expr, lowerProgramClassNew(lowerer, expr, no, loc), type), type, loc }; + } + } if (ts.isPropertyAccessExpression(expr.expression) && expr.expression.name.text === "Segmenter" && lowerer.isStdlibGlobal(expr.expression.expression, "Intl")) { const args = expr.arguments ?? []; diff --git a/packages/compiler/src/frontend/lowering/lower-containers.ts b/packages/compiler/src/frontend/lowering/lower-containers.ts index 1cadd8c5..3b66f87b 100644 --- a/packages/compiler/src/frontend/lowering/lower-containers.ts +++ b/packages/compiler/src/frontend/lowering/lower-containers.ts @@ -1,3 +1,4 @@ +import { objectEnumerationReceiver } from "./object-enumeration-receiver.js"; import { InternalCompilerError } from "../../errors.js"; /* Container-surface call lowering: array methods (including the HOF family * map/filter/forEach with their synthesized helper functions), Map/Set @@ -2751,7 +2752,17 @@ export function tryLowerNumericIndexRead(lowerer: Lowerer, operand: IrExpr, loc: } const receiver = lowerer.lowerExpr(access.expression); const argNode = call.arguments[0]!; - const fnArg = lowerer.lowerExpr(argNode); + let fnArg = lowerer.lowerExpr(argNode); + // Reusable JS comparators often accept checked values. Adapt the array's + // element ABI just as other array callbacks do before sorting. + if (fnArg.type.kind === "func" && fnArg.type.params.length <= 2 && + fnArg.type.params.every((param) => lowerer.coercibleValue(elem, param))) { + // Arithmetic on inferred JS values retains a checked result because + // it can produce BigInt. The sort callback still requires a number. + const result = fnArg.type.ret.kind === "dyn" && isJsSourceFile(argNode.getSourceFile()) ? F64 : fnArg.type.ret; + const expected = funcOf(fnArg.type.params.map(() => elem), result); + if (!typeEquals(fnArg.type, expected)) fnArg = lowerer.coerceToExpected(fnArg, expected); + } // The comparator receives exactly (a, b); declaring a prefix is // ordinary TS. Its result must be number (the spec coerces arbitrary // results — no lowering for that). @@ -6001,7 +6012,8 @@ function mapFromSeedValue(lowerer: Lowerer, seed: IrExpr, mapT: IrType & { kind: shape: IrRecordShape,): IrExpr { const resultT = lowerer.irTypeOf(call); if (resultT.kind !== "array") lowerer.badType(call, lowerer.typeOf(call)); // defensive - return objectIterOverIndexShape(lowerer, call, member, argIr, shape, lowerer.lowerExpr(call.arguments[0]!), resultT, locOf(call)); + return objectIterOverIndexShape(lowerer, call, member, argIr, shape, + objectEnumerationReceiver(lowerer, lowerer.lowerExpr(call.arguments[0]!), argIr, locOf(call)), resultT, locOf(call)); } /** The construction core, receiver/result pre-resolved — `node` anchors diff --git a/packages/compiler/src/frontend/lowering/lower-exprs.ts b/packages/compiler/src/frontend/lowering/lower-exprs.ts index 405b99e5..d5438f7f 100644 --- a/packages/compiler/src/frontend/lowering/lower-exprs.ts +++ b/packages/compiler/src/frontend/lowering/lower-exprs.ts @@ -20,7 +20,7 @@ import { ARRAY_METHODS, builtinConstLit, builtinFenceHintOf, builtinModuleConstO import { UNSUPPORTED, blockedBindingUseDiag, requiresDynamicPackageDiag, unsupportedDiag } from "../../diagnostics/diagnostic.js"; import { PoisonError, dynUndefinedExpr, jsFuncNameOf, neverTaintedJsType, nodeThrowExpr, own } from "./lowerer.js"; import { lowerCollectionSpread, lowerNpmStaticSafeIndexRead, lowerSafeIndexRead, strCharsCall, tryLowerNumericIndexRead } from "./lower-containers.js"; -import { arrayValueRead, arrayValueStore } from "./array-values.js"; +import { arrayValueRead, arrayValueStore, arrayValueType } from "./array-values.js"; import { lowerOptionalStringIndex } from "./string-index.js"; import { tryLowerIndexedComparison } from "./indexed-comparison.js"; import { npmStaticPackageOfPath } from "../npm-static.js"; @@ -32,7 +32,7 @@ import { lowerSocketInstanceOf, lowerTlsRootCertificates } from "./lower-server. import { findGenericMethodOn, lowerStaticFieldRead, staticFieldWriteTarget, storedClassValueType } from "./lower-classes.js"; import { bindingNeverReassigned, funcTypeFromParamShapes, implicitMonoFile, lowerTaggedTemplate, nullishGenericBindingUnitOf, objLitGenericFnInfoOf, objLitGenericFnNodeOf, requireObjLitGenericReceiver } from "./lower-calls.js"; import { mixinFnOfCallee } from "./lower-mixins.js"; -import { isConstAssertionTypeNode, isGenericCallableMemberType, isParseArgsDynTypeName, underConstAssertion, unitOnlyUnion } from "../type-mapper.js"; +import { isConstAssertionTypeNode, isGenericCallableMemberType, isParseArgsDynTypeName, underConstAssertion, unitOnlyUnion, withUnitArm } from "../type-mapper.js"; import { lowerYield } from "./lower-generators.js"; import { errorToStringCall } from "./error-methods.js"; import { lowerStreamProperty, lowerStreamStateProperty, streamSidesOf } from "./lower-stream.js"; @@ -50,6 +50,7 @@ import { coerceStringSearchValue, defaultAfterUndefined, lowerStaticallyUndefine import { recordTextCodecClass } from "../../ir/ir.js"; import { classSymbolKeyOf } from "./symbol-fields.js"; import { lowerClassMethodValue } from "./class-method-values.js"; +import { classInstanceOf } from "./class-dynamic-dispatch.js"; import { lowerGlobalValue } from "./lower-global-value.js"; import { lowerClassPrototypeData } from "./class-prototypes.js"; import { lowerArrayIsArrayValue } from "./lower-builtin-values.js"; @@ -674,8 +675,13 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { // every function and constructor global is "function" (the harness's // `typeof queueMicrotask === 'function'` probes). Shadowing locals // have non-stdlib symbols and keep the ordinary path. - if (ts.isIdentifier(expr.expression)) { - const sym = lowerer.checker.getSymbolAtLocation(expr.expression); + let tested = expr.expression; + while (ts.isParenthesizedExpression(tested)) tested = tested.expression; + if (ts.isIdentifier(tested)) { + const sym = lowerer.checker.getSymbolAtLocation(tested); + if (!sym && !lowerer.dynamic) { + return { kind: "strLit", value: "undefined", type: STRING, loc }; + } if (lowerer.isStdlibSymbol(sym)) { const t = lowerer.typeOf(expr.expression); if ( @@ -1523,6 +1529,10 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { if (ts.isConditionalExpression(expr)) return lowerTernary(lowerer, expr); if (ts.isPropertyAccessExpression(expr)) { + // Expando statics may still be absent even when JS inference sees + // the eventual assignment's scalar type. Preserve the checked value. + const staticField = lowerStaticFieldRead(lowerer, expr); + if (staticField) return staticField; const prototypeData = lowerClassPrototypeData(lowerer, expr); if (prototypeData) return prototypeData; // `super.x`: the base chain's GETTER, called directly (super @@ -1951,6 +1961,11 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { expr, ); } + if (expr.name.text === "length" && (recv.type.kind === "array" || recv.type.kind === "string")) { + return recv.type.kind === "array" + ? { kind: "arrIntrinsic", method: "length", receiver: recv, args: [], type: F64, loc } + : { kind: "strIntrinsic", method: "length", receiver: recv, args: [], type: F64, loc }; + } } // `f.name` / `f.length` / own properties on a FUNCTION-typed JS // value (the mustCall wrapper's function-instance members): read @@ -2177,8 +2192,12 @@ function lowerExprInner(lowerer: Lowerer, expr: ts.Expression): IrExpr { `reading the abstract property '${expr.name.text}' through a '${lowerer.checker.typeToString(lowerer.typeOf(expr.expression))}'-typed receiver (abstract property declarations are erased at runtime, so no shared slot exists — type the receiver as the concrete class, or declare an abstract getter instead)`, ); } - if (isDynTypedRefType(recvLowered.type)) { - const info = lowerer.classes.get(recvLowered.type.className); + // Array reads may add an undefined arm even when the checker still + // reports a class. Its property bag must remain reachable, while + // an absent element takes the normal checked-value TypeError. + const nativeReceiver = lowerer.stripUndefinedArm(recvLowered.type); + if (isDynTypedRefType(nativeReceiver)) { + const info = lowerer.classes.get(nativeReceiver.className); if (info && !info.def.runtime && !info.builtinError && !info.builtinEmitter && !info.builtinStream) { return lowerer.maybeNarrow({ kind: "dynKeyGet", value: lowerer.coerceToExpected(recvLowered, DYN), @@ -4733,7 +4752,7 @@ export function lowerOptionalNumber( // (`pkg.workspaces.packages["0"]` — the lowering world types the // unknown-rooted chain `any`); a non-dyn lowering falls through to // the fences below (re-lowering is pure IR construction). - if (receiverIr?.kind === "dyn" || receiverIr === null) { + if (receiverIr?.kind === "dyn" || receiverIr?.kind === "union" || receiverIr === null) { const obj = lowerer.lowerExpr(expr.expression); // A generic mapped type can stay unresolved at this body use even // though its instantiated parameter has a concrete record ABI. @@ -4744,7 +4763,7 @@ export function lowerOptionalNumber( return lowerer.lowerRecordKeyRead(expr, obj.type.shapeId, shape); } } - if (obj.type.kind === "dyn") { + if (obj.type.kind === "dyn" || isDynTypedRefType(obj.type)) { const rawKey = lowerer.lowerExpr(expr.argumentExpression); // Number, bool, and DYN keys stringify (ToPropertyKey) — the // dyn-keyed read `catchWarning[warning.name]` where the property @@ -4755,7 +4774,7 @@ export function lowerOptionalNumber( if (key.type.kind === "string" || key.type.kind === "dyn") { const opt = hasOptionalChainGuard(expr.expression); return lowerer.maybeNarrow( - { kind: "dynKeyGet", key, ...(opt ? { optional: true as const } : {}), value: obj, type: DYN, loc: locOf(expr) }, + { kind: "dynKeyGet", key, ...(opt ? { optional: true as const } : {}), value: lowerer.coerceToExpected(obj, DYN), type: DYN, loc: locOf(expr) }, expr, ); } @@ -5217,6 +5236,10 @@ export function lowerOptionalNumber( } if (ts.isPropertyAccessExpression(expr)) { const target = lowerer.fieldTarget(expr); + if (target?.container === "class" && target.fieldType.kind === "union" && + lowerer.unions.get(target.fieldType.unionId)?.arms.some(isUnitType)) { + return lowerer.fieldGetExpr(target, locOf(expr), expr); + } if (target?.container !== "recordOvf") return null; // A JS file-scope object-literal global is record-shaped to the // checker but stored in the checked-dynamic tree to preserve object @@ -5796,9 +5819,9 @@ export function lowerElementCompound(lowerer: Lowerer, expr: ts.BinaryExpression // type (static `any` — mapTypeOf answers null without --dynamic) // probes the receiver's own lowered world: a dyn value takes the same // write, anything else falls through to the fences. - if (receiverIr?.kind === "dyn" || receiverIr === null) { + if (receiverIr?.kind === "dyn" || receiverIr?.kind === "object" || receiverIr === null) { const obj = receiverIr !== null ? lowerer.lowerExpr(target.expression) : tryLowerExpression(lowerer, target.expression); - if (obj !== null && obj.type.kind === "dyn") { + if (obj !== null && (obj.type.kind === "dyn" || isDynTypedRefType(obj.type))) { const loc = locOf(expr); const litKey = recordKeyLiteralText(target.argumentExpression); let key: IrExpr = @@ -5817,7 +5840,7 @@ export function lowerElementCompound(lowerer: Lowerer, expr: ts.BinaryExpression `storing '${lowerer.fmt(value.type)}' values in a checked-dynamic object (the value cannot convert into the checked-dynamic tree)`, ); } - return { kind: "exprStmt", expr: { kind: "libCall", fn: "dyn.keySet", args: [obj, key, value], type: VOID, loc }, loc }; + return { kind: "exprStmt", expr: { kind: "libCall", fn: "dyn.keySet", args: [lowerer.coerceToExpected(obj, DYN), key, value], type: VOID, loc }, loc }; } } if (receiverIr?.kind !== "array") { @@ -5849,7 +5872,7 @@ export function lowerElementCompound(lowerer: Lowerer, expr: ts.BinaryExpression if (arr.type.kind !== "array") { lowerer.unsupported("SC1090", target.expression, "assignment through a possibly missing nested array receiver"); } - const index = lowerer.lowerExpr(target.argumentExpression); + const index = lowerOptionalNumber(lowerer, lowerer.lowerExpr(target.argumentExpression), locOf(target.argumentExpression), target.argumentExpression); if (index.type.kind !== "f64") { lowerer.unsupported("SC1090", target.argumentExpression, "indexing with non-number keys"); } @@ -5860,7 +5883,13 @@ export function lowerElementCompound(lowerer: Lowerer, expr: ts.BinaryExpression while (ts.isParenthesizedExpression(literal) || ts.isSatisfiesExpression(literal)) literal = literal.expression; const value = ts.isObjectLiteralExpression(literal) || ts.isArrayLiteralExpression(literal) ? lowerer.lowerExprExpecting(literal, arr.type.elem) : lowerer.lowerExpr(expr.right); - return arrayValueStore(lowerer, arr, index, value, arr.type.elem, locOf(expr)); + // Validate the payload before emitting a native store. A dormant JS + // method can contradict its JSDoc element type; retain its call-time + // refusal instead of leaving invalid IR in a newly reachable method. + const expected = value.type.kind === "undefinedT" || + value.type.kind === "union" && lowerer.armTag(value.type.unionId, UNDEFINED_T) >= 0 + ? arrayValueType(lowerer, arr.type.elem) : arr.type.elem; + return arrayValueStore(lowerer, arr, index, lowerer.coerceInto(expr.right, value, expected), arr.type.elem, locOf(expr)); } /** Environment property names use ToPrimitive with the string hint, so @@ -6282,7 +6311,10 @@ export function lowerPrefixUnary(lowerer: Lowerer, expr: ts.PrefixUnaryExpressio if (raw.type.kind === "bigint") { return { kind: "libCall", fn: "bigint.neg", args: [raw], type: BIGINT_T, loc }; } - const operand = lowerOptionalNumber(lowerer, raw, loc, expr.operand); + // Match the checked numeric boundary used by JS binary arithmetic. + const operand = raw.type.kind === "dyn" && isJsSourceFile(expr.getSourceFile()) + ? { kind: "dynCheck" as const, value: raw, type: F64, loc } + : lowerOptionalNumber(lowerer, raw, loc, expr.operand); if (operand.type.kind !== "f64") lowerer.unsupported("SC1043", expr); if (operand.kind === "numLit") return { ...operand, value: -operand.value, loc }; return { kind: "unary", op: "-", operand, type: F64, loc }; @@ -6859,7 +6891,7 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr const reference = scalarSide.type; if ((isDynTypedRefType(reference) || reference.kind === "record" || reference.kind === "array" || reference.kind === "bytes" || reference.kind === "func" || reference.kind === "regex" || reference.kind === "url" || - reference.kind === "bigint" || reference.kind === "symbol" || reference.kind === "set" || DYN_HANDLE_KINDS.has(reference.kind)) && + reference.kind === "bigint" || reference.kind === "symbol" || reference.kind === "set" || reference.kind === "union" || DYN_HANDLE_KINDS.has(reference.kind)) && lowerer.dynConvertible(reference)) { const boxed: IrExpr = { kind: "dynFrom", value: scalarSide, type: DYN, loc: scalarSide.loc, @@ -7175,6 +7207,15 @@ export function lowerBinary(lowerer: Lowerer, expr: ts.BinaryExpression): IrExpr } const optionalStringPlus = lowerOptionalStringPlus(); if (optionalStringPlus) return optionalStringPlus; + if (isJsSourceFile(expr.getSourceFile())) { + const primitive = (type: IrType): boolean => + type.kind === "f64" || type.kind === "string" || type.kind === "bool" || isUnitType(type) || + (type.kind === "union" && (lowerer.unions.get(type.unionId)?.arms.every(primitive) ?? false)); + if ((left.type.kind === "union" || right.type.kind === "union") && primitive(left.type) && primitive(right.type)) { + lowerer.runtimeOptionalArithmeticTypes.set(expr, DYN); + return { kind: "libCall", fn: "dyn.add", args: [lowerer.coerceToExpected(left, DYN), lowerer.coerceToExpected(right, DYN)], type: DYN, loc }; + } + } lowerer.unsupported("SC1043", expr); break; case ts.SyntaxKind.MinusToken: @@ -7477,15 +7518,17 @@ function lowerLogicalPair(lowerer: Lowerer, expr: ts.BinaryExpression, left: IrE // Test the deciding operand in its own representation. The result // contains only the values that can survive short-circuit evaluation. let target = lowerer.mapTypeOf(lowerer.typeOf(expr)); - // Unchecked array reads carry undefined even when the checker sees - // a required value. The RHS can return it for either operator; the - // LHS can return it only for &&. Preserve it through later chain pairs. - if (target !== null && !isUnitType(target) && ( - (right.type.kind === "union" && lowerer.armTag(right.type.unionId, UNDEFINED_T) >= 0) || - (op === ts.SyntaxKind.AmpersandAmpersandToken && left.type.kind === "union" && - lowerer.armTag(left.type.unionId, UNDEFINED_T) >= 0) - )) { - target = lowerer.runtimeOptionalType(target); + // Index reads can be absent without noUncheckedIndexedAccess. Their + // explicit nullish fallback still runs, even if the checker discarded it. + if (target !== null) { + const returned = [right.type, ...(op === ts.SyntaxKind.AmpersandAmpersandToken ? [left.type] : [])]; + for (const type of returned) { + const arms = type.kind === "union" ? lowerer.unions.get(type.unionId)?.arms ?? [] : [type]; + for (const arm of arms) { + if (arm.kind === "undefinedT" && !isUnitType(target)) target = lowerer.runtimeOptionalType(target); + if (arm.kind === "nullT") target = withUnitArm(target, arm.kind, lowerer.unions) ?? target; + } + } } // A broad-JSDoc npm-static body can leave the CHECKER result `any` // even after declaration-backed specialization has recovered both @@ -8518,6 +8561,9 @@ function lowerLogicalPair(lowerer: Lowerer, expr: ts.BinaryExpression, left: IrE // (name strings are user-writable), so `u instanceof TypeError` keeps // the fence. Reads past the narrow bridge through maybeNarrow's // validated %Error extraction. SEMANTICS.md 67. + if (left.type.kind === "dyn" && !target.def.runtime && !target.builtinError && !target.builtinEmitter && !target.builtinStream && !target.localClass) { + return classInstanceOf(lowerer, left, target, loc); + } if (left.type.kind === "dyn" && target.def.name === "%Error") { return { kind: "dynTest", test: "error", value: left, type: BOOL, loc }; } @@ -10013,10 +10059,15 @@ function representedClassFieldTarget( if (access.name.text === "stackTraceLimit" && lowerer.isStdlibGlobal(access.expression, "Error")) { return { container: "errorStackLimit", obj: { kind: "numLit", value: 0, type: F64, loc: locOf(access) }, field: "stackTraceLimit", fieldType: F64 }; } - const receiverIr = lowerer.mapTypeOf(lowerer.typeOf(access.expression)); + const stored = ts.isIdentifier(access.expression) + ? lowerer.peekLocal(access.expression)?.type ?? lowerer.globalOf(access.expression)?.type : undefined; + const receiverIr = stored?.kind === "record" ? stored : lowerer.mapTypeOf(lowerer.typeOf(access.expression)); if (receiverIr?.kind === "object") { return classFieldTarget(lowerer, access.expression, receiverIr, access.name.text); } + if (receiverIr?.kind === "union" && hasClassPayload(lowerer, receiverIr)) { + return representedClassFieldTarget(lowerer, access.expression, access.name.text, lowerer.lowerExpr(access.expression)); + } if (receiverIr?.kind === "record") { const shape = lowerer.shapes.get(receiverIr.shapeId); const fieldType = shape?.fields.find((f) => f.name === access.name.text)?.type; diff --git a/packages/compiler/src/frontend/lowering/lower-island.ts b/packages/compiler/src/frontend/lowering/lower-island.ts index 15024cd2..be2a658c 100644 --- a/packages/compiler/src/frontend/lowering/lower-island.ts +++ b/packages/compiler/src/frontend/lowering/lower-island.ts @@ -3395,7 +3395,15 @@ export function lowerStaticReadableStreamReaderCall( ts.isSpreadElement(call.arguments[0]!) ) { const spread = call.arguments[0]! as ts.SpreadElement; - const src = lowerer.lowerExpr(spread.expression); + let src = lowerer.lowerExpr(spread.expression); + // JavaScript array call results retain checked storage so aliases stay + // shared. This read-only fold can extract the inferred numeric array. + if (src.type.kind === "dyn") { + const inferred = lowerer.mapTypeOf(lowerer.typeOf(spread.expression)); + if (inferred?.kind === "array" && inferred.elem.kind === "f64") { + src = lowerer.coerceInto(spread.expression, src, inferred); + } + } if (src.type.kind !== "array" || src.type.elem.kind !== "f64") { lowerer.unsupported( "SC1090", diff --git a/packages/compiler/src/frontend/lowering/lower-modules.ts b/packages/compiler/src/frontend/lowering/lower-modules.ts index 95bf4432..4fc05ac7 100644 --- a/packages/compiler/src/frontend/lowering/lower-modules.ts +++ b/packages/compiler/src/frontend/lowering/lower-modules.ts @@ -19,6 +19,7 @@ import { ENTRY_NAME, PoisonError, boundIdentifiersOf, dynFallbackType, dynUndefi import { isNativeFfiRequire, builtinMemberRequireDecl, builtinNamespaceDestructureModuleOf, createRequireBindingDecl, createRequireNamespaceDecl, createRequireProgramModuleDecl, createRequireSpecOf, isPromisifyCall, registerBuiltinCallableAlias, stripTypeCasts } from "./lower-builtins.js"; import { bindingContextualGenericFnNodeOf, bindingGenericFnAliasInfoOf, bindingGenericFnInfoOf, bindingGenericFnNodeOf, bindingNeverReassigned, deadUnmappableBinding, funcTypeFromParamShapes, implicitLocalFnInfoOf, implicitLocalFnNodeOf, nullishGenericBindingUnitOf, registerOverloadedCallableAlias } from "./lower-calls.js"; import { hasJsTypeAnnotation, isVarDeclared, numericIteratorSourceOf, provenanceElidedConstDecl } from "./lower-stmts.js"; +import { objectFactorySignature } from "./object-factory-new.js"; import { streamClassAliasDecl } from "./lower-stream.js"; import { stdlibGlobalAliasDecl, stdlibGlobalAliasNameOf, stdlibGlobalNameOf } from "./surfaces.js"; import { isNativeBuiltinValueInitializer } from "./lower-builtin-values.js"; @@ -1640,7 +1641,10 @@ export function collectGlobals(lowerer: Lowerer, sf: ts.SourceFile, topStmts: ts // and closures created in the init body capture it normally. // References from separately-declared functions cascade to // their own per-site runtime fences. - if (isJsSourceFile(sf) && !lowerer.mapTypeOf(lowerer.typeOf(nameNode))) continue; + const factoryType = !isLet && isJsSourceFile(sf) && !decl.type && !hasJsTypeAnnotation(decl) && + ts.isIdentifier(decl.name) && nameNode === decl.name && decl.initializer && ts.isNewExpression(decl.initializer) + ? objectFactorySignature(lowerer, decl.initializer)?.returnType : undefined; + if (!factoryType && isJsSourceFile(sf) && !lowerer.mapTypeOf(lowerer.typeOf(nameNode))) continue; // `var p1 = import("./m")` at file scope: the global holds the // island promise/handle — the import expression's only // production — whatever the checker's namespace type mapped to @@ -1655,7 +1659,7 @@ export function collectGlobals(lowerer: Lowerer, sf: ts.SourceFile, topStmts: ts // exactly the local rule (uncheckedOverloadHandleCall). (uncheckedOverloadHandleCall(lowerer, decl.initializer) ? JSVAL : null) : null) : null; - let type = handleT ?? lowerer.irTypeOf(nameNode); + let type = handleT ?? factoryType ?? lowerer.irTypeOf(nameNode); // Calls through checked function values retain their object // identity, including symbol keys absent from inferred records. if (isJsSourceFile(sf) && !decl.type && !hasJsTypeAnnotation(decl) && type.kind === "record" && @@ -1674,6 +1678,10 @@ export function collectGlobals(lowerer: Lowerer, sf: ts.SourceFile, topStmts: ts // checked factory. Keep that callable boxed instead of asking // a fixed-signature adapter to reconstruct its hidden rest ABI. if (!decl.type && !hasJsTypeAnnotation(decl) && type.kind === "func" && type.rest === true && type.restAbi === undefined) type = DYN; + // JavaScript call results may retain a checked array supplied by + // the caller. A typed global here would copy it on extraction. + if (isJsSourceFile(sf) && !decl.type && !hasJsTypeAnnotation(decl) && type.kind === "array" && + decl.initializer && ts.isCallExpression(decl.initializer)) type = DYN; // Unannotated JavaScript aliases retain an existing native // checked object instead of copying it into an inferred record. if (isJsSourceFile(sf) && !decl.type && !hasJsTypeAnnotation(decl) && diff --git a/packages/compiler/src/frontend/lowering/lower-stmts.ts b/packages/compiler/src/frontend/lowering/lower-stmts.ts index be02a2db..57b82afa 100644 --- a/packages/compiler/src/frontend/lowering/lower-stmts.ts +++ b/packages/compiler/src/frontend/lowering/lower-stmts.ts @@ -23,6 +23,9 @@ import { bindingContextualGenericFnNodeOf, bindingGenericFnAliasInfoOf, bindingG import { isMixinFnBinding, mixinResultBindingClassOf } from "./lower-mixins.js"; import type { ClassInfo, ClassIteratorInfo } from "./lower-classes.js"; import { isCompiledPrototypeMember } from "./class-prototypes.js"; +import { classStaticDataFor } from "./class-static-data.js"; +import { objectFactorySignature } from "./object-factory-new.js"; +import { lowerClassCallbackAssign } from "./class-callbacks.js"; import { genericIfaceBindingKeepsClass, staticFieldWriteTarget } from "./lower-classes.js"; import { lowerStreamUnderscoreAssign, streamClassAliasDecl } from "./lower-stream.js"; import { lowerHttpResPropertyAssignment, lowerHttpServerTimeoutAssignment, lowerServerCloseOverrideAssignment } from "./lower-server.js"; @@ -37,7 +40,7 @@ import { isNativeProxyInitializer } from "./expressions/native-proxy.js"; import { tryLowerExpression } from "./expressions/try-lower-expression.js"; import { lowerUnionFieldWrite } from "./expressions/union-field-write.js"; import { UNSUPPORTED, checkerPanicDiag, isCheckerPanic, requiresDynamicDiag } from "../../diagnostics/diagnostic.js"; -import { isParseArgsDynTypeName, isUnitOnlyTsType, unitOnlyUnion } from "../type-mapper.js"; +import { isParseArgsDynTypeName, isUnitOnlyTsType, jsOpenObjectType, unitOnlyUnion } from "../type-mapper.js"; import { canonicalBuiltinModule } from "../builtin-modules.js"; import { isRelativeSpecifier } from "../workspace-registry.js"; import { probeNodeRequireRefusal } from "../npm.js"; @@ -3334,9 +3337,9 @@ export function isParseArgsDynCheckerType(lowerer: Lowerer, type: ts.Type): bool ); } -/** For-loop initializers evaluate left to right in the loop's head scope. - * The backend copies each captured lexical binding before the condition - * and before every update. Var declarations assign function-scoped slots. */ +/** For-loop initializers evaluate declarators from left to right. The LLVM + * loop emitter keeps their scope alive and freshens every captured let + * binding before the first condition and each subsequent update. */ export function lowerVarDeclList(lowerer: Lowerer, list: ts.VariableDeclarationList): IrStmt | null { if ((list.flags & ts.NodeFlags.Using) !== 0) { lowerer.unsupported("SC1090", list, "'using' declarations (dispose-at-scope-exit semantics)"); @@ -3840,9 +3843,9 @@ export function lowerVarDecl(lowerer: Lowerer, decl: ts.VariableDeclaration, isL type = DYN; } const preservesObjectIdentity = init.type.kind === "dyn" && - (inferredObjectType?.kind === "record" || - (isJsSourceFile(decl.getSourceFile()) && inferredObjectType?.kind === "date") || - (isJsSourceFile(decl.getSourceFile()) && inferredObjectType?.kind === "func") || + (inferredObjectType?.kind === "record" || inferredObjectType?.kind === "array" || + (isJsSourceFile(decl.getSourceFile()) && inferredObjectType !== null && + (inferredObjectType.kind === "date" || inferredObjectType.kind === "func" || jsOpenObjectType(decl, inferredObjectType, lowerer.shapes, lowerer.unions).kind === "dyn")) || (isJsSourceFile(decl.getSourceFile()) && inferredObjectType !== null && (isUnitType(inferredObjectType) || inferredObjectType.kind === "union" && lowerer.unions.get(inferredObjectType.unionId)?.arms.every(isUnitType)))) && !decl.type && !hasJsTypeAnnotation(decl); @@ -3975,6 +3978,9 @@ export function lowerVarDecl(lowerer: Lowerer, decl: ts.VariableDeclaration, isL } if (!type) lowerer.badType(decl.name, lowerer.typeOf(decl.name)); let settledType: IrType = type; + if (!isLet && isJsSourceFile(decl.getSourceFile()) && !hasJsTypeAnnotation(decl) && decl.initializer && ts.isNewExpression(decl.initializer)) { + settledType = objectFactorySignature(lowerer, decl.initializer)?.returnType ?? settledType; + } const arithmeticType = decl.initializer ? lowerer.runtimeOptionalArithmeticTypes.get(decl.initializer) : undefined; const isStringArithmeticUnion = (t: IrType): boolean => { if (t.kind !== "union") return false; @@ -5189,13 +5195,20 @@ function isStrictDelete(node: ts.DeleteExpression): boolean { // SUBCLASS name (`D.x = v` creates an OWN property on D in JS — // different storage) and through class VALUES (the same dynamic // story) are named fences, never a silently-wrong global write. - if (!expr.left.questionDotToken && ts.isIdentifier(expr.left.expression)) { + if (!expr.left.questionDotToken && (ts.isIdentifier(expr.left.expression) || expr.left.expression.kind === ts.SyntaxKind.ThisKeyword)) { // The receiver must BE the class exactly (its name, or a // const binding holding a class expression) — a general class // VALUE could hold a subclass, where JS creates an own // property instead of writing this storage. const classInfo = lowerer.exactClassOfReceiver(expr.left.expression); if (classInfo) { + const data = classStaticDataFor(lowerer, classInfo, expr.left.name.text, locOf(expr.left)); + if (data) { + const loc = locOf(expr); + const key: IrExpr = { kind: "strLit", value: expr.left.name.text, type: STRING, loc }; + const value = lowerer.lowerExprExpecting(expr.right, DYN); + return { kind: "exprStmt", expr: { kind: "libCall", fn: "dyn.keySet", args: [data, key, value], type: VOID, loc }, loc }; + } const found = lowerer.findStaticOn(classInfo, expr.left.name.text); if (found?.field !== undefined) { if (found.declarer !== classInfo) { @@ -5320,6 +5333,8 @@ function isStrictDelete(node: ts.DeleteExpression): boolean { const value = lowerer.lowerExprExpecting(expr.right, target.fieldType); return lowerer.fieldSetStmt(target, value, locOf(expr), expr.left); } + const callback = lowerClassCallbackAssign(lowerer, expr); + if (callback) return callback; // A write to an ABSTRACT property through an abstract-typed // receiver: the read fence's write twin (the declaration is // erased at runtime — no shared slot exists to write). @@ -7185,7 +7200,9 @@ function isStrictDelete(node: ts.DeleteExpression): boolean { return forValues(tmp.id, [write, ...body]); } const local = lowerer.declareLocal(decl.name, decl.name.text, elemValueT, isLet); - if (!typeEquals(elemValueT, sourceT.elem)) { + const declaredElement = lowerer.mapTypeOf(lowerer.typeOf(decl.name)); + if (!typeEquals(elemValueT, sourceT.elem) || + (declaredElement !== null && lowerer.runtimeOptionalWidening(elemValueT, declaredElement) !== null)) { const root = lowerer.runtimeOptionalRootOf(local); lowerer.runtimeOptionalLocals.add(root); lowerer.runtimeOptionalStorageLocals.add(root); diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index dcb592ff..41b850e8 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -1,4 +1,5 @@ import { everyExprChild, everyStmtChild } from "../../ir/traverse.js"; +import { sanitizeUnregisteredClassTypes } from "./sanitize-class-types.js"; import { buildUnionNarrow } from "./union-narrow.js"; import { planUnionRetag, buildUnionRetag, planRecordUnionWrap, buildRecordUnionWrap } from "./union-retag.js"; import type { WidthLift } from "./width-lift.js"; @@ -1674,6 +1675,9 @@ export class Lowerer { /** Inferred JS methods participating in an override chain keep a vtable * ABI instead of call-site specialization. Filled before class collection. */ readonly virtualJsMethods = new Set(); + /** Largest fixed argument list below a JS method, discovered before any + * base vtable signature is collected. Unused checked slots carry undefined. */ + readonly virtualJsMethodArity = new Map(); /** The class whose members are lowering — `super` binds lexically to it * (arrows inside methods lower within this window, so they see it too). */ currentClass: ClassInfo | null = null; @@ -3450,8 +3454,12 @@ export class Lowerer { } } } - node.arguments.forEach((arg, i) => { - const callbackSlot = sig.params[i]?.type; + const restAt = sig.params.findIndex((shape) => + shape.mode === "rest" || shape.mode === "dynRest" || shape.mode === "islandRest" || shape.mode === "arguments"); + node.arguments.forEach((arg, argumentIndex) => { + const i = restAt >= 0 && argumentIndex >= restAt ? restAt : argumentIndex; + const shape = sig.params[i]; + const callbackSlot = shape?.type; if (callbackSlot?.kind === "func" && !ts.isSpreadElement(arg)) { const optionalCallbackParams = callbackSlot.params.flatMap((type, index) => type.kind === "union" && this.armTag(type.unionId, UNDEFINED_T) >= 0 ? [index] : []); @@ -3459,27 +3467,28 @@ export class Lowerer { changed = true; } } - if (ts.isSpreadElement(arg) || !mayBeOptional(arg)) return; - const restAt = sig.params.findIndex((shape) => shape.mode === "rest"); - if (restAt >= 0 && i >= restAt) { - // Rest always supplies an array. An absent source value - // widens its elements, never the array parameter itself. + if (ts.isSpreadElement(arg) || !mayBeOptional(arg) || !shape) return; + if (i === restAt) { + // A rest binding always receives an array. An unchecked + // argument can make its elements undefined, not the pack. + // Keep every virtual implementation on the same array ABI. for (const target of familyBySymbol.get(symbol) ?? [symbol]) { - const shape = signatureBySymbol.get(target)?.params[restAt]; - if (shape?.type.kind !== "array") continue; - const element = addUndefined(shape.type.elem); - if (typeEquals(element, shape.type.elem)) continue; - shape.type = arrayOf(element); - const parameter = functionDeclBySymbol.get(target)?.parameters[restAt]; - const bound = parameter && ts.isIdentifier(parameter.name) ? symbolOf(parameter.name) : null; - if (bound) this.runtimeOptionalBindingTypes.set(bound, shape.type); + const rest = signatureBySymbol.get(target)?.params[i]; + if (rest?.mode !== "rest" || rest.type.kind !== "array") continue; + const widened = arrayOf(addUndefined(rest.type.elem)); + if (typeEquals(rest.type, widened)) continue; + rest.type = widened; + const parameter = functionDeclBySymbol.get(target)?.parameters[i]; + if (parameter && ts.isIdentifier(parameter.name)) { + const bound = symbolOf(parameter.name); + if (bound) this.runtimeOptionalBindingTypes.set(bound, widened); + } changed = true; } return; } - if (!sig.params[i]) return; const set = optionalParams.get(symbol) ?? new Set(); - sig.params[i]!.type = addUndefined(sig.params[i]!.type); + shape.type = addUndefined(shape.type); const before = set.size; set.add(i); optionalParams.set(symbol, set); @@ -3836,9 +3845,6 @@ export class Lowerer { // slots), uniformly across params/locals/globals/fields/body types so // every producer and consumer agrees. Programs with no unregistered // reference are untouched — byte-stability holds. - if (this.diags.length === 0) { - this.sanitizeUnregisteredClassTypes([functions, this.globalsList, artifacts.classes, artifacts.records, artifacts.unions]); - } const module: IrModule | null = this.diags.length > 0 ? null @@ -3854,6 +3860,7 @@ export class Lowerer { entry: ENTRY_NAME, ...(this.ffiImports.length > 0 ? { ffiImports: [...this.ffiImports] } : {}), }; + if (module) sanitizeUnregisteredClassTypes(module, (name) => this.classes.has(name)); return { module, diagnostics: this.diags, @@ -3866,40 +3873,6 @@ export class Lowerer { }; } - /** The unregistered-class type sweep (run()'s last step before the - * module assembles): every `{kind:"object"}` TYPE naming a class with - * no registered ClassInfo is rewritten IN PLACE to the f64 dummy. - * classval types are exempt (they emit the class-independent - * `ScrClassObj *` — inert-but-valid storage, the validator's own - * stance), and only type objects rewrite — node-level classNames - * (`new`, upcasts) cannot reach here (their lowerings fence without a - * registered class), so the validator still backstops those. */ - sanitizeUnregisteredClassTypes(roots: unknown[]): void { - const isUnregisteredObjectType = (v: unknown): boolean => - typeof v === "object" && v !== null && - (v as { kind?: unknown }).kind === "object" && - typeof (v as { className?: unknown }).className === "string" && - !this.classes.has((v as { className: string }).className); - const sweep = (node: unknown): void => { - if (node === null || typeof node !== "object") return; - if (Array.isArray(node)) { - node.forEach((item, i) => { - if (isUnregisteredObjectType(item)) node[i] = F64; - else sweep(item); - }); - return; - } - const rec = node as Record; - for (const key of Object.keys(rec)) { - if (key === "loc") continue; - const v = rec[key]; - if (isUnregisteredObjectType(v)) rec[key] = F64; - else sweep(v); - } - }; - for (const root of roots) sweep(root); - } - /** True when `t` (recursively) names a class instance type with no * registered ClassInfo — the shape of a JS class whose collection fenced. * Used by run()'s global pruning; shapes/unions recurse with a seen-set @@ -4313,7 +4286,7 @@ export class Lowerer { // turn queue more instances. Continue to the joint fixpoint; the initial // queue above is the only portion whose discovery order differed from // historical emit order. - const classDispatch = new ClassDynamicDispatch(); + const classDispatch = new ClassDynamicDispatch(extraRoots !== undefined); for (;;) { drainInstances(); const dispatchChanged = classDispatch.process(this, [ @@ -5355,6 +5328,12 @@ export class Lowerer { return { kind: "promiseVoidWiden", value: expr, type: expected, loc: expr.loc }; } if (expected.kind === "dyn" && expr.type.kind !== "dyn") { + // Native void calls still have a JavaScript value: undefined. Keep + // the call's effects before exposing that value to checked JS code. + if (expr.type.kind === "void") { + return { kind: "seqExpr", stmts: [{ kind: "exprStmt", expr, loc: expr.loc }], + result: dynUndefinedExpr(expr.loc), type: DYN, loc: expr.loc }; + } // An error-HIERARCHY object (builtin subclass or user `extends // Error` class) upcasts to the %Error root first — the caughtToDyn // encoding (scr_dyn_from_error) carries name/message/code and the @@ -5795,6 +5774,7 @@ export class Lowerer { const sameFamily = (src.kind === "record" && arm.kind === "record") || (src.kind === "array" && arm.kind === "array") || + (src.kind === "object" && arm.kind === "object") || // Tuples already lift into ordinary array slots. Consider that // same conversion when the array is an arm of a union too. (src.kind === "record" && this.shapes.get(src.shapeId)?.tuple === true && arm.kind === "array") || @@ -7824,6 +7804,17 @@ export class Lowerer { * lowering goes through here (via lowerExprExpecting) or calls this * directly when the expression was already lowered. */ coerceInto(node: ts.Node, expr: IrExpr, expected: IrType): IrExpr { + // A union destination can accept the binding's tagged storage directly. + // The checker's single-arm flow type can be stale after a callback writes + // the binding. Extracting that arm and wrapping it again would discard the + // actual tag and could read a different record layout. Retag the stored + // value instead; ordinary coercion checks any excluded destination arms. + // Explicit assertions keep their own conversion and validation. + if (expected.kind === "union" && expr.kind === "unionNarrow") { + let source = node; + while (ts.isParenthesizedExpression(source) || ts.isSatisfiesExpression(source)) source = source.expression; + if (ts.isIdentifier(source) || ts.isShorthandPropertyAssignment(source)) expr = expr.value; + } // A fresh literal can retain a wider runtime-optional field after its // initial contextual layout was chosen. Its known discriminator still // selects the destination arm; validate that payload before wrapping. @@ -7859,6 +7850,15 @@ export class Lowerer { } } let e = this.coerceToExpected(expr, expected); + // Existing JavaScript arrays retain mutations and identity across checked + // slots. Fresh literals have no prior identity and use checked storage + // directly, so later writes can change their inferred element type. + let literal = node; + while (ts.isParenthesizedExpression(literal)) literal = literal.expression; + if (isJsSourceFile(node.getSourceFile()) && !ts.isArrayLiteralExpression(literal) && + e.kind === "dynFrom" && e.value.type.kind === "array") { + e = { ...e, liveRef: true }; + } // An 'any' value PROVABLY null/undefined (the unit literal itself, or // a read of a binding nothing ever assigns a non-unit value) flowing // implicitly into a primitive slot: the validated exit refuses units @@ -8170,7 +8170,12 @@ export class Lowerer { } } } - const e = this.lowerExpr(node); + let e = this.lowerExpr(node); + if (expected?.kind === "union" && this.armTag(expected.unionId, UNDEFINED_T) >= 0) { + // A destination that accepts undefined must retain an unchecked + // read's storage value even when the checker still calls it present. + e = this.runtimeOptionalSourceValue(node, e) ?? e; + } return expected ? this.coerceInto(node, e, expected) : e; } diff --git a/packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts b/packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts new file mode 100644 index 00000000..d3e596a9 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/object-enumeration-receiver.ts @@ -0,0 +1,34 @@ +import { BOOL, STRING, type IrExpr, type IrType, type SrcLoc, isUnitType, typeEquals } from "../../ir/ir.js"; +import { varRef } from "../../ir/build.js"; +import type { Lowerer } from "./lowerer.js"; + +/** Array/property reads retain a missing-value arm even when the checker + * promises a record. Object enumeration must check it before reading fields. */ +export function objectEnumerationReceiver( + lowerer: Lowerer, receiver: IrExpr, expected: IrType & { kind: "record" }, loc: SrcLoc, +): IrExpr { + if (receiver.type.kind !== "union") return receiver; + const source = receiver.type; + const arms = lowerer.unions.get(source.unionId)?.arms; + const tag = lowerer.armTag(source.unionId, expected); + if (tag < 0 || !arms?.every((arm) => isUnitType(arm) || typeEquals(arm, expected))) return receiver; + const key = `obj.receiver:${source.unionId}:${expected.shapeId}`; + let name = lowerer.arrHofHelpers.get(key); + if (name === undefined) { + name = `%obj.receiver.${lowerer.arrHofHelpers.size}`; + lowerer.arrHofHelpers.set(key, name); + const value = varRef("value.0", source, loc); + lowerer.liftedFns.push({ + name, params: [{ localId: "value.0", name: "value", type: source }], + returnType: expected, locals: [{ id: "value.0", name: "value", type: source, mutable: false }], + body: [{ + kind: "if", cond: { kind: "unionIsTag", unionId: source.unionId, tag, value, negated: false, type: BOOL, loc }, + then: [{ kind: "return", value: { kind: "unionNarrow", unionId: source.unionId, tag, value, type: expected, loc }, loc }], + else_: [{ kind: "throw", value: { kind: "libCall", fn: "error.new", + args: [{ kind: "strLit", value: "Cannot convert undefined or null to object", type: STRING, loc }], + type: { kind: "object", className: "%TypeError" }, loc }, loc }], loc, + }], loc, + }); + } + return { kind: "call", callee: name, args: [receiver], type: expected, loc }; +} diff --git a/packages/compiler/src/frontend/lowering/object-factory-new.ts b/packages/compiler/src/frontend/lowering/object-factory-new.ts new file mode 100644 index 00000000..c0434082 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/object-factory-new.ts @@ -0,0 +1,58 @@ +import * as ts from "../ts7/adapter.js"; +import type { IrExpr } from "../../ir/ir.js"; +import { isJsSourceFile, locOf } from "../program.js"; +import type { Lowerer } from "./lowerer.js"; +import { bindingNeverReassigned, type FnSig } from "./lower-calls.js"; + +/** An ordinary constructor's explicit object return replaces its receiver. + * Without observing this or new.target, it can use the function's call ABI. */ +export function objectFactorySignature(lowerer: Lowerer, expr: ts.NewExpression): FnSig | null { + if (!ts.isIdentifier(expr.expression) || lowerer.peekLocal(expr.expression)) return null; + // This probe also sees ordinary classes during global collection. Leave + // their deferred diagnostics for the class declaration's lowering. + const wasCollecting = lowerer.collecting; + let symbol: ts.Symbol | null = null; + lowerer.collecting = true; + try { + if (!lowerer.isTopLevelFnSymbol(expr.expression)) return null; + symbol = lowerer.resolveValueSymbol(expr.expression); + } finally { + lowerer.collecting = wasCollecting; + } + if (!symbol) return null; + const declaration = lowerer.checker.declarationsOf(symbol).find((node): node is ts.FunctionDeclaration => + ts.isFunctionDeclaration(node) && node.body !== undefined, + ); + if (!declaration?.body || !bindingNeverReassigned(lowerer, symbol, declaration) || !isJsSourceFile(declaration.getSourceFile()) || declaration.asteriskToken || + declaration.modifiers?.some((m) => m.kind === ts.SyntaxKind.AsyncKeyword)) return null; + const last = declaration.body.statements.at(-1); + if (!last || !ts.isReturnStatement(last) || !last.expression || !ts.isObjectLiteralExpression(last.expression)) return null; + let safe = true; + const visit = (node: ts.Node, nestedArrow = false): void => { + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { safe = false; return; } + if (ts.isFunctionLike(node)) { + if (!ts.isArrowFunction(node)) return; + nestedArrow = true; + } + if (node.kind === ts.SyntaxKind.ThisKeyword || ts.isMetaProperty(node)) safe = false; + if (!nestedArrow && ts.isReturnStatement(node) && (!node.expression || !ts.isObjectLiteralExpression(node.expression))) safe = false; + ts.forEachChild(node, (child) => visit(child, nestedArrow)); + }; + for (const parameter of declaration.parameters) visit(parameter); + visit(declaration.body); + if (!safe) return null; + const signature = lowerer.fnSigOf(expr.expression); + return signature && signature.returnType.kind !== "void" ? signature : null; +} + +export function lowerObjectFactoryNew(lowerer: Lowerer, expr: ts.NewExpression): IrExpr | null { + const signature = objectFactorySignature(lowerer, expr); + if (!signature) return null; + const loc = locOf(expr); + lowerer.noteEdge(signature.name); + return { + kind: "call", callee: signature.name, + args: lowerer.completeArgs(expr.arguments ?? [], signature.params, loc, expr), + type: signature.returnType, loc, + }; +} diff --git a/packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts b/packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts new file mode 100644 index 00000000..ad7c2389 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/sanitize-class-types.test.ts @@ -0,0 +1,61 @@ +import { expect, test } from "vitest"; +import { F64, type IrExpr, type IrModule, type IrType } from "../../ir/ir.js"; +import { sanitizeUnregisteredClassTypes } from "./sanitize-class-types.js"; + +test("fenced instance slots are erased throughout signatures, storage and nested expressions", () => { + const loc = { file: "classes.js", start: 0, end: 1 }; + const missing: IrType = { kind: "object", className: "Fenced" }; + const kept: IrType = { kind: "object", className: "Registered" }; + const constructor: IrType = { kind: "classval", className: "Fenced" }; + const composite: IrType = { kind: "func", rest: true, restAbi: "typed", argumentsAll: true, + params: [{ kind: "array", elem: missing }, { kind: "set", elem: missing }, + { kind: "map", key: missing, value: { kind: "promise", inner: missing } }], + ret: { kind: "generator", async: true, yieldT: missing, retT: missing, nextT: missing } }; + const ref = (type: IrType): IrExpr => ({ kind: "varRef", localId: "x", type, loc }); + const module: IrModule = { + irVersion: 13, sourceFile: loc.file, entry: "main", + globals: [{ id: "g", name: "g", type: composite, mutable: false }], + classes: [{ name: "Registered", fields: [{ name: "value", type: missing }], + localCaptures: [{ localId: "x", name: "x", type: missing }], loc }], + records: [{ id: "r", fields: [{ name: "value", type: missing }, + { name: "next", type: { kind: "union", unionId: "u" } }], indexValue: missing }], + unions: [{ id: "u", arms: [missing, { kind: "record", shapeId: "r" }, kept, constructor] }], + functions: [{ name: "main", returnType: missing, loc, + params: [{ localId: "x", name: "x", type: missing }], + locals: [{ id: "x", name: "x", type: missing, mutable: false }], + captures: [{ localId: "y", name: "y", type: missing }], + classCaptures: [{ localId: "z", name: "z", type: missing, slot: 3 }], + generator: { yieldT: missing, nextT: missing, resultType: { kind: "record", shapeId: "r" } }, + body: [{ kind: "if", cond: ref(kept), else_: null, loc, then: [{ kind: "exprStmt", loc, + expr: { kind: "seqExpr", type: missing, loc, + stmts: [{ kind: "exprStmt", expr: ref(composite), loc }], + result: { kind: "new", className: "Fenced", args: [ref(missing)], type: missing, loc } } }] }], + }], + }; + // An independent structural oracle checks every nested slot and leaves + // class-value types and node-level names intact, even on invalid nodes. + const expected: unknown = JSON.parse(JSON.stringify(module, (_key, value: unknown) => { + const type = value as { kind?: string; className?: string } | null; + return type?.kind === "object" && type.className === "Fenced" ? F64 : value; + })); + sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); + expect(module).toEqual(expected); + expect(module.functions[0]!.params[0]!.type).toBe(F64); + expect(module.unions![0]!.arms[2]).toBe(kept); + expect(module.unions![0]!.arms[3]).toBe(constructor); + expect(missing).toEqual({ kind: "object", className: "Fenced" }); + sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); + expect(module).toEqual(expected); +}); + +test("modules without optional tables and registered class types retain their structure", () => { + const loc = { file: "registered.ts", start: 0, end: 1 }; + const type: IrType = { kind: "object", className: "Registered" }; + const module: IrModule = { irVersion: 13, sourceFile: loc.file, entry: "main", functions: [{ + name: "main", returnType: type, params: [], locals: [], body: [], loc, + }] }; + const before = structuredClone(module); + sanitizeUnregisteredClassTypes(module, () => true); + expect(module).toEqual(before); + expect(module.functions[0]!.returnType).toBe(type); +}); diff --git a/packages/compiler/src/frontend/lowering/sanitize-class-types.ts b/packages/compiler/src/frontend/lowering/sanitize-class-types.ts new file mode 100644 index 00000000..d90c5147 --- /dev/null +++ b/packages/compiler/src/frontend/lowering/sanitize-class-types.ts @@ -0,0 +1,68 @@ +import { F64, type IrModule, type IrType } from "../../ir/ir.js"; +import { everyStmtList } from "../../ir/traverse.js"; + +/** Replace inert instance slots left by runtime-fenced class declarations. + * Construction of an unregistered class already traps, but its remaining + * storage types must agree across declarations, signatures and expressions. + * Keep class values and node-level class names for the validator to check. + * Traverse typed IR directly: boxing a whole module into unknown recursively + * copies its records and makes this pass costly in the native compiler. */ +export function sanitizeUnregisteredClassTypes(module: IrModule, hasClass: (name: string) => boolean): void { + const rewrite = (type: IrType): IrType => { + switch (type.kind) { + case "object": + return hasClass(type.className) ? type : F64; + case "array": + case "set": + type.elem = rewrite(type.elem); + break; + case "map": + type.key = rewrite(type.key); + type.value = rewrite(type.value); + break; + case "func": + for (let i = 0; i < type.params.length; i++) type.params[i] = rewrite(type.params[i]!); + type.ret = rewrite(type.ret); + break; + case "promise": + type.inner = rewrite(type.inner); + break; + case "generator": + type.yieldT = rewrite(type.yieldT); + type.retT = rewrite(type.retT); + type.nextT = rewrite(type.nextT); + break; + } + // Record and union references use IDs. Visit each definition below, + // without following those edges through recursive type graphs. + return type; + }; + for (const global of module.globals ?? []) global.type = rewrite(global.type); + for (const cls of module.classes ?? []) { + for (const field of cls.fields) field.type = rewrite(field.type); + for (const capture of cls.localCaptures ?? []) capture.type = rewrite(capture.type); + } + for (const record of module.records ?? []) { + for (const field of record.fields) field.type = rewrite(field.type); + if (record.indexValue) record.indexValue = rewrite(record.indexValue); + } + for (const union of module.unions ?? []) { + for (let i = 0; i < union.arms.length; i++) union.arms[i] = rewrite(union.arms[i]!); + } + for (const fn of module.functions) { + fn.returnType = rewrite(fn.returnType); + for (const param of fn.params) param.type = rewrite(param.type); + for (const local of fn.locals) local.type = rewrite(local.type); + for (const capture of fn.captures ?? []) capture.type = rewrite(capture.type); + for (const capture of fn.classCaptures ?? []) capture.type = rewrite(capture.type); + if (fn.generator) { + fn.generator.yieldT = rewrite(fn.generator.yieldT); + fn.generator.nextT = rewrite(fn.generator.nextT); + // resultType is a record reference; its fields are visited above. + } + everyStmtList(fn.body, { + expr: (expr) => { expr.type = rewrite(expr.type); return true; }, + stmt: () => true, + }); + } +} diff --git a/packages/compiler/src/frontend/lowering/static-this.ts b/packages/compiler/src/frontend/lowering/static-this.ts index 8064dfe1..b2610409 100644 --- a/packages/compiler/src/frontend/lowering/static-this.ts +++ b/packages/compiler/src/frontend/lowering/static-this.ts @@ -9,6 +9,7 @@ export function rejectStaticThis( root: ts.Node, message: (keyword: "this" | "super") => string, includeRoot = false, + allowThis = false, ): void { const visit = (node: ts.Node): void => { if ( @@ -19,7 +20,7 @@ export function rejectStaticThis( ) { return; } - if (node.kind === ts.SyntaxKind.ThisKeyword || node.kind === ts.SyntaxKind.SuperKeyword) { + if ((node.kind === ts.SyntaxKind.ThisKeyword && !allowThis) || node.kind === ts.SyntaxKind.SuperKeyword) { const keyword = node.kind === ts.SyntaxKind.ThisKeyword ? "this" : "super"; lowerer.unsupported("SC1090", node, message(keyword)); } @@ -28,3 +29,27 @@ export function rejectStaticThis( if (includeRoot) visit(root); else root.forEachChild(visit); } + +export function hasStaticThis(root: ts.Node): boolean { + let found = false; + const visit = (node: ts.Node): void => { + if (node.kind === ts.SyntaxKind.ThisKeyword) found = true; + if ((ts.isFunctionLike(node) && !ts.isArrowFunction(node)) || ts.isClassDeclaration(node) || ts.isClassExpression(node)) return; + node.forEachChild(visit); + }; + root.forEachChild(visit); + return found; +} + +/** Private names belong to the lexical class, independently of any + * same-spelled member in a receiver's subclass. */ +export function hasLexicalPrivateReference(root: ts.Node): boolean { + let found = false; + const visit = (node: ts.Node): void => { + if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) return; + if (ts.isPrivateIdentifier(node)) found = true; + node.forEachChild(visit); + }; + root.forEachChild(visit); + return found; +} diff --git a/packages/compiler/src/frontend/npm-static-declaration-syntax.ts b/packages/compiler/src/frontend/npm-static-declaration-syntax.ts index 593fbf60..8a9a5873 100644 --- a/packages/compiler/src/frontend/npm-static-declaration-syntax.ts +++ b/packages/compiler/src/frontend/npm-static-declaration-syntax.ts @@ -45,6 +45,47 @@ export interface NpmStaticOverloadRewrite { insertions: readonly { offset: number; length: number }[]; } +/** TypeScript truncates JSDoc inner/instance namepaths (`Owner~Item`, + * `Owner#Item`) to Owner. That unrelated nominal type must not dictate a + * native layout. Keep the unresolved atom checked, preserving containers, + * string literal types, executable source and every diagnostic offset. */ +export function applyNpmStaticJsDocNamepaths(sourceFile: ts.SourceFile, source: string): string | null { + const replacements = new Map(); + const visit = (node: ts.Node): void => { + for (const doc of node.jsDoc ?? []) for (const tag of doc.tags ?? []) { + const expression = tag.typeExpression; + if (expression?.kind !== ts.SyntaxKind.JSDocTypeExpression) continue; + const start = expression.getStart(sourceFile); + if (source[start] !== "{") continue; + let depth = 1; + let quote = ""; + for (let i = start + 1; i < tag.getEnd() && depth > 0; i++) { + const char = source[i]!; + if (quote !== "") { + if (char === "\\") i++; + else if (char === quote) quote = ""; + continue; + } + if (char === '"' || char === "'" || char === "`") { quote = char; continue; } + if (char === "{") { depth++; continue; } + if (char === "}") { depth--; continue; } + const name = /^[$A-Z_a-z][$\w]*(?:[.#~][$A-Z_a-z][$\w]*)*/.exec(source.slice(i, tag.getEnd()))?.[0]; + if (!name) continue; + if (name.includes("~") || name.includes("#")) replacements.set(i, name.length); + i += name.length - 1; + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + if (replacements.size === 0) return null; + let text = source; + for (const [offset, length] of replacements) { + text = text.slice(0, offset) + "*" + " ".repeat(length - 1) + text.slice(offset + length); + } + return text; +} + /** Recover nullable class storage erased by a JavaScript bundle. A null * field whose writes construct one named class or call one static factory (or reset to null) * keeps that class's native methods instead of an opaque checked value. diff --git a/packages/compiler/src/frontend/npm-static-declarations.test.ts b/packages/compiler/src/frontend/npm-static-declarations.test.ts index 168a6b3f..b963602b 100644 --- a/packages/compiler/src/frontend/npm-static-declarations.test.ts +++ b/packages/compiler/src/frontend/npm-static-declarations.test.ts @@ -3,6 +3,7 @@ import { applyNpmStaticDeclarationProperties, applyNpmStaticDeclarationOverloads, applyNpmStaticFindReturnWidening, + applyNpmStaticJsDocNamepaths, applyNpmStaticNullableClassFields, npmStaticDeclarationReexports, npmStaticRuntimeClassTargets, @@ -10,6 +11,34 @@ import { parseNpmStaticDeclarationOverloads, } from "./npm-static-declarations.js"; +describe("npm-static JSDoc namepaths", () => { + test("keeps unsupported nominal names checked without changing source offsets", () => { + const source = `class Owner { + /** @param {Array} [items=[]] @returns {Owner#Result | null} */ + collect(items = []) { return items; } + }`; + const text = applyNpmStaticJsDocNamepaths("index.js", source); + expect(text).toBe(source.replace("Owner~Item", "* ").replace("Owner#Result", "* ")); + expect(text?.length).toBe(source.length); + expect(applyNpmStaticJsDocNamepaths("index.js", text!)).toBeNull(); + }); + + test("leaves strings, prose, runtime code and valid types alone", () => { + const source = `const text = "/** @type {Owner~Item} */"; + // @type {Owner~Item} + /** See Owner~Item. @param {'Owner~Item' | "Owner#Item" | Owner.Item} value */ + function keep(value) { return "Owner~Item"; }`; + expect(applyNpmStaticJsDocNamepaths("index.js", source)).toBeNull(); + }); + + test("handles nested record types and repeated names", () => { + const source = `/** @type {{ first: Outer.Inner~Value[], second: Outer.Inner~Value, literal: 'a}~b' }} */ + const values = {};`; + const text = applyNpmStaticJsDocNamepaths("index.js", source); + expect(text).toBe(source.replaceAll("Outer.Inner~Value", "* ")); + }); +}); + const declarations = ` export class Chainy { name(): string; diff --git a/packages/compiler/src/frontend/npm-static-declarations.ts b/packages/compiler/src/frontend/npm-static-declarations.ts index ff5d9b55..5268d4c2 100644 --- a/packages/compiler/src/frontend/npm-static-declarations.ts +++ b/packages/compiler/src/frontend/npm-static-declarations.ts @@ -13,6 +13,10 @@ export function applyNpmStaticFindReturnWidening(path: string, source: string): return syntax.applyNpmStaticFindReturnWidening(nodeFrontendServices().parse(path, source, "js"), source); } +export function applyNpmStaticJsDocNamepaths(path: string, source: string): string | null { + return syntax.applyNpmStaticJsDocNamepaths(nodeFrontendServices().parse(path, source, "js"), source); +} + export function parseNpmStaticDeclarationOverloads(path: string, source: string): NpmStaticDeclarationOverloads { return syntax.parseNpmStaticDeclarationOverloads(nodeFrontendServices().parse(path, source, "ts")); } diff --git a/packages/compiler/src/frontend/npm-static-subtyping.test.ts b/packages/compiler/src/frontend/npm-static-subtyping.test.ts new file mode 100644 index 00000000..184ded97 --- /dev/null +++ b/packages/compiler/src/frontend/npm-static-subtyping.test.ts @@ -0,0 +1,51 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { checkPreflight, loadProgram } from "./program-node.js"; + +const source = ` +export class Base { run() {} } +export class Derived extends Base { + /** @param {number} value */ + run(value) { console.log(value); } +} +export class Other { + /** @param {number} value */ + run(value) { console.log(value); } +} +/** @template T */ +export class Box { + /** @param {T} value */ + constructor(value) { this.value = value; } + run() {} +} +/** @extends {Box} */ +export class NumberBox extends Box { run(value) { console.log(value); } } +`; + +test.each([ + ["accepts an inherited JavaScript method with extra parameters", "function take(value: Base) {} take(new Derived());", false], + ["accepts a base in a union", "function take(value: Base | null) {} take(new Derived());", false], + ["accepts a constructor parameter", "class Use { constructor(value: Base) {} } new Use(new Derived());", false], + ["retains unrelated structural errors", "function take(value: Base) {} take(new Other());", true], + ["retains extra intersection requirements", "function take(value: Base & { required: string }) {} take(new Derived());", true], + ["retains incompatible generic arguments", "function take(value: Box) {} take(new NumberBox(1));", true], + ["accepts matching generic arguments", "function take(value: Box) {} take(new NumberBox(1));", false], + ["retains TypeScript subclass errors", "class Typed extends Base { run(value: number) {} } function take(value: Base) {} take(new Typed());", true], +] as const)("%s", (_name, consumer, error) => { + const dir = mkdtempSync("/tmp/scriptc-subtyping-"); + try { + const pkg = join(dir, "node_modules", "runtime-classes"); + mkdirSync(pkg, { recursive: true }); + writeFileSync(join(pkg, "package.json"), '{"name":"runtime-classes","type":"module","main":"index.js"}'); + writeFileSync(join(pkg, "index.js"), source); + writeFileSync(join(dir, "package.json"), '{"type":"module"}'); + writeFileSync(join(dir, "main.ts"), 'import { Base, Derived, Other, Box, NumberBox } from "runtime-classes";\n' + consumer); + const load = loadProgram(join(dir, "main.ts"), { npmStatic: ["runtime-classes"] }); + try { + const diagnostics = checkPreflight(load); + if (error) expect(diagnostics.some((d) => d.code === "SC0001" && d.message.includes("not assignable")), JSON.stringify(diagnostics)).toBe(true); + else expect(diagnostics).toEqual([]); + } finally { load.dispose(); } + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/packages/compiler/src/frontend/npm-static-subtyping.ts b/packages/compiler/src/frontend/npm-static-subtyping.ts new file mode 100644 index 00000000..b9de1838 --- /dev/null +++ b/packages/compiler/src/frontend/npm-static-subtyping.ts @@ -0,0 +1,47 @@ +import * as ts from "./ts7/adapter.js"; +import { npmStaticPackageOfPath } from "./npm-static.js"; + +/** JS subclasses can extend inferred fields and method arities. The native + * class collector validates their shared layout and dispatch ABI; a consumer + * upcast should not fail solely on the checker's structural override model. */ +export function isNpmStaticSubclassArgument(program: ts.Program, diagnostic: ts.Diagnostic): boolean { + if (diagnostic.code !== 2345 || diagnostic.fileName === undefined) return false; + const source = program.getSourceFile(diagnostic.fileName); + if (!source) return false; + const checker = program.getTypeChecker(); + const runtimeClass = (type: ts.Type): boolean => { + const symbol = type.getSymbol(); + return symbol !== undefined && checker.declarationsOf(symbol).some((declaration) => + (ts.isClassDeclaration(declaration) || ts.isClassExpression(declaration)) && + /\.[cm]?js$/.test(declaration.getSourceFile().fileName) && + npmStaticPackageOfPath(declaration.getSourceFile().fileName) !== null, + ); + }; + const subtype = (actual: ts.Type, expected: ts.Type): boolean => { + if (!runtimeClass(actual)) return false; + const targets = expected.isUnionType() ? ts.constituentTypes(expected) : [expected]; + const seen = new Set(); + const visit = (type: ts.Type): boolean => { + const target = type.isTypeReference() ? type.getTarget() : type; + if (!target?.isClassOrInterface() || seen.has(target)) return false; + seen.add(target); + for (const base of checker.getBaseTypes(type.isClassOrInterface() ? type : target)) { + if (targets.some((expected) => runtimeClass(expected) && base.getSymbol() === expected.getSymbol() && checker.isTypeAssignableTo(base, expected))) return true; + if (visit(base)) return true; + } + return false; + }; + return visit(actual); + }; + let accepted = false; + ts.walkPreorder(source, (node) => { + if (node.end < diagnostic.pos || node.getStart(source) > diagnostic.end) return "skip"; + if (!ts.isCallExpression(node) && !ts.isNewExpression(node)) return undefined; + const index = node.arguments?.findIndex((argument) => argument.getStart(source) === diagnostic.pos && argument.end === diagnostic.end) ?? -1; + if (index < 0) return undefined; + const parameter = checker.getResolvedSignature(node)?.getParameters()[index]; + if (parameter && node.arguments) accepted = subtype(checker.getTypeAtLocation(node.arguments[index]!), checker.getTypeOfSymbol(parameter)); + return undefined; + }); + return accepted; +} diff --git a/packages/compiler/src/frontend/npm-static.ts b/packages/compiler/src/frontend/npm-static.ts index 6fa6f38f..b1cc3816 100644 --- a/packages/compiler/src/frontend/npm-static.ts +++ b/packages/compiler/src/frontend/npm-static.ts @@ -379,8 +379,10 @@ export function npmStaticFsShadow(services: FrontendServices): NpmStaticFsShadow if (hit !== undefined) return hit ?? undefined; let rewritten: string | null = null; try { - const source = trackedReadFile(path); - if (source !== null) { + const original = trackedReadFile(path); + if (original !== null) { + const namepaths = services.jsDocNamepaths(path, original); + const source = namepaths ?? original; const classFields = services.nullableClassFields(path, source); const findWidened = services.findReturnWidening(path, classFields?.text ?? source); const propertyProjected = services.declarationProperties( @@ -397,7 +399,7 @@ export function npmStaticFsShadow(services: FrontendServices): NpmStaticFsShadow if (answer !== null && typeof answer === "object") { reportNpmStaticOffender(target.pkg, answer.degrade); } else { - rewritten = answer ?? projected?.text ?? propertyProjected?.text ?? findWidened?.text ?? classFields?.text ?? null; + rewritten = answer ?? projected?.text ?? propertyProjected?.text ?? findWidened?.text ?? classFields?.text ?? namepaths; } } } catch { diff --git a/packages/compiler/src/frontend/pipeline-native.ts b/packages/compiler/src/frontend/pipeline-native.ts index 453d94ec..9110d869 100644 --- a/packages/compiler/src/frontend/pipeline-native.ts +++ b/packages/compiler/src/frontend/pipeline-native.ts @@ -1,6 +1,6 @@ import { runFrontend, type Frontend } from "./pipeline.js"; import { loadProgram } from "./program.js"; -import { FrontendServices } from "./services.js"; +import { FrontendServices, type ComptimeEvaluator } from "./services.js"; import { createNativeTs7Api } from "./ts7/native-api.js"; /** Native callers supply the installed TS7 executable explicitly. The @@ -11,13 +11,15 @@ export function runNativeFrontend( executable: string, npmStatic?: readonly string[] | "auto" | "lib", externalTypes?: Readonly>, + evaluateComptime?: ComptimeEvaluator, + libraryNpmStatic: readonly string[] = [], ): Frontend { - const services = new FrontendServices((options) => createNativeTs7Api({ ...options, executable })); + const services = new FrontendServices((options) => createNativeTs7Api({ ...options, executable }), process.cwd(), evaluateComptime); try { const frontend = runFrontend(entryPath, (path, options) => loadProgram(path, services, { npmStatic: options.npmStatic ?? [], externalTypes: Object.entries(options.externalTypes ?? {}), - }), npmStatic, externalTypes); + }), npmStatic, externalTypes, libraryNpmStatic); return { ...frontend, dispose: () => { diff --git a/packages/compiler/src/frontend/pipeline.ts b/packages/compiler/src/frontend/pipeline.ts index 5dc57696..1fc52fbc 100644 --- a/packages/compiler/src/frontend/pipeline.ts +++ b/packages/compiler/src/frontend/pipeline.ts @@ -59,6 +59,14 @@ export interface Frontend { dispose: () => void; } +/** Hosts own transport lifetimes; shared compiler stages own the frontend. */ +export type FrontendFactory = ( + entryPath: string, + npmStatic?: readonly string[] | "auto" | "lib", + externalTypes?: Readonly>, + libraryNpmStatic?: readonly string[], +) => Frontend; + /** --npm-static=auto (and library mode's mandatory twin): one throwaway * load finds every bare npm import the program's own modules make, then * the eligibility heuristics (npm-static.ts) pick the packages whose @@ -112,6 +120,9 @@ function detectAutoPackages( // never npm candidates. Auto keeps its original path (the // @types/node answer skips them below), byte-for-byte. if (mode === "lib" && canonicalBuiltinModule(spec) !== null) continue; + // Explicit attempts name the runtime package. Types-first resolution + // may instead identify its @types twin, which has no runtime to admit. + if (judged?.has(packageNameOfSpecifier(spec))) continue; const npm = resolveNpmImport(sf.fileName, spec); if (npm !== null && isNodeTypesPath(npm.typesFile)) continue; if (npm === null) { @@ -175,6 +186,7 @@ export function runFrontend( loadProgram: ProgramLoader, npmStatic?: readonly string[] | "auto" | "lib", externalTypes?: Readonly>, + libraryNpmStatic: readonly string[] = [], ): Frontend { // A preflight or package probe may throw after its host has opened. Track // only live loads, releasing each immediately on normal fallback; this @@ -194,7 +206,7 @@ export function runFrontend( }; }; try { - return loadFrontend(entryPath, trackedLoader, npmStatic, externalTypes); + return loadFrontend(entryPath, trackedLoader, npmStatic, externalTypes, libraryNpmStatic); } catch (error) { for (const load of active) { // Preserve the original failure and attempt every remaining cleanup. @@ -209,6 +221,7 @@ function loadFrontend( loadProgram: ProgramLoader, npmStatic?: readonly string[] | "auto" | "lib", externalTypes?: Readonly>, + libraryNpmStatic: readonly string[] = [], ): Frontend { // Resolver package/workspace metadata is intentionally shared across the // several load attempts of ONE auto-detection fixpoint, but never across @@ -217,7 +230,9 @@ function loadFrontend( clearResolveCaches(); const statuses: NpmStaticStatus[] = []; const npmSites = new Map(); - const judged = new Set(); + // Explicit library attempts bypass only the auto-selection heuristics. + // The shared preflight/lowering refusal and dependency-closure checks remain. + const judged = new Set(libraryNpmStatic); let requested: string[] = []; let reusableScout: ReturnType | null = null; let reusablePreflight: ScrDiagnostic[] | null = null; @@ -228,7 +243,7 @@ function loadFrontend( const scoutPreflight = checkPreflight(scout); requested = npmStatic === "lib" - ? detectAutoPackages(scout, statuses, "lib", judged, npmSites) + ? [...libraryNpmStatic, ...detectAutoPackages(scout, statuses, "lib", judged, npmSites)] : detectAutoPackages(scout, statuses); // With no package to opt in, the scout already IS the final frontend: // same roots, resolution posture, preflight, and module order. Retain it diff --git a/packages/compiler/src/frontend/program.test.ts b/packages/compiler/src/frontend/program.test.ts index 15562b75..3d9cfd14 100644 --- a/packages/compiler/src/frontend/program.test.ts +++ b/packages/compiler/src/frontend/program.test.ts @@ -2,8 +2,28 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, test, vi } from "vitest"; -import { checkPreflight, loadProgram, makeCycleAdmission, type CycleEdge } from "./program-node.js"; +import { checkPreflight, isNodeEsmFile, loadProgram, makeCycleAdmission, type CycleEdge } from "./program-node.js"; import * as ts from "./ts7/ast.js"; +import { AstNode } from "./ts7/ast-node.js"; + +test("ambiguous module classification bounds ancestor work on deep expressions", () => { + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-module-depth-")); + const entry = join(directory, "main.js"); + const depth = 1_000; + writeFileSync(join(directory, "package.json"), "{}"); + writeFileSync(entry, Array.from({ length: depth }, () => "1").join(" + ") + ";\n"); + const load = loadProgram(entry); + try { + const parents = vi.spyOn(AstNode.prototype, "parent", "get"); + try { + expect(isNodeEsmFile(load.entry)).toBe(false); + expect(parents.mock.calls.length).toBeLessThan(depth * 4); + } finally { parents.mockRestore(); } + } finally { + load.dispose(); + rmSync(directory, { recursive: true, force: true }); + } +}); function requireOrderDiagnostics(source: string, dependency = "exports.value = 'ready';\n") { const dir = mkdtempSync(join(tmpdir(), "scriptc-require-order-")); diff --git a/packages/compiler/src/frontend/program.ts b/packages/compiler/src/frontend/program.ts index e57ed92d..31b9647c 100644 --- a/packages/compiler/src/frontend/program.ts +++ b/packages/compiler/src/frontend/program.ts @@ -54,6 +54,7 @@ import { isNodeModulesPath, nearestInvalidPackageJsonPath, nearestPackageType, n import { probeNodeImportRefusal, probeNodeRequireRefusal } from "./npm.js"; import { isNpmStaticPackage, npmStaticActive, npmStaticFsShadow, npmStaticPackageOfPath, reportNpmStaticOffender, setNpmStaticDeclarationOverloads, setNpmStaticPackages } from "./npm-static.js"; import { isPrunedNpmReexport, planNpmStaticReexports } from "./npm-static-prune.js"; +import { isNpmStaticSubclassArgument } from "./npm-static-subtyping.js"; import { npmStaticDeclarationReexports, npmStaticRuntimeClassTargets, parseNpmStaticDeclarationOverloads, parseNpmStaticDeclarationProperties } from "./npm-static-declaration-syntax.js"; import type { FrontendServices } from "./services.js"; import type { NpmStaticDeclarationOverloads, NpmStaticDeclarationProperties, NpmStaticOverloadSignature } from "./npm-static-declaration-syntax.js"; @@ -1392,10 +1393,12 @@ function nodeEsmSyntaxMarker7(sf: ts.SourceFile): ts.Node | null { found = node; return "stop"; } + // Only await syntax needs an ancestor walk. Doing this for every node + // makes a deeply nested expression take quadratic time to classify. if ( - !insideFunctionLike7(node) && (ts.isAwaitExpression(node) || - (ts.isForOfStatement(node) && node.awaitModifier !== undefined)) + (ts.isForOfStatement(node) && node.awaitModifier !== undefined)) && + !insideFunctionLike7(node) ) { found = node; return "stop"; @@ -2384,6 +2387,7 @@ function preflight7(load: LoadResult): { !nodeModulesJsSuppressed(d) && !namespaceCalleeSuppressed(p, d) && !inferredJsDiagnosticSuppressed(p, d) && + !isNpmStaticSubclassArgument(p, d) && !workspaceImplicitAnySuppressed(p, d) && !jsdocTypeSuppressed(p, d, commentDup), ); diff --git a/packages/compiler/src/frontend/provenance-core.ts b/packages/compiler/src/frontend/provenance-core.ts new file mode 100644 index 00000000..66ea43ca --- /dev/null +++ b/packages/compiler/src/frontend/provenance-core.ts @@ -0,0 +1,452 @@ +/* `--provenance-sources` — the provenance-assisted static compilation + * pipeline (EXPERIMENTAL prototype; the registry doc in + * provenance-registry.ts states the thesis). + * + * For every bare npm specifier the entry's module graph imports, this + * asks the npm registry for the package's PROVENANCE ATTESTATION + * (GET registry.npmjs.org/-/npm/v1/attestations/@ — the + * SLSA predicate names the exact {repository, commit} the published dist + * was built from), fetches that source tree once (content-addressed cache + * under ~/.cache/scriptc/provenance/), locates the package + * directory inside it (monorepos publish from subdirectories), and maps + * the published entry to its TypeScript source (dist/lib/build targets + * rewritten to their src twins). Mapped packages compile as ordinary + * program modules — real types, real statements, the static frontier — + * and everything that cannot be mapped FALLS BACK to the island path + * with a note; a fallback is never a build failure. + * + * PRODUCTION GAPS, deliberately unbuilt in the prototype: no sigstore + * bundle verification (the attestation is trusted as served), no + * dist-tarball ↔ source build reproduction (the behavior differential is + * the honest check today), source-entry mapping is heuristic (exports + * targets rewritten dist→src), and transitive dependency versions + * resolve from the DRIVER's installed tree rather than the package's own + * lockfile. + * + * Offline/test hook: SCRIPTC_PROVENANCE_MANIFEST= pre-seeds + * {"packages": {"": {"dir": "", "commit"?, "repo"?}}} + * — those packages skip the network entirely (harness fixtures ride + * this); unlisted packages still take the live pipeline. */ + +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { homedir, tmpdir } from "node:os"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { sourceImportsOfFile } from "./module-syntax.js"; +import type { SourceFile } from "./ts7/ast-types.js"; +import type { Ts7SourceKind } from "./ts7/source-parser.js"; +import { resolveExports, resolveRelativeModule } from "./resolve.js"; +import { packageNameOfSpecifier as packageNameOf } from "./workspace-registry.js"; +import type { ProvenancePackageSource, ProvenanceSources } from "./provenance-registry.js"; + +export type ProvenanceParser = (path: string, source: string, kind: Ts7SourceKind) => SourceFile; + +const NODE_IMPORT_CONDITIONS = new Set(["import", "node", "default"]); + +const NODE_BUILTINS = new Set(builtinModules); + +/** Hard cap on packages one compile will source-map (prototype guard). */ +const MAX_PACKAGES = 16; + +function isFile(path: string): boolean { + try { + return statSync(path).isFile(); + } catch { + return false; + } +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +function readJson(path: string): Record | null { + try { + return JSON.parse(readFileSync(path, "utf8")) as Record; + } catch { + return null; + } +} + +/* ── the bare-import prescan ───────────────────────────────────────────── + * The provenance pipeline runs BEFORE the program loads (tsgo needs the + * "paths" mapping at creation), so the bare specifiers come from a light + * parse walk of the entry's RELATIVE import closure — the same specifier + * collection shapes npm.ts scans embedded modules with, over the shared + * native TypeScript syntax service. */ + +function moduleSpecifiersLite(source: string, fileName: string, parseSourceFile: ProvenanceParser): { spec: string; typeOnly: boolean }[] { + return sourceImportsOfFile(parseSourceFile(fileName, source, "ts")); +} + +/** Every bare (non-relative, non-builtin, non-"#") VALUE specifier the + * relative closure of `roots` imports, in first-encounter order. */ +function bareImportsOf(roots: readonly string[], parseSourceFile: ProvenanceParser): string[] { + const seenFiles = new Set(); + const bare: string[] = []; + const bareSeen = new Set(); + const queue = [...roots]; + while (queue.length > 0) { + const file = resolve(queue.shift()!); + if (seenFiles.has(file)) continue; + seenFiles.add(file); + let text: string; + try { + text = readFileSync(file, "utf8"); + } catch { + continue; + } + for (const { spec, typeOnly } of moduleSpecifiersLite(text, file, parseSourceFile)) { + if (typeOnly) continue; + if (spec.startsWith("./") || spec.startsWith("../")) { + const dep = resolveRelativeModule(file, spec); + if (dep !== null && !dep.endsWith(".json") && !dep.endsWith(".d.ts")) queue.push(dep); + continue; + } + if (spec.startsWith("#") || spec.startsWith("node:")) continue; + if (NODE_BUILTINS.has(packageNameOf(spec))) continue; + if (!bareSeen.has(spec)) { + bareSeen.add(spec); + bare.push(spec); + } + } + } + return bare; +} + +/* ── installed-package lookup (name/version/published entry) ──────────── */ + +interface InstalledPackage { + dir: string; + name: string; + version: string; + pkgJson: Record; +} + +/** node_modules/ walking up from `fromDir`, realpath-free (the + * published package.json is all this needs). */ +function findInstalled(fromDir: string, name: string): InstalledPackage | null { + for (let dir = fromDir; ; ) { + const candidate = join(dir, "node_modules", name); + const pkgJson = readJson(join(candidate, "package.json")); + if (pkgJson !== null && typeof pkgJson["version"] === "string") { + return { + dir: candidate, + name: typeof pkgJson["name"] === "string" ? pkgJson["name"] : name, + version: pkgJson["version"], + pkgJson, + }; + } + const parent = dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +/* ── attestation → {repo, commit} ──────────────────────────────────────── */ + +interface Attested { + repo: string; + commit: string; +} + +/** GET the npm attestation set and extract the SLSA provenance + * predicate's resolved source dependency. Throws with a one-line reason + * on every failure shape (no attestation, network, malformed). */ +async function fetchAttestation(name: string, version: string): Promise { + const url = `https://registry.npmjs.org/-/npm/v1/attestations/${encodeURIComponent(`${name}@${version}`).replace(/%40/g, "@").replace(/%2F/gi, "/")}`; + const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (res.status === 404) throw new Error("no provenance attestation published"); + if (!res.ok) throw new Error(`attestation fetch failed (HTTP ${res.status})`); + const body = (await res.json()) as { attestations?: { predicateType?: string; bundle?: { dsseEnvelope?: { payload?: string } } }[] }; + for (const att of body.attestations ?? []) { + if (!att.predicateType?.startsWith("https://slsa.dev/provenance")) continue; + const payload = att.bundle?.dsseEnvelope?.payload; + if (payload === undefined) continue; + const stmt = JSON.parse(Buffer.from(payload, "base64").toString("utf8")) as { + predicate?: { buildDefinition?: { resolvedDependencies?: { uri?: string; digest?: { gitCommit?: string } }[] } }; + }; + for (const dep of stmt.predicate?.buildDefinition?.resolvedDependencies ?? []) { + const commit = dep.digest?.gitCommit; + if (typeof commit === "string" && commit !== "" && typeof dep.uri === "string") { + return { repo: dep.uri, commit }; + } + } + } + throw new Error("attestation set carries no SLSA provenance predicate"); +} + +/* ── source fetch (content-addressed by the attested commit) ──────────── */ + +function cacheRoot(): string { + return process.env["SCRIPTC_PROVENANCE_CACHE"] ?? join(homedir(), ".cache", "scriptc", "provenance"); +} + +/** The cached source tree for an attested commit, fetching it once from + * codeload (github repos only in the prototype). Returns the tree root. */ +async function fetchSourceTree(repo: string, commit: string): Promise { + const dest = join(cacheRoot(), commit); + if (isDirectory(dest)) return dest; + const m = /github\.com\/([^/]+)\/([^/@#]+)/.exec(repo); + if (m === null) throw new Error(`source repository is not a github URL (${repo})`); + const url = `https://codeload.github.com/${m[1]}/${m[2]}/tar.gz/${commit}`; + const res = await fetch(url, { signal: AbortSignal.timeout(120_000) }); + if (!res.ok) throw new Error(`source fetch failed (HTTP ${res.status} for ${url})`); + const bytes = Buffer.from(await res.arrayBuffer()); + mkdirSync(cacheRoot(), { recursive: true }); + const tmp = mkdtempSync(join(tmpdir(), "scriptc-provenance-")); + try { + const tarball = join(tmp, "src.tgz"); + writeFileSync(tarball, bytes); + const extractDir = join(tmp, "tree"); + mkdirSync(extractDir); + execFileSync("tar", ["-xzf", tarball, "-C", extractDir, "--strip-components=1"]); + try { + renameSync(extractDir, dest); + } catch { + // A parallel compile won the rename — its tree is the same content. + if (!isDirectory(dest)) throw new Error("source cache rename failed"); + } + } finally { + rmSync(tmp, { recursive: true, force: true }); + } + return dest; +} + +/* ── package dir + source-entry mapping ────────────────────────────────── */ + +/** The directory inside `tree` whose package.json names `name`: the root, + * then the conventional monorepo layouts, then a bounded scan. */ +function locatePackageDir(tree: string, name: string): string | null { + const nameOf = (dir: string): string | null => { + const pkg = readJson(join(dir, "package.json")); + return pkg !== null && typeof pkg["name"] === "string" ? pkg["name"] : null; + }; + if (nameOf(tree) === name) return tree; + const bare = name.startsWith("@") ? name.split("/")[1]! : name; + const candidates = [ + join(tree, "packages", bare), + join(tree, "packages", name), + join(tree, "packages", name.replace("@", "").replace("/", "-")), + join(tree, "packages", `babel-${bare}`), + ]; + for (const c of candidates) { + if (nameOf(c) === name) return c; + } + // Bounded breadth-first scan (depth ≤ 3, node_modules/.git skipped). + const queue: { dir: string; depth: number }[] = [{ dir: tree, depth: 0 }]; + while (queue.length > 0) { + const { dir, depth } = queue.shift()!; + if (depth > 0 && nameOf(dir) === name) return dir; + if (depth >= 3) continue; + let entries: string[]; + try { + entries = readdirSync(dir).filter((entry) => isDirectory(join(dir, entry))).sort(); + } catch { + continue; + } + for (const e of entries) { + if (e === "node_modules" || e.startsWith(".")) continue; + queue.push({ dir: join(dir, e), depth: depth + 1 }); + } + } + return null; +} + +/** The published dist target for `subpath` — "exports" with the import + * condition, else module/main/types fields (root subpath only). */ +function publishedTargetOf(pkgJson: Record, subpath: string): string | null { + if (pkgJson["exports"] !== undefined) { + return resolveExports(pkgJson["exports"], subpath, NODE_IMPORT_CONDITIONS); + } + if (subpath !== ".") return subpath; + for (const field of ["module", "main", "types"]) { + const v = pkgJson[field]; + if (typeof v === "string" && v !== "") return v; + } + return "index.js"; +} + +/** dist target → source file, heuristically: the built path's leading + * dist/lib/build segment rewrites to src (or drops), an esm/cjs/dts flavor + * segment is skipped for a shared TypeScript source tree, extensions + * rewrite to TypeScript twins, and the root entry falls back to the + * conventional src/index.ts homes. First existing candidate wins. */ +function mapEntryToSource(pkgDir: string, target: string, subpath: string): string | null { + const rel = target.replace(/^\.\//, ""); + const stems = new Set([rel]); + const distRe = /^(dist|lib|build|out|output|dist-node|dist-src)\//; + if (distRe.test(rel)) { + const withoutDist = rel.replace(distRe, ""); + if (/^(esm|cjs|dts)\//.test(withoutDist)) { + stems.add(`src/${withoutDist.replace(/^(esm|cjs|dts)\//, "")}`); + } + stems.add(rel.replace(distRe, "src/")); + stems.add(withoutDist); + } else { + stems.add(`src/${rel}`); + } + const candidates: string[] = []; + for (const stem of stems) { + const base = stem.replace(/\.d\.(ts|mts|cts)$/, ".$1").replace(/\.(js|mjs|cjs)$/, ""); + if (/\.(ts|tsx|mts|cts)$/.test(base)) candidates.push(base); + else { + candidates.push(`${base}.ts`, `${base}.mts`, `${base}.cts`, `${base}.tsx`); + candidates.push(join(base, "index.ts")); + } + } + if (subpath === ".") { + candidates.push("src/index.ts", "src/index.mts", "index.ts", "src/index.tsx"); + } + for (const c of candidates) { + const abs = join(pkgDir, c); + if (isFile(abs)) return abs; + } + return null; +} + +/* ── the pipeline ─────────────────────────────────────────────────────── */ + +interface ManifestEntry { + dir: string; + commit?: string; + repo?: string; +} + +function readManifest(): Map { + const path = process.env["SCRIPTC_PROVENANCE_MANIFEST"]; + const out = new Map(); + if (path === undefined || path === "") return out; + const manifest = readJson(resolve(path)); + const packages = manifest?.["packages"]; + if (packages === null || typeof packages !== "object") return out; + for (const [name, raw] of Object.entries(packages as Record)) { + if (raw === null || typeof raw !== "object") continue; + const e = raw as { dir?: unknown; commit?: unknown; repo?: unknown }; + if (typeof e.dir !== "string") continue; + const dir = isAbsolute(e.dir) ? e.dir : resolve(dirname(resolve(path)), e.dir); + out.set(name, { + dir, + ...(typeof e.commit === "string" ? { commit: e.commit } : {}), + ...(typeof e.repo === "string" ? { repo: e.repo } : {}), + }); + } + return out; +} + +/** Resolves provenance sources for everything the entry's module graph + * imports (one transitive round per newly-mapped tree: source imports of + * OTHER packages try the pipeline too). Never throws for a package + * failure — those become notes and the package keeps its island path. */ +export async function resolveProvenanceSourcesWithParser(entryPath: string, parseSourceFile: ProvenanceParser): Promise { + const entry = resolve(entryPath); + const manifest = readManifest(); + const packages: ProvenancePackageSource[] = []; + const notes: string[] = []; + /** package name → mapped package (or null after a noted failure). */ + const processed = new Map(); + + /** All specifiers seen so far, grouped by package name. */ + const specifiersByPackage = new Map>(); + const enqueue = (specs: readonly string[]): string[] => { + const newNames: string[] = []; + for (const spec of specs) { + const name = packageNameOf(spec); + let set = specifiersByPackage.get(name); + if (set === undefined) { + specifiersByPackage.set(name, (set = new Set())); + newNames.push(name); + } + set.add(spec); + } + return newNames; + }; + + const mapOne = async (name: string): Promise => { + if (processed.has(name)) return; + if (processed.size >= MAX_PACKAGES) { + processed.set(name, null); + notes.push(`${name}: skipped — provenance package limit (${MAX_PACKAGES}) reached; island path used`); + return; + } + processed.set(name, null); // claimed; overwritten on success + const installed = findInstalled(dirname(entry), name); + if (installed === null) { + notes.push(`${name}: not installed under the entry's node_modules; island path used`); + return; + } + let dir: string; + let repo: string; + let commit: string; + const seeded = manifest.get(name); + try { + if (seeded !== undefined) { + dir = seeded.dir; + repo = seeded.repo ?? "(manifest)"; + commit = seeded.commit ?? "(manifest)"; + if (!isDirectory(dir)) throw new Error(`manifest dir does not exist (${dir})`); + } else { + const attested = await fetchAttestation(installed.name, installed.version); + repo = attested.repo; + commit = attested.commit; + const tree = await fetchSourceTree(repo, commit); + const located = locatePackageDir(tree, installed.name); + if (located === null) { + throw new Error(`package directory not found inside the attested source tree (${tree})`); + } + dir = located; + } + const entries: Record = {}; + for (const spec of specifiersByPackage.get(name) ?? new Set()) { + const parts = spec.split("/"); + const nameLen = spec.startsWith("@") ? 2 : 1; + const subpath = parts.length === nameLen ? "." : `./${parts.slice(nameLen).join("/")}`; + const target = publishedTargetOf(installed.pkgJson, subpath); + const source = target === null ? null : mapEntryToSource(dir, target, subpath); + if (source === null) { + notes.push( + `${name}@${installed.version}: no source mapping for '${spec}' (published target: ${target ?? "unexported"}); island path used`, + ); + continue; + } + entries[spec] = source; + } + if (Object.keys(entries).length === 0) return; + const srcPkg = readJson(join(dir, "package.json")); + const sourceVersion = typeof srcPkg?.["version"] === "string" ? srcPkg["version"] : undefined; + const pkg: ProvenancePackageSource = { + name: installed.name, + version: installed.version, + ...(sourceVersion !== undefined && sourceVersion !== installed.version ? { sourceVersion } : {}), + repo, + commit, + dir, + entries, + }; + if (pkg.sourceVersion !== undefined) { + notes.push( + `${name}: source tree's package.json says ${pkg.sourceVersion}, installed is ${installed.version} (release tooling that bumps at publish) — the behavior differential is the check`, + ); + } + packages.push(pkg); + processed.set(name, pkg); + // One transitive round: the mapped source's own bare imports try + // the pipeline too (versions resolve from the DRIVER's tree — a + // prototype heuristic; production wants the package's lockfile). + const inner = enqueue(bareImportsOf(Object.values(entries), parseSourceFile)); + for (const n of inner) await mapOne(n); + } catch (e) { + notes.push(`${name}@${installed.version}: ${e instanceof Error ? e.message : String(e)}; island path used`); + } + }; + + for (const name of enqueue(bareImportsOf([entry], parseSourceFile))) { + await mapOne(name); + } + return { packages, notes }; +} diff --git a/packages/compiler/src/frontend/provenance.ts b/packages/compiler/src/frontend/provenance.ts index 73c50e05..8defcaf8 100644 --- a/packages/compiler/src/frontend/provenance.ts +++ b/packages/compiler/src/frontend/provenance.ts @@ -1,453 +1,7 @@ -/* `--provenance-sources` — the provenance-assisted static compilation - * pipeline (EXPERIMENTAL prototype; the registry doc in - * provenance-registry.ts states the thesis). - * - * For every bare npm specifier the entry's module graph imports, this - * asks the npm registry for the package's PROVENANCE ATTESTATION - * (GET registry.npmjs.org/-/npm/v1/attestations/@ — the - * SLSA predicate names the exact {repository, commit} the published dist - * was built from), fetches that source tree once (content-addressed cache - * under ~/.cache/scriptc/provenance/), locates the package - * directory inside it (monorepos publish from subdirectories), and maps - * the published entry to its TypeScript source (dist/lib/build targets - * rewritten to their src twins). Mapped packages compile as ordinary - * program modules — real types, real statements, the static frontier — - * and everything that cannot be mapped FALLS BACK to the island path - * with a note; a fallback is never a build failure. - * - * PRODUCTION GAPS, deliberately unbuilt in the prototype: no sigstore - * bundle verification (the attestation is trusted as served), no - * dist-tarball ↔ source build reproduction (the behavior differential is - * the honest check today), source-entry mapping is heuristic (exports - * targets rewritten dist→src), and transitive dependency versions - * resolve from the DRIVER's installed tree rather than the package's own - * lockfile. - * - * Offline/test hook: SCRIPTC_PROVENANCE_MANIFEST= pre-seeds - * {"packages": {"": {"dir": "", "commit"?, "repo"?}}} - * — those packages skip the network entirely (harness fixtures ride - * this); unlisted packages still take the live pipeline. */ - -import { execFile } from "node:child_process"; -import { readFileSync, readdirSync, statSync } from "node:fs"; -import { mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; -import { builtinModules } from "node:module"; -import { homedir, tmpdir } from "node:os"; -import { dirname, isAbsolute, join, resolve } from "node:path"; -import { promisify } from "node:util"; -import { sourceImportsOfFile } from "./module-syntax.js"; import { parseSourceFile } from "./ts7/source-parser-node.js"; -import { resolveExports, resolveRelativeModule } from "./resolve.js"; -import { packageNameOfSpecifier as packageNameOf } from "./workspace-registry.js"; -import type { ProvenancePackageSource, ProvenanceSources } from "./provenance-registry.js"; +import { resolveProvenanceSourcesWithParser } from "./provenance-core.js"; +import type { ProvenanceSources } from "./provenance-registry.js"; -const NODE_IMPORT_CONDITIONS = new Set(["import", "node", "default"]); - -const execFileAsync = promisify(execFile); - -const NODE_BUILTINS = new Set(builtinModules); - -/** Hard cap on packages one compile will source-map (prototype guard). */ -const MAX_PACKAGES = 16; - -function isFile(path: string): boolean { - try { - return statSync(path).isFile(); - } catch { - return false; - } -} - -function isDirectory(path: string): boolean { - try { - return statSync(path).isDirectory(); - } catch { - return false; - } -} - -function readJson(path: string): Record | null { - try { - return JSON.parse(readFileSync(path, "utf8")) as Record; - } catch { - return null; - } -} - -/* ── the bare-import prescan ───────────────────────────────────────────── - * The provenance pipeline runs BEFORE the program loads (tsgo needs the - * "paths" mapping at creation), so the bare specifiers come from a light - * parse walk of the entry's RELATIVE import closure — the same specifier - * collection shapes npm.ts scans embedded modules with, over the shared - * native TypeScript syntax service. */ - -function moduleSpecifiersLite(source: string, fileName: string): { spec: string; typeOnly: boolean }[] { - return sourceImportsOfFile(parseSourceFile(fileName, source, "ts")); -} - -/** Every bare (non-relative, non-builtin, non-"#") VALUE specifier the - * relative closure of `roots` imports, in first-encounter order. */ -function bareImportsOf(roots: readonly string[]): string[] { - const seenFiles = new Set(); - const bare: string[] = []; - const bareSeen = new Set(); - const queue = [...roots]; - while (queue.length > 0) { - const file = resolve(queue.shift()!); - if (seenFiles.has(file)) continue; - seenFiles.add(file); - let text: string; - try { - text = readFileSync(file, "utf8"); - } catch { - continue; - } - for (const { spec, typeOnly } of moduleSpecifiersLite(text, file)) { - if (typeOnly) continue; - if (spec.startsWith("./") || spec.startsWith("../")) { - const dep = resolveRelativeModule(file, spec); - if (dep !== null && !dep.endsWith(".json") && !dep.endsWith(".d.ts")) queue.push(dep); - continue; - } - if (spec.startsWith("#") || spec.startsWith("node:")) continue; - if (NODE_BUILTINS.has(packageNameOf(spec))) continue; - if (!bareSeen.has(spec)) { - bareSeen.add(spec); - bare.push(spec); - } - } - } - return bare; -} - -/* ── installed-package lookup (name/version/published entry) ──────────── */ - -interface InstalledPackage { - dir: string; - name: string; - version: string; - pkgJson: Record; -} - -/** node_modules/ walking up from `fromDir`, realpath-free (the - * published package.json is all this needs). */ -function findInstalled(fromDir: string, name: string): InstalledPackage | null { - for (let dir = fromDir; ; ) { - const candidate = join(dir, "node_modules", name); - const pkgJson = readJson(join(candidate, "package.json")); - if (pkgJson !== null && typeof pkgJson["version"] === "string") { - return { - dir: candidate, - name: typeof pkgJson["name"] === "string" ? pkgJson["name"] : name, - version: pkgJson["version"], - pkgJson, - }; - } - const parent = dirname(dir); - if (parent === dir) return null; - dir = parent; - } -} - -/* ── attestation → {repo, commit} ──────────────────────────────────────── */ - -interface Attested { - repo: string; - commit: string; -} - -/** GET the npm attestation set and extract the SLSA provenance - * predicate's resolved source dependency. Throws with a one-line reason - * on every failure shape (no attestation, network, malformed). */ -async function fetchAttestation(name: string, version: string): Promise { - const url = `https://registry.npmjs.org/-/npm/v1/attestations/${encodeURIComponent(`${name}@${version}`).replace(/%40/g, "@").replace(/%2F/gi, "/")}`; - const res = await fetch(url, { signal: AbortSignal.timeout(30_000) }); - if (res.status === 404) throw new Error("no provenance attestation published"); - if (!res.ok) throw new Error(`attestation fetch failed (HTTP ${res.status})`); - const body = (await res.json()) as { attestations?: { predicateType?: string; bundle?: { dsseEnvelope?: { payload?: string } } }[] }; - for (const att of body.attestations ?? []) { - if (!att.predicateType?.startsWith("https://slsa.dev/provenance")) continue; - const payload = att.bundle?.dsseEnvelope?.payload; - if (payload === undefined) continue; - const stmt = JSON.parse(Buffer.from(payload, "base64").toString("utf8")) as { - predicate?: { buildDefinition?: { resolvedDependencies?: { uri?: string; digest?: { gitCommit?: string } }[] } }; - }; - for (const dep of stmt.predicate?.buildDefinition?.resolvedDependencies ?? []) { - const commit = dep.digest?.gitCommit; - if (typeof commit === "string" && commit !== "" && typeof dep.uri === "string") { - return { repo: dep.uri, commit }; - } - } - } - throw new Error("attestation set carries no SLSA provenance predicate"); -} - -/* ── source fetch (content-addressed by the attested commit) ──────────── */ - -function cacheRoot(): string { - return process.env["SCRIPTC_PROVENANCE_CACHE"] ?? join(homedir(), ".cache", "scriptc", "provenance"); -} - -/** The cached source tree for an attested commit, fetching it once from - * codeload (github repos only in the prototype). Returns the tree root. */ -async function fetchSourceTree(repo: string, commit: string): Promise { - const dest = join(cacheRoot(), commit); - if (isDirectory(dest)) return dest; - const m = /github\.com\/([^/]+)\/([^/@#]+)/.exec(repo); - if (m === null) throw new Error(`source repository is not a github URL (${repo})`); - const url = `https://codeload.github.com/${m[1]}/${m[2]}/tar.gz/${commit}`; - const res = await fetch(url, { signal: AbortSignal.timeout(120_000) }); - if (!res.ok) throw new Error(`source fetch failed (HTTP ${res.status} for ${url})`); - const bytes = Buffer.from(await res.arrayBuffer()); - await mkdir(cacheRoot(), { recursive: true }); - const tmp = await mkdtemp(join(tmpdir(), "scriptc-provenance-")); - try { - const tarball = join(tmp, "src.tgz"); - await writeFile(tarball, bytes); - const extractDir = join(tmp, "tree"); - await mkdir(extractDir); - await execFileAsync("tar", ["-xzf", tarball, "-C", extractDir, "--strip-components=1"]); - try { - await rename(extractDir, dest); - } catch { - // A parallel compile won the rename — its tree is the same content. - if (!isDirectory(dest)) throw new Error("source cache rename failed"); - } - } finally { - await rm(tmp, { recursive: true, force: true }); - } - return dest; -} - -/* ── package dir + source-entry mapping ────────────────────────────────── */ - -/** The directory inside `tree` whose package.json names `name`: the root, - * then the conventional monorepo layouts, then a bounded scan. */ -function locatePackageDir(tree: string, name: string): string | null { - const nameOf = (dir: string): string | null => { - const pkg = readJson(join(dir, "package.json")); - return pkg !== null && typeof pkg["name"] === "string" ? pkg["name"] : null; - }; - if (nameOf(tree) === name) return tree; - const bare = name.startsWith("@") ? name.split("/")[1]! : name; - const candidates = [ - join(tree, "packages", bare), - join(tree, "packages", name), - join(tree, "packages", name.replace("@", "").replace("/", "-")), - join(tree, "packages", `babel-${bare}`), - ]; - for (const c of candidates) { - if (nameOf(c) === name) return c; - } - // Bounded breadth-first scan (depth ≤ 3, node_modules/.git skipped). - const queue: { dir: string; depth: number }[] = [{ dir: tree, depth: 0 }]; - while (queue.length > 0) { - const { dir, depth } = queue.shift()!; - if (depth > 0 && nameOf(dir) === name) return dir; - if (depth >= 3) continue; - let entries: string[]; - try { - entries = readdirSync(dir).filter((entry) => isDirectory(join(dir, entry))).sort(); - } catch { - continue; - } - for (const e of entries) { - if (e === "node_modules" || e.startsWith(".")) continue; - queue.push({ dir: join(dir, e), depth: depth + 1 }); - } - } - return null; -} - -/** The published dist target for `subpath` — "exports" with the import - * condition, else module/main/types fields (root subpath only). */ -function publishedTargetOf(pkgJson: Record, subpath: string): string | null { - if (pkgJson["exports"] !== undefined) { - return resolveExports(pkgJson["exports"], subpath, NODE_IMPORT_CONDITIONS); - } - if (subpath !== ".") return subpath; - for (const field of ["module", "main", "types"]) { - const v = pkgJson[field]; - if (typeof v === "string" && v !== "") return v; - } - return "index.js"; -} - -/** dist target → source file, heuristically: the built path's leading - * dist/lib/build segment rewrites to src (or drops), an esm/cjs/dts flavor - * segment is skipped for a shared TypeScript source tree, extensions - * rewrite to TypeScript twins, and the root entry falls back to the - * conventional src/index.ts homes. First existing candidate wins. */ -function mapEntryToSource(pkgDir: string, target: string, subpath: string): string | null { - const rel = target.replace(/^\.\//, ""); - const stems = new Set([rel]); - const distRe = /^(dist|lib|build|out|output|dist-node|dist-src)\//; - if (distRe.test(rel)) { - const withoutDist = rel.replace(distRe, ""); - if (/^(esm|cjs|dts)\//.test(withoutDist)) { - stems.add(`src/${withoutDist.replace(/^(esm|cjs|dts)\//, "")}`); - } - stems.add(rel.replace(distRe, "src/")); - stems.add(withoutDist); - } else { - stems.add(`src/${rel}`); - } - const candidates: string[] = []; - for (const stem of stems) { - const base = stem.replace(/\.d\.(ts|mts|cts)$/, ".$1").replace(/\.(js|mjs|cjs)$/, ""); - if (/\.(ts|tsx|mts|cts)$/.test(base)) candidates.push(base); - else { - candidates.push(`${base}.ts`, `${base}.mts`, `${base}.cts`, `${base}.tsx`); - candidates.push(join(base, "index.ts")); - } - } - if (subpath === ".") { - candidates.push("src/index.ts", "src/index.mts", "index.ts", "src/index.tsx"); - } - for (const c of candidates) { - const abs = join(pkgDir, c); - if (isFile(abs)) return abs; - } - return null; -} - -/* ── the pipeline ─────────────────────────────────────────────────────── */ - -interface ManifestEntry { - dir: string; - commit?: string; - repo?: string; -} - -function readManifest(): Map { - const path = process.env["SCRIPTC_PROVENANCE_MANIFEST"]; - const out = new Map(); - if (path === undefined || path === "") return out; - const manifest = readJson(resolve(path)); - const packages = manifest?.["packages"]; - if (packages === null || typeof packages !== "object") return out; - for (const [name, raw] of Object.entries(packages as Record)) { - if (raw === null || typeof raw !== "object") continue; - const e = raw as { dir?: unknown; commit?: unknown; repo?: unknown }; - if (typeof e.dir !== "string") continue; - const dir = isAbsolute(e.dir) ? e.dir : resolve(dirname(resolve(path)), e.dir); - out.set(name, { - dir, - ...(typeof e.commit === "string" ? { commit: e.commit } : {}), - ...(typeof e.repo === "string" ? { repo: e.repo } : {}), - }); - } - return out; -} - -/** Resolves provenance sources for everything the entry's module graph - * imports (one transitive round per newly-mapped tree: source imports of - * OTHER packages try the pipeline too). Never throws for a package - * failure — those become notes and the package keeps its island path. */ -export async function resolveProvenanceSources(entryPath: string): Promise { - const entry = resolve(entryPath); - const manifest = readManifest(); - const packages: ProvenancePackageSource[] = []; - const notes: string[] = []; - /** package name → mapped package (or null after a noted failure). */ - const processed = new Map(); - - /** All specifiers seen so far, grouped by package name. */ - const specifiersByPackage = new Map>(); - const enqueue = (specs: readonly string[]): string[] => { - const newNames: string[] = []; - for (const spec of specs) { - const name = packageNameOf(spec); - let set = specifiersByPackage.get(name); - if (set === undefined) { - specifiersByPackage.set(name, (set = new Set())); - newNames.push(name); - } - set.add(spec); - } - return newNames; - }; - - const mapOne = async (name: string): Promise => { - if (processed.has(name)) return; - if (processed.size >= MAX_PACKAGES) { - processed.set(name, null); - notes.push(`${name}: skipped — provenance package limit (${MAX_PACKAGES}) reached; island path used`); - return; - } - processed.set(name, null); // claimed; overwritten on success - const installed = findInstalled(dirname(entry), name); - if (installed === null) { - notes.push(`${name}: not installed under the entry's node_modules; island path used`); - return; - } - let dir: string; - let repo: string; - let commit: string; - const seeded = manifest.get(name); - try { - if (seeded !== undefined) { - dir = seeded.dir; - repo = seeded.repo ?? "(manifest)"; - commit = seeded.commit ?? "(manifest)"; - if (!isDirectory(dir)) throw new Error(`manifest dir does not exist (${dir})`); - } else { - const attested = await fetchAttestation(installed.name, installed.version); - repo = attested.repo; - commit = attested.commit; - const tree = await fetchSourceTree(repo, commit); - const located = locatePackageDir(tree, installed.name); - if (located === null) { - throw new Error(`package directory not found inside the attested source tree (${tree})`); - } - dir = located; - } - const entries: Record = {}; - for (const spec of specifiersByPackage.get(name) ?? []) { - const parts = spec.split("/"); - const nameLen = spec.startsWith("@") ? 2 : 1; - const subpath = parts.length === nameLen ? "." : `./${parts.slice(nameLen).join("/")}`; - const target = publishedTargetOf(installed.pkgJson, subpath); - const source = target === null ? null : mapEntryToSource(dir, target, subpath); - if (source === null) { - notes.push( - `${name}@${installed.version}: no source mapping for '${spec}' (published target: ${target ?? "unexported"}); island path used`, - ); - continue; - } - entries[spec] = source; - } - if (Object.keys(entries).length === 0) return; - const srcPkg = readJson(join(dir, "package.json")); - const sourceVersion = typeof srcPkg?.["version"] === "string" ? srcPkg["version"] : undefined; - const pkg: ProvenancePackageSource = { - name: installed.name, - version: installed.version, - ...(sourceVersion !== undefined && sourceVersion !== installed.version ? { sourceVersion } : {}), - repo, - commit, - dir, - entries, - }; - if (pkg.sourceVersion !== undefined) { - notes.push( - `${name}: source tree's package.json says ${pkg.sourceVersion}, installed is ${installed.version} (release tooling that bumps at publish) — the behavior differential is the check`, - ); - } - packages.push(pkg); - processed.set(name, pkg); - // One transitive round: the mapped source's own bare imports try - // the pipeline too (versions resolve from the DRIVER's tree — a - // prototype heuristic; production wants the package's lockfile). - const inner = enqueue(bareImportsOf(Object.values(entries))); - for (const n of inner) await mapOne(n); - } catch (e) { - notes.push(`${name}@${installed.version}: ${e instanceof Error ? e.message : String(e)}; island path used`); - } - }; - - for (const name of enqueue(bareImportsOf([entry]))) { - await mapOne(name); - } - return { packages, notes }; +export function resolveProvenanceSources(entryPath: string): Promise { + return resolveProvenanceSourcesWithParser(entryPath, parseSourceFile); } diff --git a/packages/compiler/src/frontend/services.ts b/packages/compiler/src/frontend/services.ts index b8ebcaa3..92185223 100644 --- a/packages/compiler/src/frontend/services.ts +++ b/packages/compiler/src/frontend/services.ts @@ -63,6 +63,10 @@ export class FrontendServices { nullableClassFields(path: string, source: string): syntax.NpmStaticOverloadRewrite | null { return syntax.applyNpmStaticNullableClassFields(this.parse(path, source, "js"), source); } + jsDocNamepaths(path: string, source: string): string | null { + if (!source.includes("~") && !source.includes("#")) return null; + return syntax.applyNpmStaticJsDocNamepaths(this.parse(path, source, "js"), source); + } findReturnWidening(path: string, source: string): syntax.NpmStaticOverloadRewrite | null { return syntax.applyNpmStaticFindReturnWidening(this.parse(path, source, "js"), source); } diff --git a/packages/compiler/src/frontend/ts7/ast-node.test.ts b/packages/compiler/src/frontend/ts7/ast-node.test.ts index 1755f219..65ee3885 100644 --- a/packages/compiler/src/frontend/ts7/ast-node.test.ts +++ b/packages/compiler/src/frontend/ts7/ast-node.test.ts @@ -5,7 +5,7 @@ import { dirname, join } from "node:path"; import { afterAll, beforeAll, expect, test } from "vitest"; import type { SourceFile, Node } from "typescript/unstable/ast"; import { AstFile, AstNode } from "./ast-node.js"; -import { AstKind, KIND_NODE_LIST, astChildNames, HEADER_OFFSET_NODES, NODE_LEN, NODE_OFFSET_PARENT } from "./ast-schema.generated.js"; +import { AstKind, KIND_NODE_LIST, astChildNames, astChildOrder, HEADER_OFFSET_NODES, NODE_LEN, NODE_OFFSET_PARENT } from "./ast-schema.generated.js"; import { walkPreorder } from "./ast.js"; import { ts7Executable } from "./rpc-api.js"; import { Ts7RpcClient } from "./rpc-client.js"; @@ -18,6 +18,22 @@ type OracleNode = Node & { index: number; id: string; text?: string; rawText?: s type OracleFile = SourceFile & { getOrCreateNodeAtIndex(index: number): OracleNode }; const { RemoteSourceFile } = require(join(sdkRoot, "dist/api/node/node.js")) as { RemoteSourceFile: new (bytes: Uint8Array, decoder: InstanceType) => OracleFile }; const { Wtf8Decoder } = require(join(sdkRoot, "dist/api/node/wtf8.js")) as { Wtf8Decoder: typeof TextDecoder }; +const { childProperties } = require(join(sdkRoot, "dist/api/node/protocol.js")) as { childProperties: Record }; + +test("generated child ordinals match every pinned TypeScript property", () => { + const names = [...new Set(Object.values(childProperties).flat())]; + for (const [kind, properties] of Object.entries(childProperties)) { + for (const name of names) { + expect(astChildOrder(Number(kind), name), `${kind}.${name}`).toBe(properties.indexOf(name)); + } + for (const name of ["", "name,body", "statementsExtra", "body ", "__proto__"]) { + expect(astChildOrder(Number(kind), name)).toBe(-1); + } + } + for (const kind of [-1, 0xffffffff, NaN, KIND_NODE_LIST]) { + for (const name of names) expect(astChildOrder(kind, name)).toBe(-1); + } +}); const cases: Record = { "main.ts": [ diff --git a/packages/compiler/src/frontend/ts7/ast-schema.generated.ts b/packages/compiler/src/frontend/ts7/ast-schema.generated.ts index c3de64f2..267d4166 100644 --- a/packages/compiler/src/frontend/ts7/ast-schema.generated.ts +++ b/packages/compiler/src/frontend/ts7/ast-schema.generated.ts @@ -684,3 +684,768 @@ export function astChildNames(kind: number): string { default: return ""; } } + +/** Wire slot for a child property, or -1 when the kind has no such child. */ +export function astChildOrder(kind: number, name: string): number { + switch (kind) { + case 167: // FirstNode + switch (name) { + case "left": return 0; + case "right": return 1; + default: return -1; + } + case 168: // ComputedPropertyName + case 171: // Decorator + case 218: // ParenthesizedExpression + case 221: // DeleteExpression + case 222: // TypeOfExpression + case 223: // VoidExpression + case 224: // AwaitExpression + case 231: // SpreadElement + case 236: // NonNullExpression + case 245: // ExpressionStatement + case 254: // ReturnStatement + case 258: // ThrowStatement + case 284: // ExternalModuleReference + case 294: // JsxSpreadAttribute + case 305: // SpreadAssignment + case 348: // PartiallyEmittedExpression + switch (name) { + case "expression": return 0; + default: return -1; + } + case 169: // TypeParameter + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "constraint": return 2; + case "expression": return 3; + case "defaultType": return 4; + default: return -1; + } + case 170: // Parameter + switch (name) { + case "modifiers": return 0; + case "dotDotDotToken": return 1; + case "name": return 2; + case "questionToken": return 3; + case "type": return 4; + case "initializer": return 5; + default: return -1; + } + case 172: // PropertySignature + case 173: // PropertyDeclaration + case 303: // PropertyAssignment + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "postfixToken": return 2; + case "type": return 3; + case "initializer": return 4; + default: return -1; + } + case 174: // MethodSignature + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "postfixToken": return 2; + case "typeParameters": return 3; + case "parameters": return 4; + case "type": return 5; + default: return -1; + } + case 175: // MethodDeclaration + switch (name) { + case "modifiers": return 0; + case "asteriskToken": return 1; + case "name": return 2; + case "postfixToken": return 3; + case "typeParameters": return 4; + case "parameters": return 5; + case "type": return 6; + case "body": return 7; + default: return -1; + } + case 176: // ClassStaticBlockDeclaration + switch (name) { + case "modifiers": return 0; + case "body": return 1; + default: return -1; + } + case 177: // Constructor + switch (name) { + case "modifiers": return 0; + case "typeParameters": return 1; + case "parameters": return 2; + case "type": return 3; + case "body": return 4; + default: return -1; + } + case 178: // GetAccessor + case 179: // SetAccessor + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "typeParameters": return 2; + case "parameters": return 3; + case "type": return 4; + case "body": return 5; + default: return -1; + } + case 180: // CallSignature + case 181: // ConstructSignature + case 185: // FunctionType + case 318: // JSDocSignature + switch (name) { + case "typeParameters": return 0; + case "parameters": return 1; + case "type": return 2; + default: return -1; + } + case 182: // IndexSignature + switch (name) { + case "modifiers": return 0; + case "parameters": return 1; + case "type": return 2; + default: return -1; + } + case 183: // FirstTypeNode + switch (name) { + case "assertsModifier": return 0; + case "parameterName": return 1; + case "type": return 2; + default: return -1; + } + case 184: // TypeReference + switch (name) { + case "typeName": return 0; + case "typeArguments": return 1; + default: return -1; + } + case 186: // ConstructorType + switch (name) { + case "modifiers": return 0; + case "typeParameters": return 1; + case "parameters": return 2; + case "type": return 3; + default: return -1; + } + case 187: // TypeQuery + switch (name) { + case "exprName": return 0; + case "typeArguments": return 1; + default: return -1; + } + case 188: // TypeLiteral + switch (name) { + case "members": return 0; + default: return -1; + } + case 189: // ArrayType + switch (name) { + case "elementType": return 0; + default: return -1; + } + case 190: // TupleType + case 207: // ObjectBindingPattern + case 208: // ArrayBindingPattern + case 210: // ArrayLiteralExpression + case 276: // NamedImports + case 280: // NamedExports + switch (name) { + case "elements": return 0; + default: return -1; + } + case 191: // OptionalType + case 192: // RestType + case 197: // ParenthesizedType + case 199: // TypeOperator + case 308: // FirstJSDocNode + case 311: // JSDocNullableType + case 312: // JSDocNonNullableType + case 313: // JSDocOptionalType + case 314: // JSDocVariadicType + switch (name) { + case "type": return 0; + default: return -1; + } + case 193: // UnionType + case 194: // IntersectionType + case 299: // HeritageClause + switch (name) { + case "types": return 0; + default: return -1; + } + case 195: // ConditionalType + switch (name) { + case "checkType": return 0; + case "extendsType": return 1; + case "trueType": return 2; + case "falseType": return 3; + default: return -1; + } + case 196: // InferType + switch (name) { + case "typeParameter": return 0; + default: return -1; + } + case 200: // IndexedAccessType + switch (name) { + case "objectType": return 0; + case "indexType": return 1; + default: return -1; + } + case 201: // MappedType + switch (name) { + case "readonlyToken": return 0; + case "typeParameter": return 1; + case "nameType": return 2; + case "questionToken": return 3; + case "type": return 4; + case "members": return 5; + default: return -1; + } + case 202: // LiteralType + switch (name) { + case "literal": return 0; + default: return -1; + } + case 203: // NamedTupleMember + switch (name) { + case "dotDotDotToken": return 0; + case "name": return 1; + case "questionToken": return 2; + case "type": return 3; + default: return -1; + } + case 204: // TemplateLiteralType + case 229: // TemplateExpression + switch (name) { + case "head": return 0; + case "templateSpans": return 1; + default: return -1; + } + case 205: // TemplateLiteralTypeSpan + switch (name) { + case "type": return 0; + case "literal": return 1; + default: return -1; + } + case 206: // LastTypeNode + switch (name) { + case "argument": return 0; + case "attributes": return 1; + case "qualifier": return 2; + case "typeArguments": return 3; + default: return -1; + } + case 209: // BindingElement + switch (name) { + case "dotDotDotToken": return 0; + case "propertyName": return 1; + case "name": return 2; + case "initializer": return 3; + default: return -1; + } + case 211: // ObjectLiteralExpression + case 293: // JsxAttributes + switch (name) { + case "properties": return 0; + default: return -1; + } + case 212: // PropertyAccessExpression + switch (name) { + case "expression": return 0; + case "questionDotToken": return 1; + case "name": return 2; + default: return -1; + } + case 213: // ElementAccessExpression + switch (name) { + case "expression": return 0; + case "questionDotToken": return 1; + case "argumentExpression": return 2; + default: return -1; + } + case 214: // CallExpression + switch (name) { + case "expression": return 0; + case "questionDotToken": return 1; + case "typeArguments": return 2; + case "arguments": return 3; + default: return -1; + } + case 215: // NewExpression + switch (name) { + case "expression": return 0; + case "typeArguments": return 1; + case "arguments": return 2; + default: return -1; + } + case 216: // TaggedTemplateExpression + switch (name) { + case "tag": return 0; + case "questionDotToken": return 1; + case "typeArguments": return 2; + case "template": return 3; + default: return -1; + } + case 217: // TypeAssertionExpression + switch (name) { + case "type": return 0; + case "expression": return 1; + default: return -1; + } + case 219: // FunctionExpression + case 263: // FunctionDeclaration + switch (name) { + case "modifiers": return 0; + case "asteriskToken": return 1; + case "name": return 2; + case "typeParameters": return 3; + case "parameters": return 4; + case "type": return 5; + case "body": return 6; + default: return -1; + } + case 220: // ArrowFunction + switch (name) { + case "modifiers": return 0; + case "typeParameters": return 1; + case "parameters": return 2; + case "type": return 3; + case "equalsGreaterThanToken": return 4; + case "body": return 5; + default: return -1; + } + case 225: // PrefixUnaryExpression + case 226: // PostfixUnaryExpression + switch (name) { + case "operand": return 0; + default: return -1; + } + case 227: // BinaryExpression + switch (name) { + case "modifiers": return 0; + case "left": return 1; + case "type": return 2; + case "operatorToken": return 3; + case "right": return 4; + default: return -1; + } + case 228: // ConditionalExpression + switch (name) { + case "condition": return 0; + case "questionToken": return 1; + case "whenTrue": return 2; + case "colonToken": return 3; + case "whenFalse": return 4; + default: return -1; + } + case 230: // YieldExpression + switch (name) { + case "asteriskToken": return 0; + case "expression": return 1; + default: return -1; + } + case 232: // ClassExpression + case 264: // ClassDeclaration + case 265: // InterfaceDeclaration + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "typeParameters": return 2; + case "heritageClauses": return 3; + case "members": return 4; + default: return -1; + } + case 234: // ExpressionWithTypeArguments + switch (name) { + case "expression": return 0; + case "typeArguments": return 1; + default: return -1; + } + case 235: // AsExpression + case 239: // SatisfiesExpression + switch (name) { + case "expression": return 0; + case "type": return 1; + default: return -1; + } + case 237: // MetaProperty + case 275: // NamespaceImport + case 281: // NamespaceExport + case 309: // JSDocNameReference + case 319: // JSDocLink + case 320: // JSDocLinkCode + case 321: // JSDocLinkPlain + switch (name) { + case "name": return 0; + default: return -1; + } + case 238: // SyntheticExpression + switch (name) { + case "tupleNameSource": return 0; + default: return -1; + } + case 240: // TemplateSpan + switch (name) { + case "expression": return 0; + case "literal": return 1; + default: return -1; + } + case 242: // Block + case 269: // ModuleBlock + switch (name) { + case "statements": return 0; + default: return -1; + } + case 244: // FirstStatement + switch (name) { + case "modifiers": return 0; + case "declarationList": return 1; + default: return -1; + } + case 246: // IfStatement + switch (name) { + case "expression": return 0; + case "thenStatement": return 1; + case "elseStatement": return 2; + default: return -1; + } + case 247: // DoStatement + switch (name) { + case "statement": return 0; + case "expression": return 1; + default: return -1; + } + case 248: // WhileStatement + case 255: // WithStatement + switch (name) { + case "expression": return 0; + case "statement": return 1; + default: return -1; + } + case 249: // ForStatement + switch (name) { + case "initializer": return 0; + case "condition": return 1; + case "incrementor": return 2; + case "statement": return 3; + default: return -1; + } + case 250: // ForInStatement + case 251: // ForOfStatement + switch (name) { + case "awaitModifier": return 0; + case "initializer": return 1; + case "expression": return 2; + case "statement": return 3; + default: return -1; + } + case 252: // ContinueStatement + case 253: // BreakStatement + switch (name) { + case "label": return 0; + default: return -1; + } + case 256: // SwitchStatement + switch (name) { + case "expression": return 0; + case "caseBlock": return 1; + default: return -1; + } + case 257: // LabeledStatement + switch (name) { + case "label": return 0; + case "statement": return 1; + default: return -1; + } + case 259: // TryStatement + switch (name) { + case "tryBlock": return 0; + case "catchClause": return 1; + case "finallyBlock": return 2; + default: return -1; + } + case 261: // VariableDeclaration + switch (name) { + case "name": return 0; + case "exclamationToken": return 1; + case "type": return 2; + case "initializer": return 3; + default: return -1; + } + case 262: // VariableDeclarationList + switch (name) { + case "declarations": return 0; + default: return -1; + } + case 266: // TypeAliasDeclaration + case 345: // JSTypeAliasDeclaration + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "typeParameters": return 2; + case "type": return 3; + default: return -1; + } + case 267: // EnumDeclaration + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "members": return 2; + default: return -1; + } + case 268: // ModuleDeclaration + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "body": return 2; + default: return -1; + } + case 270: // CaseBlock + switch (name) { + case "clauses": return 0; + default: return -1; + } + case 271: // NamespaceExportDeclaration + switch (name) { + case "modifiers": return 0; + case "name": return 1; + default: return -1; + } + case 272: // ImportEqualsDeclaration + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "moduleReference": return 2; + default: return -1; + } + case 273: // ImportDeclaration + case 346: // JSImportDeclaration + switch (name) { + case "modifiers": return 0; + case "importClause": return 1; + case "moduleSpecifier": return 2; + case "attributes": return 3; + default: return -1; + } + case 274: // ImportClause + switch (name) { + case "name": return 0; + case "namedBindings": return 1; + default: return -1; + } + case 277: // ImportSpecifier + case 282: // ExportSpecifier + switch (name) { + case "propertyName": return 0; + case "name": return 1; + default: return -1; + } + case 278: // ExportAssignment + switch (name) { + case "modifiers": return 0; + case "type": return 1; + case "expression": return 2; + default: return -1; + } + case 279: // ExportDeclaration + switch (name) { + case "modifiers": return 0; + case "exportClause": return 1; + case "moduleSpecifier": return 2; + case "attributes": return 3; + default: return -1; + } + case 283: // MissingDeclaration + switch (name) { + case "modifiers": return 0; + default: return -1; + } + case 285: // JsxElement + switch (name) { + case "openingElement": return 0; + case "children": return 1; + case "closingElement": return 2; + default: return -1; + } + case 286: // JsxSelfClosingElement + case 287: // JsxOpeningElement + switch (name) { + case "tagName": return 0; + case "typeArguments": return 1; + case "attributes": return 2; + default: return -1; + } + case 288: // JsxClosingElement + switch (name) { + case "tagName": return 0; + default: return -1; + } + case 289: // JsxFragment + switch (name) { + case "openingFragment": return 0; + case "children": return 1; + case "closingFragment": return 2; + default: return -1; + } + case 292: // JsxAttribute + case 306: // EnumMember + switch (name) { + case "name": return 0; + case "initializer": return 1; + default: return -1; + } + case 295: // JsxExpression + switch (name) { + case "dotDotDotToken": return 0; + case "expression": return 1; + default: return -1; + } + case 296: // JsxNamespacedName + switch (name) { + case "namespace": return 0; + case "name": return 1; + default: return -1; + } + case 297: // CaseClause + case 298: // DefaultClause + switch (name) { + case "expression": return 0; + case "statements": return 1; + default: return -1; + } + case 300: // CatchClause + switch (name) { + case "variableDeclaration": return 0; + case "block": return 1; + default: return -1; + } + case 301: // ImportAttributes + switch (name) { + case "attributes": return 0; + default: return -1; + } + case 302: // ImportAttribute + switch (name) { + case "name": return 0; + case "value": return 1; + default: return -1; + } + case 304: // ShorthandPropertyAssignment + switch (name) { + case "modifiers": return 0; + case "name": return 1; + case "postfixToken": return 2; + case "type": return 3; + case "equalsToken": return 4; + case "objectAssignmentInitializer": return 5; + default: return -1; + } + case 307: // SourceFile + switch (name) { + case "statements": return 0; + case "endOfFileToken": return 1; + default: return -1; + } + case 315: // JSDoc + switch (name) { + case "comment": return 0; + case "tags": return 1; + default: return -1; + } + case 317: // JSDocTypeLiteral + switch (name) { + case "jsdocPropertyTags": return 0; + default: return -1; + } + case 322: // FirstJSDocTagNode + case 325: // JSDocDeprecatedTag + case 326: // JSDocPublicTag + case 327: // JSDocPrivateTag + case 328: // JSDocProtectedTag + case 329: // JSDocReadonlyTag + case 330: // JSDocOverrideTag + switch (name) { + case "tagName": return 0; + case "comment": return 1; + default: return -1; + } + case 323: // JSDocAugmentsTag + case 324: // JSDocImplementsTag + switch (name) { + case "tagName": return 0; + case "className": return 1; + case "comment": return 2; + default: return -1; + } + case 331: // JSDocCallbackTag + case 338: // JSDocTypedefTag + switch (name) { + case "tagName": return 0; + case "typeExpression": return 1; + case "name": return 2; + case "comment": return 3; + default: return -1; + } + case 332: // JSDocOverloadTag + case 334: // JSDocReturnTag + case 335: // JSDocThisTag + case 336: // JSDocTypeTag + case 341: // JSDocThrowsTag + case 342: // JSDocSatisfiesTag + switch (name) { + case "tagName": return 0; + case "typeExpression": return 1; + case "comment": return 2; + default: return -1; + } + case 333: // JSDocParameterTag + case 340: // JSDocPropertyTag + switch (name) { + case "tagName": return 0; + case "name": return 1; + case "typeExpression": return 2; + case "comment": return 3; + default: return -1; + } + case 337: // JSDocTemplateTag + switch (name) { + case "tagName": return 0; + case "constraint": return 1; + case "typeParameters": return 2; + case "comment": return 3; + default: return -1; + } + case 339: // JSDocSeeTag + switch (name) { + case "tagName": return 0; + case "nameExpression": return 1; + case "comment": return 2; + default: return -1; + } + case 343: // LastJSDocTagNode + switch (name) { + case "tagName": return 0; + case "importClause": return 1; + case "moduleSpecifier": return 2; + case "attributes": return 3; + case "comment": return 4; + default: return -1; + } + case 344: // SyntaxList + switch (name) { + case "children": return 0; + default: return -1; + } + case 349: // SyntheticReferenceExpression + switch (name) { + case "expression": return 0; + case "thisArg": return 1; + default: return -1; + } + default: return -1; + } +} diff --git a/packages/compiler/src/frontend/ts7/ast-wire.ts b/packages/compiler/src/frontend/ts7/ast-wire.ts index 52ec9e9d..a1d453c2 100644 --- a/packages/compiler/src/frontend/ts7/ast-wire.ts +++ b/packages/compiler/src/frontend/ts7/ast-wire.ts @@ -5,7 +5,7 @@ import { HEADER_OFFSET_PARSE_OPTIONS, HEADER_OFFSET_STRING_TABLE, HEADER_OFFSET_STRING_TABLE_OFFSETS, HEADER_OFFSET_STRUCTURED_DATA, HEADER_SIZE, KIND_NODE_LIST, NODE_LEN, NODE_OFFSET_DATA, NODE_OFFSET_END, NODE_OFFSET_FLAGS, NODE_OFFSET_KIND, NODE_OFFSET_NEXT, NODE_OFFSET_PARENT, - NODE_OFFSET_POS, PROTOCOL_VERSION, astChildNames, + NODE_OFFSET_POS, PROTOCOL_VERSION, astChildOrder, } from "./ast-schema.generated.js"; export interface AstFileReference { @@ -208,17 +208,8 @@ export class AstWireFile { } namedChild(index: number, name: string): number { - const names = astChildNames(this.kind(index)); - let order = 0; - let start = 0; - while (start < names.length) { - const comma = names.indexOf(",", start); - const end = comma < 0 ? names.length : comma; - if (names.slice(start, end) === name) return this.childAtOrder(index, order); - order++; - start = end + 1; - } - return 0; + const order = astChildOrder(this.kind(index), name); + return order < 0 ? 0 : this.childAtOrder(index, order); } private childAtOrder(index: number, order: number): number { diff --git a/packages/compiler/src/frontend/tsc-codes.ts b/packages/compiler/src/frontend/tsc-codes.ts index eb7964e5..ffaf3791 100644 --- a/packages/compiler/src/frontend/tsc-codes.ts +++ b/packages/compiler/src/frontend/tsc-codes.ts @@ -87,6 +87,9 @@ export const JS_RELAXED_TSC_CODES: ReadonlySet = new Set([ 2308, 2614, // 2740's sibling elaboration ("Property X is missing in type Y") 2741, + // JavaScript subclasses may widen fields or add method parameters. + // Class lowering checks native storage and override ABIs at each use. + 2415, // unresolvable modules: Node throws WHEN the require executes — the // binding types as any and reached uses fence (never a silent pass) 2307, 2792, diff --git a/packages/compiler/src/frontend/type-mapper.ts b/packages/compiler/src/frontend/type-mapper.ts index b9ec237b..6f9acaa0 100644 --- a/packages/compiler/src/frontend/type-mapper.ts +++ b/packages/compiler/src/frontend/type-mapper.ts @@ -1040,17 +1040,25 @@ function mapTypeInner(type: ts.Type, ctx: TypeMapperCtx): IrType | null { // user aliases with the same names on the normal structural path. { const parseArgsSym = widened.getAliasSymbol() ?? widened.getSymbol(); - if ( - parseArgsSym && - PARSE_ARGS_DYN_TYPES.has(parseArgsSym.name) && - checker.declarationsOf(parseArgsSym).some( - (d) => - ctx.isStdlibFile(d.getSourceFile()) && - isDeclaredInAmbientModule(d as ts.Declaration, "util"), - ) - ) { - return DYN; - } + const belongs = (declaration: ts.Node): boolean => { + if (!ctx.isStdlibFile(declaration.getSourceFile())) return false; + let family = false; + for (let node: ts.Node | undefined = declaration; node; node = node.parent) { + if ((ts.isTypeAliasDeclaration(node) || ts.isInterfaceDeclaration(node)) && + PARSE_ARGS_DYN_TYPES.has(node.name.text)) family = true; + if (ts.isModuleDeclaration(node) && ts.isStringLiteral(node.name)) { + return family && (node.name.text === "util" || node.name.text === "node:util"); + } + } + return false; + }; + if (parseArgsSym && PARSE_ARGS_DYN_TYPES.has(parseArgsSym.name) && + checker.declarationsOf(parseArgsSym).some(belongs)) return DYN; + // ReturnType and instantiated conditional types can erase the alias. + // Require every member to retain the util declaration provenance so + // unrelated records with similar property names keep their own layout. + const members = checker.getPropertiesOfType(widened); + if (members.length > 0 && members.every((member) => checker.declarationsOf(member).some(belongs))) return DYN; } // The lib's BOXED wrapper interfaces used as TYPES (`const n: Number = // 5`): every value such a slot can hold IS the primitive — `new @@ -3542,13 +3550,19 @@ export function isUnitOnlyTsType(t: ts.Type, resolveTypeParam?: TypeParamResolve * admits, `(() => void) | undefined`) so the interned union is IDENTICAL * to what mapping the checker's own `T | undefined` produces. */ export function withUndefinedArm(t: IrType, unions: UnionRegistry): IrType | null { + return withUnitArm(t, "undefinedT", unions); +} + +/** Preserve a nullish runtime value that JavaScript inference omitted. */ +export function withUnitArm(t: IrType, kind: "nullT" | "undefinedT", unions: UnionRegistry): IrType | null { + const unit: IrType = { kind }; if (t.kind === "jsval") return t; if (t.kind === "union") { const def = unions.get(t.unionId); if (!def) return null; if (def.arms.some((a) => a.kind === "date")) return null; - if (def.arms.some((a) => a.kind === "undefinedT")) return t; - const arms = [...def.arms, UNDEFINED_T]; + if (def.arms.some((a) => a.kind === kind)) return t; + const arms = [...def.arms, unit]; arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); return { kind: "union", unionId: unions.transform(def, arms) }; } @@ -3560,7 +3574,7 @@ export function withUndefinedArm(t: IrType, unions: UnionRegistry): IrType | nul ) { return null; } - const arms = [t, UNDEFINED_T]; + const arms = [t, unit]; arms.sort((a, b) => (typeKey(a) < typeKey(b) ? -1 : 1)); return { kind: "union", unionId: unions.intern(arms) }; } diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 3bec71dd..1dcc12ce 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,3 +1,10 @@ +import { compilationTiming, type CompilationTiming } from "./timing.js"; +import { prepareExecutableModule } from "./executable/prepare.js"; +import { prepareLibrary, libraryLocalizeSymbols, libraryWasmExports, libraryWasmRefusal } from "./library/prepare.js"; +import { analyzeWithFrontend } from "./frontend/analysis.js"; +import { llvmRefusalDiag, targetRefusalDiag } from "./backend/target-diagnostics.js"; +import type { CompileOptions, CompileSourceOptions, CompileRequestOptions, CompileSourceResult, CompileResult, CompileExecutableResult, CompileRequestResult, CompileLibraryOptions, CompileLibraryResult, AnalyzeOptions, AnalyzeResult } from "./compile-types.js"; +export type { CompileOutputKind, CompileBaseOptions, CompileOptions, CompileSourceOptions, CompileRequestOptions, CompileArtifact, CompileFailure, CompileSourceResult, CompileResult, CompileExecutableResult, CompileRequestResult, CompileLibraryOptions, CompileLibraryResult, AnalyzeOptions, AnalyzeResult } from "./compile-types.js"; import { compilePackedLibrary } from "./backend/library-pack.js"; import { InternalCompilerError } from "./errors.js"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; @@ -18,7 +25,7 @@ import { emitNativeArtifact, NativeCodegenError } from "./backend/native-codegen import { privateSiblingPath } from "./backend/build-cache.js"; import { nativeCodegenTarget, nativeCodegenTargetRefusal } from "./backend/targets.js"; import { windowsSubsystemLinkerArgs, type WindowsSubsystem } from "./backend/targets.js"; -import { createNativeLinkInfo, type NativeLinkInfo } from "./backend/native-link-info.js"; +import { createNativeLinkInfo } from "./backend/native-link-info.js"; import { RuntimePackError } from "./backend/runtime-pack.js"; import { createNativeLinkPlan } from "./backend/link-plan.js"; import { @@ -29,35 +36,23 @@ import { } from "./backend/linker.js"; import { splitLlvmLibraryProgram, splitLlvmProgram } from "./backend/llvm/split.js"; import { emitLibraryIdentityLines, replaceLibraryIdentity, stripLibraryIdentity } from "./backend/library-identity-markers.js"; -import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, nativeCodegenDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; -import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./library/int-infer.js"; -import { loadLibraryProfile, profileRemediation, profileTeaching, type LibraryProfile } from "./library/library-profile.js"; -import { clearFenceEvalCaches, decorateLibraryRefusals, evaluateLibraryFences } from "./library/fence-eval.js"; -import { assembleTrapTeaching } from "./library/trap-teaching.js"; +import { ffiNativeBuildDiag, iceDiag, nativeCodegenDiag, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; +import { loadLibraryProfile, type LibraryProfile } from "./library/library-profile.js"; +import { clearFenceEvalCaches, decorateLibraryRefusals } from "./library/fence-eval.js"; import { - buildSidecar, canonicalModuleGraph, - canonicalPath, clearSidecarCaches, compilerReleaseVersion, libraryIdentityHashes, updateSidecarIdentity, - type SidecarIntegerSlotFacts, - type SidecarIrRecordPattern, - type SidecarIrTypePattern, } from "./library/sidecar.js"; -import { validateSidecar } from "./library/sidecar-validate.js"; -import type { EntryExportInfo } from "./frontend/lib-exports.js"; -import type { ContractFacts } from "./frontend/lib-contract.js"; -import { moduleRuntimeFeatures, moduleLibAsyncSurface, moduleLibNondeterministicSurface, moduleEmbedsBuiltin, moduleUsesFetch, type IrFfiImport, type IrLibSection, type IrModule, type IrRecordShape, type IrType, type SrcLoc } from "./ir/ir.js"; +import { moduleRuntimeFeatures, type IrModule, type SrcLoc } from "./ir/ir.js"; import { serializeModule } from "./ir/serialize.js"; import { validateModule } from "./ir/validate.js"; import { loadProgram } from "./frontend/program-node.js"; -import { runFrontend } from "./frontend/pipeline.js"; +import { runFrontend, type FrontendFactory } from "./frontend/pipeline.js"; import { provenanceSources } from "./frontend/provenance-registry.js"; import { clearResolveCaches } from "./frontend/resolve.js"; -import type { LowerResult } from "./frontend/lowering/lowerer.js"; -import type { CoverageInput } from "./coverage/report.js"; import { loadFfiProfile, type FfiProfile } from "./ffi/ffi-manifest.js"; import { executableLinkFeatures } from "./backend/executable-features.js"; import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; @@ -176,234 +171,6 @@ export { } from "./frontend/provenance-registry.js"; export * as ir from "./ir/ir.js"; -export type CompileOutputKind = "ir" | "llvm" | "asm" | "obj" | "exe"; - -export interface CompileBaseOptions { - /** Primary artifact path. The CLI supplies the output-kind default. */ - outPath: string; - /** Where generated intermediates and compatibility side artifacts land. */ - outDir: string; - /** @deprecated This option no longer controls output cleanup; sibling artifacts are retained. */ - defaultOutputPath?: boolean; - /** Compatibility-only additive IR side artifact for executable builds. - * The CLI's deprecated --emit-ir flag supplies this option. */ - emitIr?: boolean; - sanitize?: boolean; - /** Embed the dynamic-island engine (--dynamic). Off = the static default: - * island constructs are diagnostics and nothing about codegen or linking - * changes. */ - dynamic?: boolean; - /** LLVM is the production code generator. */ - backend?: "llvm"; - /** Native optimization posture. Release is the shipped -O2 default; dev - * uses -O0, source line tables, and stable multi-TU object caching for - * large LLVM programs. Darwin executables include an adjacent .dSYM. */ - optimization?: "release" | "dev"; - /** Remove symbol/debug payload from an executable at link time. */ - strip?: boolean; - /** Windows PE executable subsystem. Console is the default; GUI suppresses - * automatic console-window creation. Only valid for Windows executables. */ - windowsSubsystem?: WindowsSubsystem; - /** --npm-static: package names whose shipped, unminified JS compiles - * STATICALLY as program modules (inference types the bodies; statements - * the lowering cannot prove become runtime fences). "auto" opts in every - * directly-imported package passing the eligibility heuristics (own - * .d.ts, unminified JS, no build-transform markers). A package whose - * preflight refuses marks itself an offender and falls back to the - * island (--dynamic) or the requires-dynamic diagnostic (static builds) - * — never a silent misbuild. Off by default: nothing changes without - * the flag. */ - npmStatic?: readonly string[] | "auto"; - /** Outbound native FFI manifest. Its signature-only TypeScript bindings - * lower to direct C ABI calls. Source outputs retain those declarations; - * archive/system-library inputs join only an executable link. */ - ffiProfilePath?: string; - /** Attach the machine-readable external link recipe to an object result. - * Valid only with outputKind "obj"; it never invokes a linker. */ - nativeLinkInfo?: boolean; -} - -/** Executable compile options. This remains the compatibility type for the - * historical compile() API, whose omitted output kind means executable. */ -export interface CompileOptions extends CompileBaseOptions { - outputKind?: "exe"; - /** Internal validation lane retained for helper-object artifact tests. - * Supported ordinary LLVM executable builds select this path automatically. */ - nativeProgramObject?: boolean; -} - -/** Source-artifact compile options, discriminated by the required kind. */ -export interface CompileSourceOptions extends CompileBaseOptions { - outputKind: Exclude; -} - -/** Internal/dynamic request shape for callers that select the kind at runtime. - * Statically executable/source callers should prefer the narrower interfaces. */ -export interface CompileRequestOptions extends CompileBaseOptions { - outputKind?: CompileOutputKind; - /** Internal validation lane for executable requests. */ - nativeProgramObject?: boolean; -} - -export type CompileArtifact = - | { kind: "ir"; path: string } - | { kind: "llvm"; path: string } - | { kind: "asm"; path: string } - | { kind: "obj"; path: string; nativeLinkInfo?: NativeLinkInfo } - | { - kind: "exe"; - path: string; - translationUnitPath: string; - backend: "llvm"; - - }; - -export type CompileFailure = { - ok: false; - diagnostics: ScrDiagnostic[]; - sourceTexts: Map; -}; - -export type CompileSourceResult = - | { ok: true; artifact: Extract } - | CompileFailure; - -/** Historical executable result shape retained for source compatibility. */ -export type CompileResult = - | { - ok: true; - binaryPath: string; - llvmPath: string; - irPath?: string; - backend: "llvm"; - - } - | CompileFailure; - -export type CompileExecutableResult = - /** The generated LLVM source is retained beside the executable. */ - | (Extract & { - artifact: Extract; - }) - | CompileFailure; - -/** Result union for callers that choose outputKind dynamically. */ -export type CompileRequestResult = CompileSourceResult | CompileExecutableResult; - -/** The LLVM backend's tier refusal as a diagnostic. SC3xxx = backend - * coverage (the program is fine — this backend doesn't compile it yet); - * the parenthesized kind tag is machine-readable for the differential - * harness's histogram. */ -function llvmRefusalDiag(err: LlvmUnsupportedError, entryPath: string): ScrDiagnostic { - return { - code: "SC3001", - message: err.message, - loc: err.loc ?? { file: entryPath, start: 0, end: 0 }, - }; -} - -/** A valid program surface that the selected execution target cannot host. - * SC3xxx stays the backend/target-coverage family: source semantics are - * valid, but this target deliberately refuses them instead of emitting a - * binary that traps later. */ -function targetRefusalDiag(target: string, surface: string, loc: SrcLoc): ScrDiagnostic { - return { - code: "SC3002", - message: `${target} target does not support ${surface}`, - loc, - }; -} - -/** APIs that require host capabilities absent from portable WASI Preview 1. - * These are target diagnostics, not backend-tier gaps: the same language IR - * (including async, generators, and the dynamic island) is otherwise valid. - * Keep the fine-grained walk first so diagnostics point at the API use; the - * embedded-module checks are the entry-anchored safety net for island code. */ -function moduleWasiUnavailableSurface(mod: IrModule): { surface: string; loc: SrcLoc } | null { - const entryLoc: SrcLoc = { file: mod.sourceFile, start: 0, end: 0 }; - const prefixes: readonly (readonly [string, string])[] = [ - ["cp.", "child processes (WASI Preview 1 has no process-spawning API)"], - ["child.", "child processes (WASI Preview 1 has no process-spawning API)"], - ["spawnRes.", "child processes (WASI Preview 1 has no process-spawning API)"], - ["net.", "network sockets (WASI Preview 1 has no socket API)"], - ["http.", "network sockets (WASI Preview 1 has no socket API)"], - ["https.", "network sockets (WASI Preview 1 has no socket API)"], - ["http2.", "network sockets (WASI Preview 1 has no socket API)"], - ["h2.", "network sockets (WASI Preview 1 has no socket API)"], - ["dgram.", "network sockets (WASI Preview 1 has no socket API)"], - ["dns.", "network sockets (WASI Preview 1 has no socket API)"], - ["tls.", "network sockets (WASI Preview 1 has no socket API)"], - ["fetch.", "network-backed fetch (WASI Preview 1 has no socket API)"], - ["fs.watch", "filesystem watching (WASI Preview 1 has no notification API)"], - ["watcher.", "filesystem watching (WASI Preview 1 has no notification API)"], - ]; - const kinds: ReadonlyMap = new Map([ - ["child", "child processes (WASI Preview 1 has no process-spawning API)"], - ["spawnRes", "child processes (WASI Preview 1 has no process-spawning API)"], - ["childStream", "child processes (WASI Preview 1 has no process-spawning API)"], - ["childWriter", "child processes (WASI Preview 1 has no process-spawning API)"], - ["netServer", "network sockets (WASI Preview 1 has no socket API)"], - ["netSocket", "network sockets (WASI Preview 1 has no socket API)"], - ["http2Session", "network sockets (WASI Preview 1 has no socket API)"], - ["http2Stream", "network sockets (WASI Preview 1 has no socket API)"], - ["dgramSocket", "network sockets (WASI Preview 1 has no socket API)"], - ["fsWatcher", "filesystem watching (WASI Preview 1 has no notification API)"], - ["httpReq", "network sockets (WASI Preview 1 has no socket API)"], - ["httpRes", "network sockets (WASI Preview 1 has no socket API)"], - ["httpClientReq", "network sockets (WASI Preview 1 has no socket API)"], - ["secureCtx", "network sockets (WASI Preview 1 has no socket API)"], - ]); - let found: { surface: string; loc: SrcLoc } | null = null; - const visit = (value: unknown, inheritedLoc: SrcLoc): void => { - if (found !== null || value === null || typeof value !== "object") return; - if (Array.isArray(value)) { - for (const item of value) visit(item, inheritedLoc); - return; - } - const node = value as { kind?: unknown; fn?: unknown; loc?: SrcLoc }; - const loc = node.loc ?? inheritedLoc; - if (typeof node.kind === "string") { - const kindSurface = kinds.get(node.kind); - if (kindSurface !== undefined) { - found = { surface: kindSurface, loc }; - return; - } - if (node.kind === "libCall" && typeof node.fn === "string") { - if (node.fn === "process.kill" || node.fn === "process.killNum" || - node.fn === "process.onSignal" || node.fn === "process.offSignal") { - found = { surface: "OS signals (WASI Preview 1 has no signal API)", loc }; - return; - } - if (node.fn === "os.networkInterfaces") { - found = { surface: "network-interface enumeration (WASI Preview 1 has no interface API)", loc }; - return; - } - for (const [prefix, surface] of prefixes) { - if (node.fn.startsWith(prefix)) { - found = { surface, loc }; - return; - } - } - } - } - for (const key of Object.keys(value)) { - visit((value as Record)[key], loc); - } - }; - visit(mod, entryLoc); - if (found !== null) return found; - - if (moduleUsesFetch(mod)) { - return { surface: "network-backed fetch (WASI Preview 1 has no socket API)", loc: entryLoc }; - } - for (const builtin of ["node:http", "node:https", "node:net", "node:tls"] as const) { - if (moduleEmbedsBuiltin(mod, builtin)) { - return { surface: `${builtin} networking (WASI Preview 1 has no socket API)`, loc: entryLoc }; - } - } - return null; -} - /** Clang may print every warning from the generated/runtime translation * units before the actionable linker failure. Keep the source diagnostic * precise by starting at the first portable linker marker; if the driver @@ -421,11 +188,6 @@ function ffiNativeBuildDetail(err: CcCompileError): string { ); } -export interface AnalyzeResult { - coverage: CoverageInput; - sourceTexts: Map; -} - /** Platform semantics depend on the output target, without compiler discovery. */ export function buildTargetPlatform(env: NodeJS.ProcessEnv = process.env): string { return configuredTargetPlatform(env); @@ -438,109 +200,13 @@ export function sourceTargetPlatform(env: NodeJS.ProcessEnv = process.env): stri return configuredTargetPlatform(env); } -export interface AnalyzeOptions { - /** Analyze as a --dynamic build (island constructs lower instead of - * producing requires-dynamic diagnostics). */ - dynamic?: boolean; - /** --npm-static (see CompileOptions.npmStatic): the analysis compiles - * opted-in packages' JS as program modules and the coverage report - * carries each package's static/fallback status. */ - npmStatic?: readonly string[] | "auto"; - /** Analyze with the outbound native bindings from this FFI manifest. */ - ffiProfilePath?: string; - /** Coverage-only external host type surfaces: exact bare module - * specifier → local declaration file. The checker uses the declarations - * to analyze project code, but imported runtime values remain explicit - * SC1010 blockers rather than being counted as executable. */ - externalTypes?: Readonly>; -} - -/** Analysis without codegen: how much of the program compiles statically. - * Unlike compile(), lowering diagnostics are data here, not failure. */ export function analyze(entryPath: string, opts: AnalyzeOptions = {}): AnalyzeResult { - let ffi: FfiProfile | null = null; - if (opts.ffiProfilePath !== undefined) { - const loaded = loadFfiProfile(opts.ffiProfilePath); - if (!loaded.ok) { - return { - coverage: { - file: entryPath, - dynamic: opts.dynamic ?? false, - stats: { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }, - diagnostics: loaded.diagnostics, - preflightFailed: true, - }, - sourceTexts: new Map(), - }; - } - ffi = loaded.profile; - } - const fe = runFrontend(entryPath, loadProgram, opts.npmStatic, opts.externalTypes); - try { - const emptyStats = { statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0 }; - - const preflight = fe.preflight; - // Import-FORM fences don't stop the analysis: the module graph is still - // computable (a fenced import contributes no edges), the imported - // bindings poison at their use sites, and the fences join the blockers - // list beside statement-level ones — the report shows a statement - // percentage instead of stopping at the import lines. Everything else — - // tsc errors, config incompatibilities, circular imports — still stops - // at preflight (no trustworthy program to lower). Builds are unchanged: - // compile() fails on every preflight diagnostic exactly as before. - const IMPORT_FENCES = new Set(["SC1010", "SC1012", "SC1013", "SC1014", "SC1015"]); - if (preflight.some((d) => !IMPORT_FENCES.has(d.code))) { - return { - coverage: { - file: entryPath, - dynamic: opts.dynamic ?? false, - stats: emptyStats, - diagnostics: preflight, - ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), - preflightFailed: true, - }, - sourceTexts: fe.sourceTexts(), - }; - } - // Coverage is whole-program by design: builds stop at what the entry - // reaches, but the analysis additionally lowers the unreached remainder - // (throwaway) so the report covers everything the source declares — with - // the unreached share in its own group. - const lowered = fe.lower({ - dynamic: opts.dynamic ?? false, - coverage: true, - targetPlatform: buildTargetPlatform(), - ...(ffi !== null ? { ffiImports: ffi.functions } : {}), - }); - const provenance = provenanceSources(); - return { - coverage: { - file: entryPath, - dynamic: opts.dynamic ?? false, - stats: lowered.stats, - // The import fences report as blockers alongside the statement-level - // ones (use sites of the fenced bindings emit matching diagnostics, - // which the report groups with these). - diagnostics: [...preflight, ...lowered.diagnostics], - ...(lowered.runtimeFences.length > 0 ? { runtimeFences: lowered.runtimeFences } : {}), - ...(lowered.unreached ? { unreached: lowered.unreached } : {}), - ...(lowered.npmBuiltins ? { npmBuiltins: lowered.npmBuiltins } : {}), - ...(lowered.npmLazyTraps ? { npmLazyTraps: lowered.npmLazyTraps } : {}), - ...(fe.npmStatic.length > 0 ? { npmStatic: fe.npmStatic } : {}), - // --provenance-sources: the per-package attribution inputs (the - // report aggregates statsByFile under each package's source dir). - ...(provenance !== null ? { provenance } : {}), - ...(lowered.statsByFile ? { statsByFile: lowered.statsByFile } : {}), - ...(lowered.provenanceElided ? { provenanceElided: lowered.provenanceElided } : {}), - preflightFailed: false, - }, - sourceTexts: fe.sourceTexts(), - }; - } finally { - fe.dispose(); - } + return analyzeWithFrontend(entryPath, opts, buildTargetPlatform(), nodeFrontend); } +const nodeFrontend: FrontendFactory = (entry, npmStatic, externalTypes, libraryNpmStatic) => + runFrontend(entry, loadProgram, npmStatic, externalTypes, libraryNpmStatic); + /** The whole pipeline: load → preflight → lower → validate → emit LLVM → link. */ function clearCompileSessionCaches(): void { clearResolveCaches(); @@ -753,60 +419,12 @@ interface PreparedExecutable { * typed IR can be reclaimed before the native optimizer needs its heap. */ async function prepareExecutableInput( entryPath: string, opts: CompileRequestOptions, ffi: FfiProfile | null, buildPlatform: string, + timing: CompilationTiming, ): Promise { const outputKind = opts.outputKind ?? "exe"; - const fe = runFrontend(entryPath, loadProgram, opts.npmStatic); - let lowered: LowerResult; - let sourceTexts: Map; - // The frontend (and its tsgo server) is released as soon as lowering - // ends — clang and the link never hold it open. - try { - const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ - ok: false, - diagnostics, - sourceTexts: fe.sourceTexts(), - }); - - if (fe.preflight.length > 0) return fail(fe.preflight); - - try { - lowered = fe.lower({ - dynamic: opts.dynamic ?? false, - targetPlatform: buildPlatform, - ...(ffi !== null ? { ffiImports: ffi.functions } : {}), - }); - } catch (e) { - // The last-resort panic fence: an upstream tsgo panic that crossed a - // checker call no statement/collection fence wrapped still becomes a - // clean failed compile (anchored at the entry), never a crashed CLI. - if (!isCheckerPanic(e)) throw e; - return fail([ - checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 }), - ]); - } - if (lowered.module === null) return fail(lowered.diagnostics); - - const validation = validateModule(lowered.module); - if (validation.length > 0) { - return fail(validation.map((v) => iceDiag(v.message, v.loc))); - } - if (buildPlatform === "wasi") { - const entryLoc: SrcLoc = { file: entryPath, start: 0, end: 0 }; - if (opts.sanitize) { - return fail([targetRefusalDiag("wasm32-wasi", "--sanitize", entryLoc)]); - } - if (ffi !== null) { - return fail([targetRefusalDiag("wasm32-wasi", "native FFI manifests", entryLoc)]); - } - const unavailable = moduleWasiUnavailableSurface(lowered.module); - if (unavailable !== null) { - return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); - } - } - sourceTexts = fe.sourceTexts(); - } finally { - fe.dispose(); - } + const prepared = prepareExecutableModule(entryPath, opts, ffi, buildPlatform, nodeFrontend, timing); + if (!prepared.ok) return prepared; + const { mod, sourceTexts } = prepared; const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const defaultSourcePaths = { @@ -819,14 +437,14 @@ async function prepareExecutableInput( if (outputKind === "ir") { await mkdir(dirname(opts.outPath), { recursive: true }); - await writeFile(opts.outPath, serializeModule(lowered.module)); + await writeFile(opts.outPath, serializeModule(mod, true)); return { ok: true, artifact: { kind: "ir", path: opts.outPath } }; } if (outputKind === "llvm" || outputKind === "asm" || outputKind === "obj") { let llvm: string; try { - llvm = emitLlvmModule(lowered.module, { + llvm = emitLlvmModule(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, pointerBits: buildPlatform === "wasi" ? 32 : 64, @@ -873,7 +491,7 @@ async function prepareExecutableInput( programObject: opts.outPath, target, features: executableNativeFeatures( - lowered.module, + mod, "llvm", opts.dynamic ?? false, opts.optimization ?? "release", @@ -892,7 +510,7 @@ async function prepareExecutableInput( const backend = "llvm" as const; let llvmSource: string; try { - llvmSource = emitLlvmModule(lowered.module, { + llvmSource = emitLlvmModule(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "", ...debugOptions, pointerBits: buildPlatform === "wasi" ? 32 : 64, @@ -903,24 +521,28 @@ async function prepareExecutableInput( if (!(err instanceof LlvmUnsupportedError)) throw err; return { ok: false, diagnostics: [llvmRefusalDiag(err, entryPath)], sourceTexts }; } + timing("llvm-emit"); await writeFile(llvmPath, llvmSource); + timing("llvm-write"); let irPath: string | undefined; if (opts.emitIr) { irPath = defaultSourcePaths.ir; - await writeFile(irPath, serializeModule(lowered.module)); + await writeFile(irPath, serializeModule(mod, true)); } const nativeFeatures = executableNativeFeatures( - lowered.module, + mod, backend, opts.dynamic ?? false, opts.optimization ?? "release", ); + timing("link-features"); const programSplit = backend === "llvm" && (opts.optimization ?? "release") === "dev" && !(opts.sanitize ?? false) && llvmSource !== null ? splitLlvmProgram(llvmSource) : null; + timing("llvm-split"); return { llvmSource, llvmPath, irPath, nativeFeatures, programSplit, sourceTexts }; } @@ -929,6 +551,7 @@ async function compileTracked( opts: CompileRequestOptions, frontendInputs: FrontendInputTracker, ): Promise { + const timing = compilationTiming(); entryPath = resolve(entryPath); const outputKind = opts.outputKind ?? "exe"; if ((opts.backend !== undefined && opts.backend !== "llvm") || @@ -1129,6 +752,7 @@ async function compileTracked( ? null : await readEarlyExecutableCache(cacheRoot, earlyCacheOptions); if (earlyHit !== null) { + timing("executable-cache-hit"); if (earlyCacheOptions === null) { throw new InternalCompilerError("executable cache hit without executable cache options"); } @@ -1139,6 +763,7 @@ async function compileTracked( await publishEarlyExecutableRoute(cacheRoot, executableCacheOptions).catch(() => undefined); if (earlyHit.executableRestored) { await pruneBuildCache(cacheRoot); + timing("complete"); return { ok: true, artifact: { @@ -1241,7 +866,8 @@ async function compileTracked( ...(earlyHit.irPath === undefined ? {} : { irPath: earlyHit.irPath }), }; } - const prepared = await prepareExecutableInput(entryPath, opts, ffi, buildPlatform); + timing("executable-cache-miss"); + const prepared = await prepareExecutableInput(entryPath, opts, ffi, buildPlatform, timing); if ("ok" in prepared) return prepared; const { llvmSource, llvmPath, irPath, nativeFeatures, programSplit, sourceTexts } = prepared; const backend = "llvm" as const; @@ -1265,6 +891,7 @@ async function compileTracked( } try { nativeProgramObject = await emitNativeProgramObject(entryPath, opts, llvmSource); + timing("native-object"); } catch (err) { if (!(err instanceof NativeCodegenError)) throw err; return { @@ -1296,6 +923,7 @@ async function compileTracked( publishedExecutable = true; }, ); + timing("native-link"); if (nativeProgramObject !== null && opts.nativeProgramObject === true) { await rename(nativeProgramObject.linkPath, nativeProgramObject.artifactPath); } @@ -1331,6 +959,7 @@ async function compileTracked( }).catch(() => undefined); } await pruneBuildCache(cacheRoot); + timing("complete"); return { ok: true, artifact: { @@ -1354,397 +983,6 @@ async function compileTracked( * emission; there is no fallback concept on this path (an out-of-tier * program under emission "llvm" is SC3001, fail-loudly). */ -export interface CompileLibraryOptions { - profilePath: string; - /** Where the archive and the kept program TU land. */ - outDir: string; - /** Archive path. Default: /.lib.a. */ - outPath?: string; - emitIr?: boolean; - sanitize?: boolean; -} - -export type CompileLibraryResult = - /** `sidecarPath` is present exactly when the profile declares a - * `sidecar` section: the contract JSON written beside the archive by - * the same invocation (ask 2). */ - | { ok: true; archivePath: string; llvmPath: string; backend: "llvm"; irPath?: string; sidecarPath?: string } - | { ok: false; diagnostics: ScrDiagnostic[]; sourceTexts: Map }; - -/** The marshalling-class fit over IR types (design §4.2 + the ratified - * integer plumbing classes): number is every f64-backed class, bool/string - * map directly, bytes is the u8 element kind. */ -function libClassFits(cls: string, t: IrType): boolean { - switch (cls) { - case "bool": - return t.kind === "bool"; - case "string": - return t.kind === "string"; - case "bytes": - return t.kind === "bytes" && t.elem === "u8"; - default: // f64 and the u8/u32/i32 plumbing classes - return t.kind === "f64"; - } -} - -/** Resolve the profile's export map against the entry module — SC4002/ - * SC4004/SC4007 from the declaration facts, SC4003 from the lowered IR - * signatures — and land the library section on the module. */ -function resolveLibrarySection( - profile: LibraryProfile, - entryInfo: Map, - mod: IrModule, - entryPath: string, -): { lib: IrLibSection } | { diagnostics: ScrDiagnostic[] } { - const diagnostics: ScrDiagnostic[] = []; - const entryLoc = { file: entryPath, start: 0, end: 0 }; - const fnByName = new Map(mod.functions.map((f) => [f.name, f])); - const exports: IrLibSection["exports"] = []; - for (const e of profile.exports) { - const info = entryInfo.get(e.export); - if (info === undefined) { - diagnostics.push( - libExportUnresolvedDiag(e.export, "the entry module has no exported function declaration by that name", entryLoc), - ); - continue; - } - if (info.generic) { - diagnostics.push(libGenericExportDiag(e.export, info.loc)); - continue; - } - if (info.async || info.generator) { - diagnostics.push(libAsyncExportDiag(e.export, info.async ? "async" : "generator", info.loc)); - continue; - } - const fn = fnByName.get(e.export); - if (fn === undefined) { - diagnostics.push( - libExportUnresolvedDiag(e.export, "the export did not lower to a compiled function", info.loc), - ); - continue; - } - if (fn.params.length !== e.params.length) { - diagnostics.push( - libUnmappableSignatureDiag( - e.export, - "signature", - `has ${fn.params.length} parameter(s) but the profile declares ${e.params.length} marshalling class(es)`, - info.loc, - ), - ); - continue; - } - let bad = false; - e.params.forEach((cls, i) => { - if (!libClassFits(cls, fn.params[i]!.type)) { - bad = true; - diagnostics.push( - libUnmappableSignatureDiag( - e.export, - `parameter ${i + 1} ('${fn.params[i]!.name}')`, - `has IR type '${fn.params[i]!.type.kind}', which does not fit the declared marshalling class '${cls}'`, - info.loc, - ), - ); - } - }); - if (e.returns === "void" ? fn.returnType.kind !== "void" : !libClassFits(e.returns, fn.returnType)) { - bad = true; - diagnostics.push( - libUnmappableSignatureDiag( - e.export, - "the return", - `has IR type '${fn.returnType.kind}', which does not fit the declared marshalling class '${e.returns}'`, - info.loc, - ), - ); - } - if (!bad) { - const resolvedExport: IrLibSection["exports"][number] = { - symbol: e.symbol, - fnName: e.export, - params: e.params, - returns: e.returns, - }; - if (e.params.includes("bytes")) { - // The wrapper's one host-contract trap (an inbound bytes length - // past the marshalling class's range) is assembled HERE, once, as - // the structured trap-teaching message: the profile's teaching for - // SC4012 (or the mode's default text), the code, the trapping - // export's C symbol exactly as the host linked it, and the - // profile's remediation when supplied — so the backend emits the - // same bytes and the sink sees one canonical message. - resolvedExport.inboundBytesTrap = assembleTrapTeaching( - profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? - "scriptc: library inbound bytes length out of range\n", - LIB_INBOUND_BYTES_TRAP_CODE, - e.symbol, - profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), - ); - } - if (e.params.includes("i64") || e.params.includes("u64")) { - // The sibling host-contract trap for inbound declared-integer - // parameters (ask 4): a value past ±(2^53−1) cannot ride f64 - // exactly, and silent rounding is a coercion the author never - // wrote. Same code (SC4012 — one host-contract story), same - // assembly-once discipline. - resolvedExport.inboundIntTrap = assembleTrapTeaching( - profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? - "scriptc: library inbound integer parameter out of range\n", - LIB_INBOUND_BYTES_TRAP_CODE, - e.symbol, - profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), - ); - } - exports.push(resolvedExport); - } - } - if (diagnostics.length > 0) return { diagnostics }; - // The runtime detected-trap overlay rows: one per family code the profile - // declares teaching or remediation text for, in the registry family's - // order. LLVM emits these rows as the program TU's overlay table, - // which the runtime uses to assemble the sink message. (SC4012 stays compile-time - // assembled into the wrapper's message above and never reaches the - // funnel's assembly path.) - const trapOverlays: IrLibSection["trapOverlays"] = []; - for (const code of LIB_RUNTIME_TRAP_CODES) { - const teaching = profileTeaching(profile, code); - const remediation = profileRemediation(profile, code); - if (teaching !== undefined || remediation !== undefined) { - trapOverlays.push({ - code, - ...(teaching !== undefined ? { teaching } : {}), - ...(remediation !== undefined ? { remediation } : {}), - }); - } - } - return { - lib: { - profileName: profile.name, - prefix: profile.prefix, - initSymbol: profile.initSymbol, - sinkRegisterSymbol: profile.sinkRegisterSymbol, - collectSymbol: profile.collectSymbol, - resultResetSymbol: profile.resultResetSymbol, - threadInstances: profile.instancePerThread, - // Host-callback channels: declaration order is the runtime slot - // assignment, and the unregistered-call trap text is assembled HERE, - // once, for consistent constant bytes (a DETECTED - // trap: the funnel classifies the "scriptc: library callback " - // prefix as SC4025 and names the entry the host called — the entry - // is runtime knowledge, so no compile-time SC4012-style assembly - // can carry it). Both fields stay absent on callback-free profiles - // (the byte-identity guarantee). - ...(profile.callbacks.length > 0 - ? { - callbackRegisterSymbol: profile.callbackRegisterSymbol!, - callbacks: profile.callbacks.map((cb, i) => ({ - name: cb.name, - slot: i, - params: [...cb.params], - returns: cb.returns, - unregisteredTrap: `scriptc: library callback '${cb.name}' invoked before registration\n`, - })), - } - : {}), - exports, - trapOverlays, - }, - }; -} - -/** The export map's integer-slot obligations (ask 4): i64/u64 params and - * returns become declared boundary slots keyed `exports..params[i]` - * / `exports..return`; the u8/u32/i32 plumbing classes contribute - * their proven inbound shapes as parameter seeds (the wrapper's coercion - * contract), tightening the intraprocedural analysis at zero declaration - * cost. Sidecar-declared slots (record fields, msg arms, helper params - * and returns) merge into the same config at sidecar build. */ -function libraryIntSlotConfig(profile: LibraryProfile): IntSlotConfig { - const cfg: IntSlotConfig = { fns: new Map(), records: new Map() }; - for (const e of profile.exports) { - const params = e.params.map((c) => (c === "i64" || c === "u64" ? c : null)); - const ret = e.returns === "i64" || e.returns === "u64" ? e.returns : null; - const paramSeeds = e.params.map((c) => (c === "u8" || c === "u32" || c === "i32" ? classSeed(c) : null)); - if (params.every((p) => p === null) && ret === null && paramSeeds.every((s) => s === null)) continue; - const slots: FnIntSlots = { - fnName: e.export, - params, - paramPaths: e.params.map((c, i) => (c === "i64" || c === "u64" ? `exports.${e.export}.params[${i}]` : null)), - ret, - retPath: ret !== null ? `exports.${e.export}.return` : null, - paramSeeds, - }; - cfg.fns.set(e.export, slots); - } - return cfg; -} - -/** Match the sidecar syntax's exact structural type projection against the - * frontend's interned IR registries. The pattern deliberately mirrors - * ShapeRegistry's identity: every field name and recursively mapped field - * type participates. Tagged payload records additionally accept omission - * of their `kind` field because the lowering may carry that discriminant - * only in the surrounding union tag. */ -function sidecarRecordMatcher( - mod: IrModule, -): (pattern: SidecarIrRecordPattern, shape: IrRecordShape) => boolean { - const records = new Map((mod.records ?? []).map((shape) => [shape.id, shape])); - const unions = new Map((mod.unions ?? []).map((union) => [union.id, union])); - - const recordMatches = ( - pattern: SidecarIrRecordPattern, - shape: IrRecordShape, - ): boolean => { - if (shape.tuple === true || shape.indexValue !== undefined) return false; - const variants = [pattern.fields]; - if (pattern.kindMayBeOmitted === true) { - variants.push(pattern.fields.filter((field) => field.name !== "kind")); - } - return variants.some( - (fields) => - fields.length === shape.fields.length && - fields.every((field) => { - const actual = shape.fields.find((candidate) => candidate.name === field.name); - return actual !== undefined && typeMatches(field.type, actual.type); - }), - ); - }; - - const unionMatches = ( - patterns: SidecarIrTypePattern[], - actual: IrType[], - ): boolean => { - if (patterns.length !== actual.length) return false; - const used = new Set(); - const visit = (index: number): boolean => { - if (index === patterns.length) return true; - for (let i = 0; i < actual.length; i++) { - if (used.has(i) || !typeMatches(patterns[index]!, actual[i]!)) continue; - used.add(i); - if (visit(index + 1)) return true; - used.delete(i); - } - return false; - }; - return visit(0); - }; - - const typeMatches = ( - pattern: SidecarIrTypePattern, - actual: IrType, - ): boolean => { - switch (pattern.kind) { - case "f64": - case "string": - case "bool": - case "nullT": - case "undefinedT": - case "dyn": - return actual.kind === pattern.kind; - case "bytes": - return actual.kind === "bytes" && actual.elem === pattern.elem; - case "array": - return actual.kind === "array" && typeMatches(pattern.elem, actual.elem); - case "record": { - if (actual.kind !== "record") return false; - const shape = records.get(actual.shapeId); - return shape !== undefined && recordMatches(pattern, shape); - } - case "union": { - if (actual.kind !== "union") return false; - const union = unions.get(actual.unionId); - return union !== undefined && unionMatches(pattern.arms, union.arms); - } - } - }; - - return (pattern, shape) => recordMatches(pattern, shape); -} - -/** Merge the sidecar-resolved integer slots (ask 4) into the inference - * config: helper slots key by function name and IR parameter index (the - * projection already shifted past the model receiver); record-field - * slots map onto every interned IR shape whose complete structural field - * signature matches the projected record's. Shapes intern structurally, - * so a same-shaped second type shares the obligation. DECLARED paths with - * the same class coalesce while retaining every source path for verdicts; - * differing classes refuse because one lowered field cannot seed or check - * two distinct class contracts without arm provenance. A - * record fact that matches no shape binds nothing: no compiled code - * constructs the type (the contract surface — init/update/subscriptions - * and every helper — is force-lowered whenever integer slots are - * declared, so this is genuine vacuity, not dead-stripping). */ -function mergeSidecarIntSlots( - cfg: IntSlotConfig, - facts: SidecarIntegerSlotFacts, - mod: IrModule, -): { ok: true; config: IntSlotConfig } | { ok: false; diagnostic: ScrDiagnostic } { - const recordMatches = sidecarRecordMatcher(mod); - for (const h of facts.helpers) { - const fn = mod.functions.find((f) => f.name === h.fnName); - const arity = Math.max(fn?.params.length ?? 0, (h.index ?? 0) + 1); - let slots = cfg.fns.get(h.fnName); - if (slots === undefined) { - slots = { - fnName: h.fnName, - params: new Array(arity).fill(null), - paramPaths: new Array(arity).fill(null), - ret: null, - retPath: null, - paramSeeds: new Array(arity).fill(null), - }; - cfg.fns.set(h.fnName, slots); - } - if (h.kind === "param") { - const i = h.index!; - while (slots.params.length <= i) { - slots.params.push(null); - slots.paramPaths.push(null); - slots.paramSeeds.push(null); - } - slots.params[i] = h.cls; - slots.paramPaths[i] = h.path; - } else { - slots.ret = h.cls; - slots.retPath = h.path; - } - } - for (const r of facts.records) { - for (const shape of mod.records ?? []) { - if (!recordMatches(r.shape, shape)) continue; - const target = shape.fields.find((f) => f.name === r.targetField); - if (target === undefined || numberCarrierKind(target.type, mod) === null) continue; - let m = cfg.records.get(shape.id); - if (m === undefined) { - m = new Map(); - cfg.records.set(shape.id, m); - } - const existing = m.get(r.targetField); - if (existing !== undefined && existing.cls !== r.cls) { - const paths = [ - ...existing.paths.map((path) => `'${path}' (${existing.cls})`), - `'${r.path}' (${r.cls})`, - ]; - return { - ok: false, - diagnostic: libSidecarDiag( - `integer slots ${paths.join(" and ")} collapse to the same lowered record field '${r.targetField}' — their proof obligations cannot be kept distinct`, - r.loc, - "kind-tagged union arms and structurally identical records may share one lowered shape — same-class declarations coalesce, but differing classes require distinct structural shapes or at most one classified slot", - ), - }; - } - if (existing === undefined) { - m.set(r.targetField, { cls: r.cls, paths: [r.path] }); - } else if (!existing.paths.includes(r.path)) { - existing.paths.push(r.path); - } - } - } - return { ok: true, config: cfg }; -} - function libraryNativeFeatures( mod: IrModule, backend: "llvm", @@ -1762,26 +1000,11 @@ function libraryNativeFeatures( zlib: features.zlib, copying: features.copying, textDecoderLegacy: features.legacyTextDecoder, + dynInvoke: features.dynInvoke, ...(mod.lib?.identity !== undefined ? { buildId: mod.lib.identity.buildId } : {}), }; } -function libraryLocalizeSymbols(profile: LibraryProfile): string[] | undefined { - return profile.localizeRuntime - ? [ - profile.initSymbol, - profile.sinkRegisterSymbol, - ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []), - ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []), - ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []), - ...(profile.sidecar !== null - ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol] - : []), - ...profile.exports.map((entry) => entry.symbol), - ] - : undefined; -} - async function compileLibraryNative( profile: LibraryProfile, llvmPath: string, @@ -1789,6 +1012,13 @@ async function compileLibraryNative( sanitize: boolean, features: EarlyLibraryNativeFeatures, ): Promise { + const wasmTarget = nativeCodegenTarget(); + if (wasmTarget?.platform === "wasi") { + await compilePackedLibrary({ + cPath: llvmPath, outPath: archivePath, optimization: profile.optimization, ...features, + }, wasmTarget, libraryWasmExports(profile)); + return; + } const localizeSymbols = libraryLocalizeSymbols(profile); let identityLlvmSource: string | undefined; let programSource: string | undefined; @@ -1845,6 +1075,7 @@ async function compileLibraryNative( zlib: features.zlib, copying: features.copying, textDecoderLegacy: features.textDecoderLegacy, + dynInvoke: features.dynInvoke, }; if (packTarget !== null) await compilePackedLibrary(archiveOptions, packTarget); else await compileExternalCLibrary(archiveOptions); @@ -1896,7 +1127,7 @@ async function emitSemanticLibraryHit( let irPath: string | undefined; if (opts.emitIr) { irPath = join(opts.outDir, `${stem}.lib.ir.json`); - await writeFile(irPath, serializeModule(mod)); + await writeFile(irPath, serializeModule(mod, true)); } await compileLibraryNative( profile, @@ -1987,11 +1218,6 @@ async function compileLibraryTracked( } if (directory === profileDir || dirname(directory) === directory) break; } - const archivePath = opts.outPath ?? join( - opts.outDir, - `${basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, "")}.lib.a`, - ); - // Mobile-target admission first — a pure env/host check, so a refused // pairing never reaches toolchain discovery. iOS targets (device and // simulator) build on darwin hosts only: the Apple SDK sysroot and the @@ -2013,6 +1239,14 @@ async function compileLibraryTracked( } const buildPlatform = buildTargetPlatform(); + const archivePath = opts.outPath ?? join( + opts.outDir, + `${basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, "")}${buildPlatform === "wasi" ? ".wasm" : ".lib.a"}`, + ); + if (buildPlatform === "wasi") { + const refusal = libraryWasmRefusal(profile, opts.sanitize ?? false); + if (refusal !== null) return { ok: false, diagnostics: [refusal], sourceTexts: new Map() }; + } // Multi-instance library mode (abi.localize_runtime) localizes per // OBJECT FORMAT: ELF and COFF archives localize from any host (cross @@ -2021,8 +1255,8 @@ async function compileLibraryTracked( // localization runs the macOS host linker, so macos and ios targets // admit darwin hosts only (the mobile admission above already refused // an ios triple off darwin). Everything else refuses before frontend/ - // backend work, naming the pairing. WASI retains the general - // library-mode refusal below. + // backend work, naming the pairing. WASI refuses localization above: + // each WebAssembly.Instance already owns its runtime state. if (profile.localizeRuntime && buildPlatform !== "wasi") { const packTarget = nativeCodegenTarget(); const driver = opts.sanitize ? resolveCc() : { @@ -2125,236 +1359,22 @@ async function compileLibraryTracked( } timing("semantic-cache-miss"); - // Bare npm specifiers in a library graph take the STATIC-OR-REFUSE - // posture: "lib" runs the same auto-detection and eligibility bar as - // the executable lane's --npm-static (own .d.ts, unminified shipped JS, - // no build-transform markers), automatically — the library path has no - // island/dynamic tier to offer (SC4006's ground), so eligibility needs - // no flag and a miss is a refusal, never a fallback. - const fe = runFrontend(entryPath, loadProgram, "lib"); - timing("frontend-load", { - entry_bytes: fe.entryText().length, - source_files: fe.sourceTexts().size, - }); - let lowered: LowerResult; - let sourceTexts: Map; - let entryInfo: Map; - let contractFacts: ContractFacts | null; - try { - // Every library refusal leaves through the ask-5 teaching decoration: - // profile text attaches by code, manifest id, or fence coverage as the - // attributed note (the SC4004/SC4005 rider generalized). - const fail = (diagnostics: ScrDiagnostic[]): CompileLibraryResult => ({ - ok: false, - diagnostics: decorateLibraryRefusals(diagnostics, profile), - sourceTexts: fe.sourceTexts(), - }); - // Library mode emits a host-embedded static archive with native trap and - // C-ABI contracts. wasm32-wasi executable modules are supported, but the - // archive/reactor contract is not; refuse before emitting a host-width - // LLVM TU or asking Zig to compile the native library runtime for WASI. - if (buildPlatform === "wasi") { - return fail([ - targetRefusalDiag( - "wasm32-wasi", - "library-mode archive builds", - { file: entryPath, start: 0, end: 0 }, - ), - ]); - } - // The npm verdicts FIRST: whatever the shared frontend would have - // served from the island — an eligibility miss, an untyped install, a - // preflight offender inside a package's files, a dropped inferred - // surface — refuses here with the package and the specific bar it - // missed. Checked before the general preflight, whose diagnostics for - // these same imports speak executable-lane teachings (SC1010/SC0001 at - // the unresolvable edge); the library answer is this one. - const npmRefused = fe.npmStatic.filter((s) => s.status === "fallback"); - if (npmRefused.length > 0) { - return fail( - npmRefused.map((s) => - libNpmIneligibleDiag( - s.package, - // The one shared offender reason that narrates the executable - // lane's fallback loses that clause here — no island exists on - // this path to serve anything. - (s.detail ?? "its static compilation was refused").replace("; the island serves the package", ""), - fe.npmImportSites.get(s.package) ?? { file: entryPath, start: 0, end: 0 }, - ), - ), - ); - } - if (fe.preflight.length > 0) return fail(fe.preflight); - contractFacts = profile.sidecar !== null ? fe.entryContract() : null; - // Ask 4, contract-surface reachability: when the sidecar declares ANY - // integer slot, the designated init/update/subscriptions exports and - // every contract helper (model-first exported function) seed lowering - // too. They are attested surface — a declared record-field or msg-arm - // class obligates EVERY write those bodies perform, and a declared - // helper param is checked at their internal call sites — so the - // attestation must cover COMPILED bodies, never a dead-stripped - // vacuity (the bug this closes: a model-slot declaration whose only - // writers were dead-stripped attested without any proof). - const contractSurfaceRoots: string[] = []; - if (profile.sidecar !== null && profile.sidecar.integerSlots.length > 0) { - const sc = profile.sidecar; - const fnNames = new Set(contractFacts!.functions.filter((f) => !f.generic).map((f) => f.name)); - for (const name of [sc.initExport, sc.updateExport, sc.subscriptionsExport]) { - if (fnNames.has(name)) contractSurfaceRoots.push(name); - } - for (const fn of contractFacts!.functions) { - if (fn.generic) continue; - const first = fn.params[0]; - if (first !== undefined && first.shape !== null && first.shape.k === "ref" && first.shape.name === sc.model) { - contractSurfaceRoots.push(fn.name); - } - } - } - // The profile's host-callback channels ride the FFI import machinery: - // each channel is a signature-only ambient binding whose direct calls - // lower to ffiCall nodes (the classes are a subset of the FFI's), and - // `libraryCallbacks` flips the recognition to the library flavor — - // SC4024 diagnostics, unused channels legal, undeclared references - // refused with the callback teaching. The library lane never loads a - // native-FFI manifest, so the channel set owns the surface outright. - const cbImports: IrFfiImport[] = profile.callbacks.map((cb) => ({ - name: cb.name, - symbol: cb.name, - params: [...cb.params], - returns: cb.returns, - })); - try { - lowered = fe.lower({ - dynamic: false, - targetPlatform: buildPlatform, - ...(cbImports.length > 0 ? { ffiImports: cbImports, libraryCallbacks: true } : {}), - // The profile-mapped exports are called from OUTSIDE the graph: - // they seed reachability beside the entry's top level (an - // executable build would dead-strip an uncalled export). A helper - // with a declared integer slot (ask 4) seeds too: its attestation - // must cover a COMPILED body, never a dead-stripped vacuity — the - // sidecar advertises the slot's class, so the proof must exist. - libRoots: [ - ...new Set([ - ...profile.exports.map((e) => e.export), - ...(profile.sidecar?.integerSlots ?? []) - .map((s) => /^helpers\.([^.]+)\.(?:params\[\d+\]|return)$/.exec(s.slot)?.[1]) - .filter((n): n is string => n !== undefined), - ...contractSurfaceRoots, - ]), - ], - }); - timing("lower", { - lib_roots: profile.exports.length + contractSurfaceRoots.length, - }); - } catch (e) { - if (!isCheckerPanic(e)) throw e; - return fail([checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 })]); - } - if (lowered.module === null) return fail(lowered.diagnostics); - entryInfo = fe.entryExports(); - sourceTexts = fe.sourceTexts(); - } finally { - fe.dispose(); - } - const mod = lowered.module!; - timing("frontend-dispose"); - + const prepared = prepareLibrary(profile, opts.profilePath, compilerReleaseVersion(), buildPlatform, nodeFrontend, timing); + if (!prepared.ok) return prepared; + const { mod, sourceTexts, sidecarJson } = prepared; const fail = (diagnostics: ScrDiagnostic[]): CompileLibraryResult => ({ - ok: false, - diagnostics: decorateLibraryRefusals(diagnostics, profile), - sourceTexts, + ok: false, diagnostics: decorateLibraryRefusals(diagnostics, profile), sourceTexts, }); - // Export resolution first (SC4002/SC4003/SC4004/SC4007 anchor at the - // mapped declaration — a mapped async export reports as SC4004, not the - // graph-wide gate), then the async_free requirement (ratified, SC4005), - // then the profile's determinism fences (ask 5, SC4008) over the same - // compiled graph the attestation scan reads: all refused before anything - // is emitted, so the narrowed library link set below is structural fact. - const resolved = resolveLibrarySection(profile, entryInfo, mod, entryPath); - if ("diagnostics" in resolved) return fail(resolved.diagnostics); - const asyncSurface = moduleLibAsyncSurface(mod); - if (asyncSurface !== null) { - return fail([libAsyncSurfaceDiag(asyncSurface.surface, asyncSurface.loc)]); - } - const fenced = evaluateLibraryFences(mod, profile); - if (fenced.length > 0) return fail(fenced); - mod.lib = resolved.lib; - - // Ask 4's declared integer slots: the export map's i64/u64 classes - // seed the config here; sidecar-declared slots (record fields, msg - // arms, helper params/returns) merge in after the projection resolves - // them below. - let intCfg = libraryIntSlotConfig(profile); - - // The ask-2 contract sidecar rides the same invocation. Identity first - // (schema §2's worked build_id definition over compiler version, profile - // bytes, and the sorted canonical module graph; source_hash per the - // profile's "module-graph" contract) — the u64 lands on the IR so native - // archive assembly emits the identity getters from the ONE value the - // sidecar records (V12's coherence by construction), then the projection into - // the schema (declaration orders from the AST) and the V1–V14 - // self-check before anything is written. - let sidecarJson: string | null = null; - if (profile.sidecar !== null) { - const rootDir = dirname(resolve(opts.profilePath)); - const modules = canonicalModuleGraph(rootDir, sourceTexts); - const { buildId, sourceHash } = libraryIdentityHashes(compilerReleaseVersion(), profile.profileBytes, modules); - mod.lib.identity = { - buildIdSymbol: profile.sidecar.buildIdSymbol, - abiVersionSymbol: profile.sidecar.abiVersionSymbol, - buildId, - abiVersion: profile.sidecar.abiVersion, - }; - const built = buildSidecar({ - profile, - facts: contractFacts!, - compilerVersion: compilerReleaseVersion(), - entry: canonicalPath(rootDir, entryPath), - buildId, - sourceHash, - deterministic: moduleLibNondeterministicSurface(mod) === null, - }); - if (!built.ok) return fail(built.diagnostics); - const violations = validateSidecar(built.doc); - if (violations.length > 0) { - // The projection above refuses every user-caused shape; a rule - // violation surviving to here is an emitter bug. - return fail(violations.map((v) => iceDiag(`sidecar self-check failed — ${v}`, { file: entryPath, start: 0, end: 0 }))); - } - sidecarJson = built.json; - const merged = mergeSidecarIntSlots(intCfg, built.integerSlotFacts, mod); - if (!merged.ok) return fail([merged.diagnostic]); - intCfg = merged.config; - } - timing("contract-sidecar", { source_files: sourceTexts.size }); - - // Ask 4: the integer-boundary inference — every value that can reach a - // profile-declared i64/u64 slot must PROVE representability, wholeness, - // and range, or the build refuses with the failed obligation, the - // observed evidence, and the author's fix (SC4021/SC4022/SC4023). Runs - // only when at least one integer slot is declared; the sidecar (already - // built above, written only on success) may then attest the classes — - // §5's invariant that an attested integer class means the proof was - // discharged holds because no artifact leaves this function otherwise. - if (hasIntSlots(intCfg)) { - const refusals = checkLibraryIntegerSlots(mod, intCfg).filter((v) => v.outcome === "refuse"); - if (refusals.length > 0) { - return fail(refusals.map((v) => libIntBoundaryDiag(v.path, v.cls, v.obligation!, v.detail!, v.fix!, v.loc))); - } - } - timing("integer-proof"); - - const validation = validateModule(mod); - if (validation.length > 0) return fail(validation.map((v) => iceDiag(v.message, v.loc))); - timing("ir-validate"); - await mkdir(opts.outDir, { recursive: true }); const stem = basename(entryPath).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); const llvmPath = join(opts.outDir, `${stem}.lib.ll`); try { - const ll = emitLlvmModule(mod, { targetTriple: process.env["SCRIPTC_TARGET"] ?? "" }); + const ll = emitLlvmModule(mod, { + targetTriple: process.env["SCRIPTC_TARGET"] ?? "", + pointerBits: buildPlatform === "wasi" ? 32 : 64, + wasi: buildPlatform === "wasi", + }); timing("llvm-emit", { output_bytes: Buffer.byteLength(ll) }); await writeFile(llvmPath, ll); timing("llvm-write"); @@ -2366,7 +1386,7 @@ async function compileLibraryTracked( let irPath: string | undefined; if (opts.emitIr) { irPath = join(opts.outDir, `${stem}.lib.ir.json`); - await writeFile(irPath, serializeModule(mod)); + await writeFile(irPath, serializeModule(mod, true)); } const nativeFeatures = libraryNativeFeatures(mod, profile.emission); diff --git a/packages/compiler/src/ir/ir.ts b/packages/compiler/src/ir/ir.ts index 23181bd6..7380361d 100644 --- a/packages/compiler/src/ir/ir.ts +++ b/packages/compiler/src/ir/ir.ts @@ -1,4 +1,5 @@ import { InternalCompilerError } from "../errors.js"; +import { everyModuleNode, everyExprChild, everyStmtChild } from "./traverse.js"; /* scriptc IR — the only interface between frontend and backends. * * Design rules (see docs/ir.md for node-by-node semantics): @@ -5738,7 +5739,7 @@ export type IrExpr = * undefined arm of its union (an optional field) is DROPPED from the * output — Node's rule for undefined-valued properties. The value is * BORROWED; the result string is owned (+1). Never throws. */ - | { kind: "jsonStringify"; value: IrExpr; type: IrType; loc: SrcLoc } + | { kind: "jsonStringify"; value: IrExpr; indent?: string; type: IrType; loc: SrcLoc } /** The dynamic-boundary check — a CHECKED cast `dynValue as T`: validate * the dyn value's JSON dyn against `type` (a non-dyn, JSON-representable * IR type) and BUILD the typed value (+1), or THROW a catchable @@ -6433,6 +6434,7 @@ export function canDynCheckTo( // Native class capsules already support checked extraction at ordinary // boundaries. Callable adapters use the same identity/brand check. if (isDynTypedRefType(t)) return true; + if (t.kind === "array" && isDynTypedRefType(t.elem)) return true; if (t.kind === "func") return canAdaptDynFuncTo(t, getRecord, getUnion); if (DYN_HANDLE_KINDS.has(t.kind)) return true; if (t.kind === "union") { @@ -6616,19 +6618,9 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: keyof RuntimeFeatures): Run qs: false, parseArgs: false, fsWatch: false, nodeTest: false, dgram: false, http: false, http2: false, tls: false, tlsCa: false, }; - const visit = (value: unknown): void => { - if (value === null || typeof value !== "object" || (stopAt !== undefined && features[stopAt])) return; - if (Array.isArray(value)) { - for (const item of value) visit(item); - return; - } - const node = value as { - kind?: unknown; fn?: unknown; method?: unknown; op?: unknown; name?: unknown; - type?: { kind?: unknown }; - value?: { type?: { kind?: unknown; ret?: { kind?: unknown } } }; - }; - const kind = node.kind; - if (kind === "libCall" && typeof node.fn === "string") { + const keepGoing = (): boolean => stopAt === undefined || !features[stopAt]; + const expr = (node: IrExpr): boolean => { + if (node.kind === "libCall") { const fn = node.fn; if (fn === "regexp.escape" || fn === "dyn.nativeRegexIs") features.regex = true; if (fn === "text.decodeLegacy") features.legacyTextDecoder = true; @@ -6665,8 +6657,8 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: keyof RuntimeFeatures): Run if (fn.startsWith("dgram.") || fn.startsWith("dns.")) features.dgram = true; if (fn.startsWith("tlsca.")) features.tlsCa = true; } else { - switch (kind) { - case "regex": case "regexLit": case "regexIntrinsic": features.regex = true; break; + switch (node.kind) { + case "regexLit": case "regexIntrinsic": features.regex = true; break; case "strIntrinsic": if (node.method === "toLowerCase" || node.method === "toUpperCase") features.regex = true; break; @@ -6676,44 +6668,49 @@ function scanRuntimeFeatures(mod: IrModule, stopAt?: keyof RuntimeFeatures): Run case "bytesIntrinsic": if (node.method === "toReversed" || node.method === "with" || node.method === "join" || node.method === "toArray") features.copying = true; break; - case "fileHandle": features.fileHandle = true; break; case "jsOp": if (node.op === "globalGet" && node.name === "fetch") features.fetch = true; break; case "dynInvoke": features.dynInvoke = true; break; case "awaitExpr": - if (node.type?.kind === "dyn") features.dynAsync = true; + if (node.type.kind === "dyn") features.dynAsync = true; break; case "dynFrom": { - const boxed = node.value?.type; - if (boxed?.kind === "promise" || (boxed?.kind === "func" && boxed.ret?.kind === "promise")) features.dynAsync = true; + const boxed = node.value.type; + if (boxed.kind === "promise" || (boxed.kind === "func" && boxed.ret.kind === "promise")) features.dynAsync = true; break; } - case "child": case "childStream": case "childWriter": case "spawnRes": features.childProcess = true; break; - case "netServer": case "netSocket": features.net = true; break; - case "http2Session": case "http2Stream": features.net = true; features.http2 = true; break; - case "httpReq": case "httpRes": case "httpClientReq": features.net = true; features.http = true; break; - case "secureCtx": features.net = true; features.http = true; features.tls = true; break; - case "symbol": features.symbol = true; break; - case "bigint": features.bigint = true; break; - case "searchParams": features.searchParams = true; break; - case "fsWatcher": features.fsWatch = true; break; - case "testCtx": features.nodeTest = true; break; - case "dgramSocket": features.dgram = true; break; } } - for (const key of Object.keys(value)) visit((value as Record)[key]); + return keepGoing(); }; - visit(mod); + const type = (node: IrType): boolean => { + switch (node.kind) { + case "regex": features.regex = true; break; + case "fileHandle": features.fileHandle = true; break; + case "child": case "childStream": case "childWriter": case "spawnRes": features.childProcess = true; break; + case "netServer": case "netSocket": features.net = true; break; + case "http2Session": case "http2Stream": features.net = true; features.http2 = true; break; + case "httpReq": case "httpRes": case "httpClientReq": features.net = true; features.http = true; break; + case "secureCtx": features.net = true; features.http = true; features.tls = true; break; + case "symbol": features.symbol = true; break; + case "bigint": features.bigint = true; break; + case "searchParams": features.searchParams = true; break; + case "fsWatcher": features.fsWatch = true; break; + case "testCtx": features.nodeTest = true; break; + case "dgramSocket": features.dgram = true; break; + } + return keepGoing(); + }; + if (keepGoing()) everyModuleNode(mod, { expr, stmt: keepGoing, type }); return features; } /** True when the module contains any regex construct — a regexLit / * regexIntrinsic node or a regex-typed slot anywhere. This is the link * switch that pulls scr_regex.c + the vendored libregexp into the binary - * (native-toolchain.ts); regex-free programs keep the historical command line. A generic - * JSON walk: `kind` discriminants live only on IR objects, so user string - * VALUES can never false-positive. */ + * (native-toolchain.ts); regex-free programs keep the historical command line. The + * typed traversal visits type slots as well as executable nodes. */ export function moduleUsesRegex(mod: IrModule): boolean { return scanRuntimeFeatures(mod, "regex").regex; } @@ -7075,10 +7072,6 @@ const LIB_MODE_REFUSED_PREFIXES: readonly [string, string][] = [ ["als.", "AsyncLocalStorage"], ["urj.", "unhandled-rejection tracking"], ["dc.", "the diagnostics_channel surface"], - // NOT the whole "dyn." family: the checked-dynamic tree (ScrDyn) is - // static-tier surface hosted by always-linked units; only defineProps - // drags the prototype-dispatch unit (scr_dyn_invoke.c → scr_async_dyn.c). - ["dyn.defineProps", "checked-dynamic prototype dispatch"], ]; /** Value/type kinds whose mere presence means an excluded unit's code (or @@ -7104,13 +7097,12 @@ const LIB_MODE_REFUSED_KINDS: ReadonlyMap = new Map([ ["httpRes", "the node:http surface"], ["httpClientReq", "the node:http surface"], ["secureCtx", "the node:tls surface"], - ["dynInvoke", "checked-dynamic prototype dispatch"], ]); /** First async/event-loop/ambient-process surface the module graph * reaches, or null when the graph is async_free (the v1 library requirement). - * The generic-walk shape of moduleUsesRegex, tracking the nearest - * enclosing `loc` so the refusal anchors at the reaching construct; the + * Typed slots use their declaration location and executable nodes use their + * own location, so the refusal anchors at the reaching construct; the * coarse moduleUses* predicates are the safety net behind the fine-grained * table (a surface reached only through a spelling the table misses still * refuses, anchored at the entry). */ @@ -7123,50 +7115,46 @@ export function moduleLibAsyncSurface(mod: IrModule): { surface: string; loc: Sr } const entryLoc: SrcLoc = { file: mod.sourceFile, start: 0, end: 0 }; let found: { surface: string; loc: SrcLoc } | null = null; - const visit = (v: unknown, loc: SrcLoc): void => { - if (found !== null || v === null || typeof v !== "object") return; - if (Array.isArray(v)) { - for (const item of v) visit(item, loc); - return; - } - const node = v as { kind?: unknown; fn?: unknown; loc?: SrcLoc }; - const here = node.loc ?? loc; - if (typeof node.kind === "string") { - const bad = LIB_MODE_REFUSED_KINDS.get(node.kind); - if (bad !== undefined) { - found = { surface: bad, loc: here }; - return; - } - if (node.kind === "libCall" && typeof node.fn === "string") { + const checkKind = (kind: string, loc: SrcLoc): boolean => { + const surface = LIB_MODE_REFUSED_KINDS.get(kind); + if (surface === undefined) return true; + found = { surface, loc }; + return false; + }; + everyModuleNode(mod, { + type: (node, loc) => checkKind(node.kind, loc), + stmt: () => true, + expr: (node) => { + if (!checkKind(node.kind, node.loc)) return false; + if (node.kind === "libCall") { for (const [prefix, surface] of LIB_MODE_REFUSED_PREFIXES) { if (node.fn.startsWith(prefix)) { - found = { surface, loc: here }; - return; + found = { surface, loc: node.loc }; + return false; } } } - } - for (const key of Object.keys(v)) visit((v as Record)[key], here); - }; - visit(mod, entryLoc); + return true; + }, + }); if (found !== null) return found; // Safety net: the coarse unit predicates, entry-anchored. Every one of // these units is excluded from library links, so a true answer that the // fine-grained table missed must still refuse. + const features = moduleRuntimeFeatures(mod); const coarse: [boolean, string][] = [ - [moduleUsesProcessEvents(mod), "process signal/exit listeners or the stdin event surface"], - [moduleUsesNet(mod), "the node:net surface"], - [moduleUsesHttpServer(mod), "the node:http surface"], - [moduleUsesHttp2(mod), "the node:http2 surface"], - [moduleUsesDgram(mod), "the node:dgram surface"], - [moduleUsesFsWatch(mod), "fs.watch"], - [moduleUsesStream(mod), "the node:stream surface"], - [moduleUsesTls(mod), "the node:tls surface"], - [moduleUsesFetch(mod), "fetch"], - [moduleUsesNodeTest(mod), "the node:test surface"], - [moduleUsesDynAsync(mod), "the checked-dynamic async surface"], - [moduleUsesDc(mod), "the diagnostics_channel surface"], - [moduleUsesDynInvoke(mod), "checked-dynamic prototype dispatch"], + [features.processEvents, "process signal/exit listeners or the stdin event surface"], + [features.net, "the node:net surface"], + [features.http, "the node:http surface"], + [features.http2, "the node:http2 surface"], + [features.dgram, "the node:dgram surface"], + [features.fsWatch, "fs.watch"], + [features.stream, "the node:stream surface"], + [features.tls, "the node:tls surface"], + [features.fetch, "fetch"], + [features.nodeTest, "the node:test surface"], + [features.dynAsync, "the checked-dynamic async surface"], + [features.dc, "the diagnostics_channel surface"], ]; for (const [on, surface] of coarse) { if (on) return { surface, loc: entryLoc }; @@ -7239,24 +7227,21 @@ export const LIB_NONDETERMINISTIC_PREFIXES: readonly [string, string][] = [ * attests `deterministic: true`). */ export function moduleLibNondeterministicSurface(mod: IrModule): string | null { let found: string | null = null; - const visit = (v: unknown): void => { - if (found !== null || v === null || typeof v !== "object") return; - if (Array.isArray(v)) { - for (const item of v) visit(item); - return; - } - const node = v as { kind?: unknown; fn?: unknown }; - if (node.kind === "libCall" && typeof node.fn === "string") { + const expr = (node: IrExpr): boolean => { + if (node.kind === "libCall") { for (const [prefix, surface] of LIB_NONDETERMINISTIC_PREFIXES) { if (node.fn.startsWith(prefix)) { found = surface; - return; + return false; } } } - for (const key of Object.keys(v)) visit((v as Record)[key]); + return everyExprChild(node, expr, stmt); }; - visit(mod); + const stmt = (node: IrStmt): boolean => everyStmtChild(node, expr, stmt); + for (const fn of mod.functions) { + if (!fn.body.every(stmt)) break; + } return found; } diff --git a/packages/compiler/src/ir/serialize.test.ts b/packages/compiler/src/ir/serialize.test.ts index bc4660cc..4e18cc81 100644 --- a/packages/compiler/src/ir/serialize.test.ts +++ b/packages/compiler/src/ir/serialize.test.ts @@ -41,6 +41,16 @@ test("ordinary strings resembling number sentinels remain strings", () => { expect(deserializeModule(serializeModule(mod)).sourceFile).toBe(mod.sourceFile); }); +test("compact artifacts preserve the full module and special numeric values", () => { + const mod = numbers([NaN, Infinity, -Infinity, -0, 0, Number.MIN_VALUE, Number.MAX_VALUE]); + mod.sourceFile = "escaped\nsource\t😀.ts"; + const compact = serializeModule(mod, true); + expect(compact).not.toContain("\n"); + expect(compact.length).toBeLessThan(serializeModule(mod).length); + expect(deserializeModule(compact)).toEqual(mod); + expect(deserializeModule(compact)).toEqual(deserializeModule(serializeModule(mod))); +}); + test("the new number format rejects documents bearing an older version", () => { const json = serializeModule(numbers([NaN])).replace(`"irVersion": ${IR_VERSION}`, `"irVersion": ${IR_VERSION - 1}`); expect(() => deserializeModule(json)).toThrow("IR version mismatch"); diff --git a/packages/compiler/src/ir/serialize.ts b/packages/compiler/src/ir/serialize.ts index c175ce8e..8dcef1cb 100644 --- a/packages/compiler/src/ir/serialize.ts +++ b/packages/compiler/src/ir/serialize.ts @@ -6,8 +6,10 @@ import type { IrModule } from "./ir.js"; // as negative infinity, so accepting a newer document would miscompile it. export const IR_VERSION = 13 as const; -export function serializeModule(mod: IrModule): string { - return JSON.stringify(mod, (_key, value) => { +/** Compiler artifacts use compact JSON to keep large graphs below the host's + * string size limit. API consumers can retain the readable default. */ +export function serializeModule(mod: IrModule, compact = false): string { + const replacer = (_key: string, value: unknown): unknown => { if (typeof value === "number" && !Number.isFinite(value)) { return { $nonfinite: Number.isNaN(value) ? "nan" : value > 0 ? "inf" : "-inf" }; } @@ -18,7 +20,8 @@ export function serializeModule(mod: IrModule): string { return { $nonfinite: "-0" }; } return value; - }, 2); + }; + return compact ? JSON.stringify(mod, replacer) : JSON.stringify(mod, replacer, 2); } export function deserializeModule(json: string): IrModule { diff --git a/packages/compiler/src/ir/traverse.test.ts b/packages/compiler/src/ir/traverse.test.ts index 8d1279a9..9fbae0be 100644 --- a/packages/compiler/src/ir/traverse.test.ts +++ b/packages/compiler/src/ir/traverse.test.ts @@ -2,8 +2,8 @@ import { createProgram } from "../frontend/ts7/program-adapter.js"; import { fileURLToPath } from "node:url"; import { afterAll, expect, test } from "vitest"; import * as ts from "../frontend/ts7/adapter.js"; -import { BOOL, F64, VOID, type IrExpr, type IrStmt } from "./ir.js"; -import { everyExpr, everyExprChild, everyStmtChild, everyStmtList, mapExprChildren, mapStmtChildren, transformStmtList } from "./traverse.js"; +import { BOOL, F64, VOID, type IrExpr, type IrModule, type IrStmt, type IrType } from "./ir.js"; +import { everyModuleNode, everyTypeChild, everyExpr, everyExprChild, everyStmtChild, everyStmtList, mapExprChildren, mapStmtChildren, transformStmtList } from "./traverse.js"; const loc = { file: "traverse.ts", start: 0, end: 1 }; const num = (value: number): IrExpr => ({ kind: "numLit", value, type: F64, loc }); @@ -122,3 +122,103 @@ test("preorder transformations traverse replacement children exactly once", () = expect(result).toEqual([{ kind: "return", value: { kind: "intrinsic", name: "console.log", args: [], type: VOID, loc }, loc }]); expect(body).toEqual([statement(1)]); }); + +// Build the typed slots from the schema too: adding a field to a declaration +// must fail this audit until module traversal accounts for it. +test("module traversal covers every declared type slot", () => { + const expected: IrType[] = []; + const active = new Set(); + function slots(node: ts.TypeNode): unknown { + if (ts.isTypeReferenceNode(node)) { + const name = node.typeName.getText(); + if (name === "IrType") { + const value: IrType = { kind: "regex" }; + expected.push(value); + return value; + } + if (name === "IrExpr") return { kind: "numLit", value: 0, loc, type: slotsType() }; + if (name === "IrStmt") return { kind: "exprStmt", loc, expr: { kind: "numLit", value: 0, loc, type: slotsType() } }; + if (name === "SrcLoc") return loc; + if (active.has(name)) throw new Error(`unexpected recursive schema reference: ${name}`); + const declaration = source.statements.find((item) => + (ts.isInterfaceDeclaration(item) || ts.isTypeAliasDeclaration(item)) && item.name.text === name); + if (!declaration) return undefined; + active.add(name); + const result = ts.isInterfaceDeclaration(declaration) ? fields(declaration.members) + : ts.isTypeAliasDeclaration(declaration) ? slots(declaration.type) : undefined; + active.delete(name); + return result; + } + if (ts.isTypeLiteralNode(node)) return fields(node.members); + if (ts.isArrayTypeNode(node)) return [slots(node.elementType)]; + if (ts.isParenthesizedTypeNode(node)) return slots(node.type); + if (ts.isIntersectionTypeNode(node)) return slots(node.types[0]!); + if (ts.isUnionTypeNode(node)) { + // Metadata unions can choose any arm: the schema puts executable and + // typed alternatives first, followed by null/unit alternatives. + return slots(node.types[0]!); + } + return undefined; + } + function slotsType(): IrType { + const value: IrType = { kind: "regex" }; + expected.push(value); + return value; + } + function fields(members: readonly ts.TypeElement[]): Record { + const result: Record = {}; + for (const member of members) { + if (!ts.isPropertySignature(member) || !member.type) continue; + const value = slots(member.type); + if (value !== undefined) result[member.name.getText()] = value; + } + return result; + } + const declaration = source.statements.find((item): item is ts.InterfaceDeclaration => + ts.isInterfaceDeclaration(item) && item.name.text === "IrModule")!; + const module = { ...fields(declaration.members), sourceFile: loc.file } as unknown as IrModule; + const seen: IrType[] = []; + expect(everyModuleNode(module, { + expr: () => true, stmt: () => true, type: (node) => { seen.push(node); return true; }, + })).toBe(true); + expect(seen).toHaveLength(expected.length); + for (const node of expected) expect(seen.filter((item) => item === node)).toHaveLength(1); + for (let stop = 0; stop < expected.length; stop++) { + let visits = 0; + expect(everyModuleNode(module, { + expr: () => true, stmt: () => true, type: () => visits++ !== stop, + })).toBe(false); + expect(visits).toBe(stop + 1); + } +}); + +test("type traversal visits all structural children and leaves named shapes as references", () => { + const alias = source.statements.find((node): node is ts.TypeAliasDeclaration => + ts.isTypeAliasDeclaration(node) && node.name.text === "IrType")!; + if (!ts.isUnionTypeNode(alias.type)) throw new Error("IrType must be a union"); + for (const variant of alias.type.types) { + if (!ts.isTypeLiteralNode(variant)) throw new Error("IrType variants must be records"); + const members = variant.members.filter(ts.isPropertySignature); + const kind = JSON.parse(members.find((member) => member.name.getText() === "kind")!.type!.getText()) as string; + const node: Record = { kind }; + const expected: IrType[] = []; + for (const member of members) { + if (!member.type) continue; + const child = ts.isTypeReferenceNode(member.type) && member.type.typeName.getText() === "IrType"; + const array = ts.isArrayTypeNode(member.type) && member.type.elementType.getText() === "IrType"; + if (!child && !array) continue; + const value: IrType = { kind: "fileHandle" }; + expected.push(value); + node[member.name.getText()] = array ? [value] : value; + } + const seen: IrType[] = []; + expect(everyTypeChild(node as IrType, (child) => { seen.push(child); return true; })).toBe(true); + expect(seen).toHaveLength(expected.length); + expected.forEach((child, i) => expect(seen[i]).toBe(child)); + for (let stop = 0; stop < expected.length; stop++) { + let count = 0; + expect(everyTypeChild(node as IrType, () => count++ !== stop)).toBe(false); + expect(count).toBe(stop + 1); + } + } +}); diff --git a/packages/compiler/src/ir/traverse.ts b/packages/compiler/src/ir/traverse.ts index ab776ef4..6d97ba68 100644 --- a/packages/compiler/src/ir/traverse.ts +++ b/packages/compiler/src/ir/traverse.ts @@ -1,4 +1,65 @@ -import type { IrExpr, IrStmt } from "./ir.js"; +import type { IrExpr, IrModule, IrStmt, IrType, SrcLoc } from "./ir.js"; + +/** The type tree contains structural children; named shapes are references. + * Module traversal visits each shape definition separately. */ +export function everyTypeChild(node: IrType, visit: (node: IrType) => boolean): boolean { + switch (node.kind) { + case "array": case "set": return visit(node.elem); + case "map": return visit(node.key) && visit(node.value); + case "func": return node.params.every(visit) && visit(node.ret); + case "promise": return visit(node.inner); + case "generator": return visit(node.yieldT) && visit(node.retT) && visit(node.nextT); + case "f64": case "bigint": case "date": case "string": case "bool": + case "regex": case "bytes": case "url": case "searchParams": case "symbol": + case "stats": case "fileHandle": case "spawnRes": case "child": + case "netServer": case "netSocket": case "http2Session": case "http2Stream": + case "dgramSocket": case "testCtx": case "httpReq": case "httpRes": + case "httpClientReq": case "childStream": case "childWriter": case "procStream": + case "fsWatcher": case "secureCtx": case "cryptoHash": case "cryptoHmac": + case "object": case "classval": case "moduleNs": case "record": case "union": + case "dyn": case "jsval": case "caught": case "undefinedT": case "nullT": case "void": + return true; + } + node satisfies never; + throw new Error("unhandled IR type"); +} + +export interface IrModuleVisitor extends IrVisitor { + type: (type: IrType, loc: SrcLoc) => boolean; +} + +/** Inspect executable nodes and every typed slot without reflecting over + * source text, locations, labels, or other metadata. A false predicate stops + * the whole walk. Results are never retained across mutable compiler passes. */ +export function everyModuleNode(mod: IrModule, visitor: IrModuleVisitor): boolean { + const type = (node: IrType, loc: SrcLoc): boolean => + visitor.type(node, loc) && everyTypeChild(node, (child) => type(child, loc)); + const expr = (node: IrExpr): boolean => + visitor.expr(node) && everyExprChild(node, expr, stmt) && type(node.type, node.loc); + const stmt = (node: IrStmt): boolean => visitor.stmt(node) && everyStmtChild(node, expr, stmt); + const entryLoc: SrcLoc = { file: mod.sourceFile, start: 0, end: 0 }; + for (const fn of mod.functions) { + for (const param of fn.params) if (!type(param.type, fn.loc)) return false; + if (!type(fn.returnType, fn.loc)) return false; + for (const local of fn.locals) if (!type(local.type, local.source?.loc ?? fn.loc)) return false; + for (const capture of fn.captures ?? []) if (!type(capture.type, fn.loc)) return false; + for (const capture of fn.classCaptures ?? []) if (!type(capture.type, fn.loc)) return false; + if (fn.generator !== undefined && (!type(fn.generator.yieldT, fn.loc) || + !type(fn.generator.nextT, fn.loc) || !type(fn.generator.resultType, fn.loc))) return false; + for (const node of fn.body) if (!stmt(node)) return false; + } + for (const cls of mod.classes ?? []) { + for (const capture of cls.localCaptures ?? []) if (!type(capture.type, cls.loc)) return false; + for (const field of cls.fields) if (!type(field.type, cls.loc)) return false; + } + for (const global of mod.globals ?? []) if (!type(global.type, global.source?.loc ?? entryLoc)) return false; + for (const record of mod.records ?? []) { + for (const field of record.fields) if (!type(field.type, entryLoc)) return false; + if (record.indexValue !== undefined && !type(record.indexValue, entryLoc)) return false; + } + for (const union of mod.unions ?? []) for (const arm of union.arms) if (!type(arm, entryLoc)) return false; + return true; +} /** Typed structural traversal of executable IR. Types, source locations, * captures, labels and other metadata are deliberately not child nodes. diff --git a/packages/compiler/src/library/cache-primitives.ts b/packages/compiler/src/library/cache-primitives.ts index c38b59f3..4ce73b37 100644 --- a/packages/compiler/src/library/cache-primitives.ts +++ b/packages/compiler/src/library/cache-primitives.ts @@ -77,7 +77,7 @@ export function frontendOutputExclusions( if (dirname(directory) === directory) break; } } - return { outputPaths: outputArtifacts, outputDirectories }; + return { outputPaths: outputArtifacts, outputDirectories: [...outputDirectories] }; } export async function readCachedFile(path: string, expected: string): Promise { diff --git a/packages/compiler/src/library/fence-eval.ts b/packages/compiler/src/library/fence-eval.ts index 01345efd..1843d84a 100644 --- a/packages/compiler/src/library/fence-eval.ts +++ b/packages/compiler/src/library/fence-eval.ts @@ -439,14 +439,14 @@ export function evaluateLibraryFences(mod: IrModule, profile: FenceProfileView): /* ── the generalized teaching rider ────────────────────────────────────── */ function surfaceMatchesDiag( - surface: { code?: string; name: string; kind: SurfaceEntryKind }, + surface: { code?: string | undefined; name: string; kind: SurfaceEntryKind }, diag: ScrDiagnostic, ): boolean { if (surface.code === undefined || surface.code !== diag.code) return false; // A diagnostic-fence entry IS the code family. Method refusals use a // receiver description instead of the manifest's prototype spelling. if (surface.kind === "diagnostic-fence" || diag.message.includes(surface.name)) return true; - for (const [prefix, receiver] of [["number.prototype.", "numbers"], ["string.prototype.", "strings"]] as const) { + for (const [prefix, receiver] of [["number.prototype.", "numbers"], ["string.prototype.", "strings"]] as [string, string][]) { if (surface.name.startsWith(prefix)) { const method = surface.name.slice(prefix.length); return diag.message.includes(`'.${method}()' on ${receiver}`); @@ -469,14 +469,14 @@ function teachingForRefusal(profile: FenceProfileView, diag: ScrDiagnostic): str for (const [key, text] of Object.entries(profile.teachings)) { if (!key.includes(".")) continue; const entry = fenceTaxonomy().byId.get(key); - if (entry !== undefined && surfaceMatchesDiag(entry, diag)) return text; + if (entry !== undefined && surfaceMatchesDiag({ code: entry.code, name: entry.name, kind: entry.kind }, diag)) return text; } // 3. A fence covering the refused surface: the teaching rides the // surface's own refusal (the non-static half of fence coverage). for (const fence of profile.fences) { if (fence.teaching === undefined) continue; for (const s of fence.surfaces) { - if (surfaceMatchesDiag(s, diag)) return fence.teaching; + if (surfaceMatchesDiag({ code: s.code, name: s.name, kind: s.kind }, diag)) return fence.teaching; } } return undefined; diff --git a/packages/compiler/src/library/int-infer.ts b/packages/compiler/src/library/int-infer.ts index 794975b5..6d15c79c 100644 --- a/packages/compiler/src/library/int-infer.ts +++ b/packages/compiler/src/library/int-infer.ts @@ -613,8 +613,7 @@ function globalEffectsOf(mod: IrModule): GlobalEffects { default: break; } - for (const key of Object.keys(e) as (keyof typeof e)[]) { - const v = e[key] as unknown; + for (const [key, v] of Object.entries(e as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && "kind" in (item as object)) { @@ -646,7 +645,7 @@ function globalEffectsOf(mod: IrModule): GlobalEffects { default: break; } - for (const v of Object.values(s) as unknown[]) { + for (const v of Object.values(s as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && typeof (item as { kind?: unknown }).kind === "string") { @@ -684,7 +683,7 @@ function globalEffectsOf(mod: IrModule): GlobalEffects { unknown.add(name); changed = true; } - for (const g of writes.get(callee) ?? []) { + for (const g of writes.get(callee) ?? new Set()) { if (!w.has(g)) { w.add(g); changed = true; @@ -936,7 +935,8 @@ class FnAnalyzer { // could select. Multiple classified fields intentionally emit // independent obligations (their classes and paths may differ). if (s.overflowOnly !== true) { - for (const slot of this.cfg.records.get(s.shapeId)?.values() ?? []) { + const recordSlots = this.cfg.records.get(s.shapeId); + for (const slot of recordSlots === undefined ? [] : [...recordSlots.values()]) { this.emitRecordSlot(v, slot, s.loc); } } @@ -1024,7 +1024,7 @@ class FnAnalyzer { const assigned = new Set(); const visitStmt = (s: IrStmt): void => { if (s.kind === "assign" || s.kind === "varDecl") assigned.add(s.localId); - for (const v of Object.values(s) as unknown[]) { + for (const v of Object.values(s as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && typeof (item as { kind?: unknown }).kind === "string") { @@ -1046,7 +1046,7 @@ class FnAnalyzer { const visitExpr = (e: IrExpr): void => { if (e.kind === "assignExpr" || e.kind === "incDec") assigned.add(e.localId); if (e.kind === "seqExpr") e.stmts.forEach(visitStmt); - for (const v of Object.values(e) as unknown[]) { + for (const v of Object.values(e as unknown as Record)) { if (Array.isArray(v)) { for (const item of v) { if (item !== null && typeof item === "object" && typeof (item as { kind?: unknown }).kind === "string") { @@ -1579,7 +1579,8 @@ class FnAnalyzer { }); } this.havocCall(e.callee, env); - if (slots?.ret != null) return classSeed(slots.ret); + const returnSlot = slots?.ret; + if (returnSlot != null) return classSeed(returnSlot); return { ...TOP }; } case "unionWrap": { @@ -1716,7 +1717,7 @@ function typeContainsFunc(t: unknown): boolean { * order for the shapes we don't model precisely). */ function childExprs(e: IrExpr): IrExpr[] { const out: IrExpr[] = []; - for (const [key, v] of Object.entries(e)) { + for (const [key, v] of Object.entries(e as unknown as Record)) { if (key === "type") continue; if (Array.isArray(v)) { for (const item of v) { diff --git a/packages/compiler/src/library/library-cache.test.ts b/packages/compiler/src/library/library-cache.test.ts index 08c9cbcc..ea5001e0 100644 --- a/packages/compiler/src/library/library-cache.test.ts +++ b/packages/compiler/src/library/library-cache.test.ts @@ -81,6 +81,7 @@ test("early library cache restores generated artifacts and metadata", async () = zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }; await publishEarlyLibraryCache(f.root, f.options, { llvmPath: f.llvmPath, @@ -133,6 +134,7 @@ test("early library cache publishes after creating a fresh output directory", as zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), }); @@ -162,6 +164,7 @@ test("early library cache hits refresh every payload's LRU time", async () => { zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), }); @@ -206,6 +209,7 @@ test("early library cache misses on source edits and newly-resolved candidates", zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), }); @@ -234,6 +238,7 @@ test("early library cache misses on source edits and newly-resolved candidates", zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: editedTracker.snapshot(), }); @@ -280,6 +285,7 @@ test("semantic library cache restores and rebases IR after a comment-only edit", zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), semantic: { mod: semanticMod, sources: new Map([[f.source, sourceBefore]]) }, @@ -338,6 +344,7 @@ test("semantic library cache refuses token and directive edits", async () => { zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), semantic: { mod: semanticMod, sources: new Map([[f.source, sourceBefore]]) }, @@ -383,6 +390,7 @@ test("semantic LLVM cache accepts comment edits that shift source lines", async zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), semantic: { mod: semanticMod, sources: new Map([[f.source, sourceBefore]]) }, @@ -420,6 +428,7 @@ test("semantic LLVM cache accepts line separator normalization", async () => { zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), semantic: { @@ -478,6 +487,7 @@ test("semantic LLVM cache rebases comment edits in multi-source graphs", async ( zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), semantic: { @@ -513,6 +523,7 @@ test("early library cache is separated by the host Node version", async () => { zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), }); @@ -545,6 +556,7 @@ test("early library cache rejects corrupted artifacts and metadata", async () => zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), }); @@ -578,6 +590,7 @@ test("disabled early library cache performs no reads or writes", async () => { zlib: false, copying: false, textDecoderLegacy: false, + dynInvoke: false, }, frontend: tracker.snapshot(), }; diff --git a/packages/compiler/src/library/library-cache.ts b/packages/compiler/src/library/library-cache.ts index 84d2d1a4..156f6f36 100644 --- a/packages/compiler/src/library/library-cache.ts +++ b/packages/compiler/src/library/library-cache.ts @@ -57,6 +57,7 @@ interface EarlyLibraryCacheStamp { export interface EarlyLibraryNativeFeatures { backend: "llvm"; + dynInvoke: boolean; regex: boolean; assert: boolean; inspect: boolean; @@ -114,6 +115,7 @@ export interface SemanticLibraryCacheHit { } const BOOLEAN_NATIVE_KEYS = [ + "dynInvoke", "regex", "assert", "inspect", diff --git a/packages/compiler/src/library/library-profile.ts b/packages/compiler/src/library/library-profile.ts index 47b08209..ac16856e 100644 --- a/packages/compiler/src/library/library-profile.ts +++ b/packages/compiler/src/library/library-profile.ts @@ -374,6 +374,8 @@ export interface LibraryProfile { /** Native optimizer posture. release preserves the production -O2 archive; * dev uses -O0 for fast iterative embedding builds. */ optimization: "release" | "dev"; + /** Explicit source-inference attempts, including packages without declarations. */ + npmStatic: string[]; prefix: string; initSymbol: string; sinkRegisterSymbol: string; @@ -440,7 +442,7 @@ class ProfileError extends Error { } function req(v: unknown, path: string, kind: "string" | "number" | "boolean"): T { - if (typeof v !== kind) { + if (!(kind === "string" ? typeof v === "string" : kind === "number" ? typeof v === "number" : typeof v === "boolean")) { throw new ProfileError(`'${path}' must be a ${kind}${v === undefined ? " (missing)" : ""}`); } return v as T; @@ -498,7 +500,7 @@ export function loadLibraryProfile( // the root would otherwise be silently inert — the exact footgun the // fence machinery refuses everywhere else. for (const k of Object.keys(p)) { - if (["profile_format", "name", "entry", "emission", "optimization", "abi", "exports", "callbacks", "sidecar", "determinism"].includes(k)) continue; + if (["profile_format", "name", "entry", "emission", "optimization", "npm_static", "abi", "exports", "callbacks", "sidecar", "determinism"].includes(k)) continue; if (k === "fences" || k === "teachings" || k === "remediations") { throw new ProfileError( `'${k}' at the profile root does nothing — the ask-5 determinism surface lives under 'determinism.${k}'; move it there`, @@ -507,6 +509,11 @@ export function loadLibraryProfile( throw new ProfileError(`unknown field '${k}' (root keys are strict: a typo here would silently change the build; remove it)`); } const name = req(p["name"], "name", "string"); + const npmStaticRaw = p["npm_static"] === undefined ? [] : p["npm_static"]; + if (!Array.isArray(npmStaticRaw) || npmStaticRaw.some((name) => typeof name !== "string" || !/^(?:@[a-z0-9_.-]+\/)?[a-z0-9_][a-z0-9_.-]*$/.test(name))) { + throw new ProfileError("'npm_static' must be an array of npm package names (not subpath specifiers)"); + } + const npmStatic = [...new Set(npmStaticRaw as string[])]; if (name === "") throw new ProfileError("'name' must be a non-empty identity string"); const entryRel = req(p["entry"], "entry", "string"); if (entryRel === "") throw new ProfileError("'entry' must name the profile's one entry module"); @@ -861,11 +868,12 @@ export function loadLibraryProfile( if (prefix === "") throw new ProfileError(`'${path}.prefix' must be a non-empty manifest id prefix`); decl.prefix = prefix; } - for (const rider of ["teaching", "remediation"] as const) { + for (const rider of ["teaching", "remediation"] as ("teaching" | "remediation")[]) { if (ff[rider] === undefined) continue; const text = req(ff[rider], `${path}.${rider}`, "string"); checkRiderText(text, `${path}.${rider}`); - decl[rider] = text; + if (rider === "teaching") decl.teaching = text; + else decl.remediation = text; } return decl; }); @@ -885,6 +893,7 @@ export function loadLibraryProfile( entry, emission, optimization, + npmStatic, prefix, initSymbol, sinkRegisterSymbol, diff --git a/packages/compiler/src/library/prepare.ts b/packages/compiler/src/library/prepare.ts new file mode 100644 index 00000000..b40e5b0b --- /dev/null +++ b/packages/compiler/src/library/prepare.ts @@ -0,0 +1,665 @@ +import { dirname, resolve } from "node:path"; +import type { CompileFailure } from "../compile-types.js"; +import { moduleWasiUnavailableSurface, targetRefusalDiag } from "../backend/target-diagnostics.js"; +import { checkerPanicDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "../diagnostics/diagnostic.js"; +import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./int-infer.js"; +import { profileRemediation, profileTeaching, type LibraryProfile } from "./library-profile.js"; +import { decorateLibraryRefusals, evaluateLibraryFences } from "./fence-eval.js"; +import { assembleTrapTeaching } from "./trap-teaching.js"; +import { buildSidecar, canonicalModuleGraph, canonicalPath, libraryIdentityHashes, type SidecarIntegerSlotFacts, type SidecarIrRecordPattern, type SidecarIrTypePattern } from "./sidecar.js"; +import { validateSidecar } from "./sidecar-validate.js"; +import type { EntryExportInfo } from "../frontend/lib-exports.js"; +import type { ContractFacts } from "../frontend/lib-contract.js"; +import type { LowerResult } from "../frontend/lowering/lowerer.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import { moduleLibAsyncSurface, moduleLibNondeterministicSurface, type IrFfiImport, type IrLibSection, type IrModule, type IrRecordShape, type IrType } from "../ir/ir.js"; +import { validateModule } from "../ir/validate.js"; + +/** The marshalling-class fit over IR types (design §4.2 + the ratified + * integer plumbing classes): number is every f64-backed class, bool/string + * map directly, bytes is the u8 element kind. */ +function libClassFits(cls: string, t: IrType): boolean { + switch (cls) { + case "bool": + return t.kind === "bool"; + case "string": + return t.kind === "string"; + case "bytes": + return t.kind === "bytes" && t.elem === "u8"; + default: // f64 and the u8/u32/i32 plumbing classes + return t.kind === "f64"; + } +} + +/** Resolve the profile's export map against the entry module — SC4002/ + * SC4004/SC4007 from the declaration facts, SC4003 from the lowered IR + * signatures — and land the library section on the module. */ +function resolveLibrarySection( + profile: LibraryProfile, + entryInfo: Map, + mod: IrModule, + entryPath: string, +): { lib: IrLibSection } | { diagnostics: ScrDiagnostic[] } { + const diagnostics: ScrDiagnostic[] = []; + const entryLoc = { file: entryPath, start: 0, end: 0 }; + const fnByName = new Map(mod.functions.map((f) => [f.name, f])); + const exports: IrLibSection["exports"] = []; + for (const e of profile.exports) { + const info = entryInfo.get(e.export); + if (info === undefined) { + diagnostics.push( + libExportUnresolvedDiag(e.export, "the entry module has no exported function declaration by that name", entryLoc), + ); + continue; + } + if (info.generic) { + diagnostics.push(libGenericExportDiag(e.export, info.loc)); + continue; + } + if (info.async || info.generator) { + diagnostics.push(libAsyncExportDiag(e.export, info.async ? "async" : "generator", info.loc)); + continue; + } + const fn = fnByName.get(e.export); + if (fn === undefined) { + diagnostics.push( + libExportUnresolvedDiag(e.export, "the export did not lower to a compiled function", info.loc), + ); + continue; + } + if (fn.params.length !== e.params.length) { + diagnostics.push( + libUnmappableSignatureDiag( + e.export, + "signature", + `has ${fn.params.length} parameter(s) but the profile declares ${e.params.length} marshalling class(es)`, + info.loc, + ), + ); + continue; + } + let bad = false; + e.params.forEach((cls, i) => { + if (!libClassFits(cls, fn.params[i]!.type)) { + bad = true; + diagnostics.push( + libUnmappableSignatureDiag( + e.export, + `parameter ${i + 1} ('${fn.params[i]!.name}')`, + `has IR type '${fn.params[i]!.type.kind}', which does not fit the declared marshalling class '${cls}'`, + info.loc, + ), + ); + } + }); + if (e.returns === "void" ? fn.returnType.kind !== "void" : !libClassFits(e.returns, fn.returnType)) { + bad = true; + diagnostics.push( + libUnmappableSignatureDiag( + e.export, + "the return", + `has IR type '${fn.returnType.kind}', which does not fit the declared marshalling class '${e.returns}'`, + info.loc, + ), + ); + } + if (!bad) { + const resolvedExport: IrLibSection["exports"][number] = { + symbol: e.symbol, + fnName: e.export, + params: e.params, + returns: e.returns, + }; + if (e.params.includes("bytes")) { + // The wrapper's one host-contract trap (an inbound bytes length + // past the marshalling class's range) is assembled HERE, once, as + // the structured trap-teaching message: the profile's teaching for + // SC4012 (or the mode's default text), the code, the trapping + // export's C symbol exactly as the host linked it, and the + // profile's remediation when supplied — so the backend emits the + // same bytes and the sink sees one canonical message. + resolvedExport.inboundBytesTrap = assembleTrapTeaching( + profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? + "scriptc: library inbound bytes length out of range\n", + LIB_INBOUND_BYTES_TRAP_CODE, + e.symbol, + profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), + ); + } + if (e.params.includes("i64") || e.params.includes("u64")) { + // The sibling host-contract trap for inbound declared-integer + // parameters (ask 4): a value past ±(2^53−1) cannot ride f64 + // exactly, and silent rounding is a coercion the author never + // wrote. Same code (SC4012 — one host-contract story), same + // assembly-once discipline. + resolvedExport.inboundIntTrap = assembleTrapTeaching( + profileTeaching(profile, LIB_INBOUND_BYTES_TRAP_CODE) ?? + "scriptc: library inbound integer parameter out of range\n", + LIB_INBOUND_BYTES_TRAP_CODE, + e.symbol, + profileRemediation(profile, LIB_INBOUND_BYTES_TRAP_CODE), + ); + } + exports.push(resolvedExport); + } + } + if (diagnostics.length > 0) return { diagnostics }; + // The runtime detected-trap overlay rows: one per family code the profile + // declares teaching or remediation text for, in the registry family's + // order. LLVM emits these rows as the program TU's overlay table, + // which the runtime uses to assemble the sink message. (SC4012 stays compile-time + // assembled into the wrapper's message above and never reaches the + // funnel's assembly path.) + const trapOverlays: IrLibSection["trapOverlays"] = []; + for (const code of LIB_RUNTIME_TRAP_CODES) { + const teaching = profileTeaching(profile, code); + const remediation = profileRemediation(profile, code); + if (teaching !== undefined || remediation !== undefined) { + trapOverlays.push({ + code, + ...(teaching !== undefined ? { teaching } : {}), + ...(remediation !== undefined ? { remediation } : {}), + }); + } + } + const lib: IrLibSection = { + profileName: profile.name, + prefix: profile.prefix, + initSymbol: profile.initSymbol, + sinkRegisterSymbol: profile.sinkRegisterSymbol, + collectSymbol: profile.collectSymbol, + resultResetSymbol: profile.resultResetSymbol, + threadInstances: profile.instancePerThread, + // Host-callback channels: declaration order is the runtime slot + // assignment, and the unregistered-call trap text is assembled HERE, + // once, for consistent constant bytes (a DETECTED + // trap: the funnel classifies the "scriptc: library callback " + // prefix as SC4025 and names the entry the host called — the entry + // is runtime knowledge, so no compile-time SC4012-style assembly + // can carry it). Both fields stay absent on callback-free profiles + // (the byte-identity guarantee). + + exports, + trapOverlays, + }; + if (profile.callbacks.length > 0) { + lib.callbackRegisterSymbol = profile.callbackRegisterSymbol!; + lib.callbacks = profile.callbacks.map((cb, slot) => ({ + name: cb.name, slot, params: [...cb.params], returns: cb.returns, + unregisteredTrap: `scriptc: library callback '${cb.name}' invoked before registration\n`, + })); + } + return { lib }; +} + +/** The export map's integer-slot obligations (ask 4): i64/u64 params and + * returns become declared boundary slots keyed `exports..params[i]` + * / `exports..return`; the u8/u32/i32 plumbing classes contribute + * their proven inbound shapes as parameter seeds (the wrapper's coercion + * contract), tightening the intraprocedural analysis at zero declaration + * cost. Sidecar-declared slots (record fields, msg arms, helper params + * and returns) merge into the same config at sidecar build. */ +function libraryIntSlotConfig(profile: LibraryProfile): IntSlotConfig { + const cfg: IntSlotConfig = { fns: new Map(), records: new Map() }; + for (const e of profile.exports) { + const params = e.params.map((c) => (c === "i64" || c === "u64" ? c : null)); + const ret = e.returns === "i64" || e.returns === "u64" ? e.returns : null; + const paramSeeds = e.params.map((c) => (c === "u8" || c === "u32" || c === "i32" ? classSeed(c) : null)); + if (params.every((p) => p === null) && ret === null && paramSeeds.every((s) => s === null)) continue; + const slots: FnIntSlots = { + fnName: e.export, + params, + paramPaths: e.params.map((c, i) => (c === "i64" || c === "u64" ? `exports.${e.export}.params[${i}]` : null)), + ret, + retPath: ret !== null ? `exports.${e.export}.return` : null, + paramSeeds, + }; + cfg.fns.set(e.export, slots); + } + return cfg; +} + +/** Match the sidecar syntax's exact structural type projection against the + * frontend's interned IR registries. The pattern deliberately mirrors + * ShapeRegistry's identity: every field name and recursively mapped field + * type participates. Tagged payload records additionally accept omission + * of their `kind` field because the lowering may carry that discriminant + * only in the surrounding union tag. */ +function sidecarRecordMatcher( + mod: IrModule, +): (pattern: SidecarIrRecordPattern, shape: IrRecordShape) => boolean { + const records = new Map((mod.records ?? []).map((shape) => [shape.id, shape])); + const unions = new Map((mod.unions ?? []).map((union) => [union.id, union])); + + const recordMatches = ( + pattern: SidecarIrRecordPattern, + shape: IrRecordShape, + ): boolean => { + if (shape.tuple === true || shape.indexValue !== undefined) return false; + const variants = [pattern.fields]; + if (pattern.kindMayBeOmitted === true) { + variants.push(pattern.fields.filter((field) => field.name !== "kind")); + } + return variants.some( + (fields) => + fields.length === shape.fields.length && + fields.every((field) => { + const actual = shape.fields.find((candidate) => candidate.name === field.name); + return actual !== undefined && typeMatches(field.type, actual.type); + }), + ); + }; + + const unionMatches = ( + patterns: SidecarIrTypePattern[], + actual: IrType[], + ): boolean => { + if (patterns.length !== actual.length) return false; + const used = new Set(); + const visit = (index: number): boolean => { + if (index === patterns.length) return true; + for (let i = 0; i < actual.length; i++) { + if (used.has(i) || !typeMatches(patterns[index]!, actual[i]!)) continue; + used.add(i); + if (visit(index + 1)) return true; + used.delete(i); + } + return false; + }; + return visit(0); + }; + + const typeMatches = ( + pattern: SidecarIrTypePattern, + actual: IrType, + ): boolean => { + switch (pattern.kind) { + case "f64": + case "string": + case "bool": + case "nullT": + case "undefinedT": + case "dyn": + return actual.kind === pattern.kind; + case "bytes": + return actual.kind === "bytes" && actual.elem === pattern.elem; + case "array": + return actual.kind === "array" && typeMatches(pattern.elem, actual.elem); + case "record": { + if (actual.kind !== "record") return false; + const shape = records.get(actual.shapeId); + return shape !== undefined && recordMatches(pattern, shape); + } + case "union": { + if (actual.kind !== "union") return false; + const union = unions.get(actual.unionId); + return union !== undefined && unionMatches(pattern.arms, union.arms); + } + } + }; + + return (pattern, shape) => recordMatches(pattern, shape); +} + +/** Merge the sidecar-resolved integer slots (ask 4) into the inference + * config: helper slots key by function name and IR parameter index (the + * projection already shifted past the model receiver); record-field + * slots map onto every interned IR shape whose complete structural field + * signature matches the projected record's. Shapes intern structurally, + * so a same-shaped second type shares the obligation. DECLARED paths with + * the same class coalesce while retaining every source path for verdicts; + * differing classes refuse because one lowered field cannot seed or check + * two distinct class contracts without arm provenance. A + * record fact that matches no shape binds nothing: no compiled code + * constructs the type (the contract surface — init/update/subscriptions + * and every helper — is force-lowered whenever integer slots are + * declared, so this is genuine vacuity, not dead-stripping). */ +function mergeSidecarIntSlots( + cfg: IntSlotConfig, + facts: SidecarIntegerSlotFacts, + mod: IrModule, +): { ok: true; config: IntSlotConfig } | { ok: false; diagnostic: ScrDiagnostic } { + const recordMatches = sidecarRecordMatcher(mod); + for (const h of facts.helpers) { + const fn = mod.functions.find((f) => f.name === h.fnName); + const arity = Math.max(fn?.params.length ?? 0, (h.index ?? 0) + 1); + let slots = cfg.fns.get(h.fnName); + if (slots === undefined) { + slots = { + fnName: h.fnName, + params: new Array(arity).fill(null), + paramPaths: new Array(arity).fill(null), + ret: null, + retPath: null, + paramSeeds: new Array(arity).fill(null), + }; + cfg.fns.set(h.fnName, slots); + } + if (h.kind === "param") { + const i = h.index!; + while (slots.params.length <= i) { + slots.params.push(null); + slots.paramPaths.push(null); + slots.paramSeeds.push(null); + } + slots.params[i] = h.cls; + slots.paramPaths[i] = h.path; + } else { + slots.ret = h.cls; + slots.retPath = h.path; + } + } + for (const r of facts.records) { + for (const shape of mod.records ?? []) { + if (!recordMatches(r.shape, shape)) continue; + const target = shape.fields.find((f) => f.name === r.targetField); + if (target === undefined || numberCarrierKind(target.type, mod) === null) continue; + let m = cfg.records.get(shape.id); + if (m === undefined) { + m = new Map(); + cfg.records.set(shape.id, m); + } + const existing = m.get(r.targetField); + if (existing !== undefined && existing.cls !== r.cls) { + const paths = [ + ...existing.paths.map((path) => `'${path}' (${existing.cls})`), + `'${r.path}' (${r.cls})`, + ]; + return { + ok: false, + diagnostic: libSidecarDiag( + `integer slots ${paths.join(" and ")} collapse to the same lowered record field '${r.targetField}' — their proof obligations cannot be kept distinct`, + r.loc, + "kind-tagged union arms and structurally identical records may share one lowered shape — same-class declarations coalesce, but differing classes require distinct structural shapes or at most one classified slot", + ), + }; + } + if (existing === undefined) { + m.set(r.targetField, { cls: r.cls, paths: [r.path] }); + } else if (!existing.paths.includes(r.path)) { + existing.paths.push(r.path); + } + } + } + return { ok: true, config: cfg }; +} + +export interface PreparedLibrary { + ok: true; + mod: IrModule; + sourceTexts: Map; + sidecarJson: string | null; +} + +export function prepareLibrary( + profile: LibraryProfile, + profilePath: string, + compilerVersion: string, + buildPlatform: string, + createFrontend: FrontendFactory, + timing: (phase: string, detail?: Record) => void = () => {}, +): PreparedLibrary | CompileFailure { + const entryPath = profile.entry; + // Bare npm specifiers in a library graph take the STATIC-OR-REFUSE + // posture: "lib" runs the same auto-detection and eligibility bar as + // the executable lane's --npm-static (own .d.ts, unminified shipped JS, + // no build-transform markers), automatically — the library path has no + // island/dynamic tier to offer. Explicit profile npm_static entries may + // also attempt source inference; any failed attempt remains a refusal. + const fe = createFrontend(entryPath, "lib", undefined, profile.npmStatic); + timing("frontend-load", { + entry_bytes: fe.entryText().length, + source_files: fe.sourceTexts().size, + }); + let lowered: LowerResult; + let sourceTexts: Map; + let entryInfo: Map; + let contractFacts: ContractFacts | null; + try { + // Every library refusal leaves through the ask-5 teaching decoration: + // profile text attaches by code, manifest id, or fence coverage as the + // attributed note (the SC4004/SC4005 rider generalized). + const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ + ok: false, + diagnostics: decorateLibraryRefusals(diagnostics, profile), + sourceTexts: fe.sourceTexts(), + }); + // The npm verdicts FIRST: whatever the shared frontend would have + // served from the island — an eligibility miss, an untyped install, a + // preflight offender inside a package's files, a dropped inferred + // surface — refuses here with the package and the specific bar it + // missed. Checked before the general preflight, whose diagnostics for + // these same imports speak executable-lane teachings (SC1010/SC0001 at + // the unresolvable edge); the library answer is this one. + const npmRefused = fe.npmStatic.filter((s) => s.status === "fallback"); + if (npmRefused.length > 0) { + return fail( + npmRefused.map((s) => + libNpmIneligibleDiag( + s.package, + // The one shared offender reason that narrates the executable + // lane's fallback loses that clause here — no island exists on + // this path to serve anything. + (s.detail ?? "its static compilation was refused").replace("; the island serves the package", ""), + fe.npmImportSites.get(s.package) ?? { file: entryPath, start: 0, end: 0 }, + ), + ), + ); + } + if (fe.preflight.length > 0) return fail(fe.preflight); + contractFacts = profile.sidecar !== null ? fe.entryContract() : null; + // Ask 4, contract-surface reachability: when the sidecar declares ANY + // integer slot, the designated init/update/subscriptions exports and + // every contract helper (model-first exported function) seed lowering + // too. They are attested surface — a declared record-field or msg-arm + // class obligates EVERY write those bodies perform, and a declared + // helper param is checked at their internal call sites — so the + // attestation must cover COMPILED bodies, never a dead-stripped + // vacuity (the bug this closes: a model-slot declaration whose only + // writers were dead-stripped attested without any proof). + const contractSurfaceRoots: string[] = []; + if (profile.sidecar !== null && profile.sidecar.integerSlots.length > 0) { + const sc = profile.sidecar; + const fnNames = new Set(contractFacts!.functions.filter((f) => !f.generic).map((f) => f.name)); + for (const name of [sc.initExport, sc.updateExport, sc.subscriptionsExport]) { + if (fnNames.has(name)) contractSurfaceRoots.push(name); + } + for (const fn of contractFacts!.functions) { + if (fn.generic) continue; + const first = fn.params[0]; + if (first !== undefined && first.shape !== null && first.shape.k === "ref" && first.shape.name === sc.model) { + contractSurfaceRoots.push(fn.name); + } + } + } + // The profile's host-callback channels ride the FFI import machinery: + // each channel is a signature-only ambient binding whose direct calls + // lower to ffiCall nodes (the classes are a subset of the FFI's), and + // `libraryCallbacks` flips the recognition to the library flavor — + // SC4024 diagnostics, unused channels legal, undeclared references + // refused with the callback teaching. The library lane never loads a + // native-FFI manifest, so the channel set owns the surface outright. + const cbImports: IrFfiImport[] = profile.callbacks.map((cb) => ({ + name: cb.name, + symbol: cb.name, + params: [...cb.params], + returns: cb.returns, + })); + const integerSlotRoots: string[] = []; + for (const slot of profile.sidecar?.integerSlots ?? []) { + const name = /^helpers\.([^.]+)\.(?:params\[\d+\]|return)$/.exec(slot.slot)?.[1]; + if (name !== undefined) integerSlotRoots.push(name); + } + try { + lowered = fe.lower({ + dynamic: false, + targetPlatform: buildPlatform, + ...(cbImports.length > 0 ? { ffiImports: cbImports } : {}), + ...(cbImports.length > 0 ? { libraryCallbacks: true } : {}), + // The profile-mapped exports are called from OUTSIDE the graph: + // they seed reachability beside the entry's top level (an + // executable build would dead-strip an uncalled export). A helper + // with a declared integer slot (ask 4) seeds too: its attestation + // must cover a COMPILED body, never a dead-stripped vacuity — the + // sidecar advertises the slot's class, so the proof must exist. + libRoots: [ + ...new Set([ + ...profile.exports.map((e) => e.export), + ...integerSlotRoots, + ...contractSurfaceRoots, + ]), + ], + }); + timing("lower", { + lib_roots: profile.exports.length + contractSurfaceRoots.length, + }); + } catch (e) { + if (!isCheckerPanic(e)) throw e; + return fail([checkerPanicDiag(e.message.split("\n", 1)[0]!, { file: entryPath, start: 0, end: 0 })]); + } + if (lowered.module === null) return fail(lowered.diagnostics); + entryInfo = fe.entryExports(); + sourceTexts = fe.sourceTexts(); + } finally { + fe.dispose(); + } + const mod = lowered.module!; + timing("frontend-dispose"); + + const fail = (diagnostics: ScrDiagnostic[]): CompileFailure => ({ + ok: false, + diagnostics: decorateLibraryRefusals(diagnostics, profile), + sourceTexts, + }); + + // Export resolution first (SC4002/SC4003/SC4004/SC4007 anchor at the + // mapped declaration — a mapped async export reports as SC4004, not the + // graph-wide gate), then the async_free requirement (ratified, SC4005), + // then the profile's determinism fences (ask 5, SC4008) over the same + // compiled graph the attestation scan reads: all refused before anything + // is emitted, so the narrowed library link set below is structural fact. + const resolved = resolveLibrarySection(profile, entryInfo, mod, entryPath); + if ("diagnostics" in resolved) return fail(resolved.diagnostics); + const asyncSurface = moduleLibAsyncSurface(mod); + if (asyncSurface !== null) { + return fail([libAsyncSurfaceDiag(asyncSurface.surface, asyncSurface.loc)]); + } + const fenced = evaluateLibraryFences(mod, profile); + if (fenced.length > 0) return fail(fenced); + mod.lib = resolved.lib; + if (buildPlatform === "wasi") { + const unavailable = moduleWasiUnavailableSurface(mod); + if (unavailable !== null) return fail([targetRefusalDiag("wasm32-wasi", unavailable.surface, unavailable.loc)]); + } + + // Ask 4's declared integer slots: the export map's i64/u64 classes + // seed the config here; sidecar-declared slots (record fields, msg + // arms, helper params/returns) merge in after the projection resolves + // them below. + let intCfg = libraryIntSlotConfig(profile); + + // The ask-2 contract sidecar rides the same invocation. Identity first + // (schema §2's worked build_id definition over compiler version, profile + // bytes, and the sorted canonical module graph; source_hash per the + // profile's "module-graph" contract) — the u64 lands on the IR so native + // archive assembly emits the identity getters from the ONE value the + // sidecar records (V12's coherence by construction), then the projection into + // the schema (declaration orders from the AST) and the V1–V14 + // self-check before anything is written. + let sidecarJson: string | null = null; + if (profile.sidecar !== null) { + const rootDir = dirname(resolve(profilePath)); + const modules = canonicalModuleGraph(rootDir, sourceTexts); + const { buildId, sourceHash } = libraryIdentityHashes(compilerVersion, profile.profileBytes, modules); + mod.lib.identity = { + buildIdSymbol: profile.sidecar.buildIdSymbol, + abiVersionSymbol: profile.sidecar.abiVersionSymbol, + buildId, + abiVersion: profile.sidecar.abiVersion, + }; + const built = buildSidecar({ + profile, + facts: contractFacts!, + compilerVersion: compilerVersion, + entry: canonicalPath(rootDir, entryPath), + buildId, + sourceHash, + deterministic: moduleLibNondeterministicSurface(mod) === null, + }); + if (!built.ok) return fail(built.diagnostics); + // Validate the serialized contract, including its omitted optional + // properties, through the same checked-value path used by consumers. + const violations = validateSidecar(JSON.parse(built.json)); + if (violations.length > 0) { + // The projection above refuses every user-caused shape; a rule + // violation surviving to here is an emitter bug. + return fail(violations.map((v) => iceDiag(`sidecar self-check failed — ${v}`, { file: entryPath, start: 0, end: 0 }))); + } + sidecarJson = built.json; + const merged = mergeSidecarIntSlots(intCfg, built.integerSlotFacts, mod); + if (!merged.ok) return fail([merged.diagnostic]); + intCfg = merged.config; + } + timing("contract-sidecar", { source_files: sourceTexts.size }); + + // Ask 4: the integer-boundary inference — every value that can reach a + // profile-declared i64/u64 slot must PROVE representability, wholeness, + // and range, or the build refuses with the failed obligation, the + // observed evidence, and the author's fix (SC4021/SC4022/SC4023). Runs + // only when at least one integer slot is declared; the sidecar (already + // built above, written only on success) may then attest the classes — + // §5's invariant that an attested integer class means the proof was + // discharged holds because no artifact leaves this function otherwise. + if (hasIntSlots(intCfg)) { + const refusals = checkLibraryIntegerSlots(mod, intCfg).filter((v) => v.outcome === "refuse"); + if (refusals.length > 0) { + return fail(refusals.map((v) => libIntBoundaryDiag(v.path, v.cls, v.obligation!, v.detail!, v.fix!, v.loc))); + } + } + timing("integer-proof"); + + const validation = validateModule(mod); + if (validation.length > 0) return fail(validation.map((v) => iceDiag(v.message, v.loc))); + timing("ir-validate"); + + return { ok: true, mod, sourceTexts, sidecarJson }; +} + +export function libraryLocalizeSymbols(profile: LibraryProfile): string[] | undefined { + return profile.localizeRuntime + ? [ + profile.initSymbol, + profile.sinkRegisterSymbol, + ...(profile.collectSymbol !== null ? [profile.collectSymbol] : []), + ...(profile.resultResetSymbol !== null ? [profile.resultResetSymbol] : []), + ...(profile.callbackRegisterSymbol !== null ? [profile.callbackRegisterSymbol] : []), + ...(profile.sidecar !== null + ? [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol] + : []), + ...profile.exports.map((entry) => entry.symbol), + ] + : undefined; +} + +export function libraryWasmExports(profile: LibraryProfile): string[] { + return [ + profile.initSymbol, "scriptc_alloc", "scriptc_free", + ...(profile.collectSymbol === null ? [] : [profile.collectSymbol]), + ...(profile.resultResetSymbol === null ? [] : [profile.resultResetSymbol]), + ...(profile.sidecar === null ? [] : [profile.sidecar.buildIdSymbol, profile.sidecar.abiVersionSymbol]), + ...profile.exports.map((entry) => entry.symbol), + ]; +} + +export function libraryWasmRefusal(profile: LibraryProfile, sanitize: boolean): ScrDiagnostic | null { + const reserved = new Set(["scriptc_alloc", "scriptc_free", "memory", "_initialize", "_start"]); + const symbols = [profile.initSymbol, profile.sinkRegisterSymbol, profile.collectSymbol, profile.resultResetSymbol, + profile.callbackRegisterSymbol, ...profile.exports.map((entry) => entry.symbol), profile.sidecar?.buildIdSymbol, profile.sidecar?.abiVersionSymbol]; + const surface = sanitize ? "sanitized library builds" + : profile.instancePerThread ? "thread-instanced libraries (instantiate separate Wasm instances instead)" + : profile.localizeRuntime ? "runtime localization (Wasm instances already isolate their runtime)" + : symbols.some((symbol) => symbol != null && reserved.has(symbol)) ? "library symbols reserved by the Wasm embedding ABI" + : profile.callbacks.some((cb) => cb.name === "panic") ? "a callback named 'panic' (reserved by the Wasm embedding ABI)" + : null; + return surface === null ? null : targetRefusalDiag("wasm32-wasi", surface, { file: profile.entry, start: 0, end: 0 }); +} diff --git a/packages/compiler/src/library/semantic-source.ts b/packages/compiler/src/library/semantic-source.ts index 54bc9845..a2892e4d 100644 --- a/packages/compiler/src/library/semantic-source.ts +++ b/packages/compiler/src/library/semantic-source.ts @@ -1,7 +1,10 @@ import { InternalCompilerError } from "../errors.js"; import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; import { extname } from "node:path"; -import ts from "typescript5"; +import type ts from "typescript5"; + +const require = createRequire(import.meta.url); interface SemanticToken { kind: "token" | "comment"; @@ -32,15 +35,16 @@ function semanticTokens(path: string, source: string): SemanticToken[] | null { // both syntax validity (including a shebang's byte-zero requirement) and // exact regular-expression spans, then keep the cheap scanner for trivia // equivalence and location mapping. - const sourceFile = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true); + const parser = require("typescript5") as typeof ts; + const sourceFile = parser.createSourceFile(path, source, parser.ScriptTarget.Latest, true); const parseDiagnostics = (sourceFile as ts.SourceFile & { parseDiagnostics: readonly ts.Diagnostic[]; }).parseDiagnostics; if (parseDiagnostics.length > 0) return null; const regexEnds = new Map(); const collectRegex = (node: ts.Node): void => { - if (ts.isRegularExpressionLiteral(node)) regexEnds.set(node.getStart(sourceFile), node.end); - ts.forEachChild(node, collectRegex); + if (parser.isRegularExpressionLiteral(node)) regexEnds.set(node.getStart(sourceFile), node.end); + parser.forEachChild(node, collectRegex); }; collectRegex(sourceFile); diff --git a/packages/compiler/src/library/sidecar-validate.ts b/packages/compiler/src/library/sidecar-validate.ts index bac7e6b0..f3b7567a 100644 --- a/packages/compiler/src/library/sidecar-validate.ts +++ b/packages/compiler/src/library/sidecar-validate.ts @@ -14,7 +14,7 @@ type Dict = Record; -const TOP_LEVEL_ORDER = [ +const TOP_LEVEL_ORDER: readonly string[] = [ "format", "wire_version", "abi_version", @@ -35,11 +35,11 @@ const TOP_LEVEL_ORDER = [ "integer_slots", "deterministic", "async_free", -] as const; +]; const HASH_RE = /^[0-9a-f]{16}$/; const TYPEREF_KINDS = new Set(["bool", "f64", "i64", "bytes", "void", "optional", "slice", "node", "value", "enum", "union"]); -const FUNCTION_CHANNELS = ["command_msg", "frame_msg", "key_msg", "pinch_msg"] as const; +const FUNCTION_CHANNELS = ["command_msg", "frame_msg", "key_msg", "pinch_msg"]; function isDict(v: unknown): v is Dict { return v !== null && typeof v === "object" && !Array.isArray(v); @@ -58,12 +58,12 @@ export function validateSidecar(doc: unknown): string[] { /* ── V1: required fields, format, top-level key order ─────────────── */ const keys = Object.keys(doc); for (const k of TOP_LEVEL_ORDER) { - if (!(k in doc)) bad("V1", `required field '${k}' is missing`); + if (!(Object.hasOwn(doc, k))) bad("V1", `required field '${k}' is missing`); } for (const k of keys) { if (!(TOP_LEVEL_ORDER as readonly string[]).includes(k)) bad("V1", `unknown top-level field '${k}' (emit only format-1 fields)`); } - const present = TOP_LEVEL_ORDER.filter((k) => k in doc); + const present = TOP_LEVEL_ORDER.filter((k) => Object.hasOwn(doc, k)); const actual = keys.filter((k) => (TOP_LEVEL_ORDER as readonly string[]).includes(k)); if (present.join(",") !== actual.join(",")) { bad("V1", "top-level keys are not emitted in the schema's §1 order"); @@ -348,7 +348,7 @@ export function validateSidecar(doc: unknown): string[] { return; } visiting.add(name); - for (const next of edges.get(name) ?? []) visit(next, [...path, name]); + for (const next of edges.get(name) ?? new Set()) visit(next, [...path, name]); visiting.delete(name); done.add(name); }; @@ -405,7 +405,7 @@ export function validateSidecar(doc: unknown): string[] { bad("V9", `channels.${ch} is ${v} but the suffix '${ch}' is ${v ? "absent from" : "present in"} abi.exports`); } } - for (const ch of ["appearance_msg", "chrome_msg"] as const) { + for (const ch of ["appearance_msg", "chrome_msg"]) { const v = channels[ch]; if (v === null) continue; if (typeof v !== "string") { @@ -450,7 +450,7 @@ export function validateSidecar(doc: unknown): string[] { while (grew) { grew = false; for (const name of [...reachable]) { - for (const next of edges.get(name) ?? []) { + for (const next of edges.get(name) ?? new Set()) { if (!reachable.has(next)) { reachable.add(next); grew = true; diff --git a/packages/compiler/src/library/sidecar.ts b/packages/compiler/src/library/sidecar.ts index 4859cedc..a69d7b0e 100644 --- a/packages/compiler/src/library/sidecar.ts +++ b/packages/compiler/src/library/sidecar.ts @@ -145,6 +145,10 @@ export interface SidecarDoc { /* ── identity hashing (schema §2 + the "module-graph" source contract) ─── */ let releaseVersion: string | null = null; +let installedReleaseVersion: string | null = null; + +/** Installed native clients read their release identity from the distribution. */ +export function setCompilerReleaseVersion(version: string): void { installedReleaseVersion = version; } /** The package version is stable within one compilation, but a long-lived * source/worktree process may observe a release stamp between compilations. */ @@ -157,6 +161,7 @@ export function clearSidecarCaches(): void { * module lives two levels below the package root in src/ and dist/ * alike). build_id input 1 and the sidecar's `compiler_version`. */ export function compilerReleaseVersion(): string { + if (installedReleaseVersion !== null) return installedReleaseVersion; if (releaseVersion === null) { const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "package.json"); releaseVersion = (JSON.parse(readFileSync(pkgPath, "utf8")) as { version: string }).version; @@ -806,7 +811,7 @@ class Projector { if (shape.k === "object") return shape.fields; if (shape.k === "array") return this.inlineRecordFields(shape.elem); if (shape.k === "union") { - const present = shape.parts.filter((part) => part.k !== "absent"); + const present = shape.parts.filter((part): boolean => part.k !== "absent"); return present.length === 1 && present.length !== shape.parts.length ? this.inlineRecordFields(present[0]!) : null; @@ -1008,7 +1013,7 @@ class Projector { case "array": return { kind: "slice", elem: this.shapeRef(shape.elem, container, member, loc, synthesizedContext) }; case "union": { - const present = shape.parts.filter((p) => p.k !== "absent"); + const present = shape.parts.filter((p): boolean => p.k !== "absent"); const absents = shape.parts.length - present.length; if (absents > 0 && present.length === 1) { const inner = this.shapeRef(present[0]!, container, member, loc, synthesizedContext); @@ -1486,8 +1491,9 @@ export function buildSidecar(input: SidecarBuildInput): SidecarBuildResult { } const r = fn.returns; if (r !== null && r.k === "ref" && r.name === config.model) return false; - if (r !== null && r.k === "tuple" && r.elems.length === 2 && r.elems[0]!.k === "ref" && (r.elems[0] as { name: string }).name === config.model) { - return true; + if (r !== null && r.k === "tuple" && r.elems.length === 2) { + const first = r.elems[0]!; + if (first.k === "ref" && first.name === config.model) return true; } throw new SidecarError( `${which} export '${exportName}' must declare its return as '${config.model}' (bare state) or a two-element tuple '[${config.model}, ...]' (state plus an effect value)`, @@ -1523,7 +1529,7 @@ export function buildSidecar(input: SidecarBuildInput): SidecarBuildResult { const exports = abiExportSuffixes(profile); const exportSet = new Set(exports); const namedChannel = (constName: "appearanceMsg" | "chromeMsg"): string | null => { - const c = facts[constName]; + const c = constName === "appearanceMsg" ? facts.appearanceMsg : facts.chromeMsg; if (c === null) return null; const payload = armByName.get(c.value); if (payload === undefined) { diff --git a/packages/compiler/src/native/arguments.test.ts b/packages/compiler/src/native/arguments.test.ts deleted file mode 100644 index e9aa2846..00000000 --- a/packages/compiler/src/native/arguments.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { expect, test } from "vitest"; -import { parseNativeArguments } from "./arguments.js"; - -test("native build arguments preserve paths and select explicit output modes", () => { - expect(parseNativeArguments([ - "build", "source with spaces.ts", "-o", "result with spaces", "--backend=llvm", "--emit=obj", - "--toolchain", "/installed/compiler.json", "--dev", "--strip", "--keep-llvm", "--ffi=bindings.json", "--npm-static=one,@scope/two", - ], "default.json")).toEqual({ - help: false, toolchainPath: "/installed/compiler.json", - build: { - entryPath: "source with spaces.ts", outputPath: "result with spaces", backend: "llvm", outputKind: "obj", - optimization: "dev", strip: true, keepLlvm: true, ffiProfilePath: "bindings.json", npmStatic: ["one", "@scope/two"], - }, - }); - expect(parseNativeArguments(["--out=result", "--npm-static", "auto", "--", "-entry.ts"], "default.json")).toMatchObject({ - toolchainPath: "default.json", build: { entryPath: "-entry.ts", outputKind: "exe", backend: "llvm", npmStatic: "auto" }, - }); - expect(parseNativeArguments(["--help"], "default.json").help).toBe(true); -}); - -test("native build rejects unknown, missing and ambiguous arguments", () => { - for (const [args, message] of [ - [[], "entry source"], - [["main.ts"], "output path"], - [["main.ts", "-o"], "requires a value"], - [["main.ts", "-o", "--dev"], "requires a value"], - [["main.ts", "-o=x", "extra.ts"], "unexpected argument"], - [["main.ts", "-o=x", "--dynamic"], "unknown native compiler option"], - [["main.ts", "-o=x", "--backend=other"], "LLVM is the only backend"], - [["main.ts", "-o=x", "--backend=c"], "LLVM is the only backend"], - [["main.ts", "-o=x", "--emit=c"], "--emit must"], - [["main.ts", "-o=x", "--emit=other"], "--emit must"], - ] as const) expect(() => parseNativeArguments(args, "toolchain.json")).toThrow(message); -}); diff --git a/packages/compiler/src/native/arguments.ts b/packages/compiler/src/native/arguments.ts deleted file mode 100644 index d2455e6a..00000000 --- a/packages/compiler/src/native/arguments.ts +++ /dev/null @@ -1,79 +0,0 @@ -import type { NativeBuildOptions } from "./driver.js"; - -export interface NativeArguments { - help: boolean; - toolchainPath: string; - build: NativeBuildOptions; -} - -export function parseNativeArguments(args: readonly string[], defaultToolchain: string): NativeArguments { - let toolchainPath = defaultToolchain; - let entryPath = ""; - let outputPath = ""; - let backend = "llvm" as const; - let outputKind: "exe" | "obj" | "llvm" = "exe"; - let optimization: "release" | "dev" = "release"; - let strip = false; - let keepLlvm = false; - let help = false; - let ffiProfilePath: string | undefined; - let npmStatic: string[] | "auto" | undefined; - let positional = false; - for (let index = 0; index < args.length; index++) { - const arg = args[index]!; - if (!positional && (arg === "--help" || arg === "-h")) { help = true; continue; } - if (!positional && arg === "--") { positional = true; continue; } - if (!positional && arg === "--dev") { optimization = "dev"; continue; } - if (!positional && arg === "--strip") { strip = true; continue; } - if (!positional && arg === "--keep-llvm") { keepLlvm = true; continue; } - if (!positional && arg.startsWith("-")) { - const equals = arg.indexOf("="); - const name = equals < 0 ? arg : arg.slice(0, equals); - if (!["-o", "--out", "--toolchain", "--backend", "--emit", "--ffi", "--npm-static"].includes(name)) { - throw new Error(`unknown native compiler option: ${name}`); - } - const value = equals < 0 ? args[++index] : arg.slice(equals + 1); - if (value === undefined || value === "" || (equals < 0 && value.startsWith("--"))) throw new Error(`${name} requires a value`); - if (name === "-o" || name === "--out") outputPath = value; - else if (name === "--toolchain") toolchainPath = value; - else if (name === "--ffi") ffiProfilePath = value; - else if (name === "--npm-static") npmStatic = value === "auto" ? "auto" : value.split(","); - else if (name === "--backend") { - if (value !== "llvm") throw new Error("LLVM is the only backend"); - backend = value; - } else if (name === "--emit") { - if (value !== "exe" && value !== "obj" && value !== "llvm") throw new Error("--emit must be exe, obj, or llvm"); - outputKind = value; - } - continue; - } - if (!positional && index === 0 && arg === "build") continue; - if (entryPath !== "") throw new Error(`unexpected argument: ${arg}`); - entryPath = arg; - } - if (!help && entryPath === "") throw new Error("a TypeScript entry source is required"); - if (!help && outputPath === "") throw new Error("an output path is required (-o )"); - return { - help, toolchainPath, - build: { - entryPath, outputPath, backend, outputKind, optimization, strip, - ...(keepLlvm ? { keepLlvm: true } : {}), - ...(ffiProfilePath === undefined ? {} : { ffiProfilePath }), - ...(npmStatic === undefined ? {} : { npmStatic }), - }, - }; -} - -export const NATIVE_HELP = `Usage: scriptc-native build -o [options] - - --backend Code generation backend (default: llvm) - --emit Output artifact (default: exe) - --dev Disable optimization and include debug information - --strip Strip executable symbols - --keep-llvm Keep the generated LLVM at .ll - --ffi Native FFI bindings and link inputs - --npm-static Compile comma-separated packages, or auto - --toolchain Native toolchain manifest (default: beside executable) - --help Show this help - -The native compiler builds static programs; unsupported statements are errors.`; diff --git a/packages/compiler/src/native/cache.test.ts b/packages/compiler/src/native/cache.test.ts new file mode 100644 index 00000000..bfe9513a --- /dev/null +++ b/packages/compiler/src/native/cache.test.ts @@ -0,0 +1,69 @@ +import { mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { contentDigest, NativeCache, openNativeCache } from "./cache.js"; + +const directories: string[] = []; +function directory(): string { + const path = mkdtempSync(join(tmpdir(), "scriptc-native-cache-test-")); + directories.push(path); + return path; +} +afterEach(() => { + vi.unstubAllEnvs(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +test("cache hits require an intact payload and digest", () => { + const cache = new NativeCache(directory()); + const key = contentDigest("program identity"); + const content = Buffer.from([0, 10, 128, 255]); + cache.write("object", key, content); + expect(cache.read("object", key)).toEqual(content); + const path = join(cache.root, "object", key); + writeFileSync(path, Buffer.from([0, 10, 128, 254])); + expect(cache.read("object", key)).toBeNull(); + cache.write("object", key, content); + rmSync(path + ".sha256"); + expect(cache.read("object", key)).toBeNull(); +}); + +test("a failed cache publication does not fail the build or replace unrelated files", () => { + const cache = new NativeCache(directory()); + const occupied = join(cache.root, "object"); + writeFileSync(occupied, "keep"); + expect(() => cache.write("object", contentDigest("key"), "payload")).not.toThrow(); + expect(readFileSync(occupied, "utf8")).toBe("keep"); +}); + +test("eviction removes oldest complete entries and leaves unrelated paths alone", () => { + const cache = new NativeCache(directory()); + const oldKey = contentDigest("old"); + const newKey = contentDigest("new"); + cache.write("frontend", oldKey, "a".repeat(700)); + cache.write("object", newKey, "b".repeat(700)); + utimesSync(join(cache.root, "frontend", oldKey), new Date(0), new Date(0)); + const unrelated = join(cache.root, "object", "unrelated"); + writeFileSync(unrelated, "keep"); + vi.stubEnv("SCRIPTC_CACHE_MAX_MB", String(1000 / (1024 * 1024))); + cache.prune(); + expect(cache.read("frontend", oldKey)).toBeNull(); + expect(cache.read("object", newKey)?.length).toBe(700); + expect(readFileSync(unrelated, "utf8")).toBe("keep"); +}); + +test("private-directory admission and cache disable options fail closed", () => { + vi.stubEnv("SCRIPTC_NO_CACHE", "0"); + vi.stubEnv("SCRIPTC_CACHE_DIR", directory()); + const inspect = vi.fn(() => false); + expect(openNativeCache(inspect)).toBeNull(); + expect(inspect).toHaveBeenCalledWith(process.env["SCRIPTC_CACHE_DIR"], false); + const rootOnly = vi.fn().mockReturnValueOnce(true).mockReturnValueOnce(false); + expect(openNativeCache(rootOnly)).toBeNull(); + expect(openNativeCache(() => true)).toBeInstanceOf(NativeCache); + vi.stubEnv("SCRIPTC_NO_CACHE", "1"); + const disabled = vi.fn(() => true); + expect(openNativeCache(disabled)).toBeNull(); + expect(disabled).not.toHaveBeenCalled(); +}); diff --git a/packages/compiler/src/native/cache.ts b/packages/compiler/src/native/cache.ts new file mode 100644 index 00000000..5452fd0d --- /dev/null +++ b/packages/compiler/src/native/cache.ts @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveBuildCacheRoot } from "../backend/cache-root.js"; + +export function contentDigest(value: string | Uint8Array): string { + const hash = createHash("sha256"); + if (typeof value === "string") hash.update(value); + else hash.update(value); + return hash.digest("hex"); +} + +/** A compiler cache contains private source and executable data. The host + * checks ownership and permissions before this class accepts a root. */ +export class NativeCache { + readonly root: string; + + constructor(root: string) { + this.root = join(root, "native-v1"); + mkdirSync(this.root, { recursive: true, mode: 0o700 }); + } + + read(family: string, key: string): Buffer | null { + try { + const path = join(this.root, family, key); + const bytes = readFileSync(path); + if (contentDigest(bytes) !== readFileSync(path + ".sha256", "utf8").trim()) return null; + return bytes; + } catch { return null; } + } + + write(family: string, key: string, bytes: string | Uint8Array): void { + let stage: string | null = null; + try { + const directory = join(this.root, family); + mkdirSync(directory, { recursive: true, mode: 0o700 }); + stage = mkdtempSync(join(directory, ".write-")); + writeFileSync(join(stage, "payload"), bytes); + writeFileSync(join(stage, "digest"), contentDigest(bytes) + "\n"); + const path = join(directory, key); + // Concurrent writers publish identical content under the same key; + // a reader between the two renames simply treats it as a miss. + renameSync(join(stage, "payload"), path); + renameSync(join(stage, "digest"), path + ".sha256"); + } catch { /* Cache failures never change a build's result. */ } + finally { + try { if (stage !== null) rmSync(stage, { recursive: true, force: true }); } catch { /* Cleanup is best effort too. */ } + } + } + + prune(): void { + try { + const configured = Number(process.env["SCRIPTC_CACHE_MAX_MB"] ?? "4096"); + const limit = (Number.isFinite(configured) && configured >= 0 ? configured : 4096) * 1024 * 1024; + const files: { path: string; size: number; time: number }[] = []; + let total = 0; + for (const family of ["frontend", "object", "sanitizer", "executable", "binary", "dsym"]) { + const directory = join(this.root, family); + let names: string[]; + try { names = readdirSync(directory); } catch { continue; } + for (const name of names) { + if (!/^[0-9a-f]{64}$/.test(name)) continue; + const path = join(directory, name); + const info = statSync(path); + if (!info.isFile()) continue; + files.push({ path, size: info.size, time: info.mtimeMs }); + total += info.size; + } + } + if (total <= limit) return; + files.sort((a, b) => a.time - b.time); + for (const file of files) { + if (total <= limit) break; + rmSync(file.path, { force: true }); + rmSync(file.path + ".sha256", { force: true }); + total -= file.size; + } + } catch { /* Eviction is best effort. */ } + } +} + +export function openNativeCache(isPrivate: (path: string, harden: boolean) => boolean): NativeCache | null { + const root = resolveBuildCacheRoot(); + if (root === null) return null; + try { + mkdirSync(root, { recursive: true, mode: 0o700 }); + if (!isPrivate(root, process.env["SCRIPTC_CACHE_DIR"] === undefined)) return null; + const cache = new NativeCache(root); + if (!isPrivate(cache.root, false)) return null; + return cache; + } catch { return null; } +} diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts new file mode 100644 index 00000000..185a88a6 --- /dev/null +++ b/packages/compiler/src/native/cli.ts @@ -0,0 +1,100 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runCli } from "../cli/command.js"; +import type { NativeCacheWarmProfile } from "../cli/host.js"; +import { selectNativeRuntimePack, stageNativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import { setDeclarationRoot } from "../frontend/dts-paths.js"; +import { FrontendServices } from "../frontend/services.js"; +import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; +import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; +import { setCompilerReleaseVersion } from "../library/sidecar.js"; +import { NativeCompiler } from "./compiler.js"; +import { openNativeCache } from "./cache.js"; +import { loadNativeToolchain } from "./toolchain.js"; +import { buildSanitizedRuntime } from "./sanitizer.js"; +import { runCompilerTask } from "./task.js"; + +declare function compilerNativePrivateDirectory(path: string, harden: boolean): boolean; + +async function main(): Promise { + const manifestPath = process.env["SCRIPTC_TOOLCHAIN"] ?? process.execPath + ".json"; + let compiler: NativeCompiler | null = null; + let cache: ReturnType = null; + const getCompiler = (): NativeCompiler => { + if (compiler !== null) return compiler; + const toolchain = loadNativeToolchain(manifestPath, process.env); + if (toolchain.declarationsRoot !== undefined) setDeclarationRoot(toolchain.declarationsRoot); + setCompilerReleaseVersion(toolchain.compilerVersion); + cache = openNativeCache((path, harden) => compilerNativePrivateDirectory(path, harden)); + compiler = new NativeCompiler(toolchain, cache); + return compiler; + }; + return runCli(process.argv.slice(2), { + version: () => loadNativeToolchain(manifestPath).compilerVersion, + sourceTargetPlatform: () => getCompiler().toolchain.target.platform, + analyze: (entry, options) => runCompilerTask(() => getCompiler().analyze(entry, options)), + compile: (entry, options) => runCompilerTask(() => getCompiler().compile(entry, options)), + compileLibrary: (options) => runCompilerTask(() => getCompiler().compileLibrary(options)), + resolveProvenanceSources: async (entry) => { + const toolchain = getCompiler().toolchain; + const services = new FrontendServices((options) => createNativeTs7Api({ ...options, executable: toolchain.ts7Executable })); + try { return await resolveProvenanceSourcesWithParser(entry, (path, source, kind) => services.parse(path, source, kind)); } + finally { services.close(); } + }, + warmNativeCaches: async (options) => { + const toolchain = getCompiler().toolchain; + if (cache === null) throw new Error("the native build cache is disabled or unavailable"); + if (!toolchain.target.supports.exe) throw new Error(`native cache warming requires an executable target; ${toolchain.target.name} supports library archives`); + const profiles: { profile: NativeCacheWarmProfile; elapsedMs: number }[] = []; + for (const profile of [...new Set(options.profiles ?? ["runtime", "tls", "dynamic"] as NativeCacheWarmProfile[])]) { + const start = performance.now(); + const stage = mkdtempSync(join(tmpdir(), "scriptc-warm-")); + try { + const pack = selectNativeRuntimePack(toolchain.runtimePackRoot, toolchain.target, toolchain.compilerVersion, { + dynamic: profile === "dynamic", fetch: profile === "tls", regex: false, + copying: false, textDecoderLegacy: false, fileHandle: false, netIsland: false, + zlib: false, assert: false, inspect: false, dynInvoke: false, dc: false, + dynAsync: false, events: false, emitter: false, symbol: false, bigint: false, + searchParams: false, qs: false, parseArgs: false, stream: false, net: false, + http: false, http2: false, dgram: false, watch: false, foreignFfi: false, + nodeTest: false, tls: false, tlsCa: false, + }, options.optimization ?? "release"); + if (options.sanitize) buildSanitizedRuntime(toolchain, pack, stage, "executable", cache); + else stageNativeRuntimeSelection(pack, stage); + } finally { rmSync(stage, { recursive: true, force: true }); } + profiles.push({ profile, elapsedMs: performance.now() - start }); + } + return { cacheRoot: cache.root, profiles }; + }, + run: async (binary) => { + const toolchain = getCompiler().toolchain; + let executable = binary; + const args: string[] = []; + if (toolchain.target.platform === "wasi") { + if (toolchain.wasiNodeRunner === undefined) throw new Error("the WASI runner is missing from this installation"); + executable = "node"; + args.push("--no-warnings", toolchain.wasiNodeRunner, binary); + } + const result = spawnSync(executable, args, { stdio: "inherit" }); + if (result.error) { + if (toolchain.target.platform === "wasi") { + throw new Error(`could not start the Node.js WASI host: ${result.error.message}; running WASI modules requires Node.js 24 or newer on PATH; use scriptc build to compile without Node`); + } + throw new Error(result.error.message); + } + if (result.signal) { + process.stderr.write(`scriptc: program killed by ${result.signal}\n`); + return 1; + } + return result.status ?? 0; + }, + }); +} + +try { process.exitCode = await main(); } +catch (error) { + process.stderr.write(`scriptc: ${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} diff --git a/packages/compiler/src/native/compiler.test.ts b/packages/compiler/src/native/compiler.test.ts new file mode 100644 index 00000000..86343262 --- /dev/null +++ b/packages/compiler/src/native/compiler.test.ts @@ -0,0 +1,173 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { afterEach, expect, test, vi } from "vitest"; +import { NativeCompiler } from "./compiler.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { runNativeFrontend } from "../frontend/pipeline-native.js"; +import { emitNativeObject } from "../backend/native-tools.js"; +import { MACOS_ARM64_TARGET } from "../backend/targets.js"; +import { VOID, type IrModule } from "../ir/ir.js"; +import { contentDigest, NativeCache } from "./cache.js"; +import { NativeExecutableCache, openNativeExecutableCache } from "./executable-cache.js"; + +vi.mock("../frontend/pipeline-native.js", () => ({ runNativeFrontend: vi.fn() })); +vi.mock("../backend/native-tools.js", async (original) => ({ + ...await original(), emitNativeObject: vi.fn(), +})); +vi.mock("node:child_process", async (original) => ({ + ...await original(), execFileSync: vi.fn(), spawnSync: vi.fn(), +})); +vi.mock("./executable-cache.js", async (original) => ({ + ...await original(), openNativeExecutableCache: vi.fn(), +})); +vi.mock("../backend/runtime-pack-native.js", () => ({ + selectNativeRuntimePack: () => ({}), + stageNativeRuntimeSelection: () => ({ runtimeObjects: [], archives: [], systemLibraries: [] }), +})); + +const directories: string[] = []; +afterEach(() => { + vi.resetAllMocks(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +test.each(["traced", "empty trace", "failed trace", "changed input", "disabled", "failed link", "missing output"] as const)( + "executable linking preserves the dependency trace contract: %s", (mode) => { + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-link-")); + directories.push(directory); + const sdk = join(directory, "sdk"); + mkdirSync(sdk); + const library = join(sdk, "libSystem.tbd"); + writeFileSync(library, "system library"); + const entry = join(directory, "entry.ts"); + const output = join(directory, "program"); + const llvmPath = join(directory, "entry.ll"); + writeFileSync(output, "previous executable"); + writeFileSync(llvmPath, "previous LLVM"); + const module: IrModule = { + irVersion: 13, sourceFile: entry, entry: "%main", + functions: [{ name: "%main", params: [], locals: [], body: [], returnType: VOID, loc: { file: entry, start: 0, end: 0 } }], + }; + vi.mocked(runNativeFrontend).mockReturnValue({ + preflight: [], entryText: () => "", entryExports: () => new Map(), + entryContract: () => { throw new Error("unexpected contract request"); }, + sourceTexts: () => new Map(), npmStatic: [], npmImportSites: new Map(), dispose: () => {}, + lower: () => ({ module, diagnostics: [], runtimeFences: [], stats: { + statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0, + } }), + }); + vi.mocked(emitNativeObject).mockImplementation((options) => writeFileSync(options.outputPath, "object")); + const cache = new NativeCache(join(directory, "cache")); + const key = contentDigest("executable link"); + const open = () => new NativeExecutableCache(cache, key, "clang", false, [library]); + vi.mocked(openNativeExecutableCache).mockImplementation(() => mode === "disabled" ? null : open()); + let tracedLinks = 0; + let ordinaryLinks = 0; + vi.mocked(spawnSync).mockImplementation((_command, args) => { + const dry = args!.includes("-###"); + if (!dry) { + expect(args).toContain("-Wl,-t"); + tracedLinks++; + if (mode === "changed input") writeFileSync(library, "replacement system library"); + if (mode !== "missing output") writeFileSync(args![args!.indexOf("-o") + 1]!, "traced executable"); + } + const failed = !dry && (mode === "failed trace" || mode === "failed link"); + return { pid: 1, status: failed ? 1 : 0, signal: null, output: [], + stdout: mode === "empty trace" ? "" : library + "\n", stderr: failed ? "trace failed" : "" }; + }); + vi.mocked(execFileSync).mockImplementation((_command, args) => { + ordinaryLinks++; + expect(args).not.toContain("-Wl,-t"); + expect(args).not.toContain("-###"); + if (mode === "failed link") throw new Error("link failed"); + writeFileSync(args![args!.indexOf("-o") + 1]!, "ordinary executable"); + return ""; + }); + const target = MACOS_ARM64_TARGET; + const compiler = new NativeCompiler({ + compilerVersion: "0.0.0", target, helper: target.helper, ts7Executable: "unused", + helperExecutable: "unused", helperPackageRoot: "unused", runtimePackRoot: "unused", + linker: "clang", linkerArgs: [], dsymutil: "unused", + }); + const options = { outDir: directory, outPath: output, backend: "llvm", optimization: "release", strip: true } as const; + const result = compiler.compile(entry, options); + expect(tracedLinks).toBe(mode === "disabled" ? 0 : 1); + expect(ordinaryLinks).toBe(mode === "traced" || mode === "missing output" ? 0 : 1); + if (mode === "failed link" || mode === "missing output") { + expect(result.ok).toBe(false); + expect(readFileSync(output, "utf8")).toBe("previous executable"); + expect(readFileSync(llvmPath, "utf8")).toBe("previous LLVM"); + } else { + expect(result).toMatchObject({ ok: true, binaryPath: output }); + expect(readFileSync(output, "utf8")).toBe(mode === "traced" ? "traced executable" : "ordinary executable"); + expect(readFileSync(llvmPath, "utf8")).toContain("define i32 @main"); + } + expect(open().restore(join(directory, "restored"))).toBe(mode === "traced"); + if (mode === "traced") { + // A second invocation must restore the actual traced executable without + // invoking either the optimizer or the linker again. + vi.mocked(emitNativeObject).mockClear(); + expect(compiler.compile(entry, options).ok).toBe(true); + expect(readFileSync(output, "utf8")).toBe("traced executable"); + expect(emitNativeObject).not.toHaveBeenCalled(); + expect(tracedLinks).toBe(1); + expect(ordinaryLinks).toBe(0); + } + expect(readdirSync(directory).filter((name) => name.startsWith(".scriptc-"))).toEqual([]); + }, +); + +test.each([false, true])("object generation releases frontend resources and preserves outputs on failure (%s)", (fail) => { + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-driver-")); + directories.push(directory); + const entry = join(directory, "entry.ts"); + const output = join(directory, "output.o"); + const llvmPath = output + ".ll"; + writeFileSync(output, "previous object"); + writeFileSync(llvmPath, "previous LLVM"); + const module: IrModule = { + irVersion: 13, sourceFile: entry, entry: "%main", + functions: [{ name: "%main", params: [], locals: [], body: [], returnType: VOID, loc: { file: entry, start: 0, end: 0 } }], + }; + const dispose = vi.fn(); + vi.mocked(runNativeFrontend).mockReturnValue({ + preflight: [], entryText: () => "", entryExports: () => new Map(), + entryContract: () => { throw new Error("unexpected contract request"); }, + sourceTexts: () => new Map(), npmStatic: [], npmImportSites: new Map(), dispose, + lower: () => ({ module, diagnostics: [], runtimeFences: [], stats: { + statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0, + } }), + }); + let emitted = ""; + vi.mocked(emitNativeObject).mockImplementation((options) => { + expect(dispose).toHaveBeenCalledTimes(1); + emitted = readFileSync(options.inputPath, "utf8"); + expect(emitted).toContain("define i32 @main"); + if (fail) throw new Error("codegen failed"); + writeFileSync(options.outputPath, "new object"); + }); + const target = MACOS_ARM64_TARGET; + const toolchain: NativeToolchain = { + compilerVersion: "0.0.0", target, helper: target.helper, ts7Executable: "unused", + helperExecutable: "unused", helperPackageRoot: "unused", runtimePackRoot: "unused", + linker: "unused", linkerArgs: [], dsymutil: "unused", + }; + const build = () => new NativeCompiler(toolchain).compile(entry, { + outDir: directory, outPath: output, backend: "llvm", outputKind: "obj", + optimization: "release", strip: true, + }); + if (fail) { + expect(build()).toMatchObject({ ok: false, diagnostics: [expect.objectContaining({ message: "codegen failed" })] }); + expect(readFileSync(output, "utf8")).toBe("previous object"); + expect(readFileSync(llvmPath, "utf8")).toBe("previous LLVM"); + } else { + expect(build()).toMatchObject({ ok: true, artifact: { kind: "obj", path: output } }); + expect(readFileSync(output, "utf8")).toBe("new object"); + expect(emitted).toContain("define i32 @main"); + expect(readFileSync(llvmPath, "utf8")).toBe("previous LLVM"); + } + expect(dispose).toHaveBeenCalledTimes(1); + expect(readdirSync(directory).sort()).toEqual(["output.o", "output.o.ll"]); +}); diff --git a/packages/compiler/src/native/compiler.ts b/packages/compiler/src/native/compiler.ts new file mode 100644 index 00000000..b4e65d3b --- /dev/null +++ b/packages/compiler/src/native/compiler.ts @@ -0,0 +1,306 @@ +import { compilationTiming } from "../timing.js"; +import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; +import type { AnalyzeOptions, AnalyzeResult, CompileFailure, CompileLibraryOptions, CompileLibraryResult, CompileRequestOptions, CompileRequestResult } from "../compile-types.js"; +import { LlvmUnsupportedError } from "../backend/llvm/emitter.js"; +import { executableLinkInputs } from "../backend/link-plan-core.js"; +import { formatNativeLinkInfo } from "../backend/native-link-info-core.js"; +import { emitNativeObject, requireNativeArtifact, runNativeTool, verifyNativeHelper } from "../backend/native-tools.js"; +import { selectNativeRuntimePack, stageNativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import { RuntimePackError } from "../backend/runtime-pack-core.js"; +import { NativeCodegenError } from "../backend/native-codegen-core.js"; +import { llvmRefusalDiag } from "../backend/target-diagnostics.js"; +import { nativeCodegenDiag } from "../diagnostics/diagnostic.js"; +import { loadFfiProfile, type FfiProfile } from "../ffi/ffi-manifest.js"; +import { analyzeWithFrontend } from "../frontend/analysis.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import { runNativeFrontend } from "../frontend/pipeline-native.js"; +import { loadLibraryProfile } from "../library/library-profile.js"; +import { libraryLocalizeSymbols, libraryWasmExports, libraryWasmRefusal } from "../library/prepare.js"; +import { clearFenceEvalCaches } from "../library/fence-eval.js"; +import { archiveNativeLibrary, localizeNativeLibrary, linkNativeWasmLibrary } from "./library.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { evaluateNativeComptime } from "./comptime.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import { splitLlvmProgram, splitLlvmLibraryProgram } from "../backend/llvm/split.js"; +import { prepareNativeExecutable, prepareNativeLibrary } from "./prepare.js"; +import { buildSanitizedRuntime, sanitizerDriver, sanitizerFlags } from "./sanitizer.js"; +import { openNativeExecutableCache } from "./executable-cache.js"; +import { nativeFileIdentity } from "./file-identity.js"; + +function failure(error: unknown, entry: string, sources: Map): CompileFailure { + return { ok: false, diagnostics: [nativeCodegenDiag("SC3004", error instanceof Error ? error.message : String(error), entry)], sourceTexts: sources }; +} + +/** Shared frontend semantics with native processes for code generation and linking. */ +export class NativeCompiler { + private readonly frontend: FrontendFactory; + + constructor(readonly toolchain: NativeToolchain, private readonly cache: NativeCache | null = null) { + const evaluator = toolchain.comptimeExecutable; + this.frontend = (entry, npmStatic, externalTypes, libraryNpmStatic) => runNativeFrontend(entry, toolchain.ts7Executable, npmStatic, externalTypes, + evaluator === undefined ? undefined : (source, timeout) => evaluateNativeComptime(source, timeout, toolchain.ts7Executable, evaluator), libraryNpmStatic); + } + + analyze(entry: string, options: AnalyzeOptions): AnalyzeResult { + return analyzeWithFrontend(resolve(entry), options, this.toolchain.target.platform, this.frontend); + } + + private emitObject(input: string, output: string, source: string, optimization: "release" | "dev", outputKind: "obj" | "asm"): void { + const toolchain = this.toolchain; + const helperOptions = { + executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, + compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, + }; + let key: string | null = null; + let identity: string | null = null; + if (this.cache !== null && process.env["SCRIPTC_LLVM_HELPER"] === undefined) { + try { + identity = nativeFileIdentity(toolchain.helperExecutable); + key = contentDigest(JSON.stringify({ schema: 1, llvm: contentDigest(readFileSync(input)), outputKind, optimization, source, + target: toolchain.target, helper: identity, + identity: readFileSync(join(toolchain.helperPackageRoot, "package.json"), "utf8"), version: toolchain.compilerVersion })); + const bytes = this.cache.read("object", key); + if (bytes !== null) { + verifyNativeHelper(helperOptions); + if (nativeFileIdentity(toolchain.helperExecutable) === identity) { + writeFileSync(output, bytes); + return; + } + key = null; + } + } catch { key = null; } + } + emitNativeObject({ + executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, + compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, + inputPath: input, outputPath: output, sourcePath: source, optimization, outputKind, + }); + if (this.cache !== null && key !== null) { + try { + if (nativeFileIdentity(toolchain.helperExecutable) === identity) this.cache.write("object", key, readFileSync(output)); + } catch { /* An updated helper or unavailable cache only prevents reuse. */ } + } + } + + private emitProgramObject(input: string, output: string, source: string, optimization: "release" | "dev", stage: string, library: boolean): void { + const llvm = optimization === "dev" && this.toolchain.target.platform !== "wasi" ? readFileSync(input, "utf8") : null; + const split = llvm === null ? null : library ? splitLlvmLibraryProgram(llvm) : splitLlvmProgram(llvm); + if (split === null) { this.emitObject(input, output, source, optimization, "obj"); return; } + const objects: string[] = []; + for (const [index, shard] of split.shards.entries()) { + const shardInput = join(stage, `shard-${index}.ll`); + const shardOutput = join(stage, `shard-${index}` + this.toolchain.target.outputSuffixes.obj); + writeFileSync(shardInput, shard.source); + this.emitObject(shardInput, shardOutput, source + "." + shard.name, optimization, "obj"); + objects.push(shardOutput); + } + const merged = localizeNativeLibrary(this.toolchain, stage, objects, [], split.publicSymbols); + renameSync(merged, output); + } + + compile(entry: string, options: CompileRequestOptions): CompileRequestResult { + const timing = compilationTiming(); + const toolchain = this.toolchain; + const target = toolchain.target; + const output = resolve(options.outPath); + const outputKind = options.outputKind ?? "exe"; + const optimization = options.optimization ?? "release"; + entry = resolve(entry); + let sourceTexts = new Map(); + let stage: string | null = null; + try { + if (entry === output) throw new Error("output path must differ from the entry source"); + if (outputKind === "exe" && !target.supports.exe) throw new NativeCodegenError("SC3002", `${target.name} supports library archives, not executables`); + if (options.sanitize && (outputKind === "asm" || outputKind === "obj")) { + throw new NativeCodegenError("SC3002", `--sanitize is not supported with --emit=${outputKind}; AddressSanitizer instrumentation parity is not available in the LLVM native helper yet`); + } + if (options.sanitize) sanitizerDriver(toolchain); + let ffi: FfiProfile | null = null; + if (options.ffiProfilePath !== undefined) { + const loaded = loadFfiProfile(resolve(options.ffiProfilePath)); + if (!loaded.ok) return { ok: false, diagnostics: loaded.diagnostics, sourceTexts }; + ffi = loaded.profile; + } + const prepared = prepareNativeExecutable(entry, options, ffi, toolchain, this.frontend, this.cache, timing); + if (!prepared.ok) return prepared; + sourceTexts = new Map(prepared.sources); + mkdirSync(dirname(output), { recursive: true }); + stage = mkdtempSync(join(dirname(output), ".scriptc-native-")); + const stagedOutput = join(stage, basename(output)); + const stem = basename(entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const llvmPath = join(options.outDir, `${stem}.ll`); + if (outputKind === "ir") { + writeFileSync(stagedOutput, prepared.ir!); + renameSync(stagedOutput, output); + return { ok: true, artifact: { kind: "ir", path: output } }; + } + const { llvm, features } = prepared; + const inputDirectory = join(stage, "input"); + mkdirSync(inputDirectory); + const llvmInput = outputKind === "llvm" ? stagedOutput : join(inputDirectory, "program.ll"); + writeFileSync(llvmInput, llvm); + timing("llvm-write"); + if (outputKind === "llvm") { + renameSync(stagedOutput, output); + return { ok: true, artifact: { kind: "llvm", path: output } }; + } + const object = outputKind === "exe" ? join(inputDirectory, "program" + target.outputSuffixes.obj) : stagedOutput; + const executablePack = outputKind === "exe" + ? selectNativeRuntimePack(toolchain.runtimePackRoot, target, toolchain.compilerVersion, features, optimization) : null; + const executableCache = executablePack === null ? null + : openNativeExecutableCache(this.cache, toolchain, llvm, options, ffi, executablePack); + const restored = executableCache?.restore(stagedOutput) ?? false; + timing(restored ? "executable-cache-hit" : "executable-cache-miss"); + if (!restored) { + if (options.sanitize) { + runNativeTool(sanitizerDriver(toolchain), [...sanitizerFlags(toolchain, optimization), "-c", llvmInput, "-o", object]); + requireNativeArtifact(object); + } else if (outputKind === "exe") this.emitProgramObject(llvmInput, object, entry, optimization, stage, false); + else this.emitObject(llvmInput, object, entry, optimization, outputKind === "asm" ? "asm" : "obj"); + timing("native-object"); + if (outputKind === "asm" || outputKind === "obj") { + const pack = outputKind === "obj" && options.nativeLinkInfo + ? selectNativeRuntimePack(toolchain.runtimePackRoot, target, toolchain.compilerVersion, features, optimization) : null; + if (pack !== null) stageNativeRuntimeSelection(pack, join(stage, "runtime")); + const info = pack === null ? undefined : formatNativeLinkInfo({ programObject: output, target, ffi }, toolchain.compilerVersion, pack); + renameSync(stagedOutput, output); + return { ok: true, artifact: { kind: outputKind, path: output, ...(info === undefined ? {} : { nativeLinkInfo: info }) } }; + } + const pack = executablePack!; + const runtime = options.sanitize + ? buildSanitizedRuntime(toolchain, pack, join(stage, "runtime"), "executable", this.cache) + : stageNativeRuntimeSelection(pack, join(stage, "runtime")); + timing("runtime-stage"); + const plan = executableLinkInputs({ + target, programObject: object, ffiLibraries: ffi?.libraries ?? [], ffiSystemLibraries: ffi?.systemLibraries ?? [], + ffiFrameworks: ffi?.frameworks ?? [], runtimeObjects: runtime.runtimeObjects, runtimeArchives: runtime.archives, + runtimeSystemLibraries: runtime.systemLibraries, optimization, strip: options.strip ?? false, + ...(options.windowsSubsystem === undefined ? {} : { windowsSubsystem: options.windowsSubsystem }), + }); + const linkArgs = [...(options.sanitize ? ["-fsanitize=address"] : toolchain.linkerArgs), ...plan.driverFlags, ...plan.inputs, + ...plan.systemLibraries.map((name) => `-l${name}`), "-o", stagedOutput]; + const cacheable = executableCache?.trace(linkArgs, stage) ?? false; + timing("link-inputs"); + // A successful dependency trace already produced this executable. + // Keep its verified output instead of asking the linker to repeat it. + if (!cacheable) runNativeTool(options.sanitize ? sanitizerDriver(toolchain) : toolchain.linker, linkArgs); + timing("native-link"); + requireNativeArtifact(stagedOutput); + if (target.platform === "darwin" && optimization === "dev" && !options.strip) { + runNativeTool(toolchain.dsymutil, [stagedOutput, "-o", stagedOutput + ".dSYM"]); + } + if (cacheable) executableCache?.publish(stagedOutput); + } + if (target.platform === "darwin" && optimization === "dev" && !options.strip) { + rmSync(output + ".dSYM", { recursive: true, force: true }); + renameSync(stagedOutput + ".dSYM", output + ".dSYM"); + } + mkdirSync(options.outDir, { recursive: true }); + writeFileSync(llvmPath, llvm); + let irPath: string | undefined; + if (options.emitIr) { + irPath = join(options.outDir, `${stem}.ir.json`); + writeFileSync(irPath, prepared.ir!); + } + renameSync(stagedOutput, output); + if (target.platform === "darwin" && (optimization !== "dev" || options.strip)) rmSync(output + ".dSYM", { recursive: true, force: true }); + return { ok: true, artifact: { kind: "exe", path: output, translationUnitPath: llvmPath, backend: "llvm" }, + binaryPath: output, llvmPath, backend: "llvm", ...(irPath === undefined ? {} : { irPath }) }; + } catch (error) { + // Keep subclass reads at the catch boundary, where the thrown object + // retains its class identity and source location. + if (error instanceof LlvmUnsupportedError) return { ok: false, diagnostics: [llvmRefusalDiag(error, entry)], sourceTexts }; + if (error instanceof NativeCodegenError) return { ok: false, diagnostics: [nativeCodegenDiag(error.diagnosticCode, error.message, entry)], sourceTexts }; + if (error instanceof RuntimePackError) return { ok: false, diagnostics: [nativeCodegenDiag(error.code === "unsupported" ? "SC3002" : "SC3003", error.message, entry)], sourceTexts }; + return failure(error, entry, sourceTexts); + } + finally { + if (stage !== null) rmSync(stage, { recursive: true, force: true }); + this.cache?.prune(); + timing("complete"); + } + } + + compileLibrary(options: CompileLibraryOptions): CompileLibraryResult { + const timing = compilationTiming(); + clearFenceEvalCaches(); + const toolchain = this.toolchain; + let sourceTexts = new Map(); + let stage: string | null = null; + try { + const loaded = loadLibraryProfile(resolve(options.profilePath)); + if (!loaded.ok) return { ok: false, diagnostics: loaded.diagnostics, sourceTexts }; + const profile = loaded.profile; + const wasm = toolchain.target.platform === "wasi"; + if (wasm) { + const refusal = libraryWasmRefusal(profile, options.sanitize ?? false); + if (refusal !== null) return { ok: false, diagnostics: [refusal], sourceTexts }; + } + if (options.sanitize) sanitizerDriver(toolchain); + const stem = basename(profile.entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const archivePath = resolve(options.outPath ?? join(options.outDir, `${stem}${wasm ? ".wasm" : ".lib.a"}`)); + const llvmPath = join(options.outDir, `${stem}.lib.ll`); + const prepared = prepareNativeLibrary(profile, options, archivePath, toolchain, this.frontend, this.cache, timing); + if (!prepared.ok) return prepared; + sourceTexts = new Map(prepared.sources); + if (archivePath === profile.entry) throw new Error("output path must differ from the entry source"); + mkdirSync(dirname(archivePath), { recursive: true }); + mkdirSync(options.outDir, { recursive: true }); + stage = mkdtempSync(join(dirname(archivePath), ".scriptc-library-")); + const llvm = prepared.llvm; + const llvmInput = join(stage, "program.ll"); + writeFileSync(llvmInput, llvm); + timing("llvm-write"); + const features = prepared.features; + const pack = selectNativeRuntimePack(toolchain.runtimePackRoot, toolchain.target, toolchain.compilerVersion, + features, profile.optimization, profile.instancePerThread ? "library-thread" : "library"); + const runtime = options.sanitize + ? buildSanitizedRuntime(toolchain, pack, join(stage, "runtime"), profile.instancePerThread ? "library-thread" : "library", this.cache) + : stageNativeRuntimeSelection(pack, join(stage, "runtime")); + timing("runtime-stage"); + const stagedArchive = join(stage, wasm ? "output.wasm" : "output.a"); + const localizeSymbols = libraryLocalizeSymbols(profile); + const program = join(stage, "program" + toolchain.target.outputSuffixes.obj); + if (options.sanitize) { + runNativeTool(sanitizerDriver(toolchain), [...sanitizerFlags(toolchain, profile.optimization), "-c", llvmInput, "-o", program]); + requireNativeArtifact(program); + } else this.emitProgramObject(llvmInput, program, profile.entry, profile.optimization, stage, true); + timing("native-object"); + if (wasm) { + linkNativeWasmLibrary({ toolchain, programObject: program, outputPath: stagedArchive, + runtime, optimization: profile.optimization, exports: libraryWasmExports(profile) }); + } else { + archiveNativeLibrary({ toolchain, programObject: program, outputPath: stagedArchive, + stage, runtime, ...(localizeSymbols === undefined ? {} : { localizeSymbols }) }); + } + timing("native-archive"); + writeFileSync(llvmPath, llvm); + let irPath: string | undefined; + if (options.emitIr) { + irPath = join(options.outDir, `${stem}.lib.ir.json`); + writeFileSync(irPath, prepared.ir!); + } + let sidecarPath: string | undefined; + if (prepared.sidecarJson !== null) { + sidecarPath = profile.sidecar!.path === null ? `${archivePath}.contract.json` : resolve(dirname(archivePath), profile.sidecar!.path); + writeFileSync(sidecarPath, prepared.sidecarJson); + } + renameSync(stagedArchive, archivePath); + return { ok: true, archivePath, llvmPath, backend: "llvm", + ...(irPath === undefined ? {} : { irPath }), ...(sidecarPath === undefined ? {} : { sidecarPath }) }; + } catch (error) { + // Keep subclass reads at the catch boundary, where the thrown object + // retains its class identity and source location. + if (error instanceof LlvmUnsupportedError) return { ok: false, diagnostics: [llvmRefusalDiag(error, options.profilePath)], sourceTexts }; + if (error instanceof NativeCodegenError) return { ok: false, diagnostics: [nativeCodegenDiag(error.diagnosticCode, error.message, options.profilePath)], sourceTexts }; + if (error instanceof RuntimePackError) return { ok: false, diagnostics: [nativeCodegenDiag(error.code === "unsupported" ? "SC3002" : "SC3003", error.message, options.profilePath)], sourceTexts }; + return failure(error, options.profilePath, sourceTexts); + } + finally { + if (stage !== null) rmSync(stage, { recursive: true, force: true }); + this.cache?.prune(); + timing("complete"); + } + } +} diff --git a/packages/compiler/src/native/comptime.test.ts b/packages/compiler/src/native/comptime.test.ts new file mode 100644 index 00000000..f3860b09 --- /dev/null +++ b/packages/compiler/src/native/comptime.test.ts @@ -0,0 +1,68 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, expect, test } from "vitest"; +import { nativeCodegenTarget } from "../backend/targets.js"; +import { ts7Executable } from "../frontend/ts7/rpc-api.js"; +import { evaluateNativeComptime } from "./comptime.js"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const target = nativeCodegenTarget({}); +const stage = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-comptime-test-")); +const evaluator = join(stage, "comptime" + (process.platform === "win32" ? ".exe" : "")); + +beforeAll(() => { + if (target === null) throw new Error("a supported native host is required"); + const runtime = join(root, target.runtimePackPackage.replace("@scriptc/", "")); + execFileSync(target.defaultLinker, [...target.defaultLinkerArgs, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", + "-I", join(root, "runtime/vendor/quickjs-ng"), join(root, "compiler/native/comptime.c"), + join(runtime, "artifacts/vendor/quickjs/libscriptc-quickjs.a"), "-lm", "-lpthread", "-o", evaluator]); +}); +afterAll(() => { rmSync(stage, { recursive: true, force: true }); }); + +function evaluate(source: string, timeout = 2000): unknown { + return evaluateNativeComptime(source, timeout, ts7Executable(), evaluator); +} + +test("native comptime erases TypeScript and returns the actual structured value", () => { + expect(evaluate(`() => { + interface Pair { value: number; text: string } + const make = (value: T): T => value; + const items: Pair[] = []; + for (let i = 0; i < 4; i++) items.push(make({ value: i * i, text: String(i) })); + return { items, zero: -0, text: "hello \\ud83c\\udf0d", lone: "\\ud800", empty: undefined }; + }`)).toEqual({ items: [0, 1, 4, 9].map((value, index) => ({ value, text: String(index) })), zero: -0, + text: "hello 🌍", lone: "\ud800", empty: undefined }); +}); + +test("native comptime retains non-finite values for shared result validation", () => { + const values = evaluate("() => [NaN, Infinity, -Infinity, -0, undefined, null, 123n]") as unknown[]; + expect(values).toEqual([NaN, Infinity, -Infinity, -0, undefined, null, 123n]); +}); + +test("callback changes to globals and serialization hooks cannot replace its result", () => { + expect(evaluate(`() => { + JSON.stringify = () => '"spoofed"'; + Array.prototype.toJSON = () => "spoofed"; + Object.prototype.toJSON = () => "spoofed"; + return { values: [1, 2], nested: { ok: true } }; + }`)).toEqual({ values: [1, 2], nested: { ok: true } }); +}); + +test("native comptime isolates callbacks and exposes no host bindings", () => { + expect(evaluate("() => { globalThis.saved = 1; return 2; }")).toBe(2); + expect(evaluate("() => [typeof saved, typeof process, typeof require, typeof console, typeof fetch]")).toEqual( + ["undefined", "undefined", "undefined", "undefined", "undefined"], + ); +}); + +test("throws, cycles, and runaway callbacks report bounded failures", () => { + expect(() => evaluate('() => { throw new Error("user failure") }')).toThrow("user failure"); + expect(() => evaluate("() => { const result = {}; result.self = result; return result; }")).toThrow("cyclic compile-time result"); + let failure: unknown; + try { evaluate("() => { while (true) {} }", 20); } catch (error) { failure = error; } + expect(failure).toMatchObject({ code: "ERR_SCRIPT_EXECUTION_TIMEOUT" }); + expect(evaluate("() => 42")).toBe(42); +}); diff --git a/packages/compiler/src/native/comptime.ts b/packages/compiler/src/native/comptime.ts new file mode 100644 index 00000000..10938904 --- /dev/null +++ b/packages/compiler/src/native/comptime.ts @@ -0,0 +1,64 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runNativeTool } from "../backend/native-tools.js"; + +/** Tagged values preserve -0, non-finite numbers, and missing properties so + * shared result validation sees the value the callback actually returned. */ +export function decodeComptimeValue(wire: unknown): unknown { + if (!Array.isArray(wire) || typeof wire[0] !== "string") throw new Error("invalid compile-time result"); + const value = wire as unknown[]; + switch (value[0]) { + case "error": { + if (value[1] === "ERR_SCRIPT_EXECUTION_TIMEOUT") throw { code: "ERR_SCRIPT_EXECUTION_TIMEOUT", message: String(value[2]) }; + throw new Error(String(value[2])); + } + case "undefined": return undefined; + case "null": return null; + case "boolean": + if (typeof value[1] === "boolean") return value[1]; + break; + case "string": + if (typeof value[1] === "string") return value[1]; + break; + case "number": + if (typeof value[1] === "string") return value[1] === "-0" ? -0 : Number(value[1]); + break; + case "bigint": return BigInt(String(value[1])); + case "function": return () => {}; + case "symbol": throw new Error("compile-time result contains a symbol"); + case "array": + if (Array.isArray(value[1])) return value[1].map((item) => decodeComptimeValue(item)); + break; + case "object": { + if (!Array.isArray(value[1])) break; + const result: Record = Object.create(null) as Record; + for (const item of value[1]) { + if (!Array.isArray(item) || typeof item[0] !== "string") throw new Error("invalid compile-time property"); + result[item[0]] = decodeComptimeValue(item[1]); + } + return result; + } + } + throw new Error("invalid compile-time result"); +} + +export function comptimeScript(source: string): string { + return `(${source})();\n`; +} + +export function evaluateNativeComptime(source: string, timeoutMs: number, ts7: string, evaluator: string): unknown { + const stage = mkdtempSync(join(tmpdir(), "scriptc-comptime-")); + try { + const sourcePath = join(stage, "eval.ts"); + const output = join(stage, "output"); + mkdirSync(output); + writeFileSync(sourcePath, comptimeScript(source)); + // Type erasure remains TypeScript's job, including nested annotations, + // type assertions, generic functions, and enums in closed callbacks. + runNativeTool(ts7, ["--ignoreConfig", "--noCheck", "--target", "esnext", "--module", "esnext", + "--moduleDetection", "legacy", "--outDir", output, sourcePath]); + const result = runNativeTool(evaluator, [join(output, "eval.js"), String(timeoutMs)]); + return decodeComptimeValue(JSON.parse(result)); + } finally { rmSync(stage, { recursive: true, force: true }); } +} diff --git a/packages/compiler/src/native/driver.test.ts b/packages/compiler/src/native/driver.test.ts deleted file mode 100644 index b2cf39b9..00000000 --- a/packages/compiler/src/native/driver.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import { afterEach, expect, test, vi } from "vitest"; -import { buildNative, type NativeToolchain } from "./driver.js"; -import { runNativeFrontend } from "../frontend/pipeline-native.js"; -import { emitNativeObject } from "../backend/native-tools.js"; -import { MACOS_ARM64_TARGET } from "../backend/targets.js"; -import { VOID, type IrModule } from "../ir/ir.js"; - -vi.mock("../frontend/pipeline-native.js", () => ({ runNativeFrontend: vi.fn() })); -vi.mock("../backend/native-tools.js", async (original) => ({ - ...await original(), emitNativeObject: vi.fn(), -})); - -const directories: string[] = []; -afterEach(() => { - vi.resetAllMocks(); - for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); -}); - -test.each([false, true])("object generation releases frontend resources and preserves outputs on failure (%s)", (fail) => { - const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-driver-")); - directories.push(directory); - const entry = join(directory, "entry.ts"); - const output = join(directory, "output.o"); - const llvmPath = output + ".ll"; - writeFileSync(output, "previous object"); - writeFileSync(llvmPath, "previous LLVM"); - const module: IrModule = { - irVersion: 13, sourceFile: entry, entry: "%main", - functions: [{ name: "%main", params: [], locals: [], body: [], returnType: VOID, loc: { file: entry, start: 0, end: 0 } }], - }; - const dispose = vi.fn(); - vi.mocked(runNativeFrontend).mockReturnValue({ - preflight: [], entryText: () => "", entryExports: () => new Map(), - entryContract: () => { throw new Error("unexpected contract request"); }, - sourceTexts: () => new Map(), npmStatic: [], npmImportSites: new Map(), dispose, - lower: () => ({ module, diagnostics: [], runtimeFences: [], stats: { - statementsTotal: 0, statementsFailed: 0, statementsIsland: 0, functionsSkipped: 0, - } }), - }); - let emitted = ""; - vi.mocked(emitNativeObject).mockImplementation((options) => { - expect(dispose).toHaveBeenCalledTimes(1); - emitted = readFileSync(options.inputPath, "utf8"); - expect(emitted).toContain("define i32 @main"); - if (fail) throw new Error("codegen failed"); - writeFileSync(options.outputPath, "new object"); - }); - const target = MACOS_ARM64_TARGET; - const toolchain: NativeToolchain = { - compilerVersion: "0.0.0", target, helper: target.helper, ts7Executable: "unused", - helperExecutable: "unused", helperPackageRoot: "unused", runtimePackRoot: "unused", - linker: "unused", linkerArgs: [], dsymutil: "unused", - }; - const build = () => buildNative({ - entryPath: entry, outputPath: output, backend: "llvm", outputKind: "obj", - optimization: "release", strip: true, keepLlvm: true, - }, toolchain); - if (fail) { - expect(build).toThrow("codegen failed"); - expect(readFileSync(output, "utf8")).toBe("previous object"); - expect(readFileSync(llvmPath, "utf8")).toBe("previous LLVM"); - } else { - expect(build()).toMatchObject({ outputPath: output, llvmPath }); - expect(readFileSync(output, "utf8")).toBe("new object"); - expect(readFileSync(llvmPath, "utf8")).toBe(emitted); - } - expect(dispose).toHaveBeenCalledTimes(1); - expect(readdirSync(directory).sort()).toEqual(["output.o", "output.o.ll"]); -}); diff --git a/packages/compiler/src/native/driver.ts b/packages/compiler/src/native/driver.ts deleted file mode 100644 index 31094f23..00000000 --- a/packages/compiler/src/native/driver.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** Static executable compiler. The TS7 parser/checker and native toolchain - * are external native processes; lowering, validation and LLVM emission run - * inside this binary. Installed paths are supplied by the distribution. */ -import { mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; -import { basename, dirname, join, resolve } from "node:path"; -import { emitLlvmModule } from "../backend/llvm/emitter.js"; -import { executableLinkFeatures } from "../backend/executable-features.js"; -import { executableLinkInputs } from "../backend/link-plan-core.js"; -import type { NativeLinkFeatures } from "../backend/native-link-info.js"; -import { stageNativeRuntimePack } from "../backend/runtime-pack-native.js"; -import { emitNativeObject, requireNativeArtifact, runNativeTool } from "../backend/native-tools.js"; -import type { NativeHelperSpec, NativeTargetSpec } from "../backend/targets.js"; -import { loadFfiProfile, type FfiProfile } from "../ffi/ffi-manifest.js"; -import { runNativeFrontend } from "../frontend/pipeline-native.js"; -import type { LowerStats } from "../frontend/lowering/lowerer.js"; -import { validateModule } from "../ir/validate.js"; - -export interface NativeToolchain { - compilerVersion: string; - ts7Executable: string; - target: NativeTargetSpec; - helper: NativeHelperSpec; - helperExecutable: string; - helperPackageRoot: string; - runtimePackRoot: string; - linker: string; - linkerArgs: string[]; - dsymutil: string; -} - -export interface NativeBuildOptions { - entryPath: string; - outputPath: string; - backend: "llvm"; - outputKind: "exe" | "obj" | "llvm"; - optimization: "release" | "dev"; - strip: boolean; - keepLlvm?: boolean; - ffiProfilePath?: string; - npmStatic?: readonly string[] | "auto"; -} - -export interface NativeBuildResult { - outputPath: string; - llvmPath?: string; - stats: LowerStats; -} - -/** Keep the program graph and its TS7 session inside the frontend phase. - * Native object generation can need several GB of its own working memory. */ -function emitNativeInput( - options: NativeBuildOptions, toolchain: NativeToolchain, - entry: string, ffi: FfiProfile | null, path: string, -): { features: NativeLinkFeatures; stats: LowerStats } { - const frontend = runNativeFrontend(entry, toolchain.ts7Executable, options.npmStatic); - try { - if (frontend.preflight.length !== 0) throw new Error(JSON.stringify(frontend.preflight)); - const lowered = frontend.lower({ dynamic: false, targetPlatform: toolchain.target.platform, ffiImports: ffi?.functions ?? [] }); - if (lowered.module === null || lowered.stats.statementsFailed !== 0 || lowered.stats.statementsIsland !== 0 || lowered.stats.functionsSkipped !== 0) { - throw new Error(JSON.stringify(lowered.diagnostics)); - } - const module = lowered.module; - const errors = validateModule(module); - if (errors.length !== 0) throw new Error(JSON.stringify(errors)); - const features = executableLinkFeatures(module, false); - const debug = options.optimization === "dev" && !options.strip; - const debugSources = debug ? frontend.sourceTexts() : new Map(); - const llvm = emitLlvmModule(module, { - targetTriple: toolchain.target.llvmTriple, - pointerBits: toolchain.target.pointerBits, - wasi: toolchain.target.platform === "wasi", - runtimeAbiMarker: true, - ...(debug ? { debugSources } : {}), - }); - writeFileSync(path, llvm); - return { features, stats: lowered.stats }; - } finally { frontend.dispose(); } -} - -export function buildNative(options: NativeBuildOptions, toolchain: NativeToolchain): NativeBuildResult { - const entry = resolve(options.entryPath); - const output = resolve(options.outputPath); - if (entry === output) throw new Error("output path must differ from the entry source"); - let ffi: FfiProfile | null = null; - if (options.ffiProfilePath !== undefined) { - const loaded = loadFfiProfile(options.ffiProfilePath); - if (!loaded.ok) throw new Error(JSON.stringify(loaded.diagnostics)); - ffi = loaded.profile; - } - mkdirSync(dirname(output), { recursive: true }); - // A sibling temporary directory keeps installation on the same volume - // and preserves the basename used by Mach-O's ad-hoc signature. - const stage = mkdtempSync(join(dirname(output), ".scriptc-native-")); - try { - const outputDirectory = join(stage, "output"); - const inputDirectory = join(stage, "input"); - mkdirSync(outputDirectory); - mkdirSync(inputDirectory); - const stagedOutput = join(outputDirectory, basename(output)); - const sourceOutput = options.outputKind === "llvm"; - const input = sourceOutput ? stagedOutput : join(inputDirectory, "program.ll"); - const prepared = emitNativeInput(options, toolchain, entry, ffi, input); - if (!sourceOutput) { - const object = options.outputKind === "obj" ? stagedOutput : join(inputDirectory, "program" + toolchain.target.outputSuffixes.obj); - emitNativeObject({ - executable: toolchain.helperExecutable, packageRoot: toolchain.helperPackageRoot, - compilerVersion: toolchain.compilerVersion, target: toolchain.target, helper: toolchain.helper, - inputPath: input, outputPath: object, sourcePath: entry, optimization: options.optimization, - }); - if (options.outputKind === "exe") { - const runtime = stageNativeRuntimePack(toolchain.runtimePackRoot, join(stage, "runtime"), - toolchain.target, toolchain.compilerVersion, prepared.features, options.optimization); - const plan = executableLinkInputs({ - target: toolchain.target, programObject: object, - ffiLibraries: ffi?.libraries ?? [], ffiSystemLibraries: ffi?.systemLibraries ?? [], - ffiFrameworks: ffi?.frameworks ?? [], - runtimeObjects: runtime.runtimeObjects, runtimeArchives: runtime.archives, - runtimeSystemLibraries: runtime.systemLibraries, optimization: options.optimization, strip: options.strip, - }); - runNativeTool(toolchain.linker, [ - ...toolchain.linkerArgs, ...plan.driverFlags, ...plan.inputs, - ...plan.systemLibraries.map((name) => `-l${name}`), "-o", stagedOutput, - ]); - requireNativeArtifact(stagedOutput); - if (toolchain.target.platform === "darwin" && options.optimization === "dev" && !options.strip) { - runNativeTool(toolchain.dsymutil, [stagedOutput, "-o", stagedOutput + ".dSYM"]); - rmSync(output + ".dSYM", { recursive: true, force: true }); - renameSync(stagedOutput + ".dSYM", output + ".dSYM"); - } - } - } - if (options.keepLlvm && !sourceOutput) renameSync(join(inputDirectory, "program.ll"), output + ".ll"); - renameSync(stagedOutput, output); - if (options.outputKind === "exe" && toolchain.target.platform === "darwin" && (options.optimization !== "dev" || options.strip)) { - rmSync(output + ".dSYM", { recursive: true, force: true }); - } - return { - outputPath: output, stats: prepared.stats, - ...(options.keepLlvm && !sourceOutput ? { llvmPath: output + ".ll" } : {}), - }; - } finally { rmSync(stage, { recursive: true, force: true }); } -} diff --git a/packages/compiler/src/native/executable-cache.test.ts b/packages/compiler/src/native/executable-cache.test.ts new file mode 100644 index 00000000..3ee551cf --- /dev/null +++ b/packages/compiler/src/native/executable-cache.test.ts @@ -0,0 +1,159 @@ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { contentDigest, NativeCache } from "./cache.js"; +import { NativeExecutableCache } from "./executable-cache.js"; + +vi.mock("node:child_process", async (original) => ({ + ...await original(), spawnSync: vi.fn(), +})); + +const directories: string[] = []; +afterEach(() => { + vi.resetAllMocks(); + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +function fixture(debug = false) { + const root = mkdtempSync(join(tmpdir(), "scriptc-native-executable-")); + directories.push(root); + const inputs = join(root, "inputs"); + const sdk = join(root, "sdk"); + const stage = join(root, "stage"); + const restored = join(root, "restored"); + for (const directory of [inputs, sdk, stage, restored]) mkdirSync(directory); + const runtime = join(inputs, "runtime.o"); + const library = join(sdk, "libSystem.tbd"); + writeFileSync(runtime, "runtime"); + writeFileSync(library, "system library"); + const cache = new NativeCache(join(root, "cache")); + const key = contentDigest("llvm and options"); + const open = () => new NativeExecutableCache(cache, key, "clang", debug, [runtime]); + vi.mocked(spawnSync).mockReturnValue({ pid: 1, status: 0, signal: null, output: [], stdout: library + "\n", stderr: "" }); + const output = join(stage, "program"); + writeFileSync(output, Buffer.from([0, 128, 255, 10])); + if (debug) { + const contents = join(output + ".dSYM", "Contents"); + mkdirSync(join(contents, "Resources", "DWARF"), { recursive: true }); + writeFileSync(join(contents, "Info.plist"), "property list"); + writeFileSync(join(contents, "Resources", "DWARF", "program"), "debug information"); + } + const publish = () => { + const entry = open(); + expect(entry.trace(["-o", output], stage)).toBe(true); + entry.publish(output); + }; + return { root, inputs, sdk, stage, output, destination: join(restored, "program"), runtime, library, cache, key, open, publish }; +} + +test.each([false, true])("restores a complete executable and its debug information (%s)", (debug) => { + const f = fixture(debug); + f.publish(); + vi.mocked(spawnSync).mockClear(); + expect(f.open().restore(f.destination)).toBe(true); + expect(readFileSync(f.destination)).toEqual(readFileSync(f.output)); + if (process.platform !== "win32") expect(statSync(f.destination).mode & 0o111).toBe(0o111); + expect(spawnSync).not.toHaveBeenCalled(); + if (debug) { + expect(readFileSync(join(f.destination + ".dSYM/Contents/Resources/DWARF/program"), "utf8")).toBe("debug information"); + expect(readFileSync(join(f.destination + ".dSYM/Contents/Info.plist"), "utf8")).toBe("property list"); + } +}); + +test.each(["runtime", "library"] as const)("invalidates %s replacement even with restored size and mtime", (input) => { + const f = fixture(); + f.publish(); + const before = statSync(f[input]); + writeFileSync(f[input] + ".new", "x".repeat(before.size)); + utimesSync(f[input] + ".new", before.atime, before.mtime); + renameSync(f[input] + ".new", f[input]); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("adding a new SDK library search candidate invalidates the previous link", () => { + const f = fixture(); + f.publish(); + writeFileSync(join(f.sdk, "libSystem.dylib"), "new candidate"); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("a library search change during tracing prevents publication", () => { + const f = fixture(); + vi.mocked(spawnSync).mockImplementation(() => { + if (vi.mocked(spawnSync).mock.calls.length === 2) writeFileSync(join(f.sdk, "new-library.tbd"), "library"); + return { pid: 1, status: 0, signal: null, output: [], stdout: f.library + "\n", stderr: "" }; + }); + const entry = f.open(); + expect(entry.trace([], f.stage)).toBe(false); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test.skipIf(process.platform === "win32")("retargeting an input symlink invalidates the previous link", () => { + const f = fixture(); + const link = join(f.inputs, "current.o"); + symlinkSync(f.runtime, link); + const open = () => new NativeExecutableCache(f.cache, f.key, "clang", false, [link]); + const entry = open(); + expect(entry.trace([], f.stage)).toBe(true); + entry.publish(f.output); + expect(open().restore(f.destination)).toBe(true); + const replacement = join(f.inputs, "replacement.o"); + writeFileSync(replacement, "runtime"); + rmSync(link); + symlinkSync(replacement, link); + expect(open().restore(f.destination)).toBe(false); +}); + +test.each(["executable", "binary", "dsym"])("corrupt %s payloads are misses", (family) => { + const f = fixture(true); + f.publish(); + const directory = join(f.cache.root, family); + const key = readdirSync(directory).find((name) => /^[0-9a-f]{64}$/.test(name))!; + writeFileSync(join(directory, key), "corrupt"); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("invalid cache metadata is rejected even with a matching integrity digest", () => { + const f = fixture(); + f.publish(); + const bytes = f.cache.read("executable", f.key)!; + const entry = JSON.parse(bytes.toString()); + entry.inputs = []; + f.cache.write("executable", f.key, JSON.stringify(entry)); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("changing an input during emission prevents cache publication", () => { + const f = fixture(); + const entry = f.open(); + expect(entry.trace([], f.stage)).toBe(true); + writeFileSync(f.runtime, "replacement runtime"); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("metadata cannot omit a required runtime input", () => { + const f = fixture(); + f.publish(); + const bytes = f.cache.read("executable", f.key)!; + const entry = JSON.parse(bytes.toString()); + entry.inputs = entry.inputs.filter((input: { path: string }) => input.path !== f.runtime); + f.cache.write("executable", f.key, JSON.stringify(entry)); + expect(f.open().restore(f.destination)).toBe(false); +}); + +test("a failed or empty link trace cannot publish a completed executable", () => { + const f = fixture(); + const entry = f.open(); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); + vi.mocked(spawnSync).mockReturnValue({ pid: 1, status: 0, signal: null, output: [], stdout: "", stderr: "" }); + expect(entry.trace([], f.stage)).toBe(false); + entry.publish(f.output); + expect(f.open().restore(f.destination)).toBe(false); + vi.mocked(spawnSync).mockReturnValue({ pid: 1, status: 1, signal: null, output: [], stdout: "", stderr: "failed" }); + expect(entry.trace([], f.stage)).toBe(false); +}); diff --git a/packages/compiler/src/native/executable-cache.ts b/packages/compiler/src/native/executable-cache.ts new file mode 100644 index 00000000..7240d7ed --- /dev/null +++ b/packages/compiler/src/native/executable-cache.ts @@ -0,0 +1,212 @@ +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { basename, delimiter, dirname, isAbsolute, join, resolve } from "node:path"; +import type { CompileRequestOptions } from "../compile-types.js"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import { driverTraceCandidates, linkTraceCandidate } from "../backend/link-trace.js"; +import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "../backend/toolchain-environment.js"; +import { runNativeTool } from "../backend/native-tools.js"; +import type { NativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { contentDigest, NativeCache } from "./cache.js"; + +interface InputIdentity { + path: string; + canonical: string; + kind: "file" | "directory"; + dev: number; + ino: number; + size: number; + mtime: number; + ctime: number; +} + +interface ExecutableEntry { + schema: "scriptc.native-executable.v1"; + inputs: InputIdentity[]; + binary: string; + symbols: string | null; +} + +function identity(path: string): InputIdentity { + path = resolve(path); + const info = statSync(path); + if (!info.isFile() && !info.isDirectory()) throw new Error(`unsupported native cache input: ${path}`); + return { path, canonical: realpathSync(path), kind: info.isFile() ? "file" : "directory", + dev: info.dev, ino: info.ino, size: info.size, mtime: info.mtimeMs, ctime: info.ctimeMs }; +} + +function stillMatches(inputs: InputIdentity[]): boolean { + try { return inputs.every((input) => JSON.stringify(identity(input.path)) === JSON.stringify(input)); } + catch { return false; } +} + +function commandPath(command: string): string { + if (command.includes("/") || command.includes("\\")) return resolve(command); + for (const directory of (process.env["PATH"] ?? "/usr/bin:/bin").split(delimiter)) { + const candidate = join(directory || process.cwd(), command); + try { if (statSync(candidate).isFile()) return candidate; } catch { /* Try the next PATH entry. */ } + } + throw new Error(`native command is unavailable: ${command}`); +} + +function toolOutput(executable: string, args: string[]): string { + const result = spawnSync(executable, args, { encoding: "utf8", stdio: "pipe" }); + if (result.error) throw result.error; + if (result.status !== 0 || result.signal !== null) throw new Error("could not trace native link inputs"); + return result.stdout + "\n" + result.stderr; +} + +function tracePaths(output: string, driver: boolean, roots: string[]): string[] { + const paths = new Set(); + for (const line of output.split(/\r?\n/)) { + for (const candidate of driver ? driverTraceCandidates(line) : linkTraceCandidate(line)) { + const option = driver ? ["-L", "-F", "--sysroot="].find((prefix) => candidate.startsWith(prefix)) : undefined; + const spelling = option === undefined ? candidate : candidate.slice(option.length); + if (!isAbsolute(spelling)) continue; + const path = resolve(spelling); + if (roots.some((root) => path === root || path.startsWith(root + "/"))) continue; + try { + const input = identity(path); + paths.add(path); + paths.add(dirname(path)); + if (input.kind === "directory" && path.endsWith(".sdk")) { + for (const name of ["SDKSettings.json", "SDKSettings.plist"]) { + const settings = join(path, name); + try { identity(settings); paths.add(settings); } catch { /* The SDK directory observes new settings files. */ } + } + } + } catch { /* Non-path trace tokens are not dependencies. */ } + } + } + return [...paths].sort(); +} + +function validEntry(value: unknown): value is ExecutableEntry { + if (value === null || typeof value !== "object") return false; + const entry = value as Partial; + return entry.schema === "scriptc.native-executable.v1" && typeof entry.binary === "string" && + /^[0-9a-f]{64}$/.test(entry.binary) && (entry.symbols === null || + (typeof entry.symbols === "string" && /^[0-9a-f]{64}$/.test(entry.symbols))) && + Array.isArray(entry.inputs) && entry.inputs.length > 0 && entry.inputs.every((input) => + input !== null && typeof input === "object" && typeof input.path === "string" && isAbsolute(input.path) && + typeof input.canonical === "string" && (input.kind === "file" || input.kind === "directory") && + [input.dev, input.ino, input.size, input.mtime, input.ctime].every((value) => typeof value === "number" && Number.isFinite(value))); +} + +function readSymbols(binary: string): Buffer { + const contents = join(binary + ".dSYM", "Contents"); + const plist = readFileSync(join(contents, "Info.plist")); + const dwarf = readFileSync(join(contents, "Resources", "DWARF", basename(binary))); + const header = Buffer.alloc(12); + header.write("SCDSYM01"); + header.writeUInt32LE(plist.length, 8); + return Buffer.concat([header, plist, dwarf]); +} + +function stageSymbols(bytes: Buffer, binary: string): void { + if (bytes.length < 12 || bytes.subarray(0, 8).toString() !== "SCDSYM01") throw new Error("invalid cached dSYM"); + const end = 12 + bytes.readUInt32LE(8); + if (end <= 12 || end >= bytes.length) throw new Error("invalid cached dSYM sizes"); + const contents = join(binary + ".dSYM", "Contents"); + const dwarf = join(contents, "Resources", "DWARF"); + mkdirSync(dwarf, { recursive: true }); + writeFileSync(join(contents, "Info.plist"), bytes.subarray(12, end)); + writeFileSync(join(dwarf, basename(binary)), bytes.subarray(end)); +} + +/** A completed executable is reusable only after the frontend has validated + * its source observations and emitted the same LLVM. Native inputs bracket + * helper verification, runtime staging and the actual link. */ +export class NativeExecutableCache { + private inputs: InputIdentity[]; + private traced = false; + + constructor(private readonly cache: NativeCache, private readonly key: string, + private readonly linker: string, private readonly debug: boolean, paths: string[]) { + this.inputs = [...new Set(paths)].sort().map(identity); + } + + restore(output: string): boolean { + try { + const metadata = this.cache.read("executable", this.key); + if (metadata === null) return false; + const entry: unknown = JSON.parse(metadata.toString("utf8")); + if (!validEntry(entry) || !stillMatches(entry.inputs) || !stillMatches(this.inputs)) return false; + if (!this.inputs.every((input) => entry.inputs.some((saved) => saved.path === input.path && + JSON.stringify(saved) === JSON.stringify(input)))) return false; + if (this.debug !== (entry.symbols !== null)) return false; + const binary = this.cache.read("binary", entry.binary); + const symbols = entry.symbols === null ? null : this.cache.read("dsym", entry.symbols); + if (binary === null || binary.length === 0 || (this.debug && symbols === null)) return false; + if (!stillMatches(entry.inputs)) return false; + if (symbols !== null) stageSymbols(symbols, output); + writeFileSync(output, binary); + chmodSync(output, 0o755); + return true; + } catch { return false; } + } + + /** Trace the real object-only link, including the SDK's transitive stubs. + * Directories from the driver's search line detect a newly added candidate. + * Private build inputs already belong to the LLVM/runtime cache identity. + * A true result means the requested output was linked successfully and its + * inputs remained stable. The caller can publish that output directly. */ + trace(args: string[], stage: string): boolean { + this.traced = false; + try { + const roots = [resolve(stage), realpathSync(stage)]; + const dry = toolOutput(this.linker, [...args, "-###"]); + const driverInputs = tracePaths(dry, true, roots).map(identity); + const linked = toolOutput(this.linker, [...args, "-Wl,-t"]); + const paths = tracePaths(linked, false, roots); + if (paths.length === 0 || !stillMatches(this.inputs) || !stillMatches(driverInputs)) return false; + this.inputs.push(...driverInputs, ...paths.map(identity)); + this.traced = true; + return true; + } catch { return false; } + } + + publish(output: string): void { + try { + if (!this.traced || !stillMatches(this.inputs)) return; + const binary = readFileSync(output); + const symbols = this.debug ? readSymbols(output) : null; + const binaryKey = contentDigest(binary); + const symbolsKey = symbols === null ? null : contentDigest(symbols); + this.cache.write("binary", binaryKey, binary); + if (symbols !== null && symbolsKey !== null) this.cache.write("dsym", symbolsKey, symbols); + if (!stillMatches(this.inputs)) return; + const entry: ExecutableEntry = { schema: "scriptc.native-executable.v1", inputs: this.inputs, binary: binaryKey, symbols: symbolsKey }; + this.cache.write("executable", this.key, JSON.stringify(entry)); + } catch { /* Build artifacts remain valid when caching is unavailable. */ } + } +} + +export function openNativeExecutableCache(cache: NativeCache | null, toolchain: NativeToolchain, + llvm: string, options: CompileRequestOptions, ffi: FfiProfile | null, pack: NativeRuntimeSelection): NativeExecutableCache | null { + // Match the established complete-cache contract: default Apple driver, + // known runtime inputs, and no wrapper or caller-supplied library searches. + if (cache === null || toolchain.target.platform !== "darwin" || options.sanitize || + process.env["SCRIPTC_LINKER"] !== undefined || process.env["SCRIPTC_LLVM_HELPER"] !== undefined || + toolchain.linkerArgs.length !== 0 || ffi !== null || !toolchainEnvironmentCachePolicy().completeArtifacts) return null; + try { + const linker = commandPath(toolchain.linker); + if (realpathSync(linker) !== "/usr/bin/clang") return null; + const effective = commandPath(runNativeTool(linker, ["-print-prog-name=clang"]).trim()); + const debug = options.optimization === "dev" && !options.strip; + const paths = [linker, effective, toolchain.helperExecutable, join(toolchain.helperPackageRoot, "package.json"), + join(toolchain.runtimePackRoot, "package.json"), join(toolchain.runtimePackRoot, "runtime-pack.json"), + ...[...pack.selected.runtime, ...pack.selected.archives, ...pack.manifest.licenses].map((artifact) => join(pack.root, artifact.path))]; + if (debug) { + const dsymutil = commandPath(toolchain.dsymutil); + if (realpathSync(dsymutil) !== "/usr/bin/dsymutil") return null; + paths.push(dsymutil); + } + const key = contentDigest(JSON.stringify({ schema: 1, llvm: contentDigest(llvm), options, toolchain, + runtimeIdentity: pack.packageText, runtimeManifest: pack.manifestText, + linker: identity(linker), effective: identity(effective), environment: toolchainEnvironmentFingerprint(), + cwd: process.cwd(), path: process.env["PATH"] ?? null })); + return new NativeExecutableCache(cache, key, linker, debug, paths); + } catch { return null; } +} diff --git a/packages/compiler/src/native/file-identity.test.ts b/packages/compiler/src/native/file-identity.test.ts new file mode 100644 index 00000000..83b5875d --- /dev/null +++ b/packages/compiler/src/native/file-identity.test.ts @@ -0,0 +1,38 @@ +import { mkdtempSync, renameSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { nativeFileIdentity } from "./file-identity.js"; + +test("native tool identity invalidates same-size replacements with preserved mtime", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-file-identity-")); + try { + const path = join(root, "compiler"); + writeFileSync(path, "original"); + utimesSync(path, new Date(0), new Date(0)); + const first = nativeFileIdentity(path); + expect(nativeFileIdentity(path)).toBe(first); + const replacement = join(root, "new"); + writeFileSync(replacement, "replaced"); + utimesSync(replacement, new Date(0), new Date(0)); + renameSync(replacement, path); + expect(nativeFileIdentity(path)).not.toBe(first); + expect(() => nativeFileIdentity(root)).toThrow("not a file"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test.skipIf(process.platform === "win32")("native tool identity follows a retargeted symlink", () => { + const root = mkdtempSync(join(tmpdir(), "scriptc-file-symlink-")); + try { + const current = join(root, "current"); + const first = join(root, "first"); + const second = join(root, "second"); + writeFileSync(first, "same"); + writeFileSync(second, "same"); + symlinkSync(first, current); + const before = nativeFileIdentity(current); + rmSync(current); + symlinkSync(second, current); + expect(nativeFileIdentity(current)).not.toBe(before); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/packages/compiler/src/native/file-identity.ts b/packages/compiler/src/native/file-identity.ts new file mode 100644 index 00000000..8bd6f19b --- /dev/null +++ b/packages/compiler/src/native/file-identity.ts @@ -0,0 +1,13 @@ +import { realpathSync, statSync } from "node:fs"; +import { resolve } from "node:path"; + +/** ctime and inode identity detect replaced or rewritten tools even when + * their size and mtime are preserved. Avoid hashing entire compiler binaries + * on every build; source observations still use their content digests. */ +export function nativeFileIdentity(path: string): string { + path = resolve(path); + const info = statSync(path); + if (!info.isFile()) throw new Error(`native compiler input is not a file: ${path}`); + return JSON.stringify({ path, canonical: realpathSync(path), dev: info.dev, ino: info.ino, + size: info.size, mtime: info.mtimeMs, ctime: info.ctimeMs }); +} diff --git a/packages/compiler/src/native/library.ts b/packages/compiler/src/native/library.ts new file mode 100644 index 00000000..b6609233 --- /dev/null +++ b/packages/compiler/src/native/library.ts @@ -0,0 +1,102 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { requireNativeArtifact, runNativeTool } from "../backend/native-tools.js"; +import { localizeElfObject, mergeAndLocalizeCoffObjects } from "../backend/object-localize.js"; +import { RuntimePackError } from "../backend/runtime-pack-core.js"; +import type { NativeRuntimePack } from "../backend/runtime-pack-native.js"; +import { executableOptimizationLinkerArgs } from "../backend/targets.js"; +import type { NativeToolchain } from "./toolchain.js"; + +function archiver(toolchain: NativeToolchain): { executable: string; args: string[] } { + if (toolchain.archiver !== undefined) return { executable: toolchain.archiver, args: toolchain.archiverArgs ?? [] }; + const zig = /(?:^|[\\/])zig(?:\.exe)?$/.test(toolchain.linker); + return zig ? { executable: toolchain.linker, args: ["ar"] } : { executable: "ar", args: [] }; +} + +/** Archive member names are untrusted package data, even after digest verification. */ +export function runtimeArchiveMembers(listing: string): string[] { + const names = listing.trim().split(/\r?\n/).filter((name) => name !== "" && name !== "__.SYMDEF" && name !== "__.SYMDEF SORTED"); + if (new Set(names).size !== names.length || names.some((name) => !/^[A-Za-z0-9_.-]+\.o$/.test(name))) { + throw new RuntimePackError("runtime vendor archive has invalid or duplicate object members", "invalid"); + } + return names; +} + +/** Localize the runtime without changing the library's declared C ABI. */ +export function localizeNativeLibrary( + toolchain: NativeToolchain, stage: string, roots: string[], support: string[], keep: string[], +): string { + const combined = join(stage, "library.localized.o"); + const platform = toolchain.target.platform; + if (platform === "win32") { + writeFileSync(combined, mergeAndLocalizeCoffObjects( + roots.map((path) => readFileSync(path)), support.map((path) => readFileSync(path)), new Set(keep), + { roots: roots.map((path) => basename(path)), support: support.map((path) => basename(path)) }, + )); + return combined; + } + const ar = archiver(toolchain); + const supportArgs: string[] = []; + if (support.length > 0) { + const archive = join(stage, "support.a"); + runNativeTool(ar.executable, [...ar.args, "rcs", archive, ...support], undefined, { ...process.env, ZERO_AR_DATE: "1" }); + supportArgs.push(archive); + } + if (platform === "darwin") { + if (process.platform !== "darwin") throw new Error("Mach-O library localization requires a macOS host"); + const symbols = join(stage, "public.syms"); + writeFileSync(symbols, keep.map((name) => `_${name}\n`).join("")); + runNativeTool(toolchain.relocatableLinker ?? "ld", ["-r", ...roots, ...supportArgs, "-o", combined, "-exported_symbols_list", symbols]); + } else if (platform === "linux") { + runNativeTool(toolchain.linker, [...toolchain.linkerArgs, "-target", toolchain.target.linkerTargetTriple, + "-nostdlib", "-r", ...roots, ...supportArgs, "-o", combined]); + writeFileSync(combined, localizeElfObject(readFileSync(combined), new Set(keep))); + } else throw new Error(`library localization is unavailable for ${toolchain.target.name}`); + requireNativeArtifact(combined); + return combined; +} + +export function archiveNativeLibrary(options: { + toolchain: NativeToolchain; + programObject: string; + outputPath: string; + stage: string; + runtime: NativeRuntimePack; + localizeSymbols?: string[]; +}): void { + const { toolchain, stage } = options; + const ar = archiver(toolchain); + const program = options.programObject; + const support = [...options.runtime.runtimeObjects]; + for (let index = 0; index < options.runtime.archives.length; index++) { + const archive = options.runtime.archives[index]!; + const members = runtimeArchiveMembers(runNativeTool(ar.executable, [...ar.args, "t", archive])); + const directory = join(stage, `vendor-${index}`); + mkdirSync(directory); + if (members.length > 0) runNativeTool(ar.executable, [...ar.args, "x", archive, ...members], directory); + support.push(...members.map((name) => join(directory, name))); + } + const objects = options.localizeSymbols === undefined ? [program, ...support] + : [localizeNativeLibrary(toolchain, stage, [program], support, options.localizeSymbols)]; + runNativeTool(ar.executable, [...ar.args, "rcs", options.outputPath, ...objects], undefined, { ...process.env, ZERO_AR_DATE: "1" }); + requireNativeArtifact(options.outputPath); +} + +export function linkNativeWasmLibrary(options: { + toolchain: NativeToolchain; + programObject: string; + outputPath: string; + runtime: NativeRuntimePack; + optimization: "release" | "dev"; + exports: string[]; +}): void { + const { toolchain, runtime } = options; + runNativeTool(toolchain.linker, [ + ...toolchain.linkerArgs, "-target", toolchain.target.linkerTargetTriple, "-mexec-model=reactor", + ...executableOptimizationLinkerArgs("wasi", options.optimization), + ...options.exports.map((name) => `-Wl,--export=${name}`), + options.programObject, ...runtime.runtimeObjects, ...runtime.archives, + ...runtime.systemLibraries.map((name) => `-l${name}`), "-o", options.outputPath, + ]); + requireNativeArtifact(options.outputPath); +} diff --git a/packages/compiler/src/native/main.ts b/packages/compiler/src/native/main.ts deleted file mode 100644 index 60686196..00000000 --- a/packages/compiler/src/native/main.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { buildNative } from "./driver.js"; -import { loadNativeToolchain } from "./toolchain.js"; -import { NATIVE_HELP, parseNativeArguments } from "./arguments.js"; - -try { - const args = parseNativeArguments(process.argv.slice(2), process.execPath + ".json"); - if (args.help) console.log(NATIVE_HELP); - else { - const result = buildNative(args.build, loadNativeToolchain(args.toolchainPath)); - console.log(JSON.stringify(result)); - } -} catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -} diff --git a/packages/compiler/src/native/prepare.test.ts b/packages/compiler/src/native/prepare.test.ts new file mode 100644 index 00000000..384a36a7 --- /dev/null +++ b/packages/compiler/src/native/prepare.test.ts @@ -0,0 +1,71 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { nativeCodegenTarget } from "../backend/targets.js"; +import { runFrontend, type FrontendFactory } from "../frontend/pipeline.js"; +import { loadProgram } from "../frontend/program-node.js"; +import { ts7Executable } from "../frontend/ts7/rpc-api.js"; +import { NativeCache } from "./cache.js"; +import { prepareNativeExecutable } from "./prepare.js"; +import type { NativeToolchain } from "./toolchain.js"; + +const scratch: string[] = []; +afterEach(() => { for (const path of scratch.splice(0)) rmSync(path, { recursive: true, force: true }); }); + +function fixture() { + const temporary = mkdtempSync(join(tmpdir(), "scriptc-native-frontend-")); + scratch.push(temporary); + const root = join(temporary, "project"); + mkdirSync(root); + const entry = join(root, "main.ts"); + const dependency = join(root, "value.ts"); + writeFileSync(entry, 'import { value } from "./value.js"; console.log(value);\n'); + writeFileSync(dependency, 'export const value = "first";\n'); + writeFileSync(join(root, "tsconfig.json"), JSON.stringify({ compilerOptions: { strict: true, types: [], target: "esnext", module: "nodenext" } })); + writeFileSync(join(root, "package.json"), '{"type":"module"}'); + const target = nativeCodegenTarget()!; + const toolchain: NativeToolchain = { + compilerVersion: "test", target, helper: target.helper, helperExecutable: "unused", helperPackageRoot: "unused", + runtimePackRoot: "unused", linker: "unused", linkerArgs: [], dsymutil: "unused", ts7Executable: ts7Executable(), + }; + const cache = new NativeCache(join(temporary, "cache")); + const options = { outPath: join(root, ".scriptc", "main"), outDir: join(root, ".scriptc"), optimization: "release" as const }; + let loads = 0; + const frontend: FrontendFactory = (path, npmStatic, externalTypes) => { + loads++; + return runFrontend(path, loadProgram, npmStatic, externalTypes); + }; + const prepare = () => prepareNativeExecutable(entry, options, null, toolchain, frontend, cache); + return { root, entry, dependency, cache, options, prepare, loads: () => loads }; +} + +test("warm frontend reuse survives output creation and invalidates imported source edits", () => { + const f = fixture(); + const first = f.prepare(); + expect(first.ok, JSON.stringify(first)).toBe(true); + if (!first.ok) return; + mkdirSync(f.options.outDir); + writeFileSync(f.options.outPath, "binary placeholder"); + writeFileSync(join(f.options.outDir, "main.ll"), first.llvm); + const warm = f.prepare(); + expect(warm).toEqual(first); + expect(f.loads()).toBe(1); + writeFileSync(f.dependency, 'export const value = "second";\n'); + const edited = f.prepare(); + expect(edited.ok).toBe(true); + if (edited.ok) expect(edited.llvm).not.toBe(first.llvm); + expect(f.loads()).toBe(2); +}); + +test("structurally invalid cached features cause a rebuild even with a valid digest", () => { + const f = fixture(); + expect(f.prepare().ok).toBe(true); + const family = join(f.cache.root, "frontend"); + const key = readdirSync(family).find((name) => /^[0-9a-f]{64}$/.test(name))!; + const payload = JSON.parse(readFileSync(join(family, key), "utf8")); + payload.input.features = {}; + f.cache.write("frontend", key, JSON.stringify(payload)); + expect(f.prepare().ok).toBe(true); + expect(f.loads()).toBe(2); +}); diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts new file mode 100644 index 00000000..a6159407 --- /dev/null +++ b/packages/compiler/src/native/prepare.ts @@ -0,0 +1,193 @@ +import type { CompilationTiming } from "../timing.js"; +import { basename, dirname, join, resolve } from "node:path"; +import type { CompileFailure, CompileLibraryOptions, CompileRequestOptions } from "../compile-types.js"; +import { emitLlvmModule } from "../backend/llvm/emitter.js"; +import { executableLinkFeatures } from "../backend/executable-features.js"; +import type { NativeLinkFeatures } from "../backend/native-link-info-core.js"; +import { prepareExecutableModule } from "../executable/prepare.js"; +import type { FfiProfile } from "../ffi/ffi-manifest.js"; +import type { FrontendFactory } from "../frontend/pipeline.js"; +import { FrontendInputTracker, frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputSnapshot, type FrontendInputExclusions, trackedReadFile } from "../frontend/input-tracker.js"; +import { provenanceSources } from "../frontend/provenance-registry.js"; +import { prepareLibrary } from "../library/prepare.js"; +import type { LibraryProfile } from "../library/library-profile.js"; +import { serializeModule } from "../ir/serialize.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import type { NativeToolchain } from "./toolchain.js"; +import { nativeFileIdentity } from "./file-identity.js"; + +export interface NativeExecutableInput { + ok: true; + llvm: string; + ir: string | null; + sidecarJson: string | null; + features: NativeLinkFeatures; + sources: [string, string][]; +} + +interface FrontendCacheEntry { + schema: "scriptc.native-frontend.v2"; + probes: FrontendInputSnapshot; + input: NativeExecutableInput; +} + +function implementationIdentity(toolchain: NativeToolchain): string { + // Release versions alone cannot distinguish two development builds. + return contentDigest(JSON.stringify({ + compiler: nativeFileIdentity(process.execPath), + checker: nativeFileIdentity(toolchain.ts7Executable), + evaluator: toolchain.comptimeExecutable === undefined ? null : nativeFileIdentity(toolchain.comptimeExecutable), + version: toolchain.compilerVersion, + })); +} + +function validFeatures(value: unknown): value is NativeLinkFeatures { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const features = value as Record; + return ["dynamic", "regex", "copying", "textDecoderLegacy", "fileHandle", "fetch", "netIsland", "zlib", + "assert", "inspect", "dynInvoke", "dc", "dynAsync", "events", "emitter", "symbol", "bigint", "searchParams", + "qs", "parseArgs", "stream", "net", "http", "http2", "dgram", "watch", "foreignFfi", "nodeTest", "tls", "tlsCa"] + .every((name) => typeof features[name] === "boolean"); +} + +function validCachedInput(value: unknown): value is FrontendCacheEntry { + if (value === null || typeof value !== "object") return false; + const entry = value as Partial; + const input = entry.input; + if (entry.schema !== "scriptc.native-frontend.v2" || !validFrontendInputSnapshot(entry.probes) || + input === undefined || input === null || typeof input !== "object") return false; + return input.ok === true && typeof input.llvm === "string" && (input.ir === null || typeof input.ir === "string") && + (input.sidecarJson === null || typeof input.sidecarJson === "string") && + validFeatures(input.features) && + Array.isArray(input.sources) && input.sources.every((pair) => Array.isArray(pair) && pair.length === 2 && + typeof pair[0] === "string" && typeof pair[1] === "string"); +} + +/** Keep the checker and typed IR out of the native optimizer's live heap. */ +export function prepareNativeExecutable( + entry: string, options: CompileRequestOptions, ffi: FfiProfile | null, + toolchain: NativeToolchain, frontend: FrontendFactory, cache: NativeCache | null, + timing: CompilationTiming = () => {}, +): NativeExecutableInput | CompileFailure { + const outputKind = options.outputKind ?? "exe"; + const stem = basename(entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const outputPaths = [resolve(options.outPath), resolve(options.outPath) + ".dSYM", + resolve(options.outDir, stem + ".ll"), resolve(options.outDir, stem + ".ir.json")]; + const directories = new Set(); + for (const output of outputPaths) { + for (let directory = dirname(output); ; directory = dirname(directory)) { + directories.add(directory); + if (dirname(directory) === directory) break; + } + } + const exclusions = { outputPaths, outputDirectories: [...directories] }; + let key: string | null = null; + if (cache !== null && provenanceSources() === null) { + try { + key = contentDigest(JSON.stringify({ schema: 1, entry, options, ffi, implementation: implementationIdentity(toolchain), + target: toolchain.target.name, declarations: toolchain.declarationsRoot ?? null, + environment: Object.entries(process.env).filter(([name]) => name.startsWith("SCRIPTC_")).sort(([a], [b]) => a.localeCompare(b)), + })); + const bytes = cache.read("frontend", key); + if (bytes !== null) { + const cached: unknown = JSON.parse(bytes.toString("utf8")); + if (validCachedInput(cached) && frontendInputsStillMatch(cached.probes, exclusions)) { + const input: NativeExecutableInput = cached.input; + timing("frontend-cache-hit"); + return input; + } + } + } catch { key = null; } + } + timing("frontend-cache-miss"); + const tracker = new FrontendInputTracker(); + const prepared = tracker.runSynchronous(() => prepareExecutableModule(entry, options, ffi, toolchain.target.platform, frontend, timing)); + if (!prepared.ok) return prepared; + const ir = outputKind === "ir" || options.emitIr ? serializeModule(prepared.mod, true) : null; + timing("ir-serialize"); + const llvm = outputKind === "ir" ? "" : emitLlvmModule(prepared.mod, { + targetTriple: toolchain.target.llvmTriple, pointerBits: toolchain.target.pointerBits, + wasi: toolchain.target.platform === "wasi", runtimeAbiMarker: outputKind === "obj" || outputKind === "exe", + ...(options.optimization === "dev" && !options.strip ? { debugSources: prepared.sourceTexts } : {}), + }); + timing("llvm-emit"); + const input: NativeExecutableInput = { ok: true, llvm, ir, sidecarJson: null, + features: executableLinkFeatures(prepared.mod, options.dynamic ?? false), sources: [...prepared.sourceTexts] }; + timing("link-features"); + const probes = tracker.snapshot(); + if (cache !== null && key !== null && probes.stable && frontendInputsStillMatch(probes, exclusions)) { + const saved: FrontendCacheEntry = { schema: "scriptc.native-frontend.v2", probes, input }; + cache.write("frontend", key, JSON.stringify(saved)); + } + timing("frontend-cache-publish"); + return input; +} + +/** Library preparation also ends before the native optimizer starts. The + * cached form contains emitted artifacts, source frames, and the sidecar. */ +export function prepareNativeLibrary( + profile: LibraryProfile, options: CompileLibraryOptions, archivePath: string, + toolchain: NativeToolchain, frontend: FrontendFactory, cache: NativeCache | null, + timing: CompilationTiming = () => {}, +): NativeExecutableInput | CompileFailure { + const stem = basename(profile.entry).replace(/\.(ts|mts|cts|js|mjs|cjs)$/, ""); + const outputPaths = [archivePath, resolve(options.outDir, stem + ".lib.ll"), resolve(options.outDir, stem + ".lib.ir.json")]; + if (profile.sidecar !== null) outputPaths.push(profile.sidecar.path === null + ? archivePath + ".contract.json" : resolve(dirname(archivePath), profile.sidecar.path)); + const directories = new Set(); + for (const output of outputPaths) { + for (let directory = dirname(output); ; directory = dirname(directory)) { + directories.add(directory); + if (dirname(directory) === directory) break; + } + } + const exclusions: FrontendInputExclusions = { outputPaths, outputDirectories: [...directories] }; + let key: string | null = null; + if (cache !== null && provenanceSources() === null) { + try { + key = contentDigest(JSON.stringify({ schema: 1, kind: "library", entry: profile.entry, profile: profile.profileBytes, + options, implementation: implementationIdentity(toolchain), target: toolchain.target.name, + declarations: toolchain.declarationsRoot ?? null, + environment: Object.entries(process.env).filter(([name]) => name.startsWith("SCRIPTC_")).sort(([a], [b]) => a.localeCompare(b)), + })); + const bytes = cache.read("frontend", key); + if (bytes !== null) { + const cached: unknown = JSON.parse(bytes.toString("utf8")); + if (validCachedInput(cached) && frontendInputsStillMatch(cached.probes, exclusions)) { + const input: NativeExecutableInput = cached.input; + timing("frontend-cache-hit"); + return input; + } + } + } catch { key = null; } + } + timing("frontend-cache-miss"); + const tracker = new FrontendInputTracker(); + const prepared = tracker.runSynchronous(() => { + for (let directory = dirname(profile.entry); ; directory = dirname(directory)) { + trackedReadFile(join(directory, "tsconfig.json")); + trackedReadFile(join(directory, "package.json")); + if (directory === dirname(resolve(options.profilePath)) || dirname(directory) === directory) break; + } + return prepareLibrary(profile, options.profilePath, toolchain.compilerVersion, toolchain.target.platform, frontend, timing); + }); + if (!prepared.ok) return prepared; + const llvm = emitLlvmModule(prepared.mod, { + targetTriple: toolchain.target.llvmTriple, pointerBits: toolchain.target.pointerBits, + wasi: toolchain.target.platform === "wasi", + }); + timing("llvm-emit"); + const input: NativeExecutableInput = { + ok: true, llvm, + ir: options.emitIr ? serializeModule(prepared.mod, true) : null, sidecarJson: prepared.sidecarJson, + features: executableLinkFeatures(prepared.mod, false), sources: [...prepared.sourceTexts], + }; + timing("link-features"); + const probes = tracker.snapshot(); + if (cache !== null && key !== null && probes.stable && frontendInputsStillMatch(probes, exclusions)) { + const saved: FrontendCacheEntry = { schema: "scriptc.native-frontend.v2", probes, input }; + cache.write("frontend", key, JSON.stringify(saved)); + } + timing("frontend-cache-publish"); + return input; +} diff --git a/packages/compiler/src/native/sanitizer.test.ts b/packages/compiler/src/native/sanitizer.test.ts new file mode 100644 index 00000000..ed5e90db --- /dev/null +++ b/packages/compiler/src/native/sanitizer.test.ts @@ -0,0 +1,57 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "vitest"; +import { RUNTIME_ABI_MARKER, RUNTIME_ABI_VERSION } from "../backend/runtime-abi.js"; +import type { RuntimePackManifest } from "../backend/runtime-pack-core.js"; +import { NATIVE_TARGETS } from "../backend/targets.js"; +import { buildSanitizedRuntime } from "./sanitizer.js"; + +test.skipIf(process.platform !== "linux" && process.platform !== "darwin")( + "sanitized vendor archives compile with host libc declarations", () => { + const target = NATIVE_TARGETS.find((item) => item.platform === process.platform && + item.architecture === process.arch && !item.name.endsWith("-musl")); + if (target === undefined) throw new Error("unsupported sanitizer test host"); + const runtimeSourceRoot = fileURLToPath(new URL("../../../runtime", import.meta.url)); + const { version } = JSON.parse(readFileSync(join(runtimeSourceRoot, "package.json"), "utf8")) as { version: string }; + const directory = mkdtempSync("/tmp/scriptc-sanitized-vendor-"); + const artifact = { path: "libscriptc-libregexp.a", sha256: "0".repeat(64), size: 0 }; + const manifest: RuntimePackManifest = { + schema: "scriptc.runtime-pack.v1", format: 1, package: target.runtimePackPackage, version, + target: { name: target.name, llvm_triple: target.llvmTriple, architecture: target.architecture, + object_format: target.objectFormat, minimum_os: target.minimumOs }, + runtime_abi: { version: RUNTIME_ABI_VERSION, marker: RUNTIME_ABI_MARKER }, + compiler: { command: "clang", identity: "test", target: target.llvmTriple }, + macros: { executable: [], excluded: [], sanitizer: "external-toolchain-required" }, + flavors: { dev: { optimization: "-O0", runtime_units: [] } }, + archives: [{ ...artifact, id: "libregexp", predicate: true }], system_libraries: [], licenses: [], + }; + try { + // Exercise the actual strict-C11 vendor sources. On Linux, their shared + // headers need the target feature defines to expose POSIX declarations. + const built = buildSanitizedRuntime({ + compilerVersion: version, target, helper: target.helper, runtimeSourceRoot, + ts7Executable: "unused", helperExecutable: "unused", helperPackageRoot: "unused", + runtimePackRoot: "unused", linker: "unused", linkerArgs: [], dsymutil: "unused", + }, { + root: directory, manifest, packageText: "", manifestText: "", flavor: "dev", + selected: { runtime: [], archives: [artifact], systemLibraries: [], features: { + dynamic: false, regex: true, copying: false, textDecoderLegacy: false, fileHandle: false, + fetch: false, netIsland: false, zlib: false, assert: false, inspect: false, dynInvoke: false, + dc: false, dynAsync: false, events: false, emitter: false, symbol: false, bigint: false, + searchParams: false, qs: false, parseArgs: false, stream: false, net: false, http: false, + http2: false, dgram: false, watch: false, foreignFfi: false, nodeTest: false, tls: false, tlsCa: false, + nativeFetch: false, netIslandEffective: false, netEffective: false, httpEffective: false, + tlsEffective: false, tlsCaEffective: false, zlibEffective: false, + } }, + }, directory, "executable", null); + expect(built.runtimeObjects).toEqual([]); + const archive = join(directory, "libscriptc-libregexp.a"); + expect(built.archives).toEqual([archive]); + expect(statSync(archive).size).toBeGreaterThan(0); + expect(execFileSync("ar", ["t", archive], { encoding: "utf8" }).trim().split("\n")) + .toEqual(expect.arrayContaining(["libregexp-libregexp.o", "libregexp-libunicode.o"])); + } finally { rmSync(directory, { recursive: true, force: true }); } + }, +); diff --git a/packages/compiler/src/native/sanitizer.ts b/packages/compiler/src/native/sanitizer.ts new file mode 100644 index 00000000..f2d89c36 --- /dev/null +++ b/packages/compiler/src/native/sanitizer.ts @@ -0,0 +1,128 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { NativeCodegenError } from "../backend/native-codegen-core.js"; +import { requireNativeArtifact, runNativeTool } from "../backend/native-tools.js"; +import type { NativeRuntimePack, NativeRuntimeSelection } from "../backend/runtime-pack-native.js"; +import type { RuntimePackMode } from "../backend/runtime-pack-core.js"; +import { QJS_ENGINE_SOURCES, LRE_SOURCES, ZLIB_SOURCES } from "../backend/vendor-inputs.js"; +import { contentDigest, type NativeCache } from "./cache.js"; +import type { NativeToolchain } from "./toolchain.js"; + +/** Sanitizers retain the documented external Clang requirement. Normal + * builds use the installed precompiled runtime pack without compiling C. */ +export function sanitizerDriver(toolchain: NativeToolchain): string { + if (toolchain.target.platform !== process.platform || toolchain.target.architecture !== process.arch || + toolchain.target.name.startsWith("ios-") || toolchain.target.name.startsWith("android-")) { + throw new NativeCodegenError("SC3002", "--sanitize requires a native host target"); + } + return process.env["SCRIPTC_CC"] ?? "clang"; +} + +export function sanitizerFlags(toolchain: NativeToolchain, optimization: "release" | "dev"): string[] { + return ["-target", toolchain.target.llvmTriple, optimization === "dev" ? "-O0" : "-O1", "-g", "-fsanitize=address"]; +} + +function sourceFingerprint(root: string): string { + const hash = createHash("sha256"); + const visit = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + if (entry.name.startsWith(".")) continue; + const path = join(directory, entry.name); + if (entry.isDirectory()) visit(path); + else if (/\.(?:c|h|inc|def)$/.test(entry.name)) hash.update(path).update("\0").update(readFileSync(path)).update("\0"); + } + }; + visit(join(root, "src")); + visit(join(root, "vendor")); + return hash.digest("hex"); +} + +/** The installed pack's matrix also selects sanitizer source units and + * feature defines, keeping native and seed runtime reachability aligned. */ +export function buildSanitizedRuntime( + toolchain: NativeToolchain, selection: NativeRuntimeSelection, stage: string, + mode: RuntimePackMode, cache: NativeCache | null, +): NativeRuntimePack { + const driver = sanitizerDriver(toolchain); + const sourceRoot = toolchain.runtimeSourceRoot; + if (sourceRoot === undefined) throw new NativeCodegenError("SC3003", "runtime development sources are missing from this installation"); + const identity = JSON.parse(readFileSync(join(sourceRoot, "package.json"), "utf8")) as { name?: string; version?: string }; + if (identity.name !== "@scriptc/runtime" || identity.version !== toolchain.compilerVersion) { + throw new NativeCodegenError("SC3003", "runtime development source version does not match this compiler"); + } + const sourceIdentity = sourceFingerprint(sourceRoot); + const compilerIdentity = runNativeTool(driver, ["--version"]); + const flags = [...sanitizerFlags(toolchain, selection.flavor), + ...toolchain.target.runtimeCompileDefines.map((define) => "-D" + define)]; + const src = join(sourceRoot, "src"); + const vendor = join(sourceRoot, "vendor"); + const quickjs = join(vendor, "quickjs-ng"); + const zlib = join(vendor, "zlib"); + const mbedtls = join(vendor, "mbedtls"); + const cacheIdentity = contentDigest(JSON.stringify({ schema: 1, sourceIdentity, compilerIdentity, driver, + target: toolchain.target, environment: process.env })); + mkdirSync(stage, { recursive: true }); + const pending: { key: string; output: string }[] = []; + const compile = (source: string, name: string, args: string[]): string => { + const output = join(stage, name); + const key = contentDigest(JSON.stringify({ cacheIdentity, source, args: [...flags, ...args] })); + const cached = cache?.read("sanitizer", key); + if (cached !== null && cached !== undefined) writeFileSync(output, cached); + else { + runNativeTool(driver, [...flags, ...args, "-c", source, "-o", output]); + requireNativeArtifact(output); + pending.push({ key, output }); + } + return output; + }; + const key = mode === "executable" ? selection.flavor : mode + "-" + selection.flavor; + const flavors = selection.manifest.flavors; + const flavor = mode === "library" + ? selection.flavor === "dev" ? flavors["library-dev"] : flavors["library-release"] + : mode === "library-thread" + ? selection.flavor === "dev" ? flavors["library-thread-dev"] : flavors["library-thread-release"] + : selection.flavor === "dev" ? flavors.dev : flavors.release; + if (flavor === undefined) throw new NativeCodegenError("SC3003", `runtime pack lacks ${key}`); + const wanted = new Set(selection.selected.runtime.map((artifact) => artifact.path)); + const common = ["-std=c11", "-pthread", "-DSCR_RC_AUDIT", "-fno-math-errno", "-fno-strict-aliasing", + "-Wno-deprecated-declarations", "-I", src, "-I", quickjs, "-I", zlib, "-I", join(mbedtls, "include"), + ...(toolchain.target.platform === "linux" ? ["-ffunction-sections", "-fdata-sections"] : [])]; + const runtimeObjects: string[] = []; + for (const unit of flavor.runtime_units) { + for (const variant of unit.variants) { + if (wanted.has(variant.path)) runtimeObjects.push(compile(join(src, unit.source), unit.source.replace(/\.c$/, ".o"), + [...common, ...variant.defines.map((define) => "-D" + define)])); + } + } + const archives: string[] = []; + for (const archive of selection.selected.archives) { + const name = basename(archive.path).replace(/^libscriptc-/, "").replace(/\.a$/, ""); + let sources: readonly string[]; + let directory: string; + let args: string[]; + if (name === "quickjs") { + sources = QJS_ENGINE_SOURCES; + directory = quickjs; + args = ["-std=gnu11", "-fvisibility=hidden", "-funsigned-char", "-DQUICKJS_NG_BUILD", "-D_GNU_SOURCE", "-DNDEBUG", "-I", quickjs]; + } else if (name === "libregexp") { + sources = LRE_SOURCES; directory = quickjs; args = ["-std=c11", "-I", quickjs]; + } else if (name === "zlib") { + sources = ZLIB_SOURCES; directory = zlib; args = ["-std=c11", "-I", zlib]; + } else if (name === "mbedtls") { + directory = join(mbedtls, "library"); + sources = readdirSync(directory).filter((file) => file.endsWith(".c") && !file.startsWith(".")).sort(); + args = ["-std=c11", "-I", join(mbedtls, "include"), "-I", directory]; + } else throw new NativeCodegenError("SC3003", `unsupported runtime vendor archive: ${name}`); + const objects = sources.map((source) => compile(join(directory, source), name + "-" + source.replace(/\.c$/, ".o"), args)); + const output = join(stage, "libscriptc-" + name + ".a"); + runNativeTool(toolchain.archiver ?? "ar", [...(toolchain.archiverArgs ?? []), "rcs", output, ...objects], undefined, { ...process.env, ZERO_AR_DATE: "1" }); + requireNativeArtifact(output); + archives.push(output); + } + if (sourceFingerprint(sourceRoot) !== sourceIdentity || runNativeTool(driver, ["--version"]) !== compilerIdentity) { + throw new NativeCodegenError("SC3004", "runtime sources or sanitizer compiler changed during the build; retry"); + } + for (const item of pending) cache?.write("sanitizer", item.key, readFileSync(item.output)); + return { runtimeObjects, archives, systemLibraries: selection.selected.systemLibraries }; +} diff --git a/packages/compiler/src/native/task.test.ts b/packages/compiler/src/native/task.test.ts new file mode 100644 index 00000000..5c6e739c --- /dev/null +++ b/packages/compiler/src/native/task.test.ts @@ -0,0 +1,18 @@ +import { expect, test } from "vitest"; +import { runCompilerTask } from "./task.js"; + +test("compiler work starts after the caller can suspend and preserves its result", async () => { + const events: string[] = []; + const result = { ok: true, diagnostics: [] }; + const pending = runCompilerTask(() => { events.push("compile"); return result; }); + events.push("caller"); + expect(events).toEqual(["caller"]); + expect(await pending).toBe(result); + expect(events).toEqual(["caller", "compile"]); +}); + +test("synchronous compiler failures reject the task without escaping its microtask", async () => { + const failure = new Error("compiler failed"); + await expect(runCompilerTask(() => { throw failure; })).rejects.toBe(failure); + expect(await runCompilerTask(() => 42)).toBe(42); +}); diff --git a/packages/compiler/src/native/task.ts b/packages/compiler/src/native/task.ts new file mode 100644 index 00000000..f7bb6bec --- /dev/null +++ b/packages/compiler/src/native/task.ts @@ -0,0 +1,11 @@ +/** Compiler traversals use the main stack; async fibers have small stacks + * intended for suspended application work. Queueing also preserves the + * promise rejection boundary for synchronous compiler failures. */ +export function runCompilerTask(work: () => T): Promise { + return new Promise((resolve, reject) => { + queueMicrotask(() => { + try { resolve(work()); } + catch (error) { reject(error); } + }); + }); +} diff --git a/packages/compiler/src/native/toolchain.ts b/packages/compiler/src/native/toolchain.ts index c5adc827..e045ac03 100644 --- a/packages/compiler/src/native/toolchain.ts +++ b/packages/compiler/src/native/toolchain.ts @@ -1,7 +1,26 @@ import { readFileSync } from "node:fs"; import { dirname, isAbsolute, join, resolve } from "node:path"; -import { NATIVE_TARGETS, type NativeTargetSpec } from "../backend/targets.js"; -import type { NativeToolchain } from "./driver.js"; +import { NATIVE_TARGETS, selectNativeTarget, type NativeTargetSpec, type NativeHelperSpec } from "../backend/targets.js"; + +export interface NativeToolchain { + compilerVersion: string; + ts7Executable: string; + target: NativeTargetSpec; + helper: NativeHelperSpec; + helperExecutable: string; + helperPackageRoot: string; + runtimePackRoot: string; + runtimeSourceRoot?: string; + linker: string; + linkerArgs: string[]; + dsymutil: string; + archiver?: string; + archiverArgs?: string[]; + relocatableLinker?: string; + declarationsRoot?: string; + comptimeExecutable?: string; + wasiNodeRunner?: string; +} /** Distribution paths are relative to this manifest, so moving an installed * compiler does not bake the seed machine's paths into the native binary. */ @@ -12,9 +31,17 @@ export interface NativeToolchainManifest { ts7: string; llvm_package: string; runtime_pack: string; + runtime_sources?: string; linker: string; linker_args: string[]; dsymutil: string; + declarations?: string; + runtime_packs?: { target: string; path: string }[]; + archiver?: string; + archiver_args?: string[]; + relocatable_linker?: string; + comptime?: string; + wasi_node_runner?: string; } function pathFrom(root: string, value: string): string { @@ -25,7 +52,7 @@ function commandFrom(root: string, value: string): string { return value.includes("/") || value.includes("\\") ? pathFrom(root, value) : value; } -export function loadNativeToolchain(path: string): NativeToolchain { +export function loadNativeToolchain(path: string, env: NodeJS.ProcessEnv = {}): NativeToolchain { const raw: unknown = JSON.parse(readFileSync(path, "utf8")); if (raw === null || typeof raw !== "object" || Array.isArray(raw)) throw new Error("invalid native toolchain manifest"); const value = raw as Record; @@ -39,18 +66,46 @@ export function loadNativeToolchain(path: string): NativeToolchain { } } const manifest = raw as NativeToolchainManifest; + for (const name of ["runtime_sources", "declarations", "archiver", "relocatable_linker", "comptime", "wasi_node_runner"]) { + if (value[name] !== undefined && (typeof value[name] !== "string" || value[name] === "")) throw new Error(`native toolchain requires a nonempty string for ${name}`); + } + const archiverArgs: unknown = value.archiver_args; + const runtimePacks: unknown = value.runtime_packs; + if (archiverArgs !== undefined && (!Array.isArray(archiverArgs) || !archiverArgs.every((arg) => typeof arg === "string"))) { + throw new Error("native toolchain requires a string array for archiver_args"); + } + if (runtimePacks !== undefined && (!Array.isArray(runtimePacks) || !runtimePacks.every((item) => { + if (item === null || typeof item !== "object") return false; + const pack = item as Record; + return typeof pack.target === "string" && typeof pack.path === "string" && pack.path !== ""; + }))) { + throw new Error("native toolchain requires target/path entries for runtime_packs"); + } const targets: readonly NativeTargetSpec[] = NATIVE_TARGETS; - const target = targets.find((item) => item.name === manifest.target); - if (target === undefined || target.platform === "wasi") throw new Error(`unsupported native compiler target: ${manifest.target}`); + const host = targets.find((item) => item.name === manifest.target); + if (host === undefined || host.platform === "wasi") throw new Error(`unsupported native compiler target: ${manifest.target}`); + const target = selectNativeTarget(env["SCRIPTC_TARGET"] ?? "", host); + if (target === null) throw new Error(`unsupported native compiler target: ${env["SCRIPTC_TARGET"]}`); const root = dirname(resolve(path)); - const helperPackageRoot = pathFrom(root, manifest.llvm_package); + const helperPackageRoot = pathFrom(root, env["SCRIPTC_LLVM_PACKAGE"] ?? manifest.llvm_package); + const runtimePath = manifest.runtime_packs?.find((pack) => pack.target === target.name)?.path + ?? (target.name === host.name ? manifest.runtime_pack : join(dirname(manifest.runtime_pack), target.runtimePackPackage.replace("@scriptc/", ""))); + const linker = env["SCRIPTC_LINKER"] ?? (target.name === host.name ? manifest.linker : target.defaultLinker); + const linkerArgs = env["SCRIPTC_LINKER"] !== undefined ? [] : target.name === host.name ? manifest.linker_args : [...target.defaultLinkerArgs]; return { compilerVersion: manifest.compiler_version, ts7Executable: pathFrom(root, manifest.ts7), - target, helper: target.helper, helperPackageRoot, - helperExecutable: join(helperPackageRoot, "bin", target.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"), - runtimePackRoot: pathFrom(root, manifest.runtime_pack), - linker: commandFrom(root, manifest.linker), linkerArgs: manifest.linker_args, - dsymutil: commandFrom(root, manifest.dsymutil), + target, helper: host.helper, helperPackageRoot, + helperExecutable: env["SCRIPTC_LLVM_HELPER"] ?? join(helperPackageRoot, "bin", host.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"), + runtimePackRoot: pathFrom(root, env["SCRIPTC_RUNTIME_PACK"] ?? runtimePath), + ...(manifest.runtime_sources === undefined ? {} : { runtimeSourceRoot: pathFrom(root, manifest.runtime_sources) }), + linker: commandFrom(root, linker), linkerArgs, + dsymutil: commandFrom(root, env["SCRIPTC_DSYMUTIL"] ?? manifest.dsymutil), + ...(manifest.declarations === undefined ? {} : { declarationsRoot: pathFrom(root, manifest.declarations) }), + ...(manifest.archiver === undefined ? {} : { archiver: commandFrom(root, manifest.archiver) }), + ...(manifest.archiver === undefined ? {} : { archiverArgs: manifest.archiver_args ?? [] }), + ...(manifest.relocatable_linker === undefined ? {} : { relocatableLinker: commandFrom(root, manifest.relocatable_linker) }), + ...(manifest.comptime === undefined ? {} : { comptimeExecutable: pathFrom(root, manifest.comptime) }), + ...(manifest.wasi_node_runner === undefined ? {} : { wasiNodeRunner: pathFrom(root, manifest.wasi_node_runner) }), }; } diff --git a/packages/compiler/src/timing.ts b/packages/compiler/src/timing.ts new file mode 100644 index 00000000..ea644926 --- /dev/null +++ b/packages/compiler/src/timing.ts @@ -0,0 +1,19 @@ +export type CompilationTiming = (phase: string, detail?: Record) => void; + +/** Opt-in phase durations shared by the Node seed and native compiler. + * The disabled path does not read the clock or write compiler output. */ +export function compilationTiming(): CompilationTiming { + if (process.env["SCRIPTC_TIMING"] !== "1") return () => {}; + const start = performance.now(); + let previous = start; + return (phase, detail = {}) => { + const now = performance.now(); + process.stderr.write(`scriptc timing ${JSON.stringify({ + phase, + phase_ms: Math.round((now - previous) * 10) / 10, + total_ms: Math.round((now - start) * 10) / 10, + ...detail, + })}\n`); + previous = now; + }; +} diff --git a/packages/compiler/test/js-override-returns.test.ts b/packages/compiler/test/js-override-returns.test.ts index c4c2a4a2..181e6a3e 100644 --- a/packages/compiler/test/js-override-returns.test.ts +++ b/packages/compiler/test/js-override-returns.test.ts @@ -58,8 +58,48 @@ console.log("after"); } }); +test("incompatible JS parameter overrides refuse at method entry", async () => { + const dir = mkdtempSync(join(tmpdir(), "scriptc-override-parameters-")); + try { + const entry = join(dir, "main.cjs"); + writeFileSync(entry, ` +// @ts-nocheck +class Base { + method(value = 1) { return 1; } + dispatch(value) { return this.method(value); } +} +class Derived extends Base { + method(value = "x") { console.log("must not execute", value); return "x"; } +} +class Leaf extends Derived { callSuper(value) { super.method(value); } } +function argument(label) { console.log("argument", label); return 2; } +const derived = new Derived(); +try { derived.method(argument("direct")); } +catch (error) { console.log("direct", String(error).includes("SC1090"), String(error).includes("different signature")); } +try { derived.dispatch(argument("virtual")); } +catch (error) { console.log("virtual", String(error).includes("SC1090")); } +try { new Leaf().callSuper(argument("super")); } +catch (error) { console.log("super", String(error).includes("SC1090")); } +console.log("after"); +`); + const { coverage } = analyze(entry, { dynamic: false }); + expect(coverage.diagnostics).toEqual([]); + expect(coverage.runtimeFences?.map((d) => [d.code, d.message])).toEqual([ + ["SC1090", expect.stringContaining("overriding method 'method' with a different signature")], + ]); + const result = await compile(entry, { backend: "llvm", dynamic: false, sanitize, outDir: dir, outPath: join(dir, "program") }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + const child = spawnSync(result.binaryPath, [], { encoding: "utf8" }); + expect(child.status).toBe(0); + expect(child.stderr).toBe(""); + expect(child.stdout).toBe("argument direct\ndirect true true\nargument virtual\nvirtual true\nargument super\nsuper true\nafter\n"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test.each([ - ["parameter types", `class Base { method(value = 1) { return 1; } } class Derived extends Base { method(value = "x") { return "x"; } }`], ["async methods", `class Base { method() { return 1; } } class Derived extends Base { async method() { return "x"; } }`], ["generators", `class Base { method() { return 1; } } class Derived extends Base { *method() { yield "x"; } }`], ["accessors", `class Base { get value() { return 1; } } class Derived extends Base { get value() { return "x"; } }`], diff --git a/packages/compiler/test/library-computed-methods.test.ts b/packages/compiler/test/library-computed-methods.test.ts new file mode 100644 index 00000000..7c7f5131 --- /dev/null +++ b/packages/compiler/test/library-computed-methods.test.ts @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compileLibrary } from "../src/index.js"; + +test("library data lookups do not reach dormant async methods", async () => { + const dir = mkdtempSync("/tmp/scriptc-library-computed-"); + try { + const entry = join(dir, "entry.mjs"); + const source = ` +class Counter { + constructor() { this.value = 7; } + *values() { yield this.value; } + *[Symbol.iterator]() { yield this.value; } + async later() { return this.value; } +} +const counter = new Counter(); +const reader = { + read(object, key) { return object[key]; }, + sum(source) { let total = 0; for (const value of source) total += value; return Number(total); } +}; +`; + writeFileSync(entry, source + 'export function result() { return Number(reader.read(counter, "value")) + reader.sum([1, 2]); }\n'); + const profilePath = join(dir, "profile.json"); + writeFileSync(profilePath, JSON.stringify({ + profile_format: 1, name: "computed-methods", entry: "entry.mjs", emission: "llvm", + abi: { prefix: "cm_", init_symbol: "cm_init", sink_register_symbol: "cm_sink", collect_symbol: "cm_collect" }, + exports: [{ export: "result", symbol: "cm_result", params: [], returns: "f64" }], + })); + const result = await compileLibrary({ profilePath, outDir: join(dir, "out") }); + expect(result.ok, JSON.stringify(result.diagnostics)).toBe(true); + if (!result.ok) return; + if (process.env["SCRIPTC_PORTABLE_ONLY"] !== "1") { + const host = join(dir, "host.c"), binary = join(dir, "host"); + writeFileSync(host, '#include \nextern void cm_init(void); extern double cm_result(void);\nint main(void) { cm_init(); printf("%.0f\\n", cm_result()); }\n'); + execFileSync("clang", [host, result.archivePath, "-lm", "-o", binary]); + const actual = execFileSync(binary, { encoding: "utf8" }); + const expected = execFileSync(process.execPath, ["--input-type=module", "-e", `import { result } from ${JSON.stringify(entry)}; console.log(result());`], { encoding: "utf8" }); + expect(actual).toBe(expected); + expect(actual).toBe("10\n"); + } + writeFileSync(entry, source + 'export function result() { return Number(counter.values().next().value); }\n'); + const refused = await compileLibrary({ profilePath, outDir: join(dir, "refused") }); + expect(refused.ok).toBe(false); + expect(refused.diagnostics.some((diagnostic) => diagnostic.code === "SC4005")).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/llvm-runtime-abi.test.ts b/packages/compiler/test/llvm-runtime-abi.test.ts index d4f31e95..ca3df93d 100644 --- a/packages/compiler/test/llvm-runtime-abi.test.ts +++ b/packages/compiler/test/llvm-runtime-abi.test.ts @@ -199,7 +199,7 @@ interface LlDeclare { /** `declare zeroext i1 @scr_x(ptr, i1 zeroext, ...)` → shape (parameter * attributes stripped; only the type words matter for the C prototype). */ function parseDeclare(text: string): LlDeclare | undefined { - const m = /^declare\s+(.+?)\s*@([A-Za-z0-9_$.]+)\((.*)\)$/.exec(text.trim()); + const m = /^declare\s+(.+?)\s*@([A-Za-z0-9_$.]+)\(([^()]*)\)(?:\s+.*)?$/.exec(text.trim()); if (!m) return undefined; const ret = m[1]!.replace(/\b(zeroext|signext|noalias|nonnull)\b/g, " ").replace(/\s+/g, " ").trim(); const parts = m[3]!.trim() === "" ? [] : m[3]!.split(",").map((p) => @@ -227,6 +227,13 @@ function checkDeclare(d: LlDeclare, protos: Map): string | undef } describe("LLVM backend declares match scr_runtime.h prototypes", () => { + test("function memory attributes do not change the declared ABI", async () => { + const declaration = parseDeclare("declare zeroext i1 @scr_dyn_typed_ref_is_key(ptr, ptr) memory(read)"); + expect(declaration).toEqual({ ret: "i1", name: "scr_dyn_typed_ref_is_key", params: ["ptr", "ptr"], variadic: false }); + const { protos } = await parseHeader(); + expect(checkDeclare(declaration!, protos)).toBeUndefined(); + }); + test("externally linkable objects reference the versioned runtime marker", async () => { const loc = { file: "/source/abi-marker.ts", start: 0, end: 0 }; const mod: IrModule = { diff --git a/packages/compiler/test/native-class-array-boundaries.test.ts b/packages/compiler/test/native-class-array-boundaries.test.ts new file mode 100644 index 00000000..ee918242 --- /dev/null +++ b/packages/compiler/test/native-class-array-boundaries.test.ts @@ -0,0 +1,53 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compile } from "../src/index.js"; + +test("checked class arrays reject unbranded and differently branded elements", async () => { + const dir = mkdtempSync("/tmp/scriptc-class-arrays-"); + try { + const entry = join(dir, "main.ts"); + writeFileSync(entry, ` +class Item { value = 1; } +class Child extends Item { extra = 2; } +class Sibling extends Item { extra = 3; } +class Other { value = 1; } +function check(input: unknown): void { + try { const items = input as Item[]; console.log('accepted', items.length); } + catch (error) { console.log('rejected', error instanceof TypeError); } +} +check([new Item()]); +check([new Child()]); +check([new Other()]); +check([{value: 1}]); +const mixed: unknown[] = []; +mixed.push(new Item()); mixed.push({value: 1}); +check(mixed); +check(null); +check([1]); +check([]); +function checkChild(input: unknown): void { + try { const child = input as Child; console.log('child', child.extra); } + catch (error) { console.log('wrong child', error instanceof TypeError); } +} +checkChild(new Child()); +checkChild(new Item()); +checkChild(new Sibling()); +checkChild({ value: 1, extra: 2 }); +`); + const result = await compile(entry, { + dynamic: false, outDir: dir, outPath: join(dir, "program"), + sanitize: process.env["SCRIPTC_SAN"] === "1", + }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + const child = spawnSync(result.binaryPath, [], {encoding: "utf8", timeout: 30_000}); + expect(child.error).toBeUndefined(); + expect({status: child.status, signal: child.signal, stdout: child.stdout, stderr: child.stderr}).toEqual({ + status: 0, signal: null, stderr: "", + stdout: "accepted 1\naccepted 1\nrejected true\nrejected true\nrejected true\nrejected true\nrejected true\naccepted 0\nchild 2\nwrong child true\nwrong child true\nwrong child true\n", + }); + } finally { + rmSync(dir, {recursive: true, force: true}); + } +}); diff --git a/packages/compiler/test/native-class-descriptors.test.ts b/packages/compiler/test/native-class-descriptors.test.ts new file mode 100644 index 00000000..4e6a3306 --- /dev/null +++ b/packages/compiler/test/native-class-descriptors.test.ts @@ -0,0 +1,70 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { analyze, compile } from "../src/index.js"; + +function fixture(source: string) { + const dir = mkdtempSync("/tmp/scriptc-class-descriptors-"); + const entry = join(dir, "main.js"); + writeFileSync(entry, source); + return { dir, entry }; +} + +test.each([ + "Object.defineProperty(value, 'x', {value: 2, writable: false});", + "Object.defineProperties(value, {x: {value: 2}});", + "Object.defineProperty(value, 'data', {get() { return this.x; }});", + "Object.defineProperties(value, {data: {get() { return this.x; }}});", + "Object.defineProperty(value, String('data'), {value: 2});", +])("keeps unsafe native descriptor changes fenced: %s", (operation) => { + const { dir, entry } = fixture(`class Value { constructor() { this.x = 1; } } const value = new Value(); ${operation}`); + try { + const { coverage } = analyze(entry, { dynamic: false }); + expect([...coverage.diagnostics, ...(coverage.runtimeFences ?? [])].some((d) => d.code === "SC2020" && d.message.includes("Object.define"))).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("incompatible JS overrides throw at entry without executing the body", async () => { + const { dir, entry } = fixture(` +class Base { copy(value = 1) { return value; } } +class Child extends Base { copy(value) { console.log('must not execute'); return value; } } +const child = new Child(); +try { child.copy(2); } catch (error) { console.log(error.message.includes("overriding method 'copy' with a different return type")); } +console.log(new Base().copy()); +`); + try { + const result = await compile(entry, { dynamic: false, outDir: dir, outPath: join(dir, "program"), sanitize: process.env["SCRIPTC_SAN"] === "1" }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + const run = spawnSync(result.binaryPath, { encoding: "utf8", timeout: 30_000 }); + expect(run.error).toBeUndefined(); + expect({ status: run.status, signal: run.signal, stdout: run.stdout, stderr: run.stderr }).toEqual({ status: 0, signal: null, stdout: "true\n1\n", stderr: "" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("replaced instance constructors refuse before evaluating constructor arguments", async () => { + const { dir, entry } = fixture(` +function argument() { console.log('must not execute'); return 2; } +class Value { + constructor(x = 1) { this.x = x; } + clone() { return new this.constructor(argument()); } +} +function replace(value) { value.constructor = Value; } +const value = new Value(); +replace(value); +try { value.clone(); } catch (error) { console.log(error.message.includes('replaced instance constructor')); } +`); + try { + const result = await compile(entry, { dynamic: false, outDir: dir, outPath: join(dir, "program"), sanitize: process.env["SCRIPTC_SAN"] === "1" }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + const run = spawnSync(result.binaryPath, { encoding: "utf8", timeout: 30_000 }); + expect(run.error).toBeUndefined(); + expect({ status: run.status, signal: run.signal, stdout: run.stdout, stderr: run.stderr }).toEqual({ status: 0, signal: null, stdout: "true\n", stderr: "" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/test/startup.test.ts b/packages/compiler/test/startup.test.ts new file mode 100644 index 00000000..ecc39eb3 --- /dev/null +++ b/packages/compiler/test/startup.test.ts @@ -0,0 +1,62 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "vitest"; + +const compiler = new URL("../src/index.ts", import.meta.url).href; +const root = fileURLToPath(new URL("../../../", import.meta.url)); +const prelude = ` + import assert from "node:assert/strict"; + import { createRequire } from "node:module"; + const require = createRequire(${JSON.stringify(compiler)}); + const legacyParser = require.resolve("typescript5"); + assert.equal(require.cache[legacyParser], undefined); +`; + +function freshProcess(source: string, args: string[] = []): void { + const result = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", prelude + source, ...args], { + cwd: root, encoding: "utf8", timeout: 60_000, + }); + expect(result.error).toBeUndefined(); + expect(result.signal).toBeNull(); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); +} + +test("ordinary compilation does not initialize the legacy parser", () => { + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-startup-")); + const entry = join(directory, "main.ts"); + writeFileSync(entry, "const answer: number = 42; console.log(answer);\n"); + try { + freshProcess(` + const { compile } = await import(${JSON.stringify(compiler)}); + assert.equal(require.cache[legacyParser], undefined, "compiler import loaded TypeScript 5"); + const result = await compile(process.argv[1], { + outputKind: "llvm", outDir: process.argv[2], outPath: process.argv[3], + }); + assert.equal(result.ok, true, JSON.stringify(result.diagnostics)); + assert.equal(require.cache[legacyParser], undefined, "ordinary compilation loaded TypeScript 5"); + `, [entry, directory, join(directory, "main.ll")]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test.each([ + ["compile-time evaluation", "frontend/comptime-node.ts", ` + assert.equal(api.evaluateNodeComptime("(): number => 21 * 2", 1000), 42); + `], + ["semantic source comparison", "library/semantic-source.ts", ` + assert.equal(api.semanticallyEqualSource("entry.ts", "const n = 1;", "// note\\nconst n = 1;"), true); + assert.equal(api.semanticallyEqualSource("entry.ts", "const n = 1;", "const n = 2;"), false); + `], +])("%s loads the legacy parser on first use", (_name, module, operation) => { + freshProcess(` + const api = await import(${JSON.stringify(new URL(`../src/${module}`, import.meta.url).href)}); + assert.equal(require.cache[legacyParser], undefined); + ${operation} + assert.ok(require.cache[legacyParser], "the deferred parser was not loaded"); + `); +}); diff --git a/packages/compiler/test/static-receiver-boundaries.test.ts b/packages/compiler/test/static-receiver-boundaries.test.ts new file mode 100644 index 00000000..1bc3c306 --- /dev/null +++ b/packages/compiler/test/static-receiver-boundaries.test.ts @@ -0,0 +1,40 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { analyze } from "../src/index.js"; + +test.each([ + ["constructor reads this", "function Make() { console.log(this); return { value: 1 }; } new Make();"], + ["constructor arrow captures this", "function Make() { return { read: () => this }; } new Make();"], + ["constructor default reads this", "function Make({value = this} = {}) { return { value }; } new Make();"], + ["constructor reads new.target", "function Make() { console.log(new.target); return { value: 1 }; } new Make();"], + ["constructor returns a primitive on one branch", "function Make(flag) { if (flag) return 1; return { value: 1 }; } new Make(true);"], + ["constructor binding reassigned", "function Make() { return { value: 1 }; } Make = function () { return { value: 2 }; }; console.log(new Make().value);"], +] as const)("object-returning constructor retains the fence when %s", (_name, source) => { + const dir = mkdtempSync("/tmp/scriptc-constructor-boundary-"); + try { + const entry = join(dir, "main.mjs"); + writeFileSync(entry, source); + const { coverage } = analyze(entry); + expect([...coverage.diagnostics, ...(coverage.runtimeFences ?? [])].length).toBeGreaterThan(0); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test.each([ + ["detached method", "class C { static value = 1; static read(): number { return this.value; } } const read = C.read; console.log(read());"], + ["polymorphic receiver", "class C { static read(): string { return this.name; } } class D extends C {} function run(value: typeof C): string { return value.read(); } console.log(run(D));"], + ["inherited private field", "class C { static #value = 1; static read(): number { return this.#value; } } class D extends C {} console.log(D.read());"], + ["shadowed private field", "class C { static #value = 1; static read(): number { return this.#value; } } class D extends C { static #value = 2; } console.log(D.read());"], + ["inherited private method", "class C { static #value(): number { return 1; } static read(): number { return this.#value(); } } class D extends C {} console.log(D.read());"], + ["shadowed private method", "class C { static #value(): number { return 1; } static read(): number { return this.#value(); } } class D extends C { static #value(): number { return 2; } } console.log(D.read());"], + ["inherited static write", "class C { static value = 1; static write(): void { this.value = 2; } } class D extends C {} D.write();"], +] as const)("receiver specialization preserves the %s boundary", (_name, source) => { + const dir = mkdtempSync("/tmp/scriptc-static-boundary-"); + try { + const entry = join(dir, "main.ts"); + writeFileSync(entry, source); + const { coverage } = analyze(entry); + expect(coverage.preflightFailed).toBe(false); + expect(coverage.diagnostics.some(d => d.code === "SC1090"), JSON.stringify(coverage.diagnostics)).toBe(true); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/packages/compiler/test/ts7/baselines/order-parity.json b/packages/compiler/test/ts7/baselines/order-parity.json index 703272d1..b5bf7e72 100644 --- a/packages/compiler/test/ts7/baselines/order-parity.json +++ b/packages/compiler/test/ts7/baselines/order-parity.json @@ -8688,6 +8688,12 @@ ], "diags": [] }, + "/tests/corpus/array-fill-unit-values.ts": { + "order": [ + "/tests/corpus/array-fill-unit-values.ts" + ], + "diags": [] + }, "/tests/corpus/array-from-checked.js": { "order": [ "/tests/corpus/array-from-checked.js" @@ -8700,12 +8706,30 @@ ], "diags": [] }, + "/tests/corpus/array-optional-number-write-index.ts": { + "order": [ + "/tests/corpus/array-optional-number-write-index.ts" + ], + "diags": [] + }, "/tests/corpus/array-refined-storage.ts": { "order": [ "/tests/corpus/array-refined-storage.ts" ], "diags": [] }, + "/tests/corpus/array-sort-unknown-comparator.ts": { + "order": [ + "/tests/corpus/array-sort-unknown-comparator.ts" + ], + "diags": [] + }, + "/tests/corpus/array-truncate-reuse.ts": { + "order": [ + "/tests/corpus/array-truncate-reuse.ts" + ], + "diags": [] + }, "/tests/corpus/async-backend-factory.mjs": { "order": [ "/tests/corpus/async-backend-factory.mjs" @@ -8859,12 +8883,24 @@ ], "diags": [] }, + "/tests/corpus/class-callback-replacement.js": { + "order": [ + "/tests/corpus/class-callback-replacement.js" + ], + "diags": [] + }, "/tests/corpus/class-checked-composites.ts": { "order": [ "/tests/corpus/class-checked-composites.ts" ], "diags": [] }, + "/tests/corpus/class-checked-upcast.ts": { + "order": [ + "/tests/corpus/class-checked-upcast.ts" + ], + "diags": [] + }, "/tests/corpus/class-computed-presence.ts": { "order": [ "/tests/corpus/class-computed-presence.ts" @@ -8877,6 +8913,12 @@ ], "diags": [] }, + "/tests/corpus/class-data-descriptors.js": { + "order": [ + "/tests/corpus/class-data-descriptors.js" + ], + "diags": [] + }, "/tests/corpus/class-function-base/main.js": { "order": [ "/tests/corpus/class-function-base/base.js", @@ -8884,6 +8926,12 @@ ], "diags": [] }, + "/tests/corpus/class-instance-constructor.js": { + "order": [ + "/tests/corpus/class-instance-constructor.js" + ], + "diags": [] + }, "/tests/corpus/class-interface-refinements.ts": { "order": [ "/tests/corpus/class-interface-refinements.ts" @@ -8896,6 +8944,18 @@ ], "diags": [] }, + "/tests/corpus/class-static-data.js": { + "order": [ + "/tests/corpus/class-static-data.js" + ], + "diags": [] + }, + "/tests/corpus/closure-nullable-union-return.ts": { + "order": [ + "/tests/corpus/closure-nullable-union-return.ts" + ], + "diags": [] + }, "/tests/corpus/collection-array-cycles.ts": { "order": [ "/tests/corpus/collection-array-cycles.ts" @@ -8962,6 +9022,12 @@ ], "diags": [] }, + "/tests/corpus/crypto-hash-optional-input.ts": { + "order": [ + "/tests/corpus/crypto-hash-optional-input.ts" + ], + "diags": [] + }, "/tests/corpus/date-checked-native.js": { "order": [ "/tests/corpus/date-checked-native.js" @@ -9058,6 +9124,12 @@ ], "diags": [] }, + "/tests/corpus/for-multiple-bindings.ts": { + "order": [ + "/tests/corpus/for-multiple-bindings.ts" + ], + "diags": [] + }, "/tests/corpus/for-multiple-lexical-bindings.ts": { "order": [ "/tests/corpus/for-multiple-lexical-bindings.ts" @@ -9088,6 +9160,18 @@ ], "diags": [] }, + "/tests/corpus/fs-remove-readonly.ts": { + "order": [ + "/tests/corpus/fs-remove-readonly.ts" + ], + "diags": [] + }, + "/tests/corpus/fs-write-string-bytes-union.ts": { + "order": [ + "/tests/corpus/fs-write-string-bytes-union.ts" + ], + "diags": [] + }, "/tests/corpus/function-constructor-kinds/main.js": { "order": [ "/tests/corpus/function-constructor-kinds/consumer.js", @@ -9246,6 +9330,60 @@ ], "diags": [] }, + "/tests/corpus/js-chained-native-results.mjs": { + "order": [ + "/tests/corpus/js-chained-native-results.mjs" + ], + "diags": [] + }, + "/tests/corpus/js-checked-array-iterator.js": { + "order": [ + "/tests/corpus/js-checked-array-iterator.js" + ], + "diags": [] + }, + "/tests/corpus/js-checked-array-reference.js": { + "order": [ + "/tests/corpus/js-checked-array-reference.js" + ], + "diags": [] + }, + "/tests/corpus/js-checked-number-negation.mjs": { + "order": [ + "/tests/corpus/js-checked-number-negation.mjs" + ], + "diags": [] + }, + "/tests/corpus/js-checked-sort-result.js": { + "order": [ + "/tests/corpus/js-checked-sort-result.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-added-parameters.js": { + "order": [ + "/tests/corpus/js-class-added-parameters.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-computed-fields.js": { + "order": [ + "/tests/corpus/js-class-computed-fields.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-fluent-overrides.js": { + "order": [ + "/tests/corpus/js-class-fluent-overrides.js" + ], + "diags": [] + }, + "/tests/corpus/js-class-subclass-properties.js": { + "order": [ + "/tests/corpus/js-class-subclass-properties.js" + ], + "diags": [] + }, "/tests/corpus/js-declared-fields.cjs": { "order": [ "/tests/corpus/js-declared-fields.cjs" @@ -9266,30 +9404,96 @@ ], "diags": [] }, + "/tests/corpus/js-derived-constructor-hoisting.js": { + "order": [ + "/tests/corpus/js-derived-constructor-hoisting.js" + ], + "diags": [] + }, + "/tests/corpus/js-dormant-array-write.js": { + "order": [ + "/tests/corpus/js-dormant-array-write.js" + ], + "diags": [] + }, + "/tests/corpus/js-dormant-overrides.js": { + "order": [ + "/tests/corpus/js-dormant-overrides.js" + ], + "diags": [] + }, "/tests/corpus/js-expando-records.js": { "order": [ "/tests/corpus/js-expando-records.js" ], "diags": [] }, + "/tests/corpus/js-inherited-parameter-abi.js": { + "order": [ + "/tests/corpus/js-inherited-parameter-abi.js" + ], + "diags": [] + }, "/tests/corpus/js-module-function-values.js": { "order": [ "/tests/corpus/js-module-function-values.js" ], "diags": [] }, + "/tests/corpus/js-nested-native-descriptors.js": { + "order": [ + "/tests/corpus/js-nested-native-descriptors.js" + ], + "diags": [] + }, + "/tests/corpus/js-nullish-inherited-fields.mjs": { + "order": [ + "/tests/corpus/js-nullish-inherited-fields.mjs" + ], + "diags": [] + }, + "/tests/corpus/js-object-returning-constructors.mjs": { + "order": [ + "/tests/corpus/js-object-returning-constructors.mjs" + ], + "diags": [] + }, + "/tests/corpus/js-object-shared-arrays.js": { + "order": [ + "/tests/corpus/js-object-shared-arrays.js" + ], + "diags": [] + }, "/tests/corpus/js-omitted-typed-arguments.js": { "order": [ "/tests/corpus/js-omitted-typed-arguments.js" ], "diags": [] }, + "/tests/corpus/js-optional-class-fields.js": { + "order": [ + "/tests/corpus/js-optional-class-fields.js" + ], + "diags": [] + }, + "/tests/corpus/js-primitive-union-addition.js": { + "order": [ + "/tests/corpus/js-primitive-union-addition.js" + ], + "diags": [] + }, "/tests/corpus/js-scalar-iterables.js": { "order": [ "/tests/corpus/js-scalar-iterables.js" ], "diags": [] }, + "/tests/corpus/js-specialized-field-method.js": { + "order": [ + "/tests/corpus/js-specialized-field-method.js" + ], + "diags": [] + }, "/tests/corpus/js-static-registries.cjs": { "order": [ "/tests/corpus/js-static-registries.cjs" @@ -9527,6 +9731,12 @@ ], "diags": [] }, + "/tests/corpus/native-class-array-boundaries.ts": { + "order": [ + "/tests/corpus/native-class-array-boundaries.ts" + ], + "diags": [] + }, "/tests/corpus/native-class-cache.js": { "order": [ "/tests/corpus/native-class-cache.js" @@ -9551,6 +9761,12 @@ ], "diags": [] }, + "/tests/corpus/native-class-type-predicates.ts": { + "order": [ + "/tests/corpus/native-class-type-predicates.ts" + ], + "diags": [] + }, "/tests/corpus/native-factory-records.js": { "order": [ "/tests/corpus/native-factory-records.js" @@ -9635,6 +9851,12 @@ ], "diags": [] }, + "/tests/corpus/object-enumeration-array-reads.ts": { + "order": [ + "/tests/corpus/object-enumeration-array-reads.ts" + ], + "diags": [] + }, "/tests/corpus/object-from-checked-entries.js": { "order": [ "/tests/corpus/object-from-checked-entries.js" @@ -9770,6 +9992,12 @@ ], "diags": [] }, + "/tests/corpus/record-optional-json-presence.ts": { + "order": [ + "/tests/corpus/record-optional-json-presence.ts" + ], + "diags": [] + }, "/tests/corpus/record-unknown-values.ts": { "order": [ "/tests/corpus/record-unknown-values.ts" @@ -9830,6 +10058,12 @@ ], "diags": [] }, + "/tests/corpus/rest-optional-array-arguments.ts": { + "order": [ + "/tests/corpus/rest-optional-array-arguments.ts" + ], + "diags": [] + }, "/tests/corpus/rest-optional-elements.ts": { "order": [ "/tests/corpus/rest-optional-elements.ts" @@ -9867,6 +10101,12 @@ ], "diags": [] }, + "/tests/corpus/static-method-receivers.ts": { + "order": [ + "/tests/corpus/static-method-receivers.ts" + ], + "diags": [] + }, "/tests/corpus/stdio-values.mjs": { "order": [ "/tests/corpus/stdio-values.mjs" @@ -10004,6 +10244,18 @@ ], "diags": [] }, + "/tests/corpus/typed-array-constructor-identity.js": { + "order": [ + "/tests/corpus/typed-array-constructor-identity.js" + ], + "diags": [] + }, + "/tests/corpus/typed-array-constructor-union.ts": { + "order": [ + "/tests/corpus/typed-array-constructor-union.ts" + ], + "diags": [] + }, "/tests/corpus/typed-array-dynamic-methods.js": { "order": [ "/tests/corpus/typed-array-dynamic-methods.js" @@ -10118,6 +10370,12 @@ ], "diags": [] }, + "/tests/corpus/util-parseargs-wrapped/main.ts": { + "order": [ + "/tests/corpus/util-parseargs-wrapped/main.ts" + ], + "diags": [] + }, "/tests/corpus/weak-map-array-buffer.ts": { "order": [ "/tests/corpus/weak-map-array-buffer.ts" diff --git a/packages/runtime-pack-common/library-matrix.mjs b/packages/runtime-pack-common/library-matrix.mjs index 59abdb9b..a4ae4040 100644 --- a/packages/runtime-pack-common/library-matrix.mjs +++ b/packages/runtime-pack-common/library-matrix.mjs @@ -3,7 +3,7 @@ export function withLibraryRuntimeFlavors(matrix) { const omitted = new Set(["scr_async.c", "scr_crypto_async.c", "scr_child.c", "scr_ffi.c"]); const optional = new Set([ "scr_regex.c", "scr_assert.c", "scr_inspect.c", "scr_console_native.c", "scr_symbol.c", "scr_bigint_assert.c", - "scr_url_params.c", "scr_events_emitter.c", "scr_dyn_handle.c", "scr_zlib.c", "scr_copying.c", + "scr_url_params.c", "scr_events_emitter.c", "scr_dyn_handle.c", "scr_zlib.c", "scr_copying.c", "scr_dyn_invoke.c", ]); const libraryUnits = [ ...matrix.runtime_units.filter((unit) => !omitted.has(unit.source) && (unit.predicate === true || optional.has(unit.source))), @@ -13,9 +13,9 @@ export function withLibraryRuntimeFlavors(matrix) { variants: unit.variants.filter((variant) => !variant.defines.includes("SCR_DYNAMIC")), })); const flavors = { ...matrix.flavors }; - if (matrix.target.object_format !== "wasm") { - for (const [flavor, spec] of Object.entries(matrix.flavors)) { - flavors[`library-${flavor}`] = { ...spec, defines: ["SCR_LIB"], runtime_units: libraryUnits }; + for (const [flavor, spec] of Object.entries(matrix.flavors)) { + flavors[`library-${flavor}`] = { ...spec, defines: ["SCR_LIB"], runtime_units: libraryUnits }; + if (matrix.target.object_format !== "wasm") { flavors[`library-thread-${flavor}`] = { ...spec, defines: ["SCR_LIB", "SCR_THREAD_INSTANCES"], runtime_units: libraryUnits }; } } diff --git a/packages/runtime-pack-common/scripts/build.mjs b/packages/runtime-pack-common/scripts/build.mjs index 42458161..338e473a 100644 --- a/packages/runtime-pack-common/scripts/build.mjs +++ b/packages/runtime-pack-common/scripts/build.mjs @@ -86,6 +86,9 @@ async function build() { const flavors = {}; for (const [flavor, flavorSpec] of Object.entries(matrix.flavors)) { process.stdout.write(`building ${packageManifest.name} ${flavor} runtime\n`); + // Zig emits DWARF by default, including descriptions of functions + // removed by section GC. Release packs must opt out explicitly. + const debugFlags = flavor.endsWith("release") ? ["-g0"] : []; const units = []; for (const unit of flavorSpec.runtime_units ?? matrix.runtime_units) { const variants = []; @@ -97,7 +100,7 @@ async function build() { ...(unit.source === "scr_tls.c" ? ["-I", join(mbedtls, "include")] : []), ...(unit.source === "scr_zlib.c" || unit.source === "scr_fetch.c" ? ["-I", zlib] : []), ]; - await compile(join(runtimeSrc, unit.source), output, [...commonFlags, flavorSpec.optimization, ...variant.defines.map((define) => `-D${define}`), ...includeFlags]); + await compile(join(runtimeSrc, unit.source), output, [...commonFlags, flavorSpec.optimization, ...debugFlags, ...variant.defines.map((define) => `-D${define}`), ...includeFlags]); variants.push({ id: variant.id, when: variant.when, defines: variant.defines, path: artifactPath(output), sha256: await sha256(output), size: (await stat(output)).size }); } units.push({ source: unit.source, predicate: unit.predicate, variants }); @@ -111,6 +114,7 @@ async function build() { const vendorTarget = [ ...config.targetArgs, ...(config.compilerFlags ?? []), + "-g0", ...(config.runtimeDefines ?? []).map((define) => `-D${define}`), ]; const requestedArchives = new Set(matrix.archives.map((entry) => entry.id)); diff --git a/packages/runtime-pack-common/test/library-matrix.test.ts b/packages/runtime-pack-common/test/library-matrix.test.ts index 8b553346..c5f80c08 100644 --- a/packages/runtime-pack-common/test/library-matrix.test.ts +++ b/packages/runtime-pack-common/test/library-matrix.test.ts @@ -31,3 +31,15 @@ test("mobile packs ship only library modes and their permitted vendor archives", ]); expect(mobile.archives.map(({ id }) => id)).toEqual(["libregexp", "zlib"]); }); + +test("Wasm packs include reactor runtimes without thread-local state or a JS engine", () => { + const wasm = createRuntimePackMatrix({ target: { object_format: "wasm" }, compileFlags: [], systemLibraries: [] }); + expect(Object.keys(wasm.flavors).sort()).toEqual(["dev", "library-dev", "library-release", "release"]); + for (const name of ["library-release", "library-dev"]) { + expect(wasm.flavors[name].defines).toEqual(["SCR_LIB"]); + const sources = wasm.flavors[name].runtime_units.map((unit) => unit.source); + expect(sources).toContain("scr_library.c"); + expect(sources).not.toContain("scr_island.c"); + expect(sources).not.toContain("scr_async.c"); + } +}); diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index ed741abd..c5a97922 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -590,9 +590,12 @@ void scr_arr_set_len(ScrArr *a, double length) { } size_t next = (size_t)length; if (next < a->len) { + /* Spare capacity is already empty. Repeatedly clearing a reused work + * array must cost only the elements removed, not its historical peak. */ + size_t dense_end = a->len < a->cap ? a->len : a->cap; + size_t dense_stop = next < dense_end ? next : dense_end; if (scr_elem_is_ref(a->elem)) { - size_t dense_stop = next < a->cap ? next : a->cap; - for (size_t i = dense_stop; i < a->cap; i++) { + for (size_t i = dense_stop; i < dense_end; i++) { if (a->present[i] != SCR_ARR_HOLE) { uint64_t old = a->data[i]; uint8_t state = a->present[i]; @@ -610,8 +613,7 @@ void scr_arr_set_len(ScrArr *a, double length) { if (state == SCR_ARR_VALUE) scr_elem_release(a, old); } } else { - size_t dense_stop = next < a->cap ? next : a->cap; - for (size_t i = dense_stop; i < a->cap; i++) { + for (size_t i = dense_stop; i < dense_end; i++) { a->present[i] = 0; a->data[i] = 0; } diff --git a/packages/runtime/src/scr_dyn_invoke.c b/packages/runtime/src/scr_dyn_invoke.c index 660cc228..7c914a87 100644 --- a/packages/runtime/src/scr_dyn_invoke.c +++ b/packages/runtime/src/scr_dyn_invoke.c @@ -799,14 +799,20 @@ static ScrDyn *scr_dyn_invoke_impl( scr_dyn_this_pop(); return r; } + ScrDyn *view = list->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(list) + ? scr_dyn_typed_ref_materialize(list) : NULL; + if (scr_exc_pending()) { scr_dyn_release(view); return NULL; } + if (view) list = view; if (list->kind != SCR_DYN_ARR) { scr_throw_error_msg(SCR_ERR_TYPE, "CreateListFromArrayLike called on non-object", strlen("CreateListFromArrayLike called on non-object")); + scr_dyn_release(view); return NULL; } scr_dyn_this_push_dyn(thisv); ScrDyn *r = scr_dyn_call(recv, list->v.arr.items, list->v.arr.len, what); scr_dyn_this_pop(); + scr_dyn_release(view); return r; } if (dyn_name_is(method, "call")) { @@ -1092,13 +1098,18 @@ static ScrDyn *scr_dyn_invoke_impl( ScrDyn *out = scr_dyn_new_arr(); for (size_t i = 0; i < len; i++) scr_dyn_arr_push(out, scr_dyn_retain(recv->v.arr.items[i])); for (size_t a = 0; a < argc; a++) { - if (args[a]->kind == SCR_DYN_ARR) { - for (size_t i = 0; i < args[a]->v.arr.len; i++) { - scr_dyn_arr_push(out, scr_dyn_retain(args[a]->v.arr.items[i])); + ScrDyn *view = args[a]->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(args[a]) + ? scr_dyn_typed_ref_materialize(args[a]) : NULL; + if (scr_exc_pending()) { scr_dyn_release(view); scr_dyn_release(out); return NULL; } + const ScrDyn *source = view ? view : args[a]; + if (source->kind == SCR_DYN_ARR) { + for (size_t i = 0; i < source->v.arr.len; i++) { + scr_dyn_arr_push(out, scr_dyn_retain(source->v.arr.items[i])); } } else { scr_dyn_arr_push(out, scr_dyn_retain(args[a])); } + scr_dyn_release(view); } return out; } @@ -1370,6 +1381,9 @@ static ScrDyn *scr_dyn_invoke_impl( * other Promise.prototype name is `then`-adjacent sugar JS doesn't * have, so the not-a-function answer IS the JS answer. */ if (recv->kind == SCR_DYN_PROMISE) { +#ifdef SCR_LIB + scr_trap("scriptc: promise dispatch is unavailable in library mode\n"); +#else if (dyn_name_is(method, "then")) { return scr_dyn_promise_then(recv->v.promise, argc >= 1 ? args[0] : NULL, argc >= 2 ? args[1] : NULL, NULL); @@ -1382,6 +1396,7 @@ static ScrDyn *scr_dyn_invoke_impl( } dyn_throw_not_fn(what); return NULL; +#endif } if (recv->kind == SCR_DYN_BYTES) { diff --git a/packages/runtime/src/scr_json.c b/packages/runtime/src/scr_json.c index 68f89128..436ad39f 100644 --- a/packages/runtime/src/scr_json.c +++ b/packages/runtime/src/scr_json.c @@ -52,6 +52,7 @@ static SCR_TL ScrDyn *scr_iterator_symbol; static ScrDyn *scr_builtin_iterator_method(const ScrDyn *value); static bool scr_builtin_iterable(const ScrDyn *value) { return value->kind == SCR_DYN_ARR || value->kind == SCR_DYN_STR || value->kind == SCR_DYN_BYTES || + (value->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(value)) || (value->kind == SCR_DYN_HANDLE && (value->v.handle.tag == SCR_DYNH_ITERATOR || scr_dyn_handle_ops_of(value)->iter_pack)); } @@ -800,6 +801,12 @@ ScrDyn *scr_dyn_map_seed_entries(const ScrDyn *src) { if (src->kind == SCR_DYN_NULL || src->kind == SCR_DYN_UNDEF) return scr_dyn_new_arr(); if (src->kind == SCR_DYN_HANDLE && src->v.handle.tag == SCR_DYNH_MAP) return scr_native_map_pack(src->v.handle.ptr); + if (src->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(src)) { + ScrDyn *view = scr_dyn_typed_ref_materialize(src); + if (!scr_exc_pending() && view->kind == SCR_DYN_ARR) return view; + scr_dyn_release(view); + if (scr_exc_pending()) return NULL; + } if (src->kind == SCR_DYN_HANDLE || src->kind == SCR_DYN_TYPED_REF || src->kind == SCR_DYN_JSVAL) { static const char message[] = "new Map(entries) over native non-array iterables has no lowering"; scr_throw_error_msg_code(SCR_ERR_ERROR, message, sizeof message - 1, "SC2020"); @@ -1656,6 +1663,15 @@ ScrDyn *scr_dyn_bytes_key_get(const ScrDyn *value, const ScrStr *key) { if (key->len == 10 && memcmp(key->data, "byteLength", 10) == 0) return scr_dyn_new_num(scr_bytes_byte_len(bytes)); if (key->len == 10 && memcmp(key->data, "byteOffset", 10) == 0) return scr_dyn_new_num(scr_bytes_byte_offset(bytes)); if (key->len == 6 && memcmp(key->data, "buffer", 6) == 0) return scr_array_buffer_from_bytes(bytes); + if (key->len == 11 && memcmp(key->data, "constructor", 11) == 0) { + /* Match the compiler's opaque JS builtin identity values. */ + char token[64]; + int length = snprintf(token, sizeof token, "[builtin %s]", value->buffer ? "Buffer" : scr_bytes_elem_name(bytes->elem)); + ScrStr *name = scr_str_new(token, (size_t)length); + ScrDyn *result = scr_dyn_new_str(name); + scr_str_release(name); + return result; + } if (key->len && !(key->len > 1 && key->data[0] == '0')) { size_t index = 0; bool digits = true; @@ -2099,6 +2115,9 @@ const ScrDynJsvalOps *scr_dyn_jsval_ops(void) { bool scr_dyn_isl_typeof_is(const ScrDyn *d, const char *name) { if (d->kind == SCR_DYN_TYPED_REF) { + if (d->v.typed_ref.type_key_len >= 7 && + memcmp(d->v.typed_ref.type_key, "object:", 7) == 0) + return strcmp(name, "object") == 0; ScrDyn *materialized = scr_dyn_typed_ref_materialize(d); bool out = scr_dyn_isl_typeof_is(materialized, name); if (materialized->kind != SCR_DYN_JSVAL) { @@ -2126,6 +2145,8 @@ bool scr_dyn_isl_typeof_is(const ScrDyn *d, const char *name) { bool scr_dyn_isl_is_array(const ScrDyn *d) { if (d->kind == SCR_DYN_TYPED_REF) { + if (d->v.typed_ref.type_key_len >= 7 && + memcmp(d->v.typed_ref.type_key, "object:", 7) == 0) return false; ScrDyn *materialized = scr_dyn_typed_ref_materialize(d); bool out = materialized->kind == SCR_DYN_ARR || scr_dyn_isl_is_array(materialized); @@ -2460,24 +2481,30 @@ static void scr_native_iterator_release(void *ptr) { static ScrDyn *scr_native_iterator_next(ScrNativeIterator *iterator) { ScrDyn *source = iterator->source; - double length = !source ? 0 : source->kind == SCR_DYN_ARR ? (double)source->v.arr.len - : source->kind == SCR_DYN_BYTES ? (double)source->v.bytes->len : scr_str_utf16_len(source->v.str); + /* Native array capsules retain the live array, so refresh its view at each + * step rather than iterating a snapshot taken when the iterator opened. */ + ScrDyn *view = source && source->kind == SCR_DYN_TYPED_REF ? scr_dyn_typed_ref_materialize(source) : NULL; + if (scr_exc_pending()) { scr_dyn_release(view); return NULL; } + ScrDyn *items = view ? view : source; + double length = !items ? 0 : items->kind == SCR_DYN_ARR ? (double)items->v.arr.len + : items->kind == SCR_DYN_BYTES ? (double)items->v.bytes->len : scr_str_utf16_len(items->v.str); bool done = !source || iterator->index >= length; ScrDyn *value; if (done) { value = scr_dyn_retain(scr_dyn_undefined()); iterator->source = NULL; scr_dyn_release(source); - } else if (source->kind == SCR_DYN_ARR) { - value = scr_dyn_arr_at(source, iterator->index++); - } else if (source->kind == SCR_DYN_BYTES) { - value = scr_dyn_new_num(scr_bytes_get(source->v.bytes, iterator->index++)); + } else if (items->kind == SCR_DYN_ARR) { + value = scr_dyn_arr_at(items, iterator->index++); + } else if (items->kind == SCR_DYN_BYTES) { + value = scr_dyn_new_num(scr_bytes_get(items->v.bytes, iterator->index++)); } else { - ScrStr *point = scr_str_cp_at(source->v.str, iterator->index); + ScrStr *point = scr_str_cp_at(items->v.str, iterator->index); iterator->index += scr_str_utf16_len(point); value = scr_dyn_new_str(point); scr_str_release(point); } + scr_dyn_release(view); if (!value) return NULL; ScrDyn *result = scr_dyn_new_obj(); scr_dyn_obj_set(result, "value", 5, value); @@ -2544,7 +2571,7 @@ static ScrDyn *scr_native_iterator_new(ScrDyn *source) { static ScrDyn *scr_builtin_iterator_checked(int kind) { ScrDyn *self = scr_dyn_this_get(); ScrDyn *result = NULL; - if (self->kind == kind) + if (self->kind == kind || (kind == SCR_DYN_ARR && self->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(self))) result = scr_native_iterator_new(self); else { static const char message[] = "Borrowing this native iterator factory has no lowering"; @@ -2589,7 +2616,7 @@ ScrDyn *scr_dyn_array_values_function(void) { } static ScrDyn *scr_builtin_iterator_method(const ScrDyn *value) { - if (value->kind == SCR_DYN_ARR) return scr_dyn_array_values_function(); + if (value->kind == SCR_DYN_ARR || (value->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(value))) return scr_dyn_array_values_function(); if (value->kind == SCR_DYN_BYTES) return scr_iterator_function(2, &scr_builtin_bytes_iterator_call, "values"); if (value->kind == SCR_DYN_STR) return scr_iterator_function(3, &scr_builtin_string_iterator_call, "[Symbol.iterator]"); if (value->kind == SCR_DYN_HANDLE && scr_dyn_handle_ops_of(value)->iter_pack) @@ -2695,6 +2722,12 @@ ScrDyn *scr_dyn_array_from_iterator(const ScrDyn *source) { } ScrDyn *scr_dyn_symbol_key_get(const ScrDyn *value, const ScrDyn *key, bool optional) { + if (value->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(value)) { + ScrDyn *view = scr_dyn_typed_ref_materialize(value); + ScrDyn *result = scr_exc_pending() ? NULL : scr_dyn_symbol_key_get(view, key, optional); + scr_dyn_release(view); + return result; + } if (value->kind == SCR_DYN_NULL || value->kind == SCR_DYN_UNDEF) { if (optional) return scr_dyn_retain(scr_dyn_undefined()); ScrStr *name = key->v.symbol.render(key->v.symbol.value); @@ -2790,6 +2823,13 @@ static bool scr_dyn_key_probe_computed(const ScrDyn *value, const ScrDyn *raw_ke scr_dyn_release(key); return out; } + if (value->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(value)) { + ScrDyn *view = scr_dyn_typed_ref_materialize(value); + bool out = !scr_exc_pending() && scr_dyn_key_probe_computed(view, key, mode); + scr_dyn_release(view); + scr_dyn_release(key); + return out; + } if (value->kind == SCR_DYN_NULL || value->kind == SCR_DYN_UNDEF || (mode == 0 && !scr_dyn_to_primitive_result_is_object(value))) { ScrStr *name = key->v.symbol.render(key->v.symbol.value); @@ -3097,6 +3137,9 @@ bool scr_dyn_truthy(const ScrDyn *d) { ScrStr *scr_dyn_typeof(const ScrDyn *d) { const char *s; if (d->kind == SCR_DYN_TYPED_REF) { + if (d->v.typed_ref.type_key_len >= 7 && + memcmp(d->v.typed_ref.type_key, "object:", 7) == 0) + return scr_str_new("object", 6); ScrDyn *materialized = scr_dyn_typed_ref_materialize(d); ScrStr *out = scr_dyn_typeof(materialized); scr_dyn_release(materialized); @@ -5967,6 +6010,12 @@ static ScrDyn *scr_dyn_objwalk(const ScrDyn *v, ScrObjWalk mode) { scr_dyn_release(names); } if (!raw) return NULL; + if (raw->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(raw)) { + ScrDyn *view = scr_dyn_typed_ref_materialize(raw); + scr_dyn_release(raw); + raw = view; + if (scr_exc_pending()) { scr_dyn_release(raw); return NULL; } + } if (raw->kind != SCR_DYN_ARR) { if (scr_dyn_to_primitive_result_is_object(raw)) scr_dyn_proxy_unsupported("array-like ownKeys results"); else { @@ -6122,6 +6171,12 @@ ScrDyn *scr_dyn_obj_keys(const ScrDyn *v) { return scr_dyn_objwalk(v, SCR_OBJWAL * Non-enumerable own names shadow names farther up the chain. */ ScrDyn *scr_dyn_for_in_keys(const ScrDyn *v) { if (scr_dyn_class_reflection_fence(v)) return NULL; + if (v->kind == SCR_DYN_TYPED_REF && scr_dyn_isl_is_array(v)) { + ScrDyn *view = scr_dyn_typed_ref_materialize(v); + ScrDyn *keys = scr_exc_pending() ? NULL : scr_dyn_for_in_keys(view); + scr_dyn_release(view); + return keys; + } switch (v->kind) { case SCR_DYN_OBJ: { ScrDyn *keys = scr_dyn_new_arr(); @@ -6331,6 +6386,24 @@ ScrDyn *scr_dyn_copy_data_properties(ScrDyn *target, const ScrDyn *src) { return scr_exc_pending() ? NULL : scr_dyn_retain(target); } +/* Compiler-owned native class views retain every property and its flags, + * including nonenumerable data, without invoking accessors. */ +ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src) { + if (!src || src->kind == SCR_DYN_UNDEF) return scr_dyn_retain(target); + ScrDyn *keys = scr_dyn_obj_own_keys(src); + if (!keys) return NULL; + for (size_t i = 0; i < keys->v.arr.len; i++) { + ScrDyn *key = keys->v.arr.items[i]; + ScrDyn *descriptor = scr_dyn_own_descriptor(src, key->v.str); + ScrDyn *result = descriptor ? scr_dyn_define_property(target, key, descriptor) : NULL; + scr_dyn_release(descriptor); + scr_dyn_release(result); + if (scr_exc_pending()) break; + } + scr_dyn_release(keys); + return scr_exc_pending() ? NULL : scr_dyn_retain(target); +} + /* Variadic Object.assign's argument pack (the `Object.assign({}, * ...arr.map(f), tail)` shape): the compiler builds one fresh dyn array * of sources — plain arguments push borrowed (+1 in), spread arguments @@ -6955,11 +7028,18 @@ static bool scr_ffi_signature_equal(const ScrDyn *entry, const ScrDyn *definitio if (!definition || definition->kind != SCR_DYN_OBJ) return false; ScrDyn *expected = scr_dyn_obj_get(entry, "arguments", 9); ScrDyn *actual = scr_dyn_obj_get(definition, "arguments", 9); - if (!actual || actual->kind != SCR_DYN_ARR || actual->v.arr.len != expected->v.arr.len || - !scr_ffi_abi_equal(scr_dyn_obj_get(entry, "return", 6), scr_dyn_obj_get(definition, "return", 6))) return false; - for (size_t i = 0; i < actual->v.arr.len; i++) - if (!scr_ffi_abi_equal(actual->v.arr.items[i], expected->v.arr.items[i])) return false; - return true; + if (!actual) return false; + // JS object properties can retain a native array by reference. Read its + // current contents when checking the requested ABI, just like a dyn array. + ScrDyn *view = actual->kind == SCR_DYN_TYPED_REF + ? scr_dyn_typed_ref_materialize(actual) : scr_dyn_retain(actual); + bool matches = view && view->kind == SCR_DYN_ARR && view->v.arr.len == expected->v.arr.len && + scr_ffi_abi_equal(scr_dyn_obj_get(entry, "return", 6), scr_dyn_obj_get(definition, "return", 6)); + if (matches) for (size_t i = 0; i < view->v.arr.len; i++) { + if (!scr_ffi_abi_equal(view->v.arr.items[i], expected->v.arr.items[i])) { matches = false; break; } + } + scr_dyn_release(view); + return matches; } static void scr_ffi_callback_remove(ScrDyn *entry, ScrDyn *catalog) { diff --git a/packages/runtime/src/scr_lib.c b/packages/runtime/src/scr_lib.c index aaf07a9b..b4728fb0 100644 --- a/packages/runtime/src/scr_lib.c +++ b/packages/runtime/src/scr_lib.c @@ -2986,6 +2986,8 @@ void scr_fs_rename(ScrStr *oldpath, ScrStr *newpath) { if (error != 0) scr_fs_rename_error(error, oldpath, newpath); } +static int scr_rm_unlink(const char *path, size_t len); + void scr_fs_rm(ScrStr *path) { /* Node's rmSync: lstat first (a missing path reports the lstat syscall), * refuse directories (Node requires `recursive`, which the scriptc @@ -3000,7 +3002,7 @@ void scr_fs_rm(ScrStr *path) { scr_fs_throw(EISDIR, "rm", path); return; } - if (unlink(path->data) != 0) scr_fs_throw(errno, "unlink", path); + if (scr_rm_unlink(path->data, path->len) != 0) scr_fs_throw(errno, "unlink", path); } void scr_fs_rmdir(ScrStr *path) { @@ -3109,6 +3111,35 @@ static void scr_rm_fail_set(ScrRmFail *f, int err, const char *op, const char *p f->path = scr_str_new(path, len); } +static int scr_rm_unlink(const char *path, size_t len) { + if (unlink(path) == 0) return 0; +#ifdef _WIN32 + /* Node's Windows removal clears a file's read-only attribute before + * retrying. Private staged runtime objects intentionally use mode 0400. */ + const int original = errno; + if (original != EACCES && original != EPERM) return -1; + ScrStr *text = scr_str_new(path, len); + WCHAR *wide = scr_fs_win_wide(text); + scr_str_release(text); + if (!wide) { errno = original; return -1; } + const DWORD attributes = GetFileAttributesW(wide); + if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_READONLY) && + SetFileAttributesW(wide, attributes & ~FILE_ATTRIBUTE_READONLY)) { + if (DeleteFileW(wide)) { free(wide); return 0; } + const DWORD error = GetLastError(); + SetFileAttributesW(wide, attributes); + free(wide); + errno = scr_fs_win_errno(error); + return -1; + } + free(wide); + errno = original; +#else + (void)len; +#endif + return -1; +} + /* Post-order tree removal for rmSync's recursive form. Stops at (and * records) the first failure, with the failing path and syscall name. */ static void scr_rm_tree_e(const char *path, size_t len, ScrRmFail *f) { @@ -3118,7 +3149,7 @@ static void scr_rm_tree_e(const char *path, size_t len, ScrRmFail *f) { return; } if (!S_ISDIR(st.st_mode)) { - if (unlink(path) != 0) scr_rm_fail_set(f, errno, "unlink", path, len); + if (scr_rm_unlink(path, len) != 0) scr_rm_fail_set(f, errno, "unlink", path, len); return; } DIR *d = opendir(path); @@ -3168,7 +3199,7 @@ static void scr_fs_rm_attempt(ScrStr *path, bool recursive, bool force, ScrRmFai scr_rm_tree_e(path->data, path->len, f); return; } - if (unlink(path->data) != 0) scr_rm_fail_set(f, errno, "unlink", path->data, path->len); + if (scr_rm_unlink(path->data, path->len) != 0) scr_rm_fail_set(f, errno, "unlink", path->data, path->len); } void scr_fs_rm_opts(ScrStr *path, bool recursive, bool force) { diff --git a/packages/runtime/src/scr_library.c b/packages/runtime/src/scr_library.c index bfe68626..723c0fec 100644 --- a/packages/runtime/src/scr_library.c +++ b/packages/runtime/src/scr_library.c @@ -45,6 +45,13 @@ static SCR_TL ScrLibSinkFn scr_library_sink = NULL; static SCR_TL void *scr_library_sink_ctx = NULL; static SCR_TL bool scr_library_poisoned = false; +#ifdef __wasi__ +/* Reactor hosts receive the same structured panic bytes as native sinks. + * A host exception may unwind out of Wasm; poison is set before delivery. */ +__attribute__((import_module("scriptc"), import_name("panic"))) +extern void scr_wasm_panic(const uint8_t *message, size_t length); +#endif + void scr_library_set_sink(ScrLibSinkFn fn, void *ctx) { /* Latest registration wins; re-registration is permitted before a trap. * Deliberately NOT poison-guarded: a pure store, touching no runtime @@ -114,7 +121,7 @@ void scr_library_callback_end(void) { * entry prologue below), and the remediation is the profile's for that * code when the program TU's overlay table declares one. */ -#if defined(__GNUC__) || defined(__clang__) +#if !defined(__wasi__) && (defined(__GNUC__) || defined(__clang__)) #define SCR_TRAP_ADDR() ((uint64_t)(uintptr_t)__builtin_return_address(0)) #else #define SCR_TRAP_ADDR() ((uint64_t)0) @@ -214,9 +221,13 @@ static _Noreturn void scr_library_trap_deliver(const char *msg, size_t len, uint msg = buf; len = n; } +#ifdef __wasi__ + scr_wasm_panic((const uint8_t *)msg, len); +#else if (scr_library_sink != NULL) { scr_library_sink(scr_library_sink_ctx, (const uint8_t *)msg, len, addr); } +#endif abort(); } diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 8fa0e4de..fdaa0c7e 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -312,6 +312,11 @@ enum { SCR_CYC_NURSERY = 0, SCR_CYC_MATURE = 1, SCR_CYC_NGENS = 2 }; typedef struct ScrCycHdr { ScrTraceFn trace; ScrCycFreeFn free_fn; +#if UINTPTR_MAX == UINT32_MAX + /* calloc aligns the block for doubles; keep the payload aligned too. + * Padding BEFORE color preserves the emitted obj-12 mark-live offset. */ + uint32_t payload_alignment; +#endif uint32_t color; /* SCR_CYC_* */ uint16_t buffered; /* 1 = sitting in its generation's candidate buffer */ uint16_t gen; /* SCR_CYC_NURSERY..SCR_CYC_MATURE (the walk filter) */ @@ -330,10 +335,12 @@ _Static_assert(sizeof(ScrCycHdr) == 32, "LLVM backend expects a 32-byte cycle he _Static_assert(offsetof(ScrCycHdr, color) == 16, "LLVM backend's inlined mark-live stores i32 0 at obj-16"); #elif UINTPTR_MAX == UINT32_MAX -_Static_assert(sizeof(ScrCycHdr) == 20, "LLVM backend expects a 20-byte cycle header"); -_Static_assert(offsetof(ScrCycHdr, color) == 8, +_Static_assert(sizeof(ScrCycHdr) == 24, "wasm32 cycle payloads require an aligned 24-byte header"); +_Static_assert(offsetof(ScrCycHdr, color) == 12, "LLVM backend's inlined mark-live stores i32 0 at obj-12"); #endif +_Static_assert(sizeof(ScrCycHdr) % _Alignof(double) == 0, + "cycle-headered payloads must preserve double alignment"); _Static_assert(sizeof(((ScrCycHdr *)0)->color) == 4, "mark-live is an i32 store: color must own all four bytes"); _Static_assert(SCR_CYC_BLACK == 0, @@ -3624,6 +3631,7 @@ bool scr_dyn_property_is_enumerable(const ScrDyn *value, const ScrStr *key); * and scalar/function/handle sources copy nothing. */ ScrDyn *scr_dyn_assign(ScrDyn *target, const ScrDyn *src); ScrDyn *scr_dyn_copy_data_properties(ScrDyn *target, const ScrDyn *src); +ScrDyn *scr_dyn_copy_property_descriptors(ScrDyn *target, const ScrDyn *src); /* Variadic Object.assign (the spread-source form): the compiler packs * every source into one fresh dyn array — pack_push retains a plain * source in (BORROWED), pack_push_spread flattens a spread source through diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5541d351..f41115fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: js-yaml: specifier: 4.3.0 version: 4.3.0 + rollup: + specifier: 4.62.2 + version: 4.62.2 three: specifier: 0.186.1 version: 0.186.1 @@ -45,10 +48,79 @@ importers: internal/compatibility: {} packages/cli: - dependencies: + devDependencies: '@scriptc/compiler': specifier: workspace:* version: link:../compiler + optionalDependencies: + '@scriptc/cli-darwin-arm64': + specifier: workspace:* + version: link:../cli-darwin-arm64 + '@scriptc/cli-darwin-x64': + specifier: workspace:* + version: link:../cli-darwin-x64 + '@scriptc/cli-linux-arm64-gnu': + specifier: workspace:* + version: link:../cli-linux-arm64-gnu + '@scriptc/cli-linux-arm64-musl': + specifier: workspace:* + version: link:../cli-linux-arm64-musl + '@scriptc/cli-linux-x64-gnu': + specifier: workspace:* + version: link:../cli-linux-x64-gnu + '@scriptc/cli-linux-x64-musl': + specifier: workspace:* + version: link:../cli-linux-x64-musl + '@scriptc/cli-win32-x64-msvc': + specifier: workspace:* + version: link:../cli-win32-x64-msvc + '@scriptc/runtime-android-arm64': + specifier: workspace:* + version: link:../runtime-android-arm64 + '@scriptc/runtime-darwin-arm64': + specifier: workspace:* + version: link:../runtime-darwin-arm64 + '@scriptc/runtime-darwin-x64': + specifier: workspace:* + version: link:../runtime-darwin-x64 + '@scriptc/runtime-ios-arm64': + specifier: workspace:* + version: link:../runtime-ios-arm64 + '@scriptc/runtime-ios-simulator-arm64': + specifier: workspace:* + version: link:../runtime-ios-simulator-arm64 + '@scriptc/runtime-linux-arm64-gnu': + specifier: workspace:* + version: link:../runtime-linux-arm64-gnu + '@scriptc/runtime-linux-arm64-musl': + specifier: workspace:* + version: link:../runtime-linux-arm64-musl + '@scriptc/runtime-linux-x64-gnu': + specifier: workspace:* + version: link:../runtime-linux-x64-gnu + '@scriptc/runtime-linux-x64-musl': + specifier: workspace:* + version: link:../runtime-linux-x64-musl + '@scriptc/runtime-wasm32-wasi': + specifier: workspace:* + version: link:../runtime-wasm32-wasi + '@scriptc/runtime-win32-x64-msvc': + specifier: workspace:* + version: link:../runtime-win32-x64-msvc + + packages/cli-darwin-arm64: {} + + packages/cli-darwin-x64: {} + + packages/cli-linux-arm64-gnu: {} + + packages/cli-linux-arm64-musl: {} + + packages/cli-linux-x64-gnu: {} + + packages/cli-linux-x64-musl: {} + + packages/cli-win32-x64-msvc: {} packages/compiler: dependencies: diff --git a/scripts/bench-builds.mjs b/scripts/bench-builds.mjs index c2d235f0..ab608c30 100644 --- a/scripts/bench-builds.mjs +++ b/scripts/bench-builds.mjs @@ -9,15 +9,29 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; -const { values } = parseArgs({ options: { iterations: { type: "string", default: "5" } } }); +const { values } = parseArgs({ options: { + iterations: { type: "string", default: "5" }, + compiler: { type: "string" }, + optimization: { type: "string", default: "dev" }, + modules: { type: "string", default: "16" }, + functions: { type: "string", default: "16" }, + timings: { type: "boolean", default: false }, + strip: { type: "boolean", default: false }, +} }); const iterations = Number(values.iterations); if (!Number.isInteger(iterations) || iterations < 1 || iterations > 100) { throw new Error("--iterations must be an integer between 1 and 100"); } +if (!["dev", "release"].includes(values.optimization)) throw new Error("--optimization must be dev or release"); +const modules = Number(values.modules); +const functions = Number(values.functions); +for (const [name, value] of [["modules", modules], ["functions", functions]]) { + if (!Number.isInteger(value) || value < 1 || value > 256) throw new Error(`--${name} must be an integer between 1 and 256`); +} if (process.env.SCRIPTC_TARGET && process.env.SCRIPTC_TARGET !== "native") { throw new Error("bench:builds runs host executables; unset SCRIPTC_TARGET"); } -const cli = fileURLToPath(new URL("../packages/cli/dist/bootstrap.js", import.meta.url)); +const cli = values.compiler ?? fileURLToPath(new URL("../packages/cli/dist/bootstrap.js", import.meta.url)); await access(cli).catch(() => { throw new Error("Build the workspace with pnpm build before running bench:builds"); }); const root = await mkdtemp(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bench-builds-")); const cache = join(root, "cache"); @@ -27,8 +41,7 @@ const env = { ...process.env, SCRIPTC_CACHE_DIR: cache }; delete env.SCRIPTC_NO_CACHE; delete env.SCRIPTC_CACHE_MAX_MB; delete env.SCRIPTC_TEST_STABLE_TOOLCHAIN; -const modules = 16; -const functions = 16; +delete env.SCRIPTC_TIMING; const samples = []; function moduleSource(module, offset) { @@ -37,8 +50,11 @@ function moduleSource(module, offset) { ).join("\n") + "\n"; } -function run(command, args) { - const result = spawnSync(command, args, { env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: 120_000 }); +function run(command, args, timings = false) { + const result = spawnSync(command, args, { + env: timings ? { ...env, SCRIPTC_TIMING: "1" } : env, + encoding: "utf8", maxBuffer: 16 * 1024 * 1024, timeout: 120_000, + }); if (result.error) throw result.error; assert.equal(result.signal, null, `${command} received ${result.signal}`); return { stdout: result.stdout, stderr: result.stderr, status: result.status }; @@ -46,7 +62,8 @@ function run(command, args) { function build(phase) { const start = performance.now(); - const result = run(process.execPath, [cli, "build", entry, "--optimization=dev", "-o", binary]); + const args = ["build", entry, `--optimization=${values.optimization}`, "-o", binary, ...(values.strip ? ["--strip"] : [])]; + const result = values.compiler ? run(cli, args, values.timings) : run(process.execPath, [cli, ...args], values.timings); const ms = Math.round((performance.now() - start) * 10) / 10; assert.equal(result.status, 0, result.stderr); // Correctness checks are outside the timed build and run after EVERY edit, @@ -54,7 +71,11 @@ function build(phase) { const oracle = run(process.execPath, [entry]); assert.equal(oracle.status, 0, oracle.stderr); assert.deepEqual(run(binary, []), oracle); - samples.push({ phase, ms }); + const timings = result.stderr.split("\n").flatMap((line) => { + const prefix = "scriptc timing "; + return line.startsWith(prefix) ? [JSON.parse(line.slice(prefix.length))] : []; + }); + samples.push({ phase, ms, ...(values.timings ? { timings } : {}) }); process.stderr.write(`${phase}: ${ms} ms\n`); } @@ -81,10 +102,12 @@ try { build("edit"); } process.stdout.write(JSON.stringify({ + compiler: values.compiler ?? "workspace CLI", node: process.version, platform: process.platform, arch: process.arch, - optimization: "dev", + optimization: values.optimization, + strip: values.strip, modules: modules + 1, functions: modules * functions, iterations, diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts new file mode 100644 index 00000000..b6ac6e2f --- /dev/null +++ b/scripts/build-native-cli.mts @@ -0,0 +1,68 @@ +/** Build the production CLI seed and its relocatable compiler assets. */ +import { execFileSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { compile } from "../packages/compiler/src/index.js"; +import { NATIVE_TARGETS, nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; +import { ts7Executable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; +import { stageNativeCliAssets } from "./native-cli-assets.mjs"; +import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const output = resolve(process.argv[2] ?? join(root, ".scriptc/native-cli")); +const target = nativeCodegenTarget(); +if (target === null || target.platform === "wasi") throw new Error("a supported native host is required to build the compiler"); +const bin = join(output, "bin"); +const lib = join(output, "lib"); +const seed = join(dirname(output), ".scriptc", basename(output) + "-seed"); +for (const dir of [bin, lib, seed]) mkdirSync(dir, { recursive: true }); +const compiler = process.env["SCRIPTC_CC"] ?? target.defaultLinker; +const compilerArgs = process.env["SCRIPTC_CC"] === undefined ? [...target.defaultLinkerArgs] : []; +const nativeSources = join(root, "packages/compiler/native"); +const functions: unknown[] = []; +const libraries: string[] = []; +for (const name of ["ts7-process", "host"]) { + const object = join(seed, name + target.outputSuffixes.obj); + execFileSync(compiler, [ + ...compilerArgs, "-std=c11", "-Wall", "-Wextra", "-Werror", "-target", target.linkerTargetTriple, + "-c", join(nativeSources, name + ".c"), "-o", object, + ], { stdio: "inherit" }); + const manifest = JSON.parse(readFileSync(join(nativeSources, name + ".ffi.json"), "utf8")) as { functions: unknown[] }; + functions.push(...manifest.functions); + libraries.push(object); +} +const ffi = join(seed, "compiler.ffi.json"); +writeFileSync(ffi, JSON.stringify({ ffi_format: 6, functions, libraries }, null, 2) + "\n"); +const executable = join(bin, "scriptc" + target.outputSuffixes.exe); +const comptime = join(lib, "scriptc-comptime" + target.outputSuffixes.exe); +execFileSync(compiler, [ + ...compilerArgs, "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-target", target.linkerTargetTriple, + "-I", join(root, "packages/runtime/vendor/quickjs-ng"), join(nativeSources, "comptime.c"), + join(root, "packages", target.runtimePackPackage.replace("@scriptc/", ""), "artifacts/vendor/quickjs/libscriptc-quickjs.a"), + "-lm", "-lpthread", "-o", comptime, +], { stdio: "inherit" }); +const result = await compile(join(root, "packages/compiler/src/native/cli.ts"), { + outDir: seed, outPath: executable, backend: "llvm", + optimization: process.env["SCRIPTC_NATIVE_OPTIMIZATION"] === "dev" ? "dev" : "release", + strip: process.env["SCRIPTC_NATIVE_KEEP_SYMBOLS"] !== "1", dynamic: false, ffiProfilePath: ffi, sanitize: process.env["SCRIPTC_SAN"] === "1", + emitIr: process.env["SCRIPTC_NATIVE_EMIT_IR"] === "1", +}); +if (!result.ok) throw new Error(result.diagnostics.map((item) => + `${item.loc ? `${item.loc.file}:${item.loc.start}: ` : ""}${item.code}: ${item.message}`, +).join("\n")); +const compilerVersion = (JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as { version: string }).version; +const assets = stageNativeCliAssets({ repository: root, output, target, ts7: ts7Executable(), compilerVersion }); +// A cross-libc seed may use Zig on the build host. The installed compiler +// selects the destination platform's ordinary linker and SDK. +const installedTarget = NATIVE_TARGETS.find((item) => item.name === target.name)!; +const manifest: NativeToolchainManifest = { + schema: "scriptc.native-toolchain.v1", compiler_version: compilerVersion, target: target.name, + ...assets, + linker: process.env["SCRIPTC_LINKER"] ?? installedTarget.defaultLinker, + linker_args: process.env["SCRIPTC_LINKER"] === undefined ? [...installedTarget.defaultLinkerArgs] : [], + dsymutil: process.env["SCRIPTC_DSYMUTIL"] ?? "dsymutil", + comptime: relative(bin, comptime), +}; +writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); +console.log(executable); diff --git a/scripts/build-native-compiler.mts b/scripts/build-native-compiler.mts deleted file mode 100644 index 2337f953..00000000 --- a/scripts/build-native-compiler.mts +++ /dev/null @@ -1,54 +0,0 @@ -/** Seed a native compiler using the Node-hosted compiler. Subsequent builds - * can invoke the resulting executable with the emitted FFI profile. */ -import { execFileSync } from "node:child_process"; -import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; -import { dirname, join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { compile } from "../packages/compiler/src/index.js"; -import { nativeCodegenTarget } from "../packages/compiler/src/backend/targets.js"; -import { ts7Executable } from "../packages/compiler/src/frontend/ts7/rpc-api.js"; -import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const output = resolve(process.argv[2] ?? join(root, ".scriptc/native")); -const target = nativeCodegenTarget(); -if (target === null || target.platform === "wasi") throw new Error("a supported native host is required to build the compiler"); -mkdirSync(output, { recursive: true }); -const executable = join(output, "scriptc-native" + target.outputSuffixes.exe); -const object = join(output, "ts7-process" + target.outputSuffixes.obj); -const ffi = join(output, "ts7-process.ffi.json"); -const nativeSources = join(root, "packages/compiler/native"); -const compiler = process.env["SCRIPTC_CC"] ?? target.defaultLinker; -const compilerArgs = process.env["SCRIPTC_CC"] === undefined ? [...target.defaultLinkerArgs] : []; -const sanitize = process.env["SCRIPTC_SAN"] === "1"; -// This object and profile are reused by native rebuilds with packaged runtimes. -// Sanitizer instrumentation belongs to the seed compiler's own build below. -execFileSync(compiler, [ - ...compilerArgs, "-std=c11", "-Wall", "-Wextra", "-Werror", "-target", target.linkerTargetTriple, - "-c", join(nativeSources, "ts7-process.c"), "-o", object, -], { stdio: "inherit" }); -writeFileSync(ffi, JSON.stringify({ - ...JSON.parse(readFileSync(join(nativeSources, "ts7-process.ffi.json"), "utf8")), libraries: [object], -}, null, 2) + "\n"); -const result = await compile(join(root, "packages/compiler/src/native/main.ts"), { - outDir: output, outPath: executable, backend: "llvm", optimization: "release", strip: true, dynamic: false, ffiProfilePath: ffi, sanitize, -}); -if (!result.ok) throw new Error(result.diagnostics.map((item) => - `${item.loc ? `${item.loc.file}:${item.loc.start}: ` : ""}${item.code}: ${item.message}`, -).join("\n")); - -// The executable path can be canonical even when the build directory was -// reached through a symlink (for example /tmp on macOS). -const relativeRoot = realpathSync(output); -const packagePath = (name: string) => relative(relativeRoot, join(root, "packages", name.replace("@scriptc/", ""))); -const manifest: NativeToolchainManifest = { - schema: "scriptc.native-toolchain.v1", - compiler_version: (JSON.parse(readFileSync(join(root, "packages/compiler/package.json"), "utf8")) as { version: string }).version, - target: target.name, ts7: relative(relativeRoot, ts7Executable()), - llvm_package: packagePath(target.helper.packageName), runtime_pack: packagePath(target.runtimePackPackage), - linker: process.env["SCRIPTC_LINKER"] ?? target.defaultLinker, - linker_args: process.env["SCRIPTC_LINKER"] === undefined ? [...target.defaultLinkerArgs] : [], - dsymutil: process.env["SCRIPTC_DSYMUTIL"] ?? "dsymutil", -}; -writeFileSync(executable + ".json", JSON.stringify(manifest, null, 2) + "\n"); -console.log(executable); diff --git a/scripts/generate-ts7-ast-schema.mjs b/scripts/generate-ts7-ast-schema.mjs index cbad3256..6079ef7a 100644 --- a/scripts/generate-ts7-ast-schema.mjs +++ b/scripts/generate-ts7-ast-schema.mjs @@ -76,6 +76,25 @@ for (const [kind, names] of Object.entries(protocol.childProperties)) { lines.push(` case ${kind}: return ${JSON.stringify(names.join(","))}; // ${SyntaxKind[kind]}`); } lines.push(' default: return "";', " }", "}", ""); +// Named child access is a hot path in every frontend pass. Generate the +// ordinal lookup directly instead of allocating substrings per access. +// Share cases with the same child layout to keep the dispatch compact. +const childLayouts = new Map(); +for (const [kind, names] of Object.entries(protocol.childProperties)) { + if (names.length === 0) continue; + const key = JSON.stringify(names); + if (!childLayouts.has(key)) childLayouts.set(key, []); + childLayouts.get(key).push(kind); +} +lines.push("/** Wire slot for a child property, or -1 when the kind has no such child. */", + "export function astChildOrder(kind: number, name: string): number {", " switch (kind) {"); +for (const [layout, kinds] of childLayouts) { + for (const kind of kinds) lines.push(` case ${kind}: // ${SyntaxKind[kind]}`); + lines.push(" switch (name) {"); + for (const [index, name] of JSON.parse(layout).entries()) lines.push(` case ${JSON.stringify(name)}: return ${index};`); + lines.push(" default: return -1;", " }"); +} +lines.push(" default: return -1;", " }", "}", ""); const output = lines.join("\n"); const target = join(root, "packages/compiler/src/frontend/ts7/ast-schema.generated.ts"); if (process.argv.includes("--check")) { diff --git a/scripts/native-cli-assets.mts b/scripts/native-cli-assets.mts new file mode 100644 index 00000000..7a2c361a --- /dev/null +++ b/scripts/native-cli-assets.mts @@ -0,0 +1,73 @@ +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative } from "node:path"; +import type { NativeTargetSpec } from "../packages/compiler/src/backend/targets.js"; +import type { NativeToolchainManifest } from "../packages/compiler/src/native/toolchain.js"; + +/** Copy release payloads explicitly; development caches and build trees never ship. */ +function copyPackage(source: string, destination: string, members: readonly string[]): void { + mkdirSync(destination, { recursive: true }); + for (const member of ["package.json", ...members]) { + cpSync(join(source, member), join(destination, member), { + recursive: true, + filter: (path) => ![".cache", "node_modules", ".DS_Store"].includes(basename(path)), + }); + } + for (const name of readdirSync(source)) { + if (/^(?:LICENSE|COPYING|NOTICE|SCRIPTC_LICENSE|THIRD_PARTY_NOTICES)(?:\..*)?$/.test(name)) { + cpSync(join(source, name), join(destination, name)); + } + } +} + +export function stageNativeCliAssets(options: { + repository: string; + output: string; + target: NativeTargetSpec; + ts7: string; + compilerVersion: string; +}): Pick { + const { repository, output, target } = options; + const bin = join(output, "bin"); + const lib = join(output, "lib"); + mkdirSync(lib, { recursive: true }); + cpSync(join(repository, "LICENSE"), join(output, "LICENSE")); + const helper = join(lib, "llvm"); + const runtime = join(lib, target.runtimePackPackage.replace("@scriptc/", "")); + copyPackage(join(repository, "packages", target.helper.packageName.replace("@scriptc/", "")), helper, ["bin"]); + copyPackage(join(repository, "packages", target.runtimePackPackage.replace("@scriptc/", "")), runtime, ["artifacts", "runtime-pack.json"]); + const typescript = join(lib, "typescript"); + copyPackage(dirname(dirname(options.ts7)), typescript, ["lib"]); + const sources = join(lib, "runtime-sources"); + copyPackage(join(repository, "packages/runtime"), sources, ["src", "vendor"]); + const declarations = join(lib, "declarations"); + mkdirSync(declarations, { recursive: true }); + for (const name of ["scriptc.d.ts", "scriptc-overrides.d.ts", "scriptc-node-fallback.d.ts"]) { + cpSync(join(repository, "packages/compiler/ambient", name), join(declarations, name)); + } + // This optional Node host runs the built WASI module, never the compiler. + const wasi = join(lib, "wasi"); + mkdirSync(join(wasi, "cli"), { recursive: true }); + writeFileSync(join(wasi, "package.json"), '{"type":"module"}\n'); + for (const name of ["wasi-paths.js", "cli/wasi-paths.js", "cli/wasi-runner.js"]) { + const source = join(repository, "packages/compiler/dist", name); + if (!existsSync(source)) throw new Error("build the compiler workspace before packaging its WASI runner"); + cpSync(source, join(wasi, name)); + } + const identity = JSON.parse(readFileSync(join(typescript, "package.json"), "utf8")) as { version: string }; + writeFileSync(join(output, "THIRD_PARTY_NOTICES"), [ + `scriptc ${options.compilerVersion} includes the existing TypeScript ${identity.version}, LLVM, and C runtime dependencies.`, + "TypeScript licensing: lib/typescript/LICENSE and lib/typescript/NOTICE.txt.", + "LLVM licensing and notices: lib/llvm/LICENSE and lib/llvm/THIRD_PARTY_NOTICES.", + "Runtime dependency licenses: lib/runtime-sources/vendor/*/LICENSE*.", + "", + ].join("\n")); + return { + ts7: relative(bin, join(typescript, "lib", basename(options.ts7))), + llvm_package: relative(bin, helper), + runtime_pack: relative(bin, runtime), + runtime_packs: [{ target: target.name, path: relative(bin, runtime) }], + runtime_sources: relative(bin, sources), + declarations: relative(bin, declarations), + wasi_node_runner: relative(bin, join(wasi, "cli/wasi-runner.js")), + }; +} diff --git a/scripts/package-native-cli.mjs b/scripts/package-native-cli.mjs new file mode 100644 index 00000000..9597d4d1 --- /dev/null +++ b/scripts/package-native-cli.mjs @@ -0,0 +1,57 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repository = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const packages = resolve(process.argv[2] ?? join(repository, "packages")); +const output = resolve(process.argv[3] ?? join(repository, ".scriptc/releases")); +mkdirSync(output, { recursive: true }); +const compiler = JSON.parse(readFileSync(join(packages, "compiler/package.json"), "utf8")); +const runtimes = Object.keys(compiler.optionalDependencies).filter((name) => name.startsWith("@scriptc/runtime-")); +const sums = []; + +for (const name of readdirSync(packages).filter((name) => name.startsWith("cli-")).sort()) { + const source = join(packages, name); + const identity = JSON.parse(readFileSync(join(source, "package.json"), "utf8")); + if (identity.version !== compiler.version) throw new Error(`native CLI version mismatch: ${name}`); + const stage = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-release-")); + try { + cpSync(join(source, "package.json"), join(stage, "package.json")); + const distribution = join(stage, "dist"); + cpSync(join(source, "dist"), distribution, { recursive: true }); + const bin = join(distribution, "bin"); + const binary = join(bin, name.includes("win32-") ? "scriptc.exe" : "scriptc"); + const manifest = JSON.parse(readFileSync(binary + ".json", "utf8")); + const packs = new Map(); + for (const packageName of runtimes) { + const directory = packageName.replace("@scriptc/", ""); + const from = join(packages, directory); + const to = join(distribution, "lib", directory); + const runtime = JSON.parse(readFileSync(join(from, "runtime-pack.json"), "utf8")); + if (runtime.version !== identity.version || runtime.package !== packageName) throw new Error(`runtime version mismatch: ${packageName}`); + for (const member of ["package.json", "runtime-pack.json", "artifacts"]) { + cpSync(join(from, member), join(to, member), { recursive: true }); + } + packs.set(runtime.target.name, { target: runtime.target.name, path: relative(bin, to) }); + } + manifest.runtime_packs = [...packs.values()]; + writeFileSync(binary + ".json", JSON.stringify(manifest, null, 2) + "\n"); + // GitHub artifact transport drops executable modes. Restore the native + // tools before validating and creating the standalone archive. + for (const path of [binary, resolve(bin, manifest.ts7), resolve(bin, manifest.comptime), + join(resolve(bin, manifest.llvm_package), "bin", name.includes("win32-") ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen")]) { + chmodSync(path, 0o755); + } + execFileSync(process.execPath, [join(repository, "scripts/verify-native-cli.mjs"), stage], { stdio: "inherit" }); + const archive = join(output, `scriptc-${identity.version}-${name.slice(4)}.tar.gz`); + execFileSync("tar", ["-czf", archive, "-C", distribution, "."], { stdio: "inherit" }); + const digest = createHash("sha256").update(readFileSync(archive)).digest("hex"); + sums.push(`${digest} ${basename(archive)}`); + console.log(archive); + } finally { rmSync(stage, { recursive: true, force: true }); } +} +if (sums.length === 0) throw new Error("no native CLI packages found"); +writeFileSync(join(output, "SHA256SUMS"), sums.join("\n") + "\n"); diff --git a/scripts/sandbox-command.mjs b/scripts/sandbox-command.mjs index 5efaa907..c29bba1a 100644 --- a/scripts/sandbox-command.mjs +++ b/scripts/sandbox-command.mjs @@ -2,15 +2,40 @@ export const MAX_INLINE_SANDBOX_COMMAND_BYTES = 768; // Outside the shell exit-code range, so an actual exit 125 stays a failure. export const REMOTE_COMMAND_PENDING = 256; +/** A transport failure cannot determine the remote command's exit status. */ +export async function waitForSandboxCommand(probe, { deadline, label, onPending }) { + let lastError; + for (;;) { + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error(`${label} did not confirm completion before its timeout`, { cause: lastError }); + } + try { + await probe(remaining); + return; + } catch (error) { + if (error.remoteExitCode !== undefined && error.remoteExitCode !== REMOTE_COMMAND_PENDING) throw error; + lastError = error; + onPending(error); + const delay = Math.min(1000, deadline - Date.now()); + if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay)); + } + } +} + export const shellQuote = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; /** Keep long shell programs out of the local CLI's argument vector. The * same script records the remote exit status for either transport. */ export function sandboxCommand(command, args, exitMarker) { const statusPath = `/tmp/${exitMarker}.status`; + const logPath = `/tmp/${exitMarker}.log`; + const pendingStatus = statusPath + ".pending"; const script = - `${[command, ...args].map(shellQuote).join(" ")}; scriptc_status=$?; ` + - `printf '%s\\n' "$scriptc_status" > ${shellQuote(statusPath)}; ` + + `( ${[command, ...args].map(shellQuote).join(" ")}; scriptc_status=$?; ` + + `printf '%s\\n' "$scriptc_status" > ${shellQuote(pendingStatus)} ) 2>&1 | tee ${shellQuote(logPath)}; ` + + `mv ${shellQuote(pendingStatus)} ${shellQuote(statusPath)}; ` + + `scriptc_status=$(cat ${shellQuote(statusPath)}); ` + `printf '\\n${exitMarker}%s\\n' "$scriptc_status"`; const scriptPath = `/tmp/${exitMarker}.sh`; const file = Buffer.byteLength(script, "utf8") > MAX_INLINE_SANDBOX_COMMAND_BYTES; @@ -18,6 +43,7 @@ export function sandboxCommand(command, args, exitMarker) { script, scriptPath, statusPath, + logPath, file, argv: file ? ["sh", scriptPath] : ["sh", "-c", script], }; diff --git a/scripts/sandbox-config.mjs b/scripts/sandbox-config.mjs index fffb8bf7..8a90d6f1 100644 --- a/scripts/sandbox-config.mjs +++ b/scripts/sandbox-config.mjs @@ -21,12 +21,20 @@ function loadLocalEnv() { export function sandboxRunnerConfig(env) { loadLocalEnv(); const source = env ?? process.env; + const sandboxTimeout = source.SCRIPTC_SANDBOX_TIMEOUT ?? "45m"; + const duration = /^(\d+)\s*(ms|s|m|h)$/.exec(sandboxTimeout.trim()); + const units = { ms: 1, s: 1000, m: 60_000, h: 3_600_000 }; + const sandboxTimeoutMs = duration ? Number(duration[1]) * units[duration[2]] : 0; + if (!Number.isSafeInteger(sandboxTimeoutMs) || sandboxTimeoutMs <= 0 || sandboxTimeoutMs > 2_147_483_647) { + throw new Error("SCRIPTC_SANDBOX_TIMEOUT must be a positive duration such as 45m or 2h"); + } return { vcpus: source.SCRIPTC_SANDBOX_VCPUS ?? "8", testWorkers: source.SCRIPTC_TEST_WORKERS ?? "4", localTestWorkers: source.SCRIPTC_LOCAL_TEST_WORKERS ?? "2", localCaseShards: source.SCRIPTC_LOCAL_CASE_SHARDS ?? "2", - sandboxTimeout: source.SCRIPTC_SANDBOX_TIMEOUT ?? "45m", + sandboxTimeout, + sandboxTimeoutMs, }; } diff --git a/scripts/sandbox-test.mjs b/scripts/sandbox-test.mjs index 67b0508e..d47ba30b 100644 --- a/scripts/sandbox-test.mjs +++ b/scripts/sandbox-test.mjs @@ -23,7 +23,7 @@ import { filterExistingWorktreePaths, workspaceResetCommand, } from "./worktree-files.mjs"; -import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, shellQuote } from "./sandbox-command.mjs"; +import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, waitForSandboxCommand } from "./sandbox-command.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const laneCaseShardedFiles = [ @@ -206,6 +206,7 @@ const { localTestWorkers, localCaseShards, sandboxTimeout, + sandboxTimeoutMs, } = sandboxRunnerConfig(); if (!["plain", "san", "both"].includes(values.lane)) { @@ -382,14 +383,14 @@ const execIn = async ( args, env = {}, task = "", - wallTimeoutMs = 15 * 60_000, + wallTimeoutMs = sandboxTimeoutMs, workdir = "/workspace", idleTimeoutMs = 90_000, ) => { const envArgs = Object.entries(env).flatMap(([key, value]) => ["--env", `${key}=${value}`]); const exitMarker = `__SCRIPTC_REMOTE_EXIT_${randomBytes(12).toString("hex")}__`; const prepared = sandboxCommand(command, args, exitMarker); - const { statusPath } = prepared; + const { statusPath, logPath } = prepared; const label = task ? `${worker.label} ${task}` : worker.label; if (prepared.file) { const localScript = join(temp, `${exitMarker}.sh`); @@ -416,6 +417,11 @@ const execIn = async ( ...prepared.argv, ]; const deadline = Date.now() + wallTimeoutMs; + const recoveredLog = async () => { + await vercel(["sandbox", "exec", "--timeout", "1m", "--workdir", workdir, worker.name, "tail", "-n", "160", logPath], { + label: `${label} recovered log`, timeoutMs: 60_000, idleTimeoutMs: 30_000, + }).catch((error) => console.warn(`[${label}] could not recover ${logPath}: ${error.message}`)); + }; try { await vercel(commandArgs, { exitMarker, @@ -426,12 +432,10 @@ const execIn = async ( } catch (error) { if (error.remoteExitCode !== undefined) throw error; console.warn(`[${label}] CLI completion was not confirmed (${error.message}); checking the remote command status...`); - for (;;) { - const remaining = deadline - Date.now(); - if (remaining <= 0) throw new Error(`${label} did not confirm completion before its timeout`, { cause: error }); - const probeMarker = `__SCRIPTC_REMOTE_PROBE_${randomBytes(12).toString("hex")}__`; - const probeScript = sandboxStatusCommand(statusPath, probeMarker, Math.min(20, Math.floor(remaining / 1000))); - try { + try { + await waitForSandboxCommand(async (remaining) => { + const probeMarker = `__SCRIPTC_REMOTE_PROBE_${randomBytes(12).toString("hex")}__`; + const probeScript = sandboxStatusCommand(statusPath, probeMarker, Math.min(20, Math.floor(remaining / 1000))); await vercel( ["sandbox", "exec", "--timeout", "1m", "--workdir", workdir, worker.name, "sh", "-c", probeScript], { @@ -441,11 +445,14 @@ const execIn = async ( timeoutMs: Math.min(60_000, remaining), }, ); - return; - } catch (probeError) { - if (probeError.remoteExitCode !== REMOTE_COMMAND_PENDING) throw probeError; - console.log(`[${label}] remote command has not recorded completion; waiting...`); - } + }, { + deadline, label, + onPending: (probeError) => console.warn(probeError.remoteExitCode === REMOTE_COMMAND_PENDING + ? `[${label}] remote command has not recorded completion; waiting...` + : `[${label}] remote status probe was not confirmed (${probeError.message}); retrying...`), + }); + } finally { + await recoveredLog(); } } }; diff --git a/scripts/sync-versions.mjs b/scripts/sync-versions.mjs index f6b27c7a..905ee7d5 100644 --- a/scripts/sync-versions.mjs +++ b/scripts/sync-versions.mjs @@ -24,6 +24,9 @@ for (const pkg of [ "runtime-win32-x64-msvc", "runtime-wasm32-wasi", "runtime-ios-arm64", "runtime-ios-simulator-arm64", "runtime-android-arm64", "compiler", + "cli-darwin-arm64", "cli-darwin-x64", + "cli-linux-x64-gnu", "cli-linux-arm64-gnu", + "cli-linux-x64-musl", "cli-linux-arm64-musl", "cli-win32-x64-msvc", "llvm-darwin-arm64", "llvm-darwin-x64", "llvm-linux-x64-gnu", "llvm-linux-arm64-gnu", "llvm-linux-x64-musl", "llvm-linux-arm64-musl", "llvm-win32-x64-msvc", diff --git a/scripts/verify-native-cli.mjs b/scripts/verify-native-cli.mjs new file mode 100644 index 00000000..58fd1aae --- /dev/null +++ b/scripts/verify-native-cli.mjs @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; + +const root = realpathSync(resolve(process.argv[2] ?? ".")); +const identity = JSON.parse(readFileSync(join(root, "package.json"), "utf8")); +const target = identity.name.replace("@scriptc/cli-", ""); +const distribution = join(root, "dist"); +const binary = join(distribution, "bin", target.startsWith("win32-") ? "scriptc.exe" : "scriptc"); +const toolchain = JSON.parse(readFileSync(binary + ".json", "utf8")); +if (toolchain.schema !== "scriptc.native-toolchain.v1" || toolchain.compiler_version !== identity.version) { + throw new Error("native CLI manifest version does not match the package"); +} +if (statSync(binary).size < 1024) throw new Error("native compiler payload is missing"); +if (!target.startsWith("win32-") && !(statSync(binary).mode & 0o111)) throw new Error("native compiler is not executable"); +const within = (path) => { + const child = relative(distribution, realpathSync(path)); + if (child === ".." || child.startsWith(".." + sep) || isAbsolute(child)) throw new Error(`distribution references external asset: ${path}`); +}; +for (const name of ["ts7", "llvm_package", "runtime_pack", "runtime_sources", "declarations", "comptime", "wasi_node_runner"]) { + const value = toolchain[name]; + if (typeof value !== "string" || isAbsolute(value)) throw new Error(`native toolchain needs a relative ${name}`); + within(resolve(dirname(binary), value)); +} +const pack = JSON.parse(readFileSync(resolve(dirname(binary), toolchain.runtime_pack, "runtime-pack.json"), "utf8")); +if (pack.version !== identity.version || pack.target.name !== toolchain.target) throw new Error("native compiler runtime identity mismatch"); +const seen = new Set(); +for (const runtime of toolchain.runtime_packs ?? []) { + if (seen.has(runtime.target)) throw new Error(`duplicate runtime target: ${runtime.target}`); + seen.add(runtime.target); + if (isAbsolute(runtime.path)) throw new Error("native runtime pack paths must be relative"); + const directory = resolve(dirname(binary), runtime.path); + within(directory); + const manifest = JSON.parse(readFileSync(join(directory, "runtime-pack.json"), "utf8")); + const packageIdentity = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + if (manifest.schema !== "scriptc.runtime-pack.v1" || manifest.target.name !== runtime.target || + manifest.version !== identity.version || packageIdentity.version !== identity.version || + packageIdentity.name !== manifest.package) throw new Error(`native runtime pack identity mismatch: ${runtime.target}`); +} +for (const name of ["LICENSE", "THIRD_PARTY_NOTICES"]) { + if (!existsSync(join(distribution, name))) throw new Error(`native compiler is missing ${name}`); +} +for (const directory of [distribution, join(distribution, "lib/runtime-sources/vendor")]) { + if (readdirSync(directory).some((name) => ["node_modules", ".cache", "seed"].includes(name))) throw new Error(`development files present in ${directory}`); +} +if (process.argv.includes("--run")) { + const version = execFileSync(binary, ["--version"], { encoding: "utf8", env: { ...process.env, PATH: "" } }).trim(); + if (version !== identity.version) throw new Error(`native command reported ${version}, expected ${identity.version}`); +} +console.log(`${identity.name}@${identity.version}: native distribution verified`); diff --git a/tests/corpus/array-fill-unit-values.ts b/tests/corpus/array-fill-unit-values.ts new file mode 100644 index 00000000..67c4a9b1 --- /dev/null +++ b/tests/corpus/array-fill-unit-values.ts @@ -0,0 +1,10 @@ +const empty = new Array(3).fill(null); +console.log(JSON.stringify(empty), empty.map((value, index) => String(index) + ":" + String(value === null)).join(",")); +const mixed: (number | null)[] = [1, 2, 3, 4]; +console.log(JSON.stringify(mixed.fill(null, 1, -1))); +console.log(JSON.stringify(mixed.fill(7, 2))); +let calls = 0; +function nil(): null { calls++; return null; } +console.log(JSON.stringify(mixed.fill(nil())), calls); +const missing = new Array(2).fill(undefined); +console.log(JSON.stringify(missing), missing.map((value, index) => String(index) + ":" + String(value === undefined)).join(",")); diff --git a/tests/corpus/array-optional-number-write-index.ts b/tests/corpus/array-optional-number-write-index.ts new file mode 100644 index 00000000..4db5a342 --- /dev/null +++ b/tests/corpus/array-optional-number-write-index.ts @@ -0,0 +1,12 @@ +const order = [2, 0, 1]; +const mapped: number[] = new Array(3).fill(-1); +order.forEach((oldIndex, newIndex) => { mapped[oldIndex] = newIndex; }); +console.log(mapped.join(",")); + +const candidates = new Map(); +candidates.set("entry", [1, 2]); +const included = new Array(3).fill(false); +for (const candidate of candidates.get("entry") ?? new Array()) { + if (included[candidate] !== true) included[candidate] = true; +} +console.log(included.join(",")); diff --git a/tests/corpus/array-sort-unknown-comparator.ts b/tests/corpus/array-sort-unknown-comparator.ts new file mode 100644 index 00000000..662f92c2 --- /dev/null +++ b/tests/corpus/array-sort-unknown-comparator.ts @@ -0,0 +1,16 @@ +class Hit { + distance: number; + label: string; + constructor(distance: number, label: string) { this.distance = distance; this.label = label; } +} +function compare(a: unknown, b: unknown): number { + return (a as Hit).distance - (b as Hit).distance; +} +const first = new Hit(3, 'far'), second = new Hit(1, 'near'), third = new Hit(1, 'tie'); +const values = [first, second, third]; +console.log('identity', values.sort(compare) === values); +for (const item of values) console.log(item.label, item.distance); +const copy = values.toSorted(compare); +console.log('copy', copy !== values, copy[0] === second); +function numeric(a: unknown, b: unknown): number { return Number(a) - Number(b); } +console.log([9, 1, 3].sort(numeric).join(',')); diff --git a/tests/corpus/array-truncate-reuse.ts b/tests/corpus/array-truncate-reuse.ts new file mode 100644 index 00000000..d87377a7 --- /dev/null +++ b/tests/corpus/array-truncate-reuse.ts @@ -0,0 +1,31 @@ +const numbers: number[] = []; +const strings: string[] = []; +for (let i = 0; i < 4096; i++) { + numbers.push(i); + strings.push("value:" + i); +} +numbers.length = 2; +strings.length = 2; +console.log(JSON.stringify(numbers), JSON.stringify(strings)); +for (let i = 0; i < 32; i++) { + numbers.length = 0; + strings.length = 0; + numbers.push(i); + strings.push("new:" + i); + if (i % 8 === 0) console.log(JSON.stringify(numbers), JSON.stringify(strings)); +} +numbers.length = 8; +strings.length = 8; +console.log(JSON.stringify(numbers), JSON.stringify(strings)); +numbers[7] = 42; +strings[7] = "last"; +numbers.length = 4; +strings.length = 4; +numbers.length = 8; +strings.length = 8; +console.log(JSON.stringify(numbers), JSON.stringify(strings)); +numbers[2 ** 24] = 99; +strings[2 ** 24] = "sparse"; +numbers.length = 1; +strings.length = 1; +console.log(JSON.stringify(numbers), JSON.stringify(strings)); diff --git a/tests/corpus/class-callback-replacement.js b/tests/corpus/class-callback-replacement.js new file mode 100644 index 00000000..7216deb2 --- /dev/null +++ b/tests/corpus/class-callback-replacement.js @@ -0,0 +1,55 @@ +class Signal { + constructor(label) { this.label = label; } + changed() { console.log('default', this.label); } + emit() { this.changed(); } + replace(callback) { this.changed = callback; } +} +class Child extends Signal { + changed() { console.log('child', this.label); } + parent() { super.changed(); } +} +const first = new Signal('a'), second = new Signal('b'), child = new Child('c'); +const original = first.changed; +first.emit(); second.emit(); child.emit(); +let calls = 0; +function callback() { calls++; console.log('callback', this.label, calls); } +first.replace(callback); child.replace(callback); +console.log('identity', first.changed === callback, child.changed === callback); +first.emit(); second.emit(); child.emit(); child.parent(); +original.call(second); +console.log('own', Object.hasOwn(first, 'changed'), Object.hasOwn(second, 'changed')); +const saved = first.changed; +first.replace(() => console.log('replacement')); +saved.call(second); +first.emit(); +function dynamic(value) { value.changed(); } +dynamic(first); dynamic(child); +first.replace(undefined); +try { first.emit(); } catch (error) { console.log('invalid', error instanceof TypeError); } +function next() { first.replace(() => console.log('next')); return 'argument'; } +first.replace((value) => console.log('selected', value)); +function invoke(value) { value.changed(next()); } +invoke(first); +first.emit(); +const fresh = new Signal('fresh'); +function install() { fresh.replace(() => console.log('installed')); return 1; } +function invokeFresh(value) { value.changed(install()); } +invokeFresh(fresh); +fresh.emit(); +class PrivateSignal { + #changed() { console.log('private'); } + emit() { this.#changed(); } +} +const hidden = new PrivateSignal(); +function shadowPrivate(value) { value['#changed'] = () => console.log('public'); } +shadowPrivate(hidden); +hidden.emit(); +let created = 0; +function temporary() { created++; return new Signal('temporary-' + created); } +temporary().emit(); +temporary().replace(callback); +console.log('created', created); +let current = new Signal('retained'); +current.replace(function () { current = new Signal('new'); console.log('receiver', this.label); }); +current.emit(); +current.emit(); diff --git a/tests/corpus/class-checked-upcast.ts b/tests/corpus/class-checked-upcast.ts new file mode 100644 index 00000000..b19df9a4 --- /dev/null +++ b/tests/corpus/class-checked-upcast.ts @@ -0,0 +1,24 @@ +class Base { + value = 2; + read(): number { return this.value; } +} +class Middle extends Base {} +class Derived extends Middle { + extra = 3; + read(): number { return this.value + this.extra; } +} +class Unrelated { value = 8; } +const derived = new Derived(); +const boxed: unknown = derived; +const base = boxed as Base; +console.log('base', base === derived, base instanceof Derived, base.read()); +base.value = 7; +console.log('shared', derived.value, base.read()); +const array: unknown = [derived]; +const bases = array as Base[]; +console.log('array', bases[0] === derived, bases[0]!.read()); +const candidate: unknown = derived; +const optional = candidate as Base | undefined; +console.log('union', optional === derived, optional!.read()); +const unrelated: unknown = new Unrelated(); +console.log('unrelated', unrelated instanceof Base); diff --git a/tests/corpus/class-data-descriptors.js b/tests/corpus/class-data-descriptors.js new file mode 100644 index 00000000..5e7053e5 --- /dev/null +++ b/tests/corpus/class-data-descriptors.js @@ -0,0 +1,56 @@ +class Point { + constructor(x) { + this.x = x; + const result = Object.defineProperty(this, 'id', { value: x + 10 }); + console.log(result === this); + Object.defineProperties(this, { + label: { value: 'point', enumerable: true, configurable: true, writable: true }, + meta: { value: { count: 1 }, configurable: true }, + }); + } +} +class Child extends Point {} +const point = new Point(2); +const child = new Child(3); +console.log(point.id, child.id, point.label, point.meta.count); +console.log(Object.keys(point).join(','), Object.hasOwn(point, 'id')); +const id = Object.getOwnPropertyDescriptor(point, 'id'); +console.log(id.value, id.writable, id.enumerable, id.configurable); +point.meta.count = 4; +point.label = 'changed'; +console.log(point.meta.count, child.meta.count, point.label); +try { point.id = 7; } catch (error) { console.log(error instanceof TypeError); } +try { Object.defineProperty(point, 'id', { value: 9 }); } catch (error) { console.log(error instanceof TypeError); } +const returned = Object.defineProperties(point, { label: { value: 'final', writable: false } }); +console.log(returned === point, point.label); +const label = Object.getOwnPropertyDescriptor(point, 'label'); +console.log(label.value, label.writable, label.enumerable, label.configurable); +/** @param {unknown} value */ +function stringify(value) { return JSON.stringify(value); } +console.log(stringify(point)); +function read(value) { return value.id + ':' + value.label; } +console.log(read(point), read(child)); +const empty = Object.getOwnPropertyDescriptor(point, 'missing'); +console.log(empty === undefined); +const events = []; +function target() { events.push('target'); return point; } +function description() { events.push('value'); return 5; } +Object.defineProperty(target(), 'extra', { value: description(), enumerable: true }); +console.log(events.join(','), point.extra); +// Snapshot/commit operations must retain nonenumerable descriptors too. +/** @param {unknown} value */ +function merge(value) { Object.assign(value, { label: 'blocked' }); } +try { merge(point); } catch (error) { console.log(error instanceof TypeError); } +console.log(point.id, Object.getOwnPropertyDescriptor(point, 'id').writable); +/** @param {unknown} value */ +function extend(value) { Object.assign(value, { extra2: 7 }); } +extend(point); +console.log(point.extra2, point.id, Object.getOwnPropertyDescriptor(point, 'id').enumerable); +class Cycle { + constructor() { this.x = 1; Object.defineProperty(this, 'self', { value: this }); } +} +function cycle() { + const value = new Cycle(); + console.log(value.self === value, Object.keys(value).join(',')); +} +cycle(); diff --git a/tests/corpus/class-instance-constructor.js b/tests/corpus/class-instance-constructor.js new file mode 100644 index 00000000..41a66b8b --- /dev/null +++ b/tests/corpus/class-instance-constructor.js @@ -0,0 +1,24 @@ +class Value { + constructor(x = 1) { this.x = x; } + clone() { return new this.constructor(this.x); } + reset() { return new this.constructor(); } +} +class Child extends Value { + constructor(x = 2, extra = 3) { super(x); this.extra = extra; } +} +class Leaf extends Child {} +const a = new Value(4), b = new Child(5), c = new Leaf(6); +for (const source of [a, b, c]) { + const copy = source.clone(); + console.log(copy !== source, copy.x, copy instanceof Value, copy instanceof Child, copy instanceof Leaf); + console.log(source.reset().x); +} +let order = ''; +function receiver() { order += 'r'; return b; } +function argument() { order += 'a'; return 7; } +const selected = new (receiver().constructor)(argument()); +console.log(order, selected.x, selected instanceof Child); +class Other { constructor(x = 9) { this.x = x; } } +/** @returns {Value | Other} */ +function choose(flag) { return new (flag ? Value : Other)(argument()); } +console.log(choose(true).x, choose(false).x, order); diff --git a/tests/corpus/class-static-data.js b/tests/corpus/class-static-data.js new file mode 100644 index 00000000..16ef87f7 --- /dev/null +++ b/tests/corpus/class-static-data.js @@ -0,0 +1,20 @@ +// @ts-nocheck +class Defaults { constructor(value = Defaults.value) { this.value = value; } } +class Child extends Defaults {} +const Alias = Defaults; +console.log(Defaults.value, Child.value, Defaults.hasOwnProperty('value')); +let effects = 0; +function initial() { effects++; return 'first'; } +Defaults.value = initial(); +console.log(effects, new Defaults().value, Child.value, Child.hasOwnProperty('value')); +Alias.value = 'second'; +console.log(Defaults.value, new Child().value); +Child.value = 'child'; +Defaults.value = 'third'; +console.log(Defaults.value, Child.value, Child.hasOwnProperty('value')); +Child.value = undefined; +console.log(Child.value, Defaults.value, Child.hasOwnProperty('value')); +Defaults.options = { scale: 2 }; +console.log(Child.options.scale); +Child.options.scale = 3; +console.log(Defaults.options.scale); diff --git a/tests/corpus/closure-nullable-union-return.ts b/tests/corpus/closure-nullable-union-return.ts new file mode 100644 index 00000000..9042cb87 --- /dev/null +++ b/tests/corpus/closure-nullable-union-return.ts @@ -0,0 +1,65 @@ +type ValueType = { kind: "string" } | { kind: "dyn" }; +type Expr = + | { kind: "literal"; value: string; type: ValueType } + | { kind: "object"; fields: { key: string; value: Expr }[]; type: ValueType } + | { kind: "call"; args: Expr[]; name: string; type: ValueType }; +const DYNAMIC: ValueType = { kind: "dyn" }; + +function assemble(groups: string[][]): Expr { + let fields: { key: string; value: Expr }[] = []; + let acc: Expr | null = null; + const flush = (): void => { + if (fields.length === 0) return; + const chunk: Expr = { kind: "object", fields, type: DYNAMIC }; + acc = acc === null ? chunk : { kind: "call", args: [acc, chunk], name: "merge", type: DYNAMIC }; + fields = []; + }; + for (const group of groups) { + for (const key of group) fields.push({ key, value: { kind: "literal", value: key, type: { kind: "string" } } }); + flush(); + if (group.length > 10) { + acc ??= { kind: "object", fields: [], type: DYNAMIC }; + acc = { kind: "call", args: [acc], name: "large", type: DYNAMIC }; + } + } + if (acc !== null) return acc; + return { kind: "object", fields: [], type: DYNAMIC }; +} + +for (const groups of [[], [["headers"]], [["a"], ["b", "c"]]]) { + const expression = assemble(groups); + console.log(expression.kind, expression.type.kind, JSON.stringify(expression)); +} + +function inspect(expression: Expr): void { + console.log(expression.kind, expression.type.kind, JSON.stringify(expression)); +} + +async function transfer(): Promise { + let value: Expr = { kind: "call", args: [], name: "initial", type: DYNAMIC }; + const replacement: Expr = { kind: "object", fields: [], type: DYNAMIC }; + const update = (): void => { value = replacement; }; + update(); + inspect(value); + let assigned: Expr = { kind: "literal", value: "initial", type: DYNAMIC }; + assigned = value; + inspect(assigned); + const values: Expr[] = [value]; + const holder: { value: Expr | null } = { value }; + inspect(values[0]!); + if (holder.value !== null) inspect(holder.value); + console.log(values[0] === replacement, holder.value === replacement); + return (value); +} + +inspect(await transfer()); + +function subset(value: Expr): Expr | null { + if (value.kind === "literal") { + const selected: Extract | null = value; + return selected; + } + return null; +} +const selected = subset({ kind: "literal", value: "subset", type: DYNAMIC }); +if (selected !== null) inspect(selected); diff --git a/tests/corpus/crypto-hash-optional-input.ts b/tests/corpus/crypto-hash-optional-input.ts new file mode 100644 index 00000000..1e161529 --- /dev/null +++ b/tests/corpus/crypto-hash-optional-input.ts @@ -0,0 +1,17 @@ +import { createHash, createHmac } from "node:crypto"; + +const environment: Record = { present: "hello" }; +for (const name of ["present", "missing"]) { + const value = environment[name]; + const text: string = value ?? ""; + console.log(createHash("sha256").update(text).digest("hex")); + console.log(createHmac("sha256", "secret").update(text).digest("hex")); + if (value !== undefined) console.log(createHash("sha256").update(value).digest("hex")); +} + +const buffers: (Buffer | undefined)[] = [Buffer.from("hello"), undefined]; +for (let index = 0; index < buffers.length; index++) { + const buffer = buffers[index] ?? Buffer.from(""); + console.log(createHash("sha256").update(buffer).digest("hex")); + console.log(createHmac("sha256", "secret").update(buffer).digest("hex")); +} diff --git a/tests/corpus/for-multiple-bindings.ts b/tests/corpus/for-multiple-bindings.ts new file mode 100644 index 00000000..c9185a74 --- /dev/null +++ b/tests/corpus/for-multiple-bindings.ts @@ -0,0 +1,24 @@ +const callbacks: (() => string)[] = []; +for (let i = 0, label = 'a'; i < 3; i++) { + callbacks.push(() => `${i}:${label}`); + label += 'b'; + if (i === 1) continue; +} +console.log(callbacks.map((fn) => fn()).join(',')); +let initial: () => number = () => -1; +const body: (() => number)[] = []; +for (let i = 0, save = (initial = () => i); i < 2; i++) { + body.push(() => i); +} +console.log(initial(), body.map((fn) => fn()).join(',')); +let sum = 0; +for (let i = 1, end = i + 3; i < end; i++) sum += i; +console.log(sum); +let escaped: () => number = () => -1; +outer: for (let i = 0, n = 3; i < n; i++) { + for (let j = 0, m = 2; j < m; j++) { + escaped = () => i + j + n + m; + if (j === 1) break outer; + } +} +console.log(escaped()); diff --git a/tests/corpus/fs-remove-readonly.ts b/tests/corpus/fs-remove-readonly.ts new file mode 100644 index 00000000..64711c10 --- /dev/null +++ b/tests/corpus/fs-remove-readonly.ts @@ -0,0 +1,18 @@ +import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = mkdtempSync(join(tmpdir(), "scriptc-readonly-rm-")); +try { + const file = join(root, "readonly.txt"); + writeFileSync(file, "keep until removed"); + chmodSync(file, 0o400); + rmSync(file); + console.log(existsSync(file)); + const nested = join(root, "tree", "nested"); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, "payload"), "read-only payload"); + chmodSync(join(nested, "payload"), 0o400); + rmSync(join(root, "tree"), { recursive: true, force: true }); + console.log(existsSync(join(root, "tree"))); +} finally { rmSync(root, { recursive: true, force: true }); } diff --git a/tests/corpus/fs-write-string-bytes-union.ts b/tests/corpus/fs-write-string-bytes-union.ts new file mode 100644 index 00000000..ef04a497 --- /dev/null +++ b/tests/corpus/fs-write-string-bytes-union.ts @@ -0,0 +1,26 @@ +import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const directory = mkdtempSync(join(tmpdir(), "scriptc-write-union-")); +const path = join(directory, "data"); +function write(value: string | Uint8Array): void { writeFileSync(path, value); } +function append(value: string | Uint8Array): void { appendFileSync(path, value); } +let order = ""; +function destination(): string { order += "path "; return path; } +function data(bytes: boolean): string | Uint8Array { + order += "data "; + return bytes ? Buffer.from([0, 128, 255]) : "é"; +} +try { + for (const value of ["text", new Uint8Array([0, 128, 255]), Buffer.from("é"), "", new Uint8Array(0)]) { + write(value); + append("!"); + append(Buffer.from([0, 255])); + console.log(readFileSync(path).toString("hex")); + } + writeFileSync(destination(), data(true)); + appendFileSync(destination(), data(false)); + console.log(order); + console.log(readFileSync(path).toString("hex")); +} finally { rmSync(directory, { recursive: true, force: true }); } diff --git a/tests/corpus/js-chained-native-results.mjs b/tests/corpus/js-chained-native-results.mjs new file mode 100644 index 00000000..4746276d --- /dev/null +++ b/tests/corpus/js-chained-native-results.mjs @@ -0,0 +1,10 @@ +const Vector = class Vector { + constructor(x = 0) { this.x = x; } + copy(value) { this.x = value.x; return this; } + add(value) { this.x += value.x; return this; } +}; +let calls = 0; +function make() { calls++; return new Vector(); } +const result = make().copy(new Vector(2)).add(new Vector(3)); +console.log(result.x, calls); +console.log(make().copy(result).add(result).x, calls); diff --git a/tests/corpus/js-checked-array-iterator.js b/tests/corpus/js-checked-array-iterator.js new file mode 100644 index 00000000..271d8f54 --- /dev/null +++ b/tests/corpus/js-checked-array-iterator.js @@ -0,0 +1,20 @@ +/** @param {unknown} value */ +function checked(value) { return value; } +function iteratorOf(source) { return source[Symbol.iterator](); } +/** @type {number[]} */ +const values = [1, 2]; +const source = checked(values); +console.log(source[Symbol.iterator] === Array.prototype.values); +console.log(Symbol.iterator in source, Object.hasOwn(source, Symbol.iterator)); +const iterator = iteratorOf(source); +console.log(iterator[Symbol.iterator]() === iterator); +console.log(iterator.next().value); +values.push(3); +console.log(iterator.next().value, iterator.next().value, iterator.next().done); +values.push(4); +console.log(iterator.next().done); +console.log(Array.from(source).join(",")); +for (const value of source) { + console.log("live", value); + if (value === 1) values.push(5); +} diff --git a/tests/corpus/js-checked-array-reference.js b/tests/corpus/js-checked-array-reference.js new file mode 100644 index 00000000..ecb2225e --- /dev/null +++ b/tests/corpus/js-checked-array-reference.js @@ -0,0 +1,45 @@ +function append(values) { values.push(3); return values; } +function pair(first, second) { console.log('same', first === second); first.push(4); } +function variadic(...values) { values[0].push(5); return values[0]; } +function fromArguments() { arguments[0].push(6); return arguments[0]; } +const values = [1, 2]; +console.log('call', append(values) === values, values.join(',')); +pair(values, values); +console.log('pair', values.join(',')); +console.log('rest', variadic(values) === values, values.join(',')); +console.log('arguments', fromArguments(values) === values, values.join(',')); +function wrap() { return values; } +console.log('return', wrap() === values); +const object = { values }; +object.values.push(7); +console.log('property', object.values === values, values.join(',')); +class Empty { visit() {} } +class Collector extends Empty { visit(target) { target.push(8); } } +function dispatch(object, target) { return object.visit(target); } +const result = dispatch(new Collector(), values); +console.log('void', result === undefined, values.join(',')); + +/** @param {unknown} input */ +function checked(input) { return input; } +const shared = checked(values); +console.log('concat', shared.concat(values).join(',')); +function add(a, b) { return a + b; } +const argumentsList = [10, 20]; +console.log('apply', checked(add).apply(null, argumentsList)); +function names(input) { + const result = []; + for (const name in input) result.push(name); + return result.join(','); +} +console.log('keys', names(shared)); +const entries = [['first', 1], ['second', 2]]; +const dictionary = Object.fromEntries(checked(entries)); +console.log('entries', dictionary.first, dictionary.second); +const sparse = [1]; +sparse.length = 3; +sparse[2] = 5; +console.log('sparse', checked(sparse).join(',')); +/** @type {any} */ +const flexible = [1, 2]; +flexible[0] = 'changed'; +console.log('fresh', flexible.join(',')); diff --git a/tests/corpus/js-checked-number-negation.mjs b/tests/corpus/js-checked-number-negation.mjs new file mode 100644 index 00000000..8989abd2 --- /dev/null +++ b/tests/corpus/js-checked-number-negation.mjs @@ -0,0 +1,4 @@ +const input = JSON.parse('[3, -4, 0, -0]'); +for (const value of input) console.log(-value, Object.is(-value, -0)); +const object = { get() { return JSON.parse('5'); } }; +console.log(-object.get()); diff --git a/tests/corpus/js-checked-sort-result.js b/tests/corpus/js-checked-sort-result.js new file mode 100644 index 00000000..6d703219 --- /dev/null +++ b/tests/corpus/js-checked-sort-result.js @@ -0,0 +1,10 @@ +function compare(a, b) { return a.value - b.value; } +class Item { + constructor(value) { this.value = value; } +} +const items = [new Item(3), new Item(1), new Item(2)]; +console.log(items.sort(compare) === items); +console.log(items[0].value, items[1].value, items[2].value); +const bigints = [new Item(3n), new Item(1n)]; +try { bigints.sort(compare); } +catch (error) { console.log(error.name); } diff --git a/tests/corpus/js-class-added-parameters.js b/tests/corpus/js-class-added-parameters.js new file mode 100644 index 00000000..210f73dd --- /dev/null +++ b/tests/corpus/js-class-added-parameters.js @@ -0,0 +1,22 @@ +class Base { + visit() { console.log("base"); } + dispatch(first, second, third) { this.visit(first, second, third); } +} +class Pair extends Base { + visit(first, second) { console.log("pair", first, second); } +} +class Triple extends Pair { + visit(first, second, third) { console.log("triple", first, second, third); } + parent(first, second) { super.visit(first, second); } +} +class Short extends Base { + visit(first) { console.log("short", first); } +} +function value(n) { console.log("arg", n); return n; } +for (const object of [new Base(), new Pair(), new Triple(), new Short()]) { + object.dispatch(value(1), value(2), value(3)); + object.visit(); +} +new Triple().parent(4, 5); +const method = new Pair().visit; +method(6, 7); diff --git a/tests/corpus/js-class-computed-fields.js b/tests/corpus/js-class-computed-fields.js new file mode 100644 index 00000000..20764751 --- /dev/null +++ b/tests/corpus/js-class-computed-fields.js @@ -0,0 +1,10 @@ +class Vector { constructor() { this.x = 1; this.y = 2; this.z = 3; } } +const value = new Vector(); +let order = ''; +function receiver() { order += 'r'; return value; } +function key() { order += 'k'; return 'y'; } +function number() { order += 'v'; return 8; } +receiver()[key()] = number(); +console.log(order, value.x, value.y, value.z); +for (const name of ['x', 'z', 'extra']) value[name] = 4; +console.log(value.x, value.z, value.extra); diff --git a/tests/corpus/js-class-fluent-overrides.js b/tests/corpus/js-class-fluent-overrides.js new file mode 100644 index 00000000..c3ffbf4e --- /dev/null +++ b/tests/corpus/js-class-fluent-overrides.js @@ -0,0 +1,28 @@ +class Base { + constructor() { this.value = 1; } + copy(source) { this.value = source.value; return this; } + dispatch(source) { return this.copy(source); } +} +class Child extends Base { + constructor() { super(); this.extra = 2; } + copy(source) { super.copy(source); this.extra = source.extra; return this; } + childOnly() { return this.extra; } +} +class Leaf extends Child { + constructor() { super(); this.leaf = 3; } + copy(source) { + super.copy(source); + if (source.leaf === 0) return (this); + this.leaf = source.leaf; + return this; + } +} +const source = new Leaf(); +source.value = 4; source.extra = 5; source.leaf = 6; +const target = new Leaf(); +console.log('direct', target.copy(source) === target, target.value, target.extra, target.leaf); +console.log('chain', target.copy(source).childOnly(), target.copy(source).leaf); +source.extra = 8; +console.log('virtual', target.dispatch(source) === target, target.extra); +source.leaf = 0; +console.log('early', target.copy(source) === target, target.leaf); diff --git a/tests/corpus/js-class-subclass-properties.js b/tests/corpus/js-class-subclass-properties.js new file mode 100644 index 00000000..19e76030 --- /dev/null +++ b/tests/corpus/js-class-subclass-properties.js @@ -0,0 +1,43 @@ +class Base { + constructor() { this.base = 1; } + read(key) { return this[key]; } + write(key, value) { this[key] = value; } + named() { return this.extra; } + attach(value) { this.extra = value; } + remember(key, value) { + if (this.cache === undefined) this.cache = {}; + this.cache[key] = value; + } + recall(key) { return this.cache[key]; } + saved() { + const cache = this.cache; + if (cache === undefined) return 'empty'; + return Object.keys(cache).join(','); + } +} +class Child extends Base { + constructor() { super(); this.extra = 2; this.storage = 3; } + get doubled() { return this.storage * 2; } + set doubled(value) { this.storage = value / 2; } +} +class Leaf extends Child { + constructor() { super(); this.leaf = 4; } + get doubled() { return this.storage * 3; } + set doubled(value) { this.storage = value / 3; } +} +const base = new Base(), child = new Child(), leaf = new Leaf(); +console.log('absent', base.read('extra'), Object.keys(base).join(',')); +base.attach({answer: 42}); +console.log('added', base.read('extra').answer, Object.keys(base).join(',')); +console.log('inherited', child.read('extra'), child.named(), leaf.read('leaf')); +child.write('extra', 9); leaf.write('leaf', 10); +child.write('doubled', 12); leaf.write('doubled', 12); +console.log('writes', child.extra, leaf.leaf, child.storage, leaf.storage); +console.log('accessors', child.read('doubled'), leaf.read('doubled')); +let order = ''; +function receiver() { order += 'r'; return leaf; } +function key() { order += 'k'; return 'leaf'; } +console.log('read', receiver()[key()], order); +base.remember('first', 1); base.remember('second', {value: 2}); +console.log('cache', base.recall('first'), base.recall('second').value, Object.keys(base.cache).join(',')); +console.log('saved', base.saved(), child.saved()); diff --git a/tests/corpus/js-derived-constructor-hoisting.js b/tests/corpus/js-derived-constructor-hoisting.js new file mode 100644 index 00000000..3e1a89b3 --- /dev/null +++ b/tests/corpus/js-derived-constructor-hoisting.js @@ -0,0 +1,14 @@ +class Base { constructor() { this.x = 1; } } +class Child extends Base { + constructor() { + before(); + super(); + let count = 2; + after(); + after(); + this.x = count; + function before() { console.log('before super'); } + function after() { count++; console.log('after', count); } + } +} +console.log(new Child().x); diff --git a/tests/corpus/js-dormant-array-write.js b/tests/corpus/js-dormant-array-write.js new file mode 100644 index 00000000..c85db380 --- /dev/null +++ b/tests/corpus/js-dormant-array-write.js @@ -0,0 +1,12 @@ +class Coordinates { + constructor() { this.value = 3; } + /** @param {number[]} values */ + toArray(values = []) { + values[0] = this.value; + values[1] = "label"; + return values; + } +} +const coordinates = new Coordinates(); +const reader = { read(source, key) { return source[key]; } }; +console.log(reader.read(coordinates, "value"), coordinates.value); diff --git a/tests/corpus/js-dormant-overrides.js b/tests/corpus/js-dormant-overrides.js new file mode 100644 index 00000000..b8739851 --- /dev/null +++ b/tests/corpus/js-dormant-overrides.js @@ -0,0 +1,11 @@ +class Shape { + constructor(size = 1) { this.size = size; } + copy(source, recursive = true) { return source.size + (recursive ? 1 : 0); } + area() { return this.size * this.size; } +} +class ColoredShape extends Shape { + constructor(size = 1) { super(size); this.color = 'red'; } + copy(source, recursive) { return source.size + (recursive ? 1 : 0); } +} +const shape = new ColoredShape(3); +console.log(shape.area(), shape.color, new Shape(2).copy(shape)); diff --git a/tests/corpus/js-inherited-parameter-abi.js b/tests/corpus/js-inherited-parameter-abi.js new file mode 100644 index 00000000..89a4c399 --- /dev/null +++ b/tests/corpus/js-inherited-parameter-abi.js @@ -0,0 +1,17 @@ +class Base { + /** @param {boolean} [force] */ + update(force) { console.log('base', force === undefined, force === true); } + forward(force) { this.update(force); } +} +class Child extends Base { + update(force) { console.log('child', force === undefined, force === false); super.update(force); } +} +const base = new Base(), child = new Child(); +base.update(); child.update(); child.update(false); child.forward(true); +class Defaults { + run(value = 5) { console.log('default', value); } +} +class NoDefault extends Defaults { + run(value) { console.log('no default', value === undefined); } +} +new Defaults().run(); new NoDefault().run(); new NoDefault().run(3); diff --git a/tests/corpus/js-nested-native-descriptors.js b/tests/corpus/js-nested-native-descriptors.js new file mode 100644 index 00000000..750133f9 --- /dev/null +++ b/tests/corpus/js-nested-native-descriptors.js @@ -0,0 +1,18 @@ +class Position { constructor(x) { this.x = x; } } +class Scene { + constructor(value) { + Object.defineProperties(this, { + position: { value, enumerable: true, configurable: true }, + hidden: { value: new Position(3) }, + }); + } +} +const position = new Position(1); +const scene = new Scene(position); +console.log(scene.position === position, scene.hidden.x, Object.keys(scene).join(',')); +position.x = 2; +console.log(scene.position.x); +const descriptor = Object.getOwnPropertyDescriptor(scene, 'position'); +console.log(descriptor.value === position, descriptor.writable, descriptor.enumerable, descriptor.configurable); +// @ts-expect-error JavaScript typeof permits an unresolvable reference. +console.log(typeof scriptcMissingHostGlobal, typeof (scriptcMissingHostGlobal)); diff --git a/tests/corpus/js-nullish-inherited-fields.mjs b/tests/corpus/js-nullish-inherited-fields.mjs new file mode 100644 index 00000000..11b0dd9a --- /dev/null +++ b/tests/corpus/js-nullish-inherited-fields.mjs @@ -0,0 +1,40 @@ +class Value { + constructor() { this.id = 7; } +} + +class Base { + /** @param {Record} values */ + load(values) { this.value = values.missing || null; } +} + +class Derived extends Base { + constructor() { + super(); + /** @type {?Value} */ + this.value = null; + } +} + +const base = new Base(); +console.log('initial', base.value === undefined); +base.load({}); +console.log('missing', base.value === null); +const value = new Value(); +base.load({ missing: value }); +console.log('present', base.value === value); +const derived = new Derived(); +console.log('derived', derived.value === null); +derived.load({ missing: value }); +console.log('identity', derived.value === value); +derived.load({}); +console.log('reset', derived.value === null); + +class Optional { + /** @param {Value} [value] */ + constructor(value) { + /** @type {Value} */ + this.value = value; + } +} +console.log('omitted', new Optional().value === undefined); +console.log('supplied', new Optional(value).value === value); diff --git a/tests/corpus/js-object-returning-constructors.mjs b/tests/corpus/js-object-returning-constructors.mjs new file mode 100644 index 00000000..16f1a63a --- /dev/null +++ b/tests/corpus/js-object-returning-constructors.mjs @@ -0,0 +1,23 @@ +function Counter(start = 2) { + let value = start; + return { + next() { return ++value; }, + current() { return value; }, + }; +} + +let calls = 0; +function initial() { calls++; return 10; } +const first = new Counter(initial()); +const second = new Counter(); +console.log(calls, first.current(), second.current()); +console.log(first.next(), first.next(), second.next()); +console.log(first === second); +function readFirst() { return first.current(); } +console.log('global', readFirst()); + +function Choice(flag = false) { + if (flag) return { value: 3 }; + return { value: 9 }; +} +console.log(new Choice(true).value, new Choice().value); diff --git a/tests/corpus/js-object-shared-arrays.js b/tests/corpus/js-object-shared-arrays.js new file mode 100644 index 00000000..1fb79aa3 --- /dev/null +++ b/tests/corpus/js-object-shared-arrays.js @@ -0,0 +1,11 @@ +const shared = [1, 2, 3]; +const a = {values: shared}, b = {nested: {values: shared}}; +console.log('identity', a.values === b.nested.values, a.values === shared); +a.values[1] = 8; +console.log('write', shared[1], b.nested.values[1]); +shared.push(4); +console.log('push', a.values.length, b.nested.values.join(',')); +b.nested.values.pop(); +console.log('pop', shared.length, a.values.join(',')); +const separate = {values: [1, 8, 3]}; +console.log('distinct', separate.values === a.values); diff --git a/tests/corpus/js-optional-class-fields.js b/tests/corpus/js-optional-class-fields.js new file mode 100644 index 00000000..f51890b4 --- /dev/null +++ b/tests/corpus/js-optional-class-fields.js @@ -0,0 +1,26 @@ +class Vector { + constructor() { this.x = 1; this.y = 2; } + get doubled() { return this.x * 2; } + set doubled(value) { this.x = value / 2; } +} +class Holder { + initialize() { this.vector = new Vector(); } +} +const holder = new Holder(); +holder.initialize(); +let calls = 0; +function receiver() { calls++; return holder; } +receiver().vector.x = 7; +holder.vector.y += 3; +holder.vector.doubled = 18; +console.log('nested', holder.vector.x, holder.vector.y, calls); + +class Positioned { + constructor() { Object.defineProperty(this, 'position', {value: new Vector(), enumerable: true}); } +} +const objects = [new Positioned()]; +console.log('array', objects[0].position.x); +objects[0].position.x = 4; +console.log('write', objects[0].position.x); +try { console.log(objects[1].position.x); } +catch (error) { console.log('missing', error instanceof TypeError, error.message); } diff --git a/tests/corpus/js-primitive-union-addition.js b/tests/corpus/js-primitive-union-addition.js new file mode 100644 index 00000000..69106639 --- /dev/null +++ b/tests/corpus/js-primitive-union-addition.js @@ -0,0 +1,9 @@ +const table = ['a', 'b']; +function pair(i = 0, j = 0) { return table[i] + table[j]; } +function chain(i = 0, j = 0, k = 0) { return table[i] + table[j] + table[k]; } +console.log(pair(0, 1), pair(0, 4), pair(4, 0), pair(4, 5)); +console.log(chain(0, 1, 0), chain(4, 5, 0), chain(4, 5, 6)); +console.log(chain(0, 4, 1)); +let calls = 0; +function take(i = 0) { calls++; return table[i]; } +console.log(take(4) + take(5) + take(0), calls); diff --git a/tests/corpus/js-specialized-field-method.js b/tests/corpus/js-specialized-field-method.js new file mode 100644 index 00000000..4a3a3a32 --- /dev/null +++ b/tests/corpus/js-specialized-field-method.js @@ -0,0 +1,8 @@ +class Vector { constructor(x = 0) { this.x = x; } dot(other) { return this.x * other.x; } } +class Plane { constructor() { this.normal = new Vector(2); } } +class Ray { + constructor() { this.direction = new Vector(3); } + distance(plane) { return plane.normal.dot(this.direction); } +} +const ray = new Ray(); +console.log(ray.distance(new Plane())); diff --git a/tests/corpus/native-class-array-boundaries.ts b/tests/corpus/native-class-array-boundaries.ts new file mode 100644 index 00000000..452528a0 --- /dev/null +++ b/tests/corpus/native-class-array-boundaries.ts @@ -0,0 +1,29 @@ +class Plane { + constant: number; + constructor(value: number) { this.constant = value; } +} +function roundtrip(value: unknown): Plane[] { return value as Plane[]; } +function optional(value: unknown): Plane[] | null { return value as Plane[] | null; } +const plane = new Plane(2); +const planes = [plane]; +const copy = roundtrip(planes); +console.log('array', copy.length, copy[0] === plane, copy[0].constant); +copy[0].constant = 4; +console.log('mutation', plane.constant, optional(planes)![0] === plane, optional(null) === null); +function boxedRecord(value: unknown) { return value; } +const boxed = boxedRecord({plane, planes}); +const view = boxed as {plane: unknown; planes: unknown[]}; +console.log('record', view.plane === plane, view.planes[0] === plane); +const callback: unknown = (value: Plane): number => value.constant; +console.log('callback', (callback as (value: Plane) => number)(plane)); + +class Tree { + children: Tree[] = []; + select: (node: Tree) => Tree[] = (node) => node.children; +} +const root = new Tree(), child = new Tree(); +root.children.push(child); +const tree: unknown = root; +const restored = tree as Tree; +console.log('recursive', restored === root, restored.select(root)[0] === child); +console.log('view', Object.keys(tree as object).sort().join(',')); diff --git a/tests/corpus/native-class-type-predicates.ts b/tests/corpus/native-class-type-predicates.ts new file mode 100644 index 00000000..80c08c7b --- /dev/null +++ b/tests/corpus/native-class-type-predicates.ts @@ -0,0 +1,15 @@ +class Opaque { + entries: Map = new Map(); + self: Opaque | null = null; +} +function inspect(value: unknown): void { + console.log(typeof value, typeof value === 'object', typeof value === 'function', Array.isArray(value)); +} +const value = new Opaque(); +value.self = value; +inspect(value); +inspect([value]); +inspect(null); +inspect(1); +inspect('text'); +value.self = null; diff --git a/tests/corpus/object-enumeration-array-reads.ts b/tests/corpus/object-enumeration-array-reads.ts new file mode 100644 index 00000000..e281e50d --- /dev/null +++ b/tests/corpus/object-enumeration-array-reads.ts @@ -0,0 +1,9 @@ +const records: { first: number; second: number }[] = [{ first: 1, second: 2 }]; +for (const record of records) console.log(Object.keys(record).join(","), Object.values(record).join(","), JSON.stringify(Object.entries(record))); +const dictionaries: Record[] = [{ b: 2, a: 1 }]; +for (const dict of dictionaries) console.log(Object.keys(dict).join(","), Object.values(dict).join(","), JSON.stringify(Object.entries(dict))); +const empty: Record[] = []; +try { console.log(Object.values(empty[0])); } +catch (error) { console.log(error instanceof TypeError, error instanceof Error ? error.message : "unexpected"); } +try { console.log(Object.keys(records[20])); } +catch (error) { console.log(error instanceof TypeError, error instanceof Error ? error.message : "unexpected"); } diff --git a/tests/corpus/record-optional-json-presence.ts b/tests/corpus/record-optional-json-presence.ts new file mode 100644 index 00000000..c47a02f4 --- /dev/null +++ b/tests/corpus/record-optional-json-presence.ts @@ -0,0 +1,32 @@ +interface Entry { + name: string; + synthesized?: true; + count?: number; +} + +function inspect(value: unknown): void { + if (value === null || typeof value !== "object" || Array.isArray(value)) return; + const entry = value as Record; + console.log(entry.name, "synthesized" in entry, Object.hasOwn(entry, "synthesized"), entry.synthesized === true); + console.log(Object.keys(entry).join(","), JSON.stringify(entry)); +} + +function make(name: string, synthesized: boolean): Entry { + const entry: Entry = { name }; + if (synthesized) entry.synthesized = true; + return entry; +} + +const entries: Entry[] = [{ name: "declared" }, { name: "inline", synthesized: true }, make("built", false), make("generated", true)]; +for (const entry of entries) inspect(JSON.parse(JSON.stringify(entry))); +const document = { entries }; +const unknownDocument: unknown = JSON.parse(JSON.stringify(document)); +if (unknownDocument !== null && typeof unknownDocument === "object") { + const dict = unknownDocument as Record; + const items = dict.entries; + if (Array.isArray(items)) for (const item of items) inspect(item); +} + +// Boxing an explicitly present undefined value must retain its key. +inspect({ name: "undefined", synthesized: undefined }); +inspect({ name: "undefined-count", count: undefined }); diff --git a/tests/corpus/rest-optional-array-arguments.ts b/tests/corpus/rest-optional-array-arguments.ts new file mode 100644 index 00000000..54642378 --- /dev/null +++ b/tests/corpus/rest-optional-array-arguments.ts @@ -0,0 +1,50 @@ +// Unchecked array reads widen rest elements while the rest array stays present. +class Collector { + first(...values: number[]): string { + return values.map(value => value === undefined ? "missing" : String(value)).join(","); + } + + later(prefix: string, ...values: number[]): string { + return prefix + values.map(value => value === undefined ? "missing" : String(value)).join(","); + } +} + +const items: number[] = [7]; +const collector = new Collector(); +console.log(collector.first(items[3])); +console.log(collector.first()); +console.log(collector.first(...items)); +console.log(collector.later("values:", 1, items[3], 2)); +console.log(collector.later("empty:")); + +function collect(...values: number[]): string { + return values.map(value => value === undefined ? "missing" : String(value)).join(","); +} +console.log(collect(1, items[3], 2)); +console.log(collect()); + +class Base { + describe(...values: number[]): string { + return "base:" + values.map(value => value === undefined ? "missing" : String(value)).join(","); + } +} +class Derived extends Base { + describe(...values: number[]): string { + return "derived:" + values.map(value => value === undefined ? "missing" : String(value)).join(","); + } +} +function describe(receiver: Base): string { + return receiver.describe(1, items[3], 2); +} +console.log(describe(new Base())); +console.log(describe(new Derived())); + +interface Node { marked?: boolean } +class Gate { + blocked(...nodes: Node[]): boolean { + return nodes.some(node => node?.marked === true); + } +} +const nodes: Node[] = [{ marked: true }]; +const gate = new Gate(); +console.log(gate.blocked(nodes[3]), gate.blocked(nodes[0]!), gate.blocked()); diff --git a/tests/corpus/static-method-receivers.ts b/tests/corpus/static-method-receivers.ts new file mode 100644 index 00000000..e8d1a9dd --- /dev/null +++ b/tests/corpus/static-method-receivers.ts @@ -0,0 +1,24 @@ +class Base { + static label = "base"; + static scale(value: number): number { return value * 2; } + static run(value: number): number { return this.scale(value) + 1; } + static describe(label: string = this.label): string { return this.name + ":" + label; } + static rename(label: string): void { this.label = label; } + static delayed(value: number): () => number { return () => this.run(value); } + static generic(value: T): T { console.log(this.run(2)); return value; } +} +class Derived extends Base { + static label = "derived"; + static scale(value: number): number { return value * 3; } +} +console.log(Base.run(4), Derived.run(4)); +console.log(Base.delayed(5)(), Derived.delayed(5)()); +console.log(Base.generic('base'), Derived.generic('derived')); +console.log(Base.describe(), Derived.describe(), Derived.describe('custom')); +Base.rename('first'); Derived.rename('second'); +console.log(Base.describe(), Derived.describe()); +class Private { + static #value = 7; + static read(): number { return this.#value; } +} +console.log(Private.read()); diff --git a/tests/corpus/typed-array-constructor-identity.js b/tests/corpus/typed-array-constructor-identity.js new file mode 100644 index 00000000..64161452 --- /dev/null +++ b/tests/corpus/typed-array-constructor-identity.js @@ -0,0 +1,19 @@ +// @ts-nocheck +/** @param {unknown} value */ +function identify(value) { + const ctor = value.constructor; + if (ctor === Uint8Array) return 'u8'; + if (ctor === Uint8ClampedArray) return 'u8c'; + if (ctor === Int8Array) return 'i8'; + if (ctor === Uint16Array) return 'u16'; + if (ctor === Int16Array) return 'i16'; + if (ctor === Uint32Array) return 'u32'; + if (ctor === Int32Array) return 'i32'; + if (ctor === Float32Array) return 'f32'; + if (ctor === Float64Array) return 'f64'; + if (ctor === Buffer) return 'buffer'; + return 'other'; +} +console.log(identify(new Uint8Array()), identify(new Uint8ClampedArray()), identify(new Int8Array())); +console.log(identify(new Uint16Array()), identify(new Int16Array()), identify(new Uint32Array()), identify(new Int32Array())); +console.log(identify(new Float32Array()), identify(new Float64Array()), identify(Buffer.from([1, 2]))); diff --git a/tests/corpus/typed-array-constructor-union.ts b/tests/corpus/typed-array-constructor-union.ts new file mode 100644 index 00000000..11782cc1 --- /dev/null +++ b/tests/corpus/typed-array-constructor-union.ts @@ -0,0 +1,19 @@ +let evaluations = 0; +function source(kind: number): number[] | Float32Array | number | undefined { + evaluations++; + if (kind === 0) return [1.5, 2.5]; + if (kind === 1) return new Float32Array([3.5, 4.5]); + if (kind === 2) return 3; + return undefined; +} +for (let i = 0; i < 4; i++) { + // @ts-expect-error JavaScript accepts every arm, although no single TS overload does. + const values = new Float32Array(source(i)); + console.log(values.length, values.join(','), evaluations); +} +function copy(values: number[] | Float32Array) { return new Uint16Array(values); } +const original = new Float32Array([10.5, 20.5]); +const cloned = copy(original); +original[0] = 100; +console.log(cloned.join(','), copy([65537, -1]).join(',')); +try { new Float32Array(source(2) === 3 ? -1 : 0); } catch (e) { console.log(e instanceof RangeError); } diff --git a/tests/corpus/typed-rest-filtered-return.ts b/tests/corpus/typed-rest-filtered-return.ts index f53fa611..46880194 100644 --- a/tests/corpus/typed-rest-filtered-return.ts +++ b/tests/corpus/typed-rest-filtered-return.ts @@ -12,3 +12,33 @@ const second = new Value(2); const result = collect(undefined, first, null, second); console.log(result.map((value) => value.value).join(","), result[0] === first, result[1] === second); console.log(collect().length); + +// Unchecked inputs widen the elements of a rest pack, never the pack itself. +const inputs: Value[] = [first]; +const missing = inputs[10]; +const fromReads = collect(inputs[0], missing, second); +console.log(fromReads.map((value) => value.value).join(","), fromReads.length); + +class Collector { + report(label: string, ...values: Value[]): void { + console.log(label, values.length, values[0] === undefined, values[1] === undefined); + for (const value of values) console.log(value === undefined ? "missing" : value.value); + } +} +const collector = new Collector(); +collector.report("first", missing, second); +collector.report("later", first, missing); +collector.report("empty"); + +class DerivedCollector extends Collector { + report(label: string, ...values: Value[]): void { + super.report(label, ...values); + } +} +function throughBase(value: Collector): void { value.report("virtual", inputs[10], first); } +throughBase(new DerivedCollector()); + +function reportValues(...values: Value[]): void { + for (const value of values) console.log("function", value === undefined ? "missing" : value.value); +} +reportValues(inputs[10], first); diff --git a/tests/corpus/util-parseargs-wrapped/main.ts b/tests/corpus/util-parseargs-wrapped/main.ts new file mode 100644 index 00000000..1375adb4 --- /dev/null +++ b/tests/corpus/util-parseargs-wrapped/main.ts @@ -0,0 +1,26 @@ +import { parseArgs } from "node:util"; + +const options = { + output: { type: "string", short: "o" }, + verbose: { type: "boolean", default: false }, + keep: { type: "boolean", default: true }, + include: { type: "string", multiple: true }, +} as const; + +type Arguments = ReturnType>; +function parse(args: string[]): Arguments { + return parseArgs({ args, options, allowPositionals: true, allowNegative: true }); +} +function print(args: string[]): void { + const { values, positionals } = parse(args); + console.log(values.output ?? "none", values.verbose, values.keep); + console.log((values.include ?? []).join(":"), positionals.join(":")); +} +print(["build", "input.ts", "-o", "program", "--verbose", "--no-keep", "--include=one", "--include", "two"]); +print([]); +try { parse(["--unknown"]); } catch (error) { console.log(error instanceof Error ? error.message.split("\n")[0] : String(error)); } + +// Similar user declarations retain their own structural field types. +type ParsedResults = { values: number; positionals: boolean }; +function local(value: ParsedResults): number { return value.positionals ? value.values : 0; } +console.log(local({ values: 42, positionals: true })); diff --git a/tests/corpus/util-parseargs-wrapped/tsconfig.json b/tests/corpus/util-parseargs-wrapped/tsconfig.json new file mode 100644 index 00000000..27d492dd --- /dev/null +++ b/tests/corpus/util-parseargs-wrapped/tsconfig.json @@ -0,0 +1,3 @@ +{ + "compilerOptions": { "strict": true, "types": ["node"] } +} diff --git a/tests/diagnostics/class-values.ts b/tests/diagnostics/class-values.ts index d2de5d0f..02c5360f 100644 --- a/tests/diagnostics/class-values.ts +++ b/tests/diagnostics/class-values.ts @@ -69,10 +69,12 @@ let pick: typeof Left | typeof Right = Left; const picked = new pick(); void picked; -// `this` in a static method names the RECEIVER class — a dynamic value. +// A polymorphic class value cannot specialize a receiver-dependent method. class UsesThis { static self(): string { return this.name; } } -console.log(UsesThis.self()); +class OtherThis extends UsesThis {} +function self(value: typeof UsesThis): string { return value.self(); } +console.log(self(OtherThis)); diff --git a/tests/ffi/catalog.mjs b/tests/ffi/catalog.mjs index 521b03fa..7c0e16ae 100644 --- a/tests/ffi/catalog.mjs +++ b/tests/ffi/catalog.mjs @@ -2,8 +2,9 @@ import { createRequire } from "node:module"; const requireModule = createRequire(import.meta.url); const ffi = requireModule("node:ffi"); console.log(ffi === requireModule("node:ffi")); +const u64Arguments = ["u64"]; const definitions = { - nativeU64: { arguments: ["u64"], return: "u64" }, + nativeU64: { arguments: u64Arguments, return: "u64" }, nativePointer: { arguments: ["pointer"], return: "pointer" }, nativeI64: { arguments: ["int64"], return: "int64" }, nativeByte: { arguments: ["bool"], return: "uint8" }, @@ -44,4 +45,9 @@ second.lib.close(); if (process.env.FFI_STATIC === "1") { try { ffi.dlopen(process.env.FFI_LIBRARY, { nativeU64: { arguments: ["i64"], return: "u64" } }); } catch { console.log("signature mismatch"); } + u64Arguments[0] = "i64"; + try { ffi.dlopen(process.env.FFI_LIBRARY, definitions); } + catch { console.log("mutated signature"); } + try { ffi.dlopen(process.env.FFI_LIBRARY, { nativeU64: { return: "u64" } }); } + catch (error) { console.log("missing arguments", error.name, error.code); } } diff --git a/tests/fixtures/npm-static/array-output-parameters.mjs b/tests/fixtures/npm-static/array-output-parameters.mjs new file mode 100644 index 00000000..dc71eb44 --- /dev/null +++ b/tests/fixtures/npm-static/array-output-parameters.mjs @@ -0,0 +1,30 @@ +class Item { + constructor(value = 1) { this.value = value; } +} +class Collector { + /** @param {Array} output */ + collect(output) { output.push(new Item(3)); } + /** @param {Array} output + * @return {Array} */ + append(output) { this.collect(output); return output; } + run() { + const output = []; + this.collect(output); + console.log('local', output.length, output[0].value); + return output; + } +} +const collector = new Collector(); +const values = []; +collector.collect(values); +const alias = collector.append(values); +console.log('shared', values.length, alias === values, values[0].value); +console.log('nested', collector.run().length); +alias.push(new Item(7)); +function readShared() { return alias.length + values[2].value; } +console.log('write-back', readShared()); +/** @returns {Array} */ +function seeded() { return [new Item(4)]; } +const typed = seeded(); +collector.collect(typed); +console.log('typed-result', typed.length, typed[0].value, typed[1].value); diff --git a/tests/fixtures/npm-static/node_modules/virtual-classes/index.d.ts b/tests/fixtures/npm-static/node_modules/virtual-classes/index.d.ts index 93b9d600..2a0fae73 100644 --- a/tests/fixtures/npm-static/node_modules/virtual-classes/index.d.ts +++ b/tests/fixtures/npm-static/node_modules/virtual-classes/index.d.ts @@ -1 +1,2 @@ export function exercise(): void; +export function structuralMethods(): void; diff --git a/tests/fixtures/npm-static/node_modules/virtual-classes/index.js b/tests/fixtures/npm-static/node_modules/virtual-classes/index.js index 1f77cdf1..b29b6962 100644 --- a/tests/fixtures/npm-static/node_modules/virtual-classes/index.js +++ b/tests/fixtures/npm-static/node_modules/virtual-classes/index.js @@ -50,3 +50,36 @@ export function exercise() { new Short().defaulted("direct"); arityCounts(); } + +class Pair { + constructor(x, y) { this.x = x; this.y = y; } + /** @param {Pair} other */ + add(other) { this.x += other.x; this.y += other.y; return this; } +} +class Triple { + constructor(x, y, z) { this.x = x; this.y = y; this.z = z; } +} +class Helpers { + /** @param {Pair} target @param {Pair} other @returns {Pair} */ + static combine(target, other) { target.x += other.x; target.y += other.y; return target; } +} +class InheritedHelpers extends Helpers {} +class OverriddenHelpers extends Helpers { + static combine(target, other) { target.x -= other.x; target.y -= other.y; return target; } +} + +export function structuralMethods() { + const pair = new Pair(1, 2); + const triple = new Triple(3, 4, 5); + console.log('instance', pair.add(triple) === pair, pair.x, pair.y); + console.log('static', Helpers.combine(triple, pair) === triple, triple.x, triple.y, triple.z); + console.log('inherited', InheritedHelpers.combine(triple, pair) === triple, triple.x); + console.log('overridden', OverriddenHelpers.combine(triple, pair) === triple, triple.x); + const combine = Helpers.combine; + console.log('static value', combine(pair, pair) === pair, pair.x, pair.y); + const add = pair.add; + try { add(pair); } catch (error) { console.log('unbound', error instanceof TypeError); } + const rows = [triple, new Triple(1, 0, 0)]; + function compare(a, b) { return a.x - b.x; } + console.log('sort', rows.sort(compare) === rows, rows[0].x, rows[1].x); +} diff --git a/tests/fixtures/npm-static/virtual-classes-cli.ts b/tests/fixtures/npm-static/virtual-classes-cli.ts index 64cc37cd..0f72a3d0 100644 --- a/tests/fixtures/npm-static/virtual-classes-cli.ts +++ b/tests/fixtures/npm-static/virtual-classes-cli.ts @@ -1,2 +1,3 @@ -import { exercise } from "virtual-classes"; +import { exercise, structuralMethods } from "virtual-classes"; exercise(); +structuralMethods(); diff --git a/tests/fixtures/self-hosting/class-types.ts b/tests/fixtures/self-hosting/class-types.ts new file mode 100644 index 00000000..8e680f14 --- /dev/null +++ b/tests/fixtures/self-hosting/class-types.ts @@ -0,0 +1,26 @@ +import { F64, typeKey, type IrExpr, type IrModule, type IrType } from "../../../packages/compiler/src/ir/ir.js"; +import { sanitizeUnregisteredClassTypes } from "../../../packages/compiler/src/frontend/lowering/sanitize-class-types.js"; + +const loc = { file: "class-types.ts", start: 0, end: 1 }; +const missing: IrType = { kind: "object", className: "Fenced" }; +const kept: IrType = { kind: "object", className: "Registered" }; +const expr: IrExpr = { kind: "varRef", localId: "x", type: missing, loc }; +const nested: IrType = { kind: "func", params: [{ kind: "array", elem: missing }], + ret: { kind: "generator", yieldT: missing, retT: kept, nextT: missing } }; +const module: IrModule = { irVersion: 13, sourceFile: loc.file, entry: "main", + functions: [{ name: "main", params: [{ localId: "x", name: "x", type: missing }], + returnType: nested, locals: [], loc, body: [{ kind: "exprStmt", expr, loc }] }], + records: [{ id: "r", fields: [{ name: "callback", type: nested }], indexValue: missing }], + unions: [{ id: "u", arms: [missing, kept, { kind: "classval", className: "Fenced" }] }], +}; +sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); +console.log(module.functions[0]!.params[0]!.type.kind, expr.type.kind); +console.log(typeKey(nested)); +for (const record of module.records ?? []) { + for (const field of record.fields) console.log(typeKey(field.type)); + if (record.indexValue) console.log(typeKey(record.indexValue)); +} +for (const union of module.unions ?? []) console.log(union.arms.map(typeKey).join(",")); +console.log(missing.kind, kept.kind, F64.kind); +sanitizeUnregisteredClassTypes(module, (name) => name === "Registered"); +console.log(expr.type.kind); diff --git a/tests/fixtures/self-hosting/frontend-services-cases.ts b/tests/fixtures/self-hosting/frontend-services-cases.ts index 520fb398..f054fbbd 100644 --- a/tests/fixtures/self-hosting/frontend-services-cases.ts +++ b/tests/fixtures/self-hosting/frontend-services-cases.ts @@ -33,6 +33,8 @@ export function runFrontendServices(services: FrontendServices, input: string, o const properties = parseNpmStaticDeclarationProperties(declarationFile); const nullable = services.nullableClassFields("package.js", request.source); const widened = services.findReturnWidening("package.js", nullable?.text ?? request.source); + const namepaths = services.jsDocNamepaths("package.js", "/** @returns {Array} */ function items() { return []; }"); + check(namepaths !== null && namepaths.includes("Array<*"), "checked JSDoc namepaths"); const fields = services.declarationProperties("package.js", widened?.text ?? nullable?.text ?? request.source, properties); const overloads = services.declarationOverloads("package.js", fields?.text ?? widened?.text ?? nullable?.text ?? request.source, methods); const rewritten = services.rewriteCjs(request.bundled, join(request.directory, "bundle.js")); diff --git a/tests/fixtures/self-hosting/native-cache.ts b/tests/fixtures/self-hosting/native-cache.ts new file mode 100644 index 00000000..2cf12e0b --- /dev/null +++ b/tests/fixtures/self-hosting/native-cache.ts @@ -0,0 +1,24 @@ +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { NativeCache, contentDigest } from "../../../packages/compiler/src/native/cache.js"; +import { NativeExecutableCache } from "../../../packages/compiler/src/native/executable-cache.js"; +import { toolchainEnvironmentCachePolicy, toolchainEnvironmentFingerprint } from "../../../packages/compiler/src/backend/toolchain-environment.js"; + +const directory = process.argv[2]!; +const linker = process.argv[3]!; +const script = process.argv[4]!; +const library = process.argv[5]!; +const output = join(directory, "program"); +const cache = new NativeCache(join(directory, "cache")); +const key = contentDigest("native executable"); +const entry = new NativeExecutableCache(cache, key, linker, false, [library]); +console.log(toolchainEnvironmentFingerprint({ LANG: "en_US.UTF-8" })); +console.log(JSON.stringify(toolchainEnvironmentCachePolicy({ LIBRARY_PATH: "/fixture" }))); +console.log(entry.trace([script, library], directory)); +writeFileSync(output, "cached program"); +entry.publish(output); +rmSync(output); +console.log(entry.restore(output)); +console.log(readFileSync(output, "utf8")); +writeFileSync(library, "changed library"); +console.log(entry.restore(output)); diff --git a/tests/fixtures/self-hosting/serialize.ts b/tests/fixtures/self-hosting/serialize.ts index 97a358e0..0146da1a 100644 --- a/tests/fixtures/self-hosting/serialize.ts +++ b/tests/fixtures/self-hosting/serialize.ts @@ -15,7 +15,7 @@ try { const statement = mod.functions[0]!.body[0]; if (statement?.kind === "exprStmt" && statement.expr.kind === "numLit") statement.expr.value = NaN; } - console.log(serializeModule(mod)); + console.log(serializeModule(mod, process.argv[3] === "compact")); } } catch (error) { if (error instanceof Error) console.log(error.name + ": " + error.message); diff --git a/tests/fixtures/three/attributes.mjs b/tests/fixtures/three/attributes.mjs new file mode 100644 index 00000000..9da5890b --- /dev/null +++ b/tests/fixtures/three/attributes.mjs @@ -0,0 +1,36 @@ +import { Float32BufferAttribute, Uint8BufferAttribute, Uint16BufferAttribute } from 'three/src/core/BufferAttribute.js'; +import { Matrix3 } from 'three/src/math/Matrix3.js'; +import { Matrix4 } from 'three/src/math/Matrix4.js'; +import { Box3 } from 'three/src/math/Box3.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; +const positions = new Float32BufferAttribute([0, 0, 0, 1, 2, 3], 3); +console.log(positions.count, positions.itemSize, positions.getX(1), positions.getY(1), positions.getZ(1)); +positions.applyMatrix4(new Matrix4().makeTranslation(2, 3, 4)); +console.log(positions.getX(0), positions.getY(0), positions.getZ(0), positions.getX(1), positions.getY(1), positions.getZ(1)); +const bounds = new Box3().setFromBufferAttribute(positions); +console.log(bounds.min.x, bounds.min.y, bounds.min.z, bounds.max.x, bounds.max.y, bounds.max.z); +const indices = new Uint16BufferAttribute([0, 1, 2], 1); +console.log(indices.count, indices.getX(1)); +console.log(Object.hasOwn(positions, 'id'), Object.keys(positions).includes('id')); +const descriptor = Object.getOwnPropertyDescriptor(positions, 'id'); +console.log(descriptor.writable, descriptor.enumerable, descriptor.configurable); +const source = new Float32Array([2, 4, 6]); +const copied = new Float32BufferAttribute(source, 3); +source[0] = 100; +console.log('copy', copied.getX(0), copied.getY(0), copied.getZ(0)); +copied.setXYZ(0, 3, 2, 1); +copied.applyMatrix3(new Matrix3().set(2, 0, 0, 0, 3, 0, 0, 0, 4)); +console.log('matrix3', copied.getX(0), copied.getY(0), copied.getZ(0)); +const vector = new Vector3().fromBufferAttribute(copied, 0); +console.log('vector', vector.x, vector.y, vector.z); +const colors = new Uint8BufferAttribute([0, 255, 0, 255], 4, true); +console.log('normalized', colors.getX(0), colors.getY(0), colors.getW(0)); +colors.setX(0, 1); +console.log('stored', colors.array[0]); +copied.addUpdateRange(0, 3); +console.log('ranges', copied.updateRanges.length, copied.updateRanges[0].start, copied.updateRanges[0].count); +copied.clearUpdateRanges(); +console.log('cleared', copied.updateRanges.length); +const before = copied.version; +copied.needsUpdate = true; +console.log('updated', copied.version === before + 1); diff --git a/tests/fixtures/three/collision.mjs b/tests/fixtures/three/collision.mjs new file mode 100644 index 00000000..342ea88c --- /dev/null +++ b/tests/fixtures/three/collision.mjs @@ -0,0 +1,34 @@ +import { Vector3, Triangle, Sphere, Ray } from 'three'; +import { Capsule } from 'three/addons/math/Capsule.js'; +import { Octree } from 'three/addons/math/Octree.js'; + +const round = n => Math.round(n * 1000000); +const tree = new Octree(); +tree.trianglesPerLeaf = 2; +for (let x = -4; x < 4; x++) { + for (let z = -4; z < 4; z++) { + tree.addTriangle(new Triangle(new Vector3(x,0,z),new Vector3(x,0,z+1),new Vector3(x+1,0,z))); + tree.addTriangle(new Triangle(new Vector3(x+1,0,z),new Vector3(x,0,z+1),new Vector3(x+1,0,z+1))); + } +} +tree.build(); +const body = new Capsule(new Vector3(0.1,0.2,0.1),new Vector3(0.1,1.3,0.1),0.35); +const candidates = []; +tree.getCapsuleTriangles(body,candidates); +console.log('candidates', candidates.length, new Set(candidates).size); +const hit = tree.capsuleIntersect(body); +if (hit) console.log('capsule',round(hit.depth),round(hit.normal.y),round(body.start.y)); +else console.log('capsule miss'); +body.translate(new Vector3(0,3,0)); +console.log('capsule miss',tree.capsuleIntersect(body) === false); +const sphere = tree.sphereIntersect(new Sphere(new Vector3(0.1,0.2,0.1),0.4)); +if(sphere) console.log('sphere',round(sphere.depth),round(sphere.normal.y)); +else console.log('sphere miss'); +const ray = tree.rayIntersect(new Ray(new Vector3(0.2,4,0.3),new Vector3(0,-1,0))); +if(ray) console.log('ray',round(ray.distance),round(ray.position.x),round(ray.position.y)); +else console.log('ray miss'); +console.log('ray miss',tree.rayIntersect(new Ray(new Vector3(8,4,8),new Vector3(0,-1,0))) === false); +const a = new Vector3(0,0,0), b = new Vector3(0,0,1), c = new Vector3(1,0,0); +console.log('inside', Triangle.containsPoint(new Vector3(0.2,0.1,0.2),a,b,c)); +console.log('outside', Triangle.containsPoint(new Vector3(2,0,2),a,b,c)); +console.log('degenerate', Triangle.containsPoint(new Vector3(),a,a,a)); diff --git a/tests/fixtures/three/geometry.mjs b/tests/fixtures/three/geometry.mjs new file mode 100644 index 00000000..99204826 --- /dev/null +++ b/tests/fixtures/three/geometry.mjs @@ -0,0 +1,39 @@ +import { BoxGeometry } from 'three/src/geometries/BoxGeometry.js'; +import { BufferGeometry } from 'three/src/core/BufferGeometry.js'; +import { Float32BufferAttribute } from 'three/src/core/BufferAttribute.js'; +import { Matrix4 } from 'three/src/math/Matrix4.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; + +const box = new BoxGeometry(2, 4, 6); +console.log('box', box.type, box.attributes.position.count, box.index.count, box.groups.length); +box.computeBoundingBox(); +box.computeBoundingSphere(); +console.log('bounds', box.boundingBox.min.x, box.boundingBox.min.y, box.boundingBox.min.z, + box.boundingBox.max.x, box.boundingBox.max.y, box.boundingBox.max.z, box.boundingSphere.radius.toFixed(4)); +box.translate(1, 2, 3); +console.log('translated', box.boundingBox.min.x, box.boundingBox.min.y, box.boundingBox.min.z); +const dense = new BoxGeometry(1, 1, 1, 2, 3, 4); +console.log('segments', dense.attributes.position.count, dense.index.count, dense.groups[5].materialIndex); + +const geometry = new BufferGeometry(); +geometry.setAttribute('position', new Float32BufferAttribute([0, 0, 0, 2, 0, 0, 0, 2, 0], 3)); +geometry.computeBoundingBox(); +geometry.computeBoundingSphere(); +geometry.computeVertexNormals(); +console.log('triangle', geometry.attributes.position.count, geometry.attributes.normal.getZ(0)); +console.log('sphere', geometry.boundingSphere.center.x, geometry.boundingSphere.center.y, geometry.boundingSphere.radius.toFixed(4)); +geometry.applyMatrix4(new Matrix4().makeScale(2, 3, 4)); +console.log('scaled', geometry.boundingBox.max.x, geometry.boundingBox.max.y, geometry.attributes.normal.getZ(1)); +geometry.setIndex([0, 1, 2]); +console.log('index16', geometry.index.array instanceof Uint16Array, geometry.index.getX(2)); +geometry.setIndex([0, 65536, 2]); +console.log('index32', geometry.index.array instanceof Uint32Array, geometry.index.getX(1)); +geometry.addGroup(0, 3, 2); +geometry.setDrawRange(1, 2); +console.log('draw', geometry.groups[0].materialIndex, geometry.drawRange.start, geometry.drawRange.count); +geometry.clearGroups(); +geometry.deleteAttribute('normal'); +console.log('removed', geometry.groups.length, geometry.hasAttribute('normal')); +const point = new Vector3(1, 2, 3), copy = point.clone(); +copy.x = 4; +console.log('clone', copy !== point, point.x, copy.x, copy.y, copy.z); diff --git a/tests/fixtures/three/materials.mjs b/tests/fixtures/three/materials.mjs new file mode 100644 index 00000000..3bbe66d9 --- /dev/null +++ b/tests/fixtures/three/materials.mjs @@ -0,0 +1,45 @@ +import { MeshNormalMaterial } from 'three/src/materials/MeshNormalMaterial.js'; +import { MeshBasicMaterial } from 'three/src/materials/MeshBasicMaterial.js'; +import { MeshStandardMaterial } from 'three/src/materials/MeshStandardMaterial.js'; +import { Plane } from 'three/src/math/Plane.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; + +const normal = new MeshNormalMaterial({wireframe: true, opacity: 0.5, transparent: true}); +console.log('normal', normal.type, normal.wireframe, normal.opacity, normal.transparent); +normal.setValues({wireframe: false, opacity: 0.8}); +normal.needsUpdate = true; +console.log('updated', normal.wireframe, normal.opacity, normal.version); +const copy = normal.clone(); +console.log('clone', copy !== normal, copy.type, copy.opacity, copy.normalScale.x); +copy.normalScale.x = 2; +console.log('independent', normal.normalScale.x, copy.normalScale.x); + +const plane = new Plane(new Vector3(1, 0, 0), 2); +normal.setValues({clippingPlanes: [plane], clipIntersection: true}); +if (normal.clippingPlanes !== null) { + console.log('clip', normal.clippingPlanes.length, normal.clippingPlanes[0] === plane, normal.clipIntersection); + normal.clippingPlanes[0].constant = 3; +} +console.log('plane', plane.constant); +normal.setValues({clippingPlanes: null}); +console.log('unclipped', normal.clippingPlanes === null); + +const basic = new MeshBasicMaterial({color: 0xff0000}); +console.log('red', basic.color.r, basic.color.g, basic.color.b); +basic.setValues({color: 0x0000ff}); +console.log('blue', basic.color.r, basic.color.g, basic.color.b); +console.log('basic-copy', basic.clone().color.b); +const standard = new MeshStandardMaterial({roughness: 0.25, metalness: 0.8, color: 0x00ff00}); +console.log('standard', standard.type, standard.roughness, standard.metalness, standard.color.g); +console.log('standard-copy', standard.clone().roughness, standard.clone().metalness); + +let count = 0; +function disposed(event) { + count++; + console.log('event', event.type, event.target === normal, this === normal); +} +normal.addEventListener('dispose', disposed); +normal.dispose(); +normal.removeEventListener('dispose', disposed); +normal.dispose(); +console.log('events', count); diff --git a/tests/fixtures/three/mesh.mjs b/tests/fixtures/three/mesh.mjs new file mode 100644 index 00000000..ccfc6f79 --- /dev/null +++ b/tests/fixtures/three/mesh.mjs @@ -0,0 +1,33 @@ +import { Mesh } from 'three/src/objects/Mesh.js'; +import { BoxGeometry } from 'three/src/geometries/BoxGeometry.js'; +import { MeshNormalMaterial } from 'three/src/materials/MeshNormalMaterial.js'; +import { Scene } from 'three/src/scenes/Scene.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; + +const geometry = new BoxGeometry(2, 4, 6); +const material = new MeshNormalMaterial(); +const mesh = new Mesh(geometry, material); +console.log('mesh', mesh.type, mesh.isMesh, mesh.geometry === geometry, mesh.material === material); +const scene = new Scene(); +scene.add(mesh); +mesh.position.set(1, 2, 3); +scene.updateMatrixWorld(); +const point = mesh.getVertexPosition(0, new Vector3()); +console.log('vertex', point.x, point.y, point.z); +console.log('world', mesh.matrixWorld.elements[12], mesh.matrixWorld.elements[13], mesh.matrixWorld.elements[14]); +const copy = mesh.clone(); +console.log('copy', copy !== mesh, copy.geometry === geometry, copy.material === material, copy.position.x); +copy.position.x = 7; +console.log('independent', mesh.position.x, copy.position.x); +const other = new Mesh(geometry, material); +console.log('fluent', other.copy(mesh) === other, other.copy(mesh).geometry === geometry); +const child = new Mesh(geometry, material); +child.position.y = 3; +mesh.add(child); +const recursive = mesh.clone(); +console.log('children', recursive.children.length, recursive.children[0] !== child, recursive.children[0].position.y); +console.log('shallow', mesh.clone(false).children.length); +const materials = [material, new MeshNormalMaterial({wireframe: true})]; +const multi = new Mesh(geometry, materials); +const multiCopy = multi.clone(); +console.log('materials', multiCopy.material !== multi.material, multiCopy.material.length, multiCopy.material[0] === material, multiCopy.material[1].wireframe); diff --git a/tests/fixtures/three/raycast-camera.mjs b/tests/fixtures/three/raycast-camera.mjs new file mode 100644 index 00000000..1b3dc778 --- /dev/null +++ b/tests/fixtures/three/raycast-camera.mjs @@ -0,0 +1,18 @@ +import { Raycaster } from 'three/src/core/Raycaster.js'; +import { Vector2 } from 'three/src/math/Vector2.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; +import { PerspectiveCamera } from 'three/src/cameras/PerspectiveCamera.js'; +import { OrthographicCamera } from 'three/src/cameras/OrthographicCamera.js'; +const ray = new Raycaster(); +const p = new PerspectiveCamera(90, 1, 0.1, 100); +p.position.set(0,0,5); p.updateMatrixWorld(); +ray.setFromCamera(new Vector2(0,0),p); +console.log('perspective', ray.ray.origin.z, ray.ray.direction.x, ray.ray.direction.y, ray.ray.direction.z, ray.camera === p); +ray.setFromCamera(new Vector2(0.5,0.5),p); +console.log('offset', ray.ray.direction.x.toFixed(4), ray.ray.direction.y.toFixed(4), ray.ray.direction.z.toFixed(4)); +const o = new OrthographicCamera(-2,2,2,-2,0.1,100); +o.position.set(0,0,5); o.updateMatrixWorld(); +ray.setFromCamera(new Vector2(0.5,0.5),o); +console.log('orthographic', ray.ray.origin.x.toFixed(4), ray.ray.origin.y.toFixed(4), ray.ray.origin.z.toFixed(4), ray.ray.direction.z, ray.camera === o); +ray.set(new Vector3(1,2,3),new Vector3(0,1,0)); +console.log('set', ray.ray.origin.x, ray.ray.origin.y, ray.ray.origin.z, ray.ray.direction.y); diff --git a/tests/fixtures/three/raycast-lines-points.mjs b/tests/fixtures/three/raycast-lines-points.mjs new file mode 100644 index 00000000..2ac7fc74 --- /dev/null +++ b/tests/fixtures/three/raycast-lines-points.mjs @@ -0,0 +1,28 @@ +import { Line } from 'three/src/objects/Line.js'; +import { LineSegments } from 'three/src/objects/LineSegments.js'; +import { LineLoop } from 'three/src/objects/LineLoop.js'; +import { Points } from 'three/src/objects/Points.js'; +import { BufferGeometry } from 'three/src/core/BufferGeometry.js'; +import { Float32BufferAttribute } from 'three/src/core/BufferAttribute.js'; +import { LineBasicMaterial } from 'three/src/materials/LineBasicMaterial.js'; +import { PointsMaterial } from 'three/src/materials/PointsMaterial.js'; +import { Raycaster } from 'three/src/core/Raycaster.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; +const geometry = new BufferGeometry(); +geometry.setAttribute('position',new Float32BufferAttribute([-1,0,-5,1,0,-5,1,2,-5,-1,2,-5],3)); +const ray = new Raycaster(new Vector3(0,0.1,0),new Vector3(0,0,-1)); +ray.params.Line.threshold = 0.2; +for (const line of [new Line(geometry,new LineBasicMaterial()), new LineSegments(geometry,new LineBasicMaterial()), new LineLoop(geometry,new LineBasicMaterial())]) { + line.updateMatrixWorld(); + const hit = ray.intersectObject(line); + console.log('line', line.type, hit.length, hit[0].distance, hit[0].index, hit[0].point.x, hit[0].point.y, hit[0].object === line); +} +const pg = new BufferGeometry(); pg.setAttribute('position',new Float32BufferAttribute([0,0,-5,1,0,-5,0,0,-3],3)); +const points = new Points(pg,new PointsMaterial()); points.updateMatrixWorld(); +ray.params.Points.threshold = 0.2; +const hits = ray.intersectObject(points); +console.log('points', hits.length, hits[0].distance, hits[1].distance, Number(hits[0].distanceToRay).toFixed(4), hits[0].index, hits[0].object === points); +ray.params.Points.threshold = 0.05; +console.log('threshold', ray.intersectObject(points).length); +pg.setIndex([0,1]); ray.params.Points.threshold = 0.2; +console.log('indexed', ray.intersectObject(points).length); diff --git a/tests/fixtures/three/raycast-mesh.mjs b/tests/fixtures/three/raycast-mesh.mjs new file mode 100644 index 00000000..5eb180c2 --- /dev/null +++ b/tests/fixtures/three/raycast-mesh.mjs @@ -0,0 +1,49 @@ +import { Mesh } from 'three/src/objects/Mesh.js'; +import { BoxGeometry } from 'three/src/geometries/BoxGeometry.js'; +import { BufferGeometry } from 'three/src/core/BufferGeometry.js'; +import { Float32BufferAttribute } from 'three/src/core/BufferAttribute.js'; +import { MeshBasicMaterial } from 'three/src/materials/MeshBasicMaterial.js'; +import { Raycaster } from 'three/src/core/Raycaster.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; +import { Scene } from 'three/src/scenes/Scene.js'; +import { FrontSide, BackSide, DoubleSide } from 'three/src/constants.js'; +const material = new MeshBasicMaterial(); +const near = new Mesh(new BoxGeometry(2, 2, 2), material); +near.position.z = -5; near.name = 'near'; +const far = new Mesh(new BoxGeometry(2, 2, 2), material); +far.position.z = -10; far.name = 'far'; +const scene = new Scene(); scene.add(far, near); scene.updateMatrixWorld(true); +const ray = new Raycaster(new Vector3(0.25,0.5,0),new Vector3(0,0,-1)); +const hits = ray.intersectObjects([far, near]); +console.log('sorted', hits.length, hits[0].distance, hits[1].distance, hits[0].object === near); +const hit = hits[0]; +console.log('hit', hit.point.x, hit.point.y, hit.point.z, hit.normal.z, hit.uv.x, hit.uv.y, hit.faceIndex, hit.face.normal.z, hit.barycoord.x + hit.barycoord.y + hit.barycoord.z); +const target = []; +console.log('target', ray.intersectObject(near, false, target) === target, target.length); +ray.intersectObject(far, false, target); +console.log('append', target.length, target[0].distance, target[1].distance); +target.length = 0; +console.log('clear', ray.intersectObject(scene, true, target) === target, target.length); +console.log('shallow', ray.intersectObject(scene, false).length); +ray.near = 5; console.log('near', ray.intersectObject(scene).length); +ray.near = 0; ray.far = 5; console.log('far', ray.intersectObject(scene).length); +ray.far = Infinity; near.layers.set(1); +console.log('layers', ray.intersectObject(scene).length); +ray.layers.enable(1); console.log('enabled', ray.intersectObject(scene).length); +near.layers.set(0); near.geometry.computeBoundingBox(); +ray.set(new Vector3(3,0,0),new Vector3(0,0,-1)); console.log('miss', ray.intersectObject(near).length); +ray.set(new Vector3(0.25,0.5,0),new Vector3(0,0,-1)); +material.side = BackSide; console.log('back', ray.intersectObject(near)[0].distance); +material.side = DoubleSide; console.log('double', ray.intersectObject(near).length); +const triangle = new BufferGeometry(); +triangle.setAttribute('position', new Float32BufferAttribute([-1,-1,-3,1,-1,-3,0,1,-3],3)); +material.side = FrontSide; +const tri = new Mesh(triangle, material); tri.updateMatrixWorld(); +const t = ray.intersectObject(tri)[0]; +console.log('triangle', t.distance, t.faceIndex, t.face.normal.z, t.uv === undefined, t.normal === undefined); +triangle.setDrawRange(0,0); console.log('drawrange', ray.intersectObject(tri).length); +triangle.setDrawRange(0,3); triangle.addGroup(0,3,1); +const multi = new Mesh(triangle,[new MeshBasicMaterial({side: BackSide}),material]); multi.updateMatrixWorld(); +console.log('group', ray.intersectObject(multi)[0].face.materialIndex); +triangle.setIndex([0,1,2]); +console.log('indexed-group', ray.intersectObject(multi)[0].face.materialIndex); diff --git a/tests/fixtures/three/scene.mjs b/tests/fixtures/three/scene.mjs new file mode 100644 index 00000000..05afb1ac --- /dev/null +++ b/tests/fixtures/three/scene.mjs @@ -0,0 +1,40 @@ +import { Scene } from 'three/src/scenes/Scene.js'; +import { Object3D } from 'three/src/core/Object3D.js'; +import { PerspectiveCamera } from 'three/src/cameras/PerspectiveCamera.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; + +const scene = new Scene(); +scene.position.set(1, 2, 3); +const parent = new Object3D(), child = new Object3D(); +parent.position.set(2, 3, 4); +child.position.set(3, 4, 5); +parent.add(child); +scene.add(parent); +scene.updateMatrixWorld(); +console.log('scene', scene.isScene, scene.children.length, child.parent === parent); +console.log('world', child.matrixWorld.elements[12], child.matrixWorld.elements[13], child.matrixWorld.elements[14]); +child.rotation.z = Math.PI / 2; +console.log('rotation', Number(child.quaternion.z).toFixed(4), Number(child.quaternion.w).toFixed(4)); +child.quaternion.setFromAxisAngle(new Vector3(0, 1, 0), Math.PI / 4); +console.log('quaternion', Number(child.rotation.y).toFixed(4)); +const descriptor = Object.getOwnPropertyDescriptor(child, 'position'); +console.log('position', descriptor.value === child.position, descriptor.writable, descriptor.enumerable, descriptor.configurable); +const names = []; +scene.name = 'scene'; parent.name = 'parent'; child.name = 'child'; +scene.traverse((object) => names.push(object.name)); +console.log('traverse', names.join(',')); +parent.remove(child); +console.log('removed', parent.children.length, child.parent === null); +scene.add(child); +scene.updateMatrixWorld(); +console.log('reparent', child.matrixWorld.elements[12], child.matrixWorld.elements[13], child.matrixWorld.elements[14]); + +const camera = new PerspectiveCamera(70, 1.5, 0.01, 10); +camera.position.z = 1; +scene.add(camera); +scene.updateMatrixWorld(true); +console.log('camera', camera.isPerspectiveCamera, camera.position.z, camera.projectionMatrix.elements[11]); +console.log('view', camera.matrixWorldInverse.elements[12], camera.matrixWorldInverse.elements[13], camera.matrixWorldInverse.elements[14]); +camera.aspect = 2; +camera.updateProjectionMatrix(); +console.log('projection', Number(camera.projectionMatrix.elements[0]).toFixed(4)); diff --git a/tests/fixtures/three/spatial.mjs b/tests/fixtures/three/spatial.mjs new file mode 100644 index 00000000..d312c534 --- /dev/null +++ b/tests/fixtures/three/spatial.mjs @@ -0,0 +1,59 @@ +import { Vector2 } from 'three/src/math/Vector2.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; +import { Euler } from 'three/src/math/Euler.js'; +import { Quaternion } from 'three/src/math/Quaternion.js'; +import { Matrix4 } from 'three/src/math/Matrix4.js'; +import { Box2 } from 'three/src/math/Box2.js'; +import { Box3 } from 'three/src/math/Box3.js'; +import { Sphere } from 'three/src/math/Sphere.js'; +import { Ray } from 'three/src/math/Ray.js'; +import { Plane } from 'three/src/math/Plane.js'; +import { Triangle } from 'three/src/math/Triangle.js'; +import { Frustum } from 'three/src/math/Frustum.js'; +import { Spherical } from 'three/src/math/Spherical.js'; +import { Cylindrical } from 'three/src/math/Cylindrical.js'; +import { generateUUID, seededRandom } from 'three/src/math/MathUtils.js'; + +const euler = new Euler(0, 0, 0); +const quaternion = new Quaternion().setFromEuler(euler); +console.log('rotation', quaternion.x, quaternion.y, quaternion.z, quaternion.w, euler.order); +Euler.DEFAULT_ORDER = 'ZYX'; +console.log('default', new Euler().order, euler.order); +Euler.DEFAULT_ORDER = 'XYZ'; +euler.set(0, Math.PI / 2, 0); +const rotated = new Vector3(1, 0, 0).applyEuler(euler); +console.log('rotated', Math.round(rotated.x), Math.round(rotated.y), Math.round(rotated.z)); + +const box = new Box3(new Vector3(-1, -1, -1), new Vector3(1, 1, 1)); +console.log('box', box.containsPoint(new Vector3()), box.containsPoint(new Vector3(2, 0, 0))); +box.expandByPoint(new Vector3(2, 3, 4)); +const size = box.getSize(new Vector3()); +console.log('size', size.x, size.y, size.z); +const sphere = new Sphere(new Vector3(), 1); +console.log('sphere', sphere.containsPoint(new Vector3(0, 0.5, 0)), sphere.intersectsBox(box)); +console.log('separated', sphere.intersectsSphere(new Sphere(new Vector3(5, 0, 0), 1))); +const ray = new Ray(new Vector3(0, 0, 8), new Vector3(0, 0, -1)); +const hit = ray.intersectBox(box, new Vector3()); +console.log('box hit', hit.x, hit.y, hit.z); +const sphereHit = ray.intersectSphere(sphere, new Vector3()); +console.log('sphere hit', sphereHit.x, sphereHit.y, sphereHit.z); +const miss = new Ray(new Vector3(10, 10, 10), new Vector3(1, 0, 0)); +console.log('miss', miss.intersectBox(box, new Vector3()) === null, miss.intersectSphere(sphere, new Vector3()) === null); +const plane = new Plane(new Vector3(0, 0, 1), -2); +const planeHit = ray.intersectPlane(plane, new Vector3()); +console.log('plane', planeHit.z, plane.distanceToPoint(new Vector3(0, 0, 5))); +const triangle = new Triangle(new Vector3(0, 0, 0), new Vector3(2, 0, 0), new Vector3(0, 2, 0)); +console.log('triangle', triangle.getArea(), triangle.getNormal(new Vector3()).z); +const barycentric = triangle.getBarycoord(new Vector3(0.5, 0.5, 0), new Vector3()); +console.log('barycentric', barycentric.x, barycentric.y, barycentric.z); +const projection = new Matrix4().makePerspective(-1, 1, 1, -1, 1, 10); +const frustum = new Frustum().setFromProjectionMatrix(projection); +console.log('frustum', frustum.containsPoint(new Vector3(0, 0, -2)), frustum.containsPoint(new Vector3(0, 0, 2))); +console.log('visible', frustum.intersectsSphere(new Sphere(new Vector3(0, 0, -3), 0.5)), frustum.intersectsSphere(new Sphere(new Vector3(20, 0, -3), 0.5))); +const box2 = new Box2().setFromPoints([new Vector2(-1, -2), new Vector2(3, 4)]); +console.log('box2', box2.min.x, box2.max.y, box2.containsPoint(new Vector2(0, 0))); +const spherical = new Spherical().setFromVector3(new Vector3(0, 0, 2)); +const cylindrical = new Cylindrical().setFromVector3(new Vector3(0, 3, 2)); +console.log('coordinates', spherical.radius, cylindrical.radius, cylindrical.y); +console.log('uuid', /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(generateUUID())); +console.log('seed', seededRandom(42).toFixed(6), seededRandom().toFixed(6)); diff --git a/tests/harness/__snapshots__/class-values.ts.txt b/tests/harness/__snapshots__/class-values.ts.txt index ef410ef3..1d502afe 100644 --- a/tests/harness/__snapshots__/class-values.ts.txt +++ b/tests/harness/__snapshots__/class-values.ts.txt @@ -72,9 +72,9 @@ class-values.ts:69:16 - error SC1090: constructing through a union of class valu | ^~~~~~~~~~ 70 | void picked; -class-values.ts:75:12 - error SC1090: 'this' in static methods (it names the RECEIVER class — a dynamic value; reference the class by name instead) are not supported yet +class-values.ts:79:56 - error SC1090: receiver-dependent static methods through a polymorphic class value is not supported yet - 74 | static self(): string { - 75 | return this.name; - | ^~~~ - 76 | } \ No newline at end of file + 78 | class OtherThis extends UsesThis {} + 79 | function self(value: typeof UsesThis): string { return value.self(); } + | ^~~~~~~~~~ + 80 | console.log(self(OtherThis)); \ No newline at end of file diff --git a/tests/harness/ffi-catalog.test.ts b/tests/harness/ffi-catalog.test.ts index ccbdf484..8a88c886 100644 --- a/tests/harness/ffi-catalog.test.ts +++ b/tests/harness/ffi-catalog.test.ts @@ -31,7 +31,7 @@ describe.each(["llvm"] as const)("static node:ffi catalog, %s", backend => { expect(native.error).toBeUndefined(); expect(native.stderr).toBe(""); expect(native.status).toBe(0); - expect(native.stdout).toBe("true\n9007199254740993n\ntrue\n0n 0n\n65 255\n-9223372036854775808n 18446744073709551615n\n" + "TypeError ERR_INVALID_ARG_VALUE\n".repeat(12) + "5\narity\narity\nsignature\nclosed\n42n\nsignature mismatch\n"); + expect(native.stdout).toBe("true\n9007199254740993n\ntrue\n0n 0n\n65 255\n-9223372036854775808n 18446744073709551615n\n" + "TypeError ERR_INVALID_ARG_VALUE\n".repeat(12) + "5\narity\narity\nsignature\nclosed\n42n\nsignature mismatch\nmutated signature\nmissing arguments Error SC2020\n"); // A separately linked Node library is only the oracle. scriptc links // the object file and must work before this dynamic library exists. if (oracle) { @@ -42,7 +42,7 @@ describe.each(["llvm"] as const)("static node:ffi catalog, %s", backend => { expect(node.error).toBeUndefined(); expect(node.stderr).toBe(native.stderr); expect(node.status).toBe(native.status); - expect(node.stdout + "signature mismatch\n").toBe(native.stdout); + expect(node.stdout + "signature mismatch\nmutated signature\nmissing arguments Error SC2020\n").toBe(native.stdout); } }); }); diff --git a/tests/harness/library-dispatch.test.ts b/tests/harness/library-dispatch.test.ts new file mode 100644 index 00000000..1c7285a0 --- /dev/null +++ b/tests/harness/library-dispatch.test.ts @@ -0,0 +1,50 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { WASI } from "node:wasi"; +import { expect, test } from "vitest"; +import { compileLibrary } from "@scriptc/compiler"; + +const fixture = join(import.meta.dirname, "../library-mode/wasm/dispatch.mjs"); +const modes = ["native", "sanitized", "wasm-dev"] as const; + +test.for(modes)("synchronous checked dispatch in a %s library matches Node", async (mode, context) => { + if (mode === "wasm-dev" ? spawnSync("zig", ["version"]).status !== 0 : process.platform !== "darwin" && process.platform !== "linux") context.skip(); + const directory = await mkdtemp("/tmp/scriptc-library-dispatch-"); + const oldTarget = process.env["SCRIPTC_TARGET"]; + try { + if (mode === "wasm-dev") process.env["SCRIPTC_TARGET"] = "wasm32-wasi"; + else delete process.env["SCRIPTC_TARGET"]; + const profilePath = join(directory, "profile.json"); + await writeFile(profilePath, JSON.stringify({ + profile_format: 1, name: "sync-dispatch", entry: fixture, emission: "llvm", + optimization: mode === "wasm-dev" ? "dev" : "release", + abi: { prefix: "app_", init_symbol: "app_init", sink_register_symbol: "app_sink" }, + exports: [{ export: "dispatch", symbol: "app_dispatch", params: [], returns: "f64" }], + })); + const result = await compileLibrary({ profilePath, outDir: directory, sanitize: mode === "sanitized" }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + const reference = spawnSync(process.execPath, ["--no-warnings", "--input-type=module", "-e", `import {dispatch} from ${JSON.stringify(fixture)}; console.log(dispatch());`], { encoding: "utf8" }); + let actual: { stdout: string; stderr: string; status: number | null }; + if (mode === "wasm-dev") { + const wasi = new WASI({ version: "preview1" }); + const instance = new WebAssembly.Instance(new WebAssembly.Module(await readFile(result.archivePath)), { + ...wasi.getImportObject(), scriptc: { panic() { throw new Error("unexpected panic"); } }, + }); + wasi.initialize(instance); + (instance.exports.app_init as Function)(); + actual = { stdout: `${(instance.exports.app_dispatch as Function)()}\n`, stderr: "", status: 0 }; + } else { + const probe = join(directory, "probe.c"), binary = join(directory, "probe"); + await writeFile(probe, '#include \nvoid app_init(void); double app_dispatch(void);\nint main(void) { app_init(); printf("%.0f\\n", app_dispatch()); app_init(); return 0; }\n'); + const link = spawnSync("clang", [probe, result.archivePath, "-lm", ...(mode === "sanitized" ? ["-fsanitize=address"] : []), "-o", binary], { encoding: "utf8" }); + expect(link.status, link.stderr).toBe(0); + actual = spawnSync(binary, [], { encoding: "utf8" }); + } + expect({ stdout: actual.stdout, stderr: actual.stderr, status: actual.status }).toEqual({ stdout: reference.stdout, stderr: reference.stderr, status: reference.status }); + } finally { + if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"]; + else process.env["SCRIPTC_TARGET"] = oldTarget; + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/harness/library-multi.test.ts b/tests/harness/library-multi.test.ts index eaf70e40..f96bb293 100644 --- a/tests/harness/library-multi.test.ts +++ b/tests/harness/library-multi.test.ts @@ -1071,22 +1071,30 @@ test.each(["aarch64-ios", "x86_64-linux-android", "armv7-linux-androideabi"])( }, ); -test.each(MOBILE_TARGETS)( - "M12: the executable lane refuses %s with the pointer to --lib", - async (target) => { +test.each(MOBILE_TARGETS.flatMap((target) => + ["darwin", "linux", "win32"].map((host) => ({ target, host })), +))( + "M12: the executable lane refuses $target on $host with the pointer to --lib", + async ({ target, host }) => { const outDir = join(cacheDir, `mobile-exe-refusal-${target}`); mkdirSync(outDir, { recursive: true }); const entry = join(outDir, "main.ts"); writeFileSync(entry, 'console.log("hi");\n'); - await withMobileTarget(target, async () => { - const result = await compile(entry, { outDir, outPath: join(outDir, "main") }); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.diagnostics[0]!.code).toBe("SC3002"); - expect(result.diagnostics[0]!.message).toContain(target); - expect(result.diagnostics[0]!.message).toContain("scriptc build --lib --profile "); - } - }); + const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform")!; + Object.defineProperty(process, "platform", { ...platformDescriptor, value: host }); + try { + await withMobileTarget(target, async () => { + const result = await compile(entry, { outDir, outPath: join(outDir, "main") }); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.diagnostics[0]!.code).toBe("SC3002"); + expect(result.diagnostics[0]!.message).toContain(target); + expect(result.diagnostics[0]!.message).toContain("scriptc build --lib --profile "); + } + }); + } finally { + Object.defineProperty(process, "platform", platformDescriptor); + } }, ); diff --git a/tests/harness/library-profile.test.ts b/tests/harness/library-profile.test.ts index 819a55be..2fcd350a 100644 --- a/tests/harness/library-profile.test.ts +++ b/tests/harness/library-profile.test.ts @@ -48,6 +48,14 @@ function expectSc4001(json: unknown, fragment: string): void { } describe("library profile validation", () => { + test("explicit npm source attempts accept package names only", () => { + const result = loadLibraryProfile(writeProfile({ ...good, npm_static: ["three", "@scope/pkg", "three"] })); + expect(result.ok).toBe(true); + if (result.ok) expect(result.profile.npmStatic).toEqual(["three", "@scope/pkg"]); + for (const npm_static of [null, "three", [null], [""], ["three/src/math"], ["../three"]]) { + expectSc4001({ ...good, npm_static }, "npm_static"); + } + }); test("well-formed profile resolves", () => { const path = writeProfile(good); const r = loadLibraryProfile(path); diff --git a/tests/harness/library-static-types.test.ts b/tests/harness/library-static-types.test.ts new file mode 100644 index 00000000..b9174b28 --- /dev/null +++ b/tests/harness/library-static-types.test.ts @@ -0,0 +1,53 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compileLibrary } from "@scriptc/compiler"; + +test.each(["sample", "@scope/sample"])("explicit library attempts use %s runtime code with installed declaration twins", async (name) => { + const dir = await mkdtemp("/tmp/scriptc-library-static-types-"); + try { + const pkg = join(dir, "node_modules", name); + const types = join(dir, "node_modules/@types", name.startsWith("@") ? "scope__sample" : name); + await mkdir(pkg, { recursive: true }); + await mkdir(types, { recursive: true }); + await writeFile(join(dir, "package.json"), '{"type":"module"}'); + await writeFile(join(pkg, "package.json"), JSON.stringify({ name, version: "1.0.0", type: "module", exports: { ".": "./index.js", "./value": "./index.js" } })); + await writeFile(join(pkg, "index.js"), "export function calculate(value) { return value * 3 + 1; }\n"); + await writeFile(join(types, "package.json"), JSON.stringify({ name: `@types/${name.startsWith("@") ? "scope__sample" : name}`, version: "1.0.0", types: "index.d.ts" })); + const declaration = "export function calculate(value: number): number;\n"; + await writeFile(join(types, "index.d.ts"), declaration); + await writeFile(join(types, "value.d.ts"), declaration); + await writeFile(join(dir, "entry.ts"), `import { calculate } from '${name}/value';\nexport function result(): number { return calculate(7); }\n`); + const profile = { + profile_format: 1, name: "static-types", entry: "entry.ts", emission: "llvm", npm_static: [name], + abi: { prefix: "st_", init_symbol: "st_init", sink_register_symbol: "st_sink", collect_symbol: "st_collect" }, + exports: [{ export: "result", symbol: "st_result", params: [], returns: "f64" }], + }; + const profilePath = join(dir, "profile.json"); + await writeFile(profilePath, JSON.stringify(profile)); + const result = await compileLibrary({ profilePath, outDir: join(dir, "out") }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + if (process.env["SCRIPTC_PORTABLE_ONLY"] !== "1") { + const source = join(dir, "host.c"), binary = join(dir, "host"); + await writeFile(source, '#include \nextern void st_init(void); extern double st_result(void);\nint main(void) { st_init(); printf("%.0f\\n", st_result()); }\n'); + execFileSync("clang", [source, result.archivePath, "-lm", "-o", binary]); + const actual = execFileSync(binary, { encoding: "utf8" }); + const reference = execFileSync(process.execPath, ["--input-type=module", "-e", `import { result } from ${JSON.stringify(join(dir, "entry.ts"))}; console.log(result());`], { encoding: "utf8" }); + expect(actual).toBe(reference); + expect(actual).toBe("22\n"); + } + // Opting in to one runtime package does not admit its untyped dependency. + const dependency = join(dir, "node_modules/unchecked-dependency"); + await mkdir(dependency, { recursive: true }); + await writeFile(join(dependency, "package.json"), '{"name":"unchecked-dependency","version":"1.0.0","type":"module","main":"index.js"}'); + await writeFile(join(dependency, "index.js"), 'export function adjust(value) { return value; }'); + await writeFile(join(pkg, "index.js"), "import { adjust } from 'unchecked-dependency'; export function calculate(value) { return adjust(value) * 3 + 1; }\n"); + const refused = await compileLibrary({ profilePath, outDir: join(dir, "refused") }); + expect(refused.ok).toBe(false); + if (!refused.ok) expect(refused.diagnostics.some((d) => d.code === "SC4020" && d.message.includes("unchecked-dependency"))).toBe(true); + expect(await readFile(join(types, "index.d.ts"), "utf8")).toBe(declaration); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/harness/runtime-tree-shaking.test.ts b/tests/harness/runtime-tree-shaking.test.ts index 1af62f35..247ba3db 100644 --- a/tests/harness/runtime-tree-shaking.test.ts +++ b/tests/harness/runtime-tree-shaking.test.ts @@ -1,9 +1,6 @@ -/* Executable runtime reachability. The source-runtime recipe intentionally - * still includes the complete historical base for direct compileC callers; - * executable section GC is what makes its unused functions/data disappear. - * These fixtures therefore pin both halves of that contract: hello has no - * reachable members from the formerly-unavoidable families, while every - * feature fixture retains an anchor and behaves exactly like Node. */ +/* Executable runtime reachability. Section GC removes unused code from the + * precompiled runtime pack. Hello must discard unreachable families, while + * each feature fixture retains its runtime anchor and matches Node. */ import { execFile } from "node:child_process"; import { createServer } from "node:http"; import { mkdirSync, statSync, writeFileSync } from "node:fs"; @@ -16,7 +13,7 @@ const execFileAsync = promisify(execFile); const repoRoot = join(import.meta.dirname, "../.."); const cacheDir = join(repoRoot, "node_modules/.cache/scriptc-tests/runtime-tree-shaking"); // The executable and symbol contracts use POSIX tools and /bin/echo. -const sourceToolchainTest = process.platform === "win32" ? test.skip : test; +const nativeToolchainTest = process.platform === "win32" ? test.skip : test; interface Fixture { name: string; @@ -25,6 +22,11 @@ interface Fixture { } const FIXTURES: Fixture[] = [ + { + name: "regex", + source: `console.log(/world/.test("hello world"));\n`, + anchor: "scr_regex_test", + }, { name: "child", source: `import { spawnSync } from "node:child_process"; @@ -89,8 +91,6 @@ async function build(name: string, source: string) { const result = await compile(sourcePath, { outPath: join(outDir, "program"), outDir, - // The C lane proves source-toolchain linking. macOS additionally runs - // the default lane below, which selects the helper/runtime-pack path. backend: "llvm", }); if (!result.ok) { @@ -110,6 +110,12 @@ async function symbols(binaryPath: string): Promise { return output("nm", [binaryPath]); } +async function expectNoReleaseDebugInfo(binaryPath: string): Promise { + if (process.platform !== "linux") return; + const sections = await output("readelf", ["--section-headers", "--wide", binaryPath]); + expect(sections).not.toMatch(/\.(?:z?debug_|stab)/); +} + async function expectNodeParity(sourcePath: string, binaryPath: string, args: string[] = []): Promise { const [node, native] = await Promise.all([ output(process.execPath, [sourcePath, ...args]), @@ -118,10 +124,11 @@ async function expectNodeParity(sourcePath: string, binaryPath: string, args: st expect(native).toBe(node); } -sourceToolchainTest("static hello strips unreachable runtime families while feature programs retain them", async () => { +nativeToolchainTest("static hello strips unreachable runtime families while feature programs retain them", async () => { const helloSource = `console.log("hello", "world");\n`; const hello = await build("hello", helloSource); await expectNodeParity(hello.sourcePath, hello.binaryPath); + await expectNoReleaseDebugInfo(hello.binaryPath); const helloSymbols = await symbols(hello.binaryPath); for (const family of [ "scr_path_win32_", @@ -139,6 +146,7 @@ sourceToolchainTest("static hello strips unreachable runtime families while feat if (fixture.name === "child" && process.platform === "win32") continue; const result = await build(fixture.name, fixture.source); await expectNodeParity(result.sourcePath, result.binaryPath); + await expectNoReleaseDebugInfo(result.binaryPath); const nativeSymbols = await symbols(result.binaryPath); expect(nativeSymbols, `${fixture.name} lost ${fixture.anchor}`).toContain(fixture.anchor); } @@ -146,14 +154,14 @@ sourceToolchainTest("static hello strips unreachable runtime families while feat // Symbol absence is the primary reachability contract. Keep a deliberately // roomy, platform-specific hello-world ceiling too: it catches losing // section GC without pinning an exact linker/SDK byte count. The canonical - // Linux C build is about 41KB and current Mach-O builds are about 70KB; + // Linux build is about 41KB and current Mach-O builds are about 70KB; // these limits leave several native pages of linker-version slack while // remaining far below the former roughly-400KB always-linked runtime. const helloSizeLimit = process.platform === "linux" ? 64 * 1024 : 96 * 1024; expect(statSync(hello.binaryPath).size).toBeLessThan(helloSizeLimit); }); -sourceToolchainTest("fetch response JSON retains the URL and parser runtime", async () => { +nativeToolchainTest("fetch response JSON retains the URL and parser runtime", async () => { const server = createServer((_request, response) => { response.setHeader("content-type", "application/json"); response.end('{"ok":true}'); @@ -171,6 +179,7 @@ console.log((await response.json()).ok); }; const result = await build(fixture.name, fixture.source); await expectNodeParity(result.sourcePath, result.binaryPath, [`http://127.0.0.1:${address.port}`]); + await expectNoReleaseDebugInfo(result.binaryPath); const nativeSymbols = await symbols(result.binaryPath); expect(nativeSymbols).toContain("scr_json_parse"); expect(nativeSymbols).toContain("scr_url_release"); @@ -179,9 +188,7 @@ console.log((await response.json()).ok); } }); -// The helper emits a native program object only on supported macOS arm64. -// Its normal backend path links the precompiled runtime pack; run the same -// reachability assertion there so the source-only C lane cannot regress it. +// Also exercise default backend selection on the Darwin shipping host. test.skipIf(process.platform !== "darwin" || process.arch !== "arm64")( "macOS helper/runtime-pack links dead-strip static hello too", async () => { diff --git a/tests/harness/sandbox-command.test.ts b/tests/harness/sandbox-command.test.ts index 6c1ab796..366b29be 100644 --- a/tests/harness/sandbox-command.test.ts +++ b/tests/harness/sandbox-command.test.ts @@ -2,26 +2,66 @@ import { execFile, execFileSync } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; -import { afterEach, expect, test } from "vitest"; -import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand } from "../../scripts/sandbox-command.mjs"; +import { afterEach, expect, test, vi } from "vitest"; +import { REMOTE_COMMAND_PENDING, sandboxCommand, sandboxStatusCommand, waitForSandboxCommand } from "../../scripts/sandbox-command.mjs"; const roots: string[] = []; const statuses: string[] = []; afterEach(async () => { + vi.useRealTimers(); await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); await Promise.all(statuses.splice(0).map((path) => rm(path, { force: true }))); }); +test("status recovery retries lost probes and pending commands until a confirmed success", async () => { + vi.useFakeTimers(); + const probe = vi.fn() + .mockRejectedValueOnce(new Error("transport disconnected")) + .mockRejectedValueOnce(Object.assign(new Error("pending"), { remoteExitCode: REMOTE_COMMAND_PENDING })) + .mockResolvedValueOnce(undefined); + const onPending = vi.fn(); + const recovery = waitForSandboxCommand(probe, { deadline: Date.now() + 5000, label: "tests", onPending }); + await vi.runAllTimersAsync(); + await expect(recovery).resolves.toBeUndefined(); + expect(probe.mock.calls).toEqual([[5000], [4000], [3000]]); + expect(onPending).toHaveBeenCalledTimes(2); +}); + +test("status recovery preserves a confirmed remote failure after a transport failure", async () => { + vi.useFakeTimers(); + const failure = Object.assign(new Error("tests failed"), { remoteExitCode: 7 }); + const probe = vi.fn().mockRejectedValueOnce(new Error("disconnected")).mockRejectedValue(failure); + const recovery = expect(waitForSandboxCommand(probe, { + deadline: Date.now() + 5000, label: "tests", onPending: vi.fn(), + })).rejects.toBe(failure); + await vi.runAllTimersAsync(); + await recovery; + expect(probe).toHaveBeenCalledTimes(2); +}); + +test("unconfirmed status probes cannot extend the original command deadline", async () => { + vi.useFakeTimers(); + const disconnected = new Error("status timeout"); + const probe = vi.fn().mockRejectedValue(disconnected); + const recovery = expect(waitForSandboxCommand(probe, { + deadline: Date.now() + 1500, label: "tests", onPending: vi.fn(), + })).rejects.toMatchObject({ message: "tests did not confirm completion before its timeout", cause: disconnected }); + await vi.runAllTimersAsync(); + await recovery; + expect(probe.mock.calls).toEqual([[1500], [500]]); +}); + test("short commands stay inline and retain the remote exit contract", async () => { const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_SHORT__`; - const prepared = sandboxCommand("sh", ["-c", "exit 7"], marker); - statuses.push(prepared.statusPath); + const prepared = sandboxCommand("sh", ["-c", "printf 'failure detail\\n' >&2; exit 7"], marker); + statuses.push(prepared.statusPath, prepared.logPath); expect(prepared.file).toBe(false); const output = execFileSync(prepared.argv[0], prepared.argv.slice(1), { encoding: "utf8" }); - expect(output).toBe(`\n${marker}7\n`); + expect(output).toBe(`failure detail\n\n${marker}7\n`); expect(await readFile(prepared.statusPath, "utf8")).toBe("7\n"); + expect(await readFile(prepared.logPath, "utf8")).toBe("failure detail\n"); }); test("uploaded scripts preserve long and shell-sensitive argument bytes", async () => { @@ -37,7 +77,7 @@ test("uploaded scripts preserve long and shell-sensitive argument bytes", async ]; const marker = `__SCRIPTC_COMMAND_TEST_${process.pid}_LONG__`; const prepared = sandboxCommand("printf", ["<%s>\n", ...args], marker); - statuses.push(prepared.statusPath); + statuses.push(prepared.statusPath, prepared.logPath); expect(prepared.file).toBe(true); expect(prepared.argv).toEqual(["sh", prepared.scriptPath]); expect(prepared.argv.every((arg) => arg.length < 128)).toBe(true); @@ -46,6 +86,7 @@ test("uploaded scripts preserve long and shell-sensitive argument bytes", async await writeFile(localScript, prepared.script); const output = execFileSync("sh", [localScript], { encoding: "utf8" }); expect(output).toBe(args.map((arg) => `<${arg}>\n`).join("") + `\n${marker}0\n`); + expect(await readFile(prepared.logPath, "utf8")).toBe(args.map((arg) => `<${arg}>\n`).join("")); await expect(readFile(sentinel)).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(prepared.statusPath, "utf8")).toBe("0\n"); }); diff --git a/tests/harness/sandbox-config.test.ts b/tests/harness/sandbox-config.test.ts index d2c8df5c..71f637ae 100644 --- a/tests/harness/sandbox-config.test.ts +++ b/tests/harness/sandbox-config.test.ts @@ -30,6 +30,7 @@ test("runner settings are read after loading the sandbox environment", () => { localTestWorkers: "3", localCaseShards: "4", sandboxTimeout: "90m", + sandboxTimeoutMs: 5_400_000, }); }); @@ -40,9 +41,19 @@ test("runner settings retain their documented defaults", () => { localTestWorkers: "2", localCaseShards: "2", sandboxTimeout: "45m", + sandboxTimeoutMs: 2_700_000, }); }); +test("the local command deadline follows the configured Sandbox duration", () => { + for (const [duration, milliseconds] of [["120m", 7_200_000], ["2h", 7_200_000], ["30 s", 30_000], ["500ms", 500]] as const) { + expect(sandboxRunnerConfig({ SCRIPTC_SANDBOX_TIMEOUT: duration }).sandboxTimeoutMs).toBe(milliseconds); + } + for (const duration of ["0m", "-1h", "forever", "1.5m", "999999999h"]) { + expect(() => sandboxRunnerConfig({ SCRIPTC_SANDBOX_TIMEOUT: duration })).toThrow("positive duration"); + } +}); + test("the managed Sandbox image is the default", () => { expect(sandboxImageConfig({})).toEqual({ custom: false, diff --git a/tests/harness/self-hosting-bootstrap.test.ts b/tests/harness/self-hosting-bootstrap.test.ts deleted file mode 100644 index 346a3007..00000000 --- a/tests/harness/self-hosting-bootstrap.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { execFile, execFileSync, spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { pathToFileURL } from "node:url"; -import { promisify } from "node:util"; -import { expect, test } from "vitest"; -import { compileC, deserializeModule, validateModule } from "@scriptc/compiler"; -import type { compile } from "@scriptc/compiler"; -import { nativeFeatures } from "./self-hosting-native-features.js"; -import { bootstrapStep } from "./self-hosting-timing.js"; -import { ts7Executable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; - -const root = join(import.meta.dirname, "../.."); -const execFileAsync = promisify(execFile); -const sanitize = process.env["SCRIPTC_SAN"] === "1"; -// ASan instruments the entire compiler graph, including its native AST and -// IR traversal. Keep a separate budget for those full self-compilation passes. -const options = { cwd: root, timeout: sanitize ? 3_600_000 : 1_200_000, maxBuffer: 16 * 1024 * 1024 }; - -function comparableStderr(text: string): string { - return sanitize - ? text.replace(/^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, "") - : text; -} - -test("the native frontend and LLVM emitter rebuild a working frontend from its TypeScript source", async () => { - const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-bootstrap-")); - const executable = (name: string) => join(directory, name + (process.platform === "win32" ? ".exe" : "")); - const frontend = join(root, "tests/fixtures/self-hosting/frontend-lowering.ts"); - const emitter = join(root, "tests/fixtures/self-hosting/llvm-emitter.ts"); - const nativeOptions = { ...options, env: { ...process.env, PATH: "" } }; - try { - const nativeSources = join(root, "packages/compiler/native"); - const object = join(directory, "ts7-process.o"); - execFileSync("clang", ["-std=c11", "-Wall", "-Wextra", "-Werror", ...(sanitize ? ["-fsanitize=address"] : []), - "-c", join(nativeSources, "ts7-process.c"), "-o", object]); - const profile = join(directory, "ffi.json"); - writeFileSync(profile, JSON.stringify({ - ...JSON.parse(readFileSync(join(nativeSources, "ts7-process.ffi.json"), "utf8")), libraries: [object], - })); - - const buildSeed = async (entry: string, name: string, ffi?: string) => { - const api = pathToFileURL(join(root, "packages/compiler/src/index.ts")).href; - const built = await execFileAsync(process.execPath, [ - "--max-old-space-size=8192", "--import", "tsx", "--input-type=module", "--eval", - `import { compile } from ${JSON.stringify(api)}; - const result = await compile(process.argv[1], { - outDir: process.argv[2], outPath: process.argv[3], backend: 'llvm', - dynamic: false, optimization: 'release', strip: true, sanitize: process.argv[4] === '1', - ffiProfilePath: process.argv[5] || undefined, emitIr: true, - }); - console.log(JSON.stringify({ ...result, sourceTexts: undefined }));`, - entry, directory, executable(name), sanitize ? "1" : "0", ffi ?? "", - ], options); - expect(built.stderr).toBe(""); - const result = JSON.parse(built.stdout) as Awaited>; - if (!result.ok) throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); - if (!("binaryPath" in result)) throw new Error("expected a native frontend executable"); - expect(result.backend).toBe("llvm"); - expect(result.irPath).toBeDefined(); - return result; - }; - const seed = await bootstrapStep("build frontend seed", () => buildSeed(frontend, "frontend-seed", profile)); - const nativeEmitter = await bootstrapStep("build LLVM emitter seed", () => buildSeed(emitter, "emitter")); - const emitterRequest = join(directory, "emitter-request.json"); - writeFileSync(emitterRequest, JSON.stringify({ debug: true, sources: [], pointerBits: 64, wasi: false, emitLibraryIdentity: true, runtimeAbiMarker: false })); - const ownIr = join(directory, "frontend-native.json"); - const self = await bootstrapStep("frontend lowers itself", () => - execFileAsync(seed.binaryPath, [ts7Executable(), frontend, ownIr, profile], nativeOptions)); - expect(self.stdout).toBe("0\n0 0\n"); - expect(comparableStderr(self.stderr)).toBe(""); - - // Compare complete IR, including layouts and helper signatures. A valid - // module alone can hide different optional-return inference or dropped - // method-registry mutations in the native compiler. - const cPath = join(directory, "frontend-native.ll"); - const emitted = await bootstrapStep("emit frontend LLVM", () => - execFileAsync(nativeEmitter.binaryPath, [ownIr, cPath, emitterRequest], nativeOptions)); - expect(emitted.stdout).toBe(""); - expect(comparableStderr(emitted.stderr)).toBe(""); - // Full compiler graphs exceed the worker's default heap. Keep both the - // structural IR comparison and reference emission in a roomy child. - const verification = await bootstrapStep("compare frontend IR and LLVM", () => execFileAsync(process.execPath, [ - "--max-old-space-size=8192", "--import", "tsx", "--input-type=module", "--eval", - `import assert from 'node:assert/strict'; - import { readFileSync } from 'node:fs'; - import { isDeepStrictEqual } from 'node:util'; - import { deserializeModule, validateModule, emitLlvmModule } from ${JSON.stringify(pathToFileURL(join(root, "packages/compiler/src/index.ts")).href)}; - import { nativeFeatures } from ${JSON.stringify(new URL("./self-hosting-native-features.ts", import.meta.url).href)}; - const module = deserializeModule(readFileSync(process.argv[1], 'utf8')); - assert.ok(module.functions.length > 1000); - assert.deepEqual(validateModule(module), []); - assert.ok(isDeepStrictEqual(module, deserializeModule(readFileSync(process.argv[2], 'utf8'))), 'native self-lowering must match the Node seed'); - assert.ok(readFileSync(process.argv[3], 'utf8') === emitLlvmModule(module, { debugSources: new Map() }), 'native LLVM emission must match Node exactly'); - console.log(JSON.stringify(nativeFeatures(module)));`, - ownIr, seed.irPath!, cPath, - ], options)); - expect(verification.stderr).toBe(""); - const features = JSON.parse(verification.stdout) as ReturnType; - const rebuilt = executable("frontend-rebuilt"); - await bootstrapStep("build second frontend generation", () => - compileC({ cPath, outPath: rebuilt, optimization: "release", strip: true, sanitize, linkInputs: [object], ...features })); - - // The rebuilt compiler consumes new source, and its emitted program - // executes. These inputs exercise the two bugs found by self-compiling: - // long nullish dispatch and optional class-method return promotion. - for (const name of ["nullish-long-chain", "class-array-optional-return"]) { - const source = join(root, "tests/corpus", name + ".ts"); - const actualIr = join(directory, name + "-native.json"); - const expectedIr = join(directory, name + "-node.json"); - const expected = await execFileAsync(process.execPath, ["--import", "tsx", - join(root, "tests/fixtures/self-hosting/frontend-lowering-node.ts"), ts7Executable(), source, expectedIr], options); - const actual = await execFileAsync(rebuilt, [ts7Executable(), source, actualIr], nativeOptions); - expect(expected.stdout).toBe("0\n0 0\n"); - expect(actual.stdout).toBe(expected.stdout); - expect(expected.stderr).toBe(""); - expect(comparableStderr(actual.stderr)).toBe(""); - const program = deserializeModule(readFileSync(actualIr, "utf8")); - expect(program).toEqual(deserializeModule(readFileSync(expectedIr, "utf8"))); - expect(validateModule(program)).toEqual([]); - const programC = join(directory, "program.ll"); - const emission = await execFileAsync(nativeEmitter.binaryPath, [actualIr, programC, emitterRequest], nativeOptions); - expect(emission.stdout).toBe(""); - expect(comparableStderr(emission.stderr)).toBe(""); - await compileC({ cPath: programC, outPath: executable("program"), sanitize, ...nativeFeatures(program) }); - const oracle = spawnSync(process.execPath, [source], options); - const native = spawnSync(executable("program"), [], options); - for (const result of [oracle, native]) { - expect(result.error).toBeUndefined(); - expect(result.signal).toBeNull(); - expect(result.status, result.stderr.toString()).toBe(0); - } - expect(native.stdout).toEqual(oracle.stdout); - expect(native.stderr).toEqual(oracle.stderr); - } - } finally { rmSync(directory, { recursive: true, force: true }); } -}, sanitize ? 7_200_000 : 2_400_000); diff --git a/tests/harness/self-hosting-class-types.test.ts b/tests/harness/self-hosting-class-types.test.ts new file mode 100644 index 00000000..b6585b3c --- /dev/null +++ b/tests/harness/self-hosting-class-types.test.ts @@ -0,0 +1,30 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compileInChild } from "./self-hosting-compiler-process.js"; + +test("native class type cleanup updates shared IR without dynamic snapshots", async () => { + const root = join(import.meta.dirname, "../.."); + const entry = join(root, "tests/fixtures/self-hosting/class-types.ts"); + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-class-types-")); + try { + const built = await compileInChild(entry, { outDir: directory, + outPath: join(directory, "class-types" + (process.platform === "win32" ? ".exe" : "")), + optimization: "dev", dynamic: false, sanitize: process.env["SCRIPTC_SAN"] === "1" }); + if (!built.ok) throw new Error(JSON.stringify(built.diagnostics)); + const options = { cwd: root, encoding: "utf8" as const, timeout: 30_000 }; + const oracle = spawnSync(process.execPath, ["--import", "tsx", entry], options); + const native = spawnSync(built.binaryPath, [], options); + for (const result of [oracle, native]) { + expect(result.error).toBeUndefined(); + expect(result.signal, result.stderr).toBeNull(); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + } + expect(native.stdout).toBe(oracle.stdout); + expect(native.stdout).toMatch(/^f64 f64\n/); + expect(native.stdout).toMatch(/object object f64\nf64\n$/); + } finally { rmSync(directory, { recursive: true, force: true }); } +}, 300_000); diff --git a/tests/harness/self-hosting-native-cache.test.ts b/tests/harness/self-hosting-native-cache.test.ts new file mode 100644 index 00000000..5ce0aa00 --- /dev/null +++ b/tests/harness/self-hosting-native-cache.test.ts @@ -0,0 +1,41 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { compileInChild } from "./self-hosting-compiler-process.js"; + +const root = join(import.meta.dirname, "../.."); +const entry = join(root, "tests/fixtures/self-hosting/native-cache.ts"); + +test("native executable cache discovery, restoration and invalidation match Node", async () => { + const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-cache-")); + try { + const built = await compileInChild(entry, { outDir: directory, + outPath: join(directory, "cache" + (process.platform === "win32" ? ".exe" : "")), + optimization: "dev", dynamic: false, sanitize: process.env["SCRIPTC_SAN"] === "1" }); + if (!built.ok) throw new Error(JSON.stringify(built.diagnostics)); + const linker = join(directory, "linker.mjs"); + writeFileSync(linker, 'console.error(JSON.stringify(process.argv[2]));\n'); + const sdk = join(directory, "sdk"); + mkdirSync(sdk); + const library = join(sdk, "system library.tbd"); + const results: string[] = []; + for (const native of [false, true]) { + const stage = join(directory, native ? "native" : "node"); + mkdirSync(stage); + writeFileSync(library, "system library"); + const args = [stage, process.execPath, linker, library]; + const result = spawnSync(native ? built.binaryPath : process.execPath, native ? args : ["--import", "tsx", entry, ...args], { + cwd: root, encoding: "utf8", timeout: 30_000, env: { ...process.env, ...(native ? { PATH: "" } : {}) }, + }); + expect(result.error).toBeUndefined(); + expect(result.signal, result.stderr).toBeNull(); + expect(result.status, result.stderr + result.stdout).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain("true\ntrue\ncached program\nfalse\n"); + results.push(result.stdout); + } + expect(results[1]).toBe(results[0]); + } finally { rmSync(directory, { recursive: true, force: true }); } +}, 300_000); diff --git a/tests/harness/self-hosting-native-driver.test.ts b/tests/harness/self-hosting-native-driver.test.ts index 90581859..70e48905 100644 --- a/tests/harness/self-hosting-native-driver.test.ts +++ b/tests/harness/self-hosting-native-driver.test.ts @@ -1,8 +1,10 @@ import { execFile, spawnSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, join, resolve } from "node:path"; import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; +import { WASI } from "node:wasi"; import { expect, test } from "vitest"; import type { NativeToolchainManifest } from "../../packages/compiler/src/native/toolchain.js"; import { bootstrapStep } from "./self-hosting-timing.js"; @@ -29,14 +31,23 @@ function absoluteCommand(command: string): string { throw new Error(`native tool is not on PATH: ${command}`); } -test("the standalone compiler builds programs and rebuilds itself with Node unavailable", async () => { +test("the production CLI relocates, builds programs, and rebuilds itself with Node unavailable", async () => { const directory = mkdtempSync(join(process.platform === "win32" ? tmpdir() : "/tmp", "scriptc-native-bootstrap-")); const executable = (name: string) => join(directory, name + (process.platform === "win32" ? ".exe" : "")); const options = { cwd: root, timeout: sanitize ? 5_400_000 : 1_800_000, maxBuffer: 16 * 1024 * 1024 }; try { - await bootstrapStep("build native compiler seed", () => - exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-compiler.mts"), directory], options)); - const seed = executable("scriptc-native"); + const distribution = join(directory, "distribution"); + await bootstrapStep("build production CLI seed", () => + exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", join(root, "scripts/build-native-cli.mts"), distribution], { + // The seed builds an instrumented second generation below. Run + // the full sanitizer proof with that generation instead of paying + // for two instrumented traversals of the complete compiler graph. + ...options, env: { ...process.env, SCRIPTC_SAN: "", SCRIPTC_NATIVE_EMIT_IR: "1" }, + })); + // All compiler assets must survive moving the complete distribution. + const relocated = join(directory, "relocated"); + renameSync(distribution, relocated); + const seed = join(relocated, "bin", "scriptc" + (process.platform === "win32" ? ".exe" : "")); const manifestPath = seed + ".json"; const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as NativeToolchainManifest; manifest.linker = absoluteCommand(manifest.linker); @@ -47,28 +58,53 @@ test("the standalone compiler builds programs and rebuilds itself with Node unav manifest.linker_args.push("--ld-path=" + absoluteCommand(linker.stdout.trim())); } if (process.platform === "darwin") manifest.dsymutil = absoluteCommand(manifest.dsymutil); + if (process.platform === "darwin") manifest.relocatable_linker = absoluteCommand("ld"); + manifest.archiver = absoluteCommand(process.platform === "win32" ? "zig" : "ar"); + manifest.archiver_args = process.platform === "win32" ? ["ar"] : []; writeFileSync(manifestPath, JSON.stringify(manifest)); - const nativeOptions = { ...options, env: { ...process.env, PATH: "" } }; - const invoke = async (compiler: string, args: string[]) => { - const result = await exec(compiler, [...args, "--toolchain", manifestPath], nativeOptions).catch((error: unknown) => { + let sanitizerCompiler: string | undefined; + if (sanitize) { + sanitizerCompiler = absoluteCommand(process.env["SCRIPTC_CC"] ?? "clang"); + if (process.platform === "linux") { + const linker = await exec(sanitizerCompiler, ["--print-prog-name=ld"], options); + const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; + const wrapper = join(directory, "sanitizer-clang"); + // Preserve the empty PATH while making Clang's child linker explicit. + writeFileSync(wrapper, `#!/bin/sh\nexec ${quote(sanitizerCompiler)} ${quote("--ld-path=" + absoluteCommand(linker.stdout.trim()))} "$@"\n`); + chmodSync(wrapper, 0o755); + sanitizerCompiler = wrapper; + } + } + const nativeOptions = { ...options, env: { ...process.env, PATH: "", SCRIPTC_TOOLCHAIN: manifestPath, + SCRIPTC_CACHE_DIR: join(directory, "cache"), ...(sanitizerCompiler === undefined ? {} : { SCRIPTC_CC: sanitizerCompiler }) } }; + const invoke = async (compiler: string, args: string[], expectedStderr = "") => { + const result = await exec(compiler, args, nativeOptions).catch((error: unknown) => { const failure = error as Error & { code?: string | number; signal?: string; stdout?: string; stderr?: string }; throw new Error(`${failure.message}\ncode=${failure.code} signal=${failure.signal}\n${failure.stderr ?? ""}\n${failure.stdout ?? ""}`, { cause: error }); }); - expect(comparableStderr(result.stderr)).toBe(""); - const built = JSON.parse(result.stdout) as { outputPath: string; llvmPath?: string; stats: { - statementsTotal: number; statementsFailed: number; statementsIsland: number; functionsSkipped: number; - } }; - expect(built.stats.statementsFailed).toBe(0); - expect(built.stats.statementsIsland).toBe(0); - expect(built.stats.functionsSkipped).toBe(0); - return built; + expect(comparableStderr(result.stderr)).toBe(expectedStderr); + return result.stdout; }; - expect((await exec(seed, ["--help"], nativeOptions)).stdout).toContain("Usage: scriptc-native"); - const checkProgram = async (compiler: string, source: string, backend: string) => { + const entry = join(root, "packages/compiler/src/native/cli.ts"); + const ffi = join(directory, ".scriptc/distribution-seed/compiler.ffi.json"); + const rebuilt = executable("scriptc-rebuilt"); + // Optimize the compiler that will process the full graph again. Small + // programs below still exercise development output. + const self = await bootstrapStep("production CLI rebuilds itself", () => + invoke(seed, ["build", entry, "-o", rebuilt, "--strip", "--keep-llvm", "--emit-ir", "--ffi", ffi, + ...(sanitize ? ["--sanitize"] : [])], + "scriptc: warning: --emit-ir is deprecated; use --emit=ir for IR as the primary output\n")); + expect(self.trim()).toBe(rebuilt); + // All command, library, and dynamic probes run inside the instrumented + // compiler in the sanitizer lane, including their failure paths. + const probe = sanitize ? rebuilt : seed; + expect(await invoke(probe, ["--help"])).toContain("scriptc build"); + expect((await invoke(probe, ["--version"])).trim()).toBe(manifest.compiler_version); + const checkProgram = async (compiler: string, source: string, extra: string[] = []) => { // This basename formerly collided with the driver's temporary object. const output = executable("program.o"); - const built = await invoke(compiler, ["build", source, "-o", output, "--backend", backend, "--dev", "--strip"]); - expect(built.outputPath).toBe(output); + const built = await invoke(compiler, ["build", source, "-o", output, "--optimization=dev", "--strip", ...extra]); + expect(built.trim()).toBe(output); const oracle = spawnSync(process.execPath, [source], options); const actual = spawnSync(output, [], nativeOptions); for (const result of [oracle, actual]) { @@ -82,41 +118,125 @@ test("the standalone compiler builds programs and rebuilds itself with Node unav const sample = join(root, "tests/corpus/class-array-optional-return.ts"); const unionSample = join(root, "tests/corpus/union-nested-layout-discriminant.ts"); const receiverSample = join(root, "tests/corpus/llvm-read-receiver-lifetime.ts"); - await checkProgram(seed, sample, "llvm"); - await checkProgram(seed, unionSample, "llvm"); - await checkProgram(seed, receiverSample, "llvm"); + await checkProgram(probe, sample); + await checkProgram(probe, unionSample); + await checkProgram(probe, receiverSample); + await checkProgram(probe, join(root, "tests/corpus/closure-nullable-union-return.ts")); + await checkProgram(probe, join(root, "tests/corpus/record-optional-json-presence.ts")); + await checkProgram(probe, join(root, "tests/corpus/1010-json-stringify-space.ts")); + await checkProgram(probe, join(root, "tests/corpus/fs-write-string-bytes-union.ts")); + + const fetchOptions = join(directory, "fetch-options.ts"); + writeFileSync(fetchOptions, 'async function probe() { const response = await fetch("https://example.invalid", { headers: { accept: "application/json" } }); console.log(response.status); } if (process.env["RUN_NATIVE_FETCH_PROBE"] === "1") await probe();\n'); + await checkProgram(probe, fetchOptions); + expect(await invoke(probe, ["coverage", fetchOptions])).toContain("(100%)"); + + const dynamic = join(directory, "dynamic.ts"); + writeFileSync(dynamic, 'const value: any = { answer: 42 }; console.log(`answer:${value.answer}`);\n'); + await checkProgram(probe, dynamic, ["--dynamic"]); + const comptime = join(directory, "comptime.ts"); + writeFileSync(comptime, 'const answer = comptime(() => [1, 2, 3].reduce((sum, value) => sum + value, 0) * 7); console.log(answer);\n'); + expect(await invoke(probe, ["run", comptime, "-o", executable("comptime"), "--strip"])).toBe("42\n"); + + const object = join(directory, "program.obj"); + const linkInfo = JSON.parse(await invoke(probe, ["build", sample, "-o", object, "--print=native-link-info"])); + expect(linkInfo.program.object).toBe(object); + expect(readFileSync(object).length).toBeGreaterThan(0); + + const profile = join(root, "tests/library-mode/contract-attest/profile.json"); + const archive = join(directory, "contract.a"); + const expectedArchive = join(directory, "contract-node.a"); + await exec(process.execPath, [join(root, "packages/cli/dist/main.js"), "build", "--lib", "--profile", profile, "-o", expectedArchive], options); + await invoke(probe, ["build", "--lib", "--profile", profile, "-o", archive]); + expect(JSON.parse(readFileSync(archive + ".contract.json", "utf8"))) + .toEqual(JSON.parse(readFileSync(expectedArchive + ".contract.json", "utf8"))); + + // The unmodified package exercises deep validator traversals. Compile it + // through the installed CLI, including its async command boundary. + const threeProfile = join(root, "tests/library-mode/wasm/three.json"); + await invoke(probe, ["build", "--lib", "--profile", threeProfile, "-o", join(directory, "three.a")]); + if (process.platform !== "win32" && spawnSync("zig", ["version"]).status === 0 && existsSync(join(root, "packages/runtime-wasm32-wasi/runtime-pack.json"))) { + const linker = join(directory, "zigcc"); + writeFileSync(linker, `#!/bin/sh\nexec '${absoluteCommand("zig").replaceAll("'", "'\\''")}' cc "$@"\n`, { mode: 0o755 }); + const output = join(directory, "three.wasm"); + const built = await exec(probe, ["build", "--lib", "--profile", threeProfile, "-o", output], { + ...nativeOptions, env: { ...nativeOptions.env, SCRIPTC_TARGET: "wasm32-wasi", SCRIPTC_LINKER: linker, + SCRIPTC_RUNTIME_PACK: join(root, "packages/runtime-wasm32-wasi"), SCRIPTC_NO_CACHE: "1" }, + }); + expect(built.stdout.trim()).toBe(output); + expect(comparableStderr(built.stderr)).toBe(""); + const vertices: number[][] = []; + const wasi = new WASI({ version: "preview1" }); + const instance = new WebAssembly.Instance(new WebAssembly.Module(readFileSync(output)), { + ...wasi.getImportObject(), scriptc: { + vertex: (...args: number[]) => vertices.push(args.map((value) => Number(value.toFixed(9)))), + panic: () => { throw new Error("unexpected Wasm panic"); }, + }, + }); + wasi.initialize(instance); + const api = instance.exports as Record number>; + api.app_init!(); + for (const time of [0, 123, 2000]) expect(api.app_frame!(time, 1.5)).toBe(24); + const oracle = await exec(process.execPath, ["--input-type=module", "--eval", ` + const vertices = []; + globalThis.vertex = (...args) => vertices.push(args.map(value => Number(value.toFixed(9)))); + const { frame } = await import(${JSON.stringify(join(root, "tests/library-mode/wasm/three.mjs"))}); + for (const time of [0, 123, 2000]) frame(time, 1.5); + console.log(JSON.stringify(vertices)); + `], options); + expect(vertices).toEqual(JSON.parse(oracle.stdout)); + expect(oracle.stderr).toBe(""); + api.app_collect!(); + } const badSource = join(directory, "bad.ts"); writeFileSync(badSource, 'const value: number = "wrong"; console.log(value);\n'); const retained = executable("retained"); writeFileSync(retained, "existing output"); - const failed = spawnSync(seed, [badSource, "-o", retained, "--toolchain", manifestPath], nativeOptions); + const failed = spawnSync(probe, ["build", badSource, "-o", retained], nativeOptions); expect(failed.status).toBe(1); expect(failed.stderr.toString()).toContain("not assignable"); expect(readFileSync(retained, "utf8")).toBe("existing output"); - const entry = join(root, "packages/compiler/src/native/main.ts"); - const profile = join(directory, "ts7-process.ffi.json"); - const rebuilt = executable("scriptc-rebuilt"); - // Optimize the compiler that will process the full graph again. Small - // programs above and below still exercise development output. - const self = await bootstrapStep("native compiler rebuilds itself", () => - invoke(seed, [entry, "-o", rebuilt, "--backend=llvm", "--strip", "--keep-llvm", "--ffi", profile])); - expect(self.stats.statementsTotal).toBeGreaterThan(10_000); - expect(self.llvmPath).toBe(rebuilt + ".ll"); - console.log("native self-build", self.stats); - await checkProgram(rebuilt, join(root, "tests/corpus/nullish-long-chain.ts"), "llvm"); - await checkProgram(rebuilt, sample, "llvm"); - await checkProgram(rebuilt, unionSample, "llvm"); - await checkProgram(rebuilt, receiverSample, "llvm"); + // The production chain owns the complete frontend and emitter proof: + // compare the Node seed's IR and LLVM with the native self-rebuild, then + // execute the next compiler generation below. Large structural checks + // run in a roomy child so Vitest's worker remains responsive. + const seedDirectory = join(directory, ".scriptc/distribution-seed"); + const comparison = await bootstrapStep("compare Node and native compiler IR and LLVM", () => + exec(process.execPath, ["--max-old-space-size=8192", "--import", "tsx", "--input-type=module", "--eval", ` + import assert from 'node:assert/strict'; + import { readFileSync } from 'node:fs'; + import { isDeepStrictEqual } from 'node:util'; + import { deserializeModule, validateModule } from ${JSON.stringify(pathToFileURL(join(root, "packages/compiler/src/index.ts")).href)}; + const expected = deserializeModule(readFileSync(process.argv[1], 'utf8')); + const actual = deserializeModule(readFileSync(process.argv[2], 'utf8')); + assert.ok(actual.functions.length > 1000); + assert.deepEqual(validateModule(actual), []); + assert.ok(isDeepStrictEqual(actual, expected), 'native self-lowering must match the Node seed'); + assert.ok(readFileSync(process.argv[3], 'utf8') === readFileSync(process.argv[4], 'utf8'), + 'native LLVM emission must match the Node seed'); + `, join(seedDirectory, "cli.ir.json"), join(directory, "cli.ir.json"), + join(seedDirectory, "cli.ll"), join(directory, "cli.ll")], options)); + expect(comparison.stdout).toBe(""); + expect(comparison.stderr).toBe(""); + await checkProgram(rebuilt, join(root, "tests/corpus/nullish-long-chain.ts")); + await checkProgram(rebuilt, sample); + await checkProgram(rebuilt, unionSample); + await checkProgram(rebuilt, receiverSample); + await checkProgram(rebuilt, join(root, "tests/corpus/1010-json-stringify-space.ts")); // Compare the LLVM used to build the second generation with its own // output. Retaining the build input avoids repeating the seed's work. - if (!self.llvmPath) throw new Error("native self-build did not retain its LLVM"); - const seedLlvm = self.llvmPath; + const seedLlvm = join(directory, "cli.ll"); const rebuiltLlvm = join(directory, "rebuilt.ll"); await bootstrapStep("rebuilt compiler emits itself", () => - invoke(rebuilt, [entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", profile])); - expect(readFileSync(seedLlvm).equals(readFileSync(rebuiltLlvm)), "native compiler generations must emit identical LLVM").toBe(true); + invoke(rebuilt, ["build", entry, "--emit=llvm", "-o", rebuiltLlvm, "--ffi", ffi])); + // Executable output adds a runtime ABI check; textual LLVM emission + // omits it. Compare all other output without changing either mode. + const withoutAbiCheck = (path: string): string => readFileSync(path, "utf8") + .replace(/^declare void @scr_runtime_abi_v\d+\(\)\n/m, "") + .replace(/^ call void @scr_runtime_abi_v\d+\(\)\n/m, ""); + expect(withoutAbiCheck(seedLlvm) === withoutAbiCheck(rebuiltLlvm), "native compiler generations must emit identical LLVM").toBe(true); } finally { rmSync(directory, { recursive: true, force: true }); } }, sanitize ? 10_800_000 : 5_400_000); diff --git a/tests/harness/self-hosting-native-frontend.test.ts b/tests/harness/self-hosting-native-frontend.test.ts index 0525f0c4..ddb49415 100644 --- a/tests/harness/self-hosting-native-frontend.test.ts +++ b/tests/harness/self-hosting-native-frontend.test.ts @@ -21,6 +21,12 @@ const execFileAsync = promisify(execFile); const sanitize = process.env["SCRIPTC_SAN"] === "1"; const runOptions = { cwd: root, timeout: 120_000, maxBuffer: 16 * 1024 * 1024, encoding: "utf8" as const }; +function comparableStderr(text: string): string { + return sanitize + ? text.replace(/^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, "") + : text; +} + const programs = [ "001-hello.ts", "101-arithmetic.ts", @@ -93,10 +99,7 @@ for (const backend of ["llvm"] as const) { expect(result.error, file + "\n" + result.stderr).toBeUndefined(); expect(result.signal, file + "\n" + result.stderr).toBeNull(); expect(result.status, file + "\n" + result.stderr).toBe(0); - const stderr = sanitize && process.platform === "linux" - ? result.stderr.replace(/^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, "") - : result.stderr; - expect(stderr, file).toBe(""); + expect(comparableStderr(result.stderr), file).toBe(""); } const expectedReport = expected.stdout.trim().split("\n").map((line) => JSON.parse(line)); const actualReport = actual.stdout.trim().split("\n").map((line) => JSON.parse(line)); @@ -135,10 +138,7 @@ for (const backend of ["llvm"] as const) { expect(result.status, file + "\n" + result.stderr).toBe(0); } expect(native.stdout, file).toBe(node.stdout); - const nativeStderr = sanitize - ? native.stderr.replace(/^==\d+==WARNING: ASan doesn't fully support makecontext\/swapcontext functions and may produce false positives in some cases!\n/gm, "") - : native.stderr; - expect(nativeStderr, file).toBe(node.stderr); + expect(comparableStderr(native.stderr), file).toBe(node.stderr); } for (const [file, npmStatic, packages] of [ @@ -222,7 +222,7 @@ for (const backend of ["llvm"] as const) { expect(ffiRun.error).toBeUndefined(); expect(ffiRun.status, ffiRun.stderr).toBe(0); expect(ffiRun.stdout).toBe("42\n"); - expect(ffiRun.stderr).toBe(""); + expect(comparableStderr(ffiRun.stderr)).toBe(""); } finally { rmSync(directory, { recursive: true, force: true }); } }, 1_200_000); } diff --git a/tests/harness/self-hosting-native-toolchain.test.ts b/tests/harness/self-hosting-native-toolchain.test.ts index 91975d19..53e368f6 100644 --- a/tests/harness/self-hosting-native-toolchain.test.ts +++ b/tests/harness/self-hosting-native-toolchain.test.ts @@ -63,8 +63,8 @@ for (const backend of ["llvm"] as const) { const toolchain = join(directory, "toolchain.json"); const config: NativeToolchainManifest = { schema: "scriptc.native-toolchain.v1", compiler_version: manifest.version, target: target.name, - ts7: "ts7", llvm_package: "llvm", runtime_pack: "pack", runtime_headers: "headers", - c_compiler: "cc", c_compiler_args: [], linker: "ld", linker_args: [], dsymutil: "dsymutil", + ts7: "ts7", llvm_package: "llvm", runtime_pack: "pack", + linker: "ld", linker_args: [], dsymutil: "dsymutil", }; writeFileSync(toolchain, JSON.stringify(config)); const input = join(directory, "request.json"); diff --git a/tests/harness/self-hosting-serialization.test.ts b/tests/harness/self-hosting-serialization.test.ts index e5716118..6742ef1e 100644 --- a/tests/harness/self-hosting-serialization.test.ts +++ b/tests/harness/self-hosting-serialization.test.ts @@ -83,6 +83,11 @@ for (const backend of ["llvm"] as const) { }; const numeric = roundTrip(numericModule()); + const compact = run(serializeModule(numeric), ["compact"]); + expect(compact.native.status).toBe(0); + expect(compact.native.stdout.toString().trimEnd()).not.toContain("\n"); + expect(deserializeModule(compact.native.stdout.toString())).toEqual(numeric); + expect(deserializeModule(compact.native.stdout.toString())).toEqual(deserializeModule(compact.oracle.stdout.toString())); const print = numeric.functions[0]!.body[1]!; expect(print.kind).toBe("exprStmt"); if (print.kind !== "exprStmt" || print.expr.kind !== "intrinsic") throw new Error("numeric IR changed"); @@ -114,6 +119,7 @@ for (const backend of ["llvm"] as const) { "tests/corpus/3089-array-find-narrowing.ts", "tests/corpus/3091-json-recursive-discriminants.ts", "tests/corpus/3094-json-replacer-traversal.ts", + "tests/corpus/1010-json-stringify-space.ts", ]) { const path = join(dir, "emitted.json"); const emitted = await compile(join(root, source), { outDir: dir, outPath: path, outputKind: "ir", dynamic: false }); diff --git a/tests/harness/static-spatial.test.ts b/tests/harness/static-spatial.test.ts new file mode 100644 index 00000000..2925cf1b --- /dev/null +++ b/tests/harness/static-spatial.test.ts @@ -0,0 +1,58 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createRequire } from "node:module"; +import { rollup } from "rollup"; +import { expect, test } from "vitest"; +import { compile } from "@scriptc/compiler"; + +const hasZig = spawnSync("zig", ["version"], { stdio: "ignore" }).status === 0; +const require = createRequire(import.meta.url); +const wasiRunner = `const {WASI}=require('node:wasi'); const {readFileSync}=require('node:fs'); +const wasi=new WASI({version:'preview1',args:[process.argv[1]],returnOnExit:true}); +const instance=new WebAssembly.Instance(new WebAssembly.Module(readFileSync(process.argv[1])),wasi.getImportObject()); +process.exitCode=wasi.start(instance);`; +function run(command: string, args: string[]) { + const result = spawnSync(command,args,{encoding:"utf8",timeout:30_000}); + if (result.error) throw result.error; + const stderr = result.stderr.split("\n").filter(line => !line.startsWith("scriptc RC audit skipped:")).join("\n"); + return { stdout: result.stdout, stderr, status: result.status, signal: result.signal }; +} + +test.for(["native", "wasm32-wasi"])("published spatial queries and output arrays run statically on %s", async (target, context) => { + if (target === "wasm32-wasi" && !hasZig) context.skip(); + const dir = await mkdtemp("/tmp/scriptc-spatial-"); + const previousTarget = process.env["SCRIPTC_TARGET"], previousCc = process.env["SCRIPTC_CC"]; + try { + if(target === "wasm32-wasi") { process.env["SCRIPTC_TARGET"] = target; process.env["SCRIPTC_CC"] = "zigcc"; } + const pkg = join(dir,"node_modules","spatial-fixture"); + await mkdir(pkg,{recursive:true}); + await writeFile(join(pkg,"package.json"),'{"name":"spatial-fixture","type":"module","main":"index.js"}'); + const entry = join(dir,"main.mjs"); + await writeFile(entry,'import "spatial-fixture";'); + for (const fixture of ["three/collision.mjs", "npm-static/array-output-parameters.mjs"]) { + const source = join(import.meta.dirname,"../fixtures",fixture); + const bundle = await rollup({ input:source, plugins:[{name:"three-cpu-entry",resolveId(id) { + if(id === "three") return require.resolve("three/src/Three.Core.js"); + if(id.startsWith("three/")) return require.resolve(id); + return null; + }}] }); + try { await bundle.write({file:join(pkg,"index.js"),format:"es"}); } + finally { await bundle.close(); } + const wasm = target === "wasm32-wasi"; + const output = join(dir,wasm ? "program.wasm" : "program"); + const result = await compile(entry,{outDir:dir,outPath:output,backend:"llvm",dynamic:false,npmStatic:["spatial-fixture"], + sanitize:!wasm && process.env["SCRIPTC_SAN"] === "1"}); + if(!result.ok) throw new Error(result.diagnostics.map(d=>`${d.code}: ${d.message}`).join("\n")); + if(wasm) expect([...(await readFile(output)).subarray(0,4)]).toEqual([0,97,115,109]); + const reference = run(process.execPath,["--no-warnings",source]); + expect(reference.status).toBe(0); + expect(run(process.execPath,["--no-warnings",entry])).toEqual(reference); + expect(wasm ? run(process.execPath,["--no-warnings","-e",wasiRunner,output]) : run(output,[])).toEqual(reference); + } + } finally { + if(previousTarget === undefined) delete process.env["SCRIPTC_TARGET"]; else process.env["SCRIPTC_TARGET"] = previousTarget; + if(previousCc === undefined) delete process.env["SCRIPTC_CC"]; else process.env["SCRIPTC_CC"] = previousCc; + await rm(dir,{recursive:true,force:true}); + } +}); diff --git a/tests/harness/three.test.ts b/tests/harness/three.test.ts index c6badd95..e87d13b6 100644 --- a/tests/harness/three.test.ts +++ b/tests/harness/three.test.ts @@ -4,7 +4,6 @@ import { join } from "node:path"; import { expect, test } from "vitest"; import { compile } from "@scriptc/compiler"; -const entry = join(import.meta.dirname, "../fixtures/three/math.mjs"); const hasZig = spawnSync("zig", ["version"], { stdio: "ignore" }).status === 0; const wasiRunner = ` const {readFileSync} = require('node:fs'); @@ -22,7 +21,10 @@ function run(command: string, args: string[]) { return { stdout: result.stdout, stderr: result.stderr, status: result.status, signal: result.signal }; } -test.for(["native", "wasm32-wasi"])("published three.js math runs statically through LLVM on %s", async (target, context) => { +const cases = ["math", "spatial", "attributes", "geometry", "scene", "materials", "mesh", "raycast-mesh", "raycast-camera", "raycast-lines-points"].flatMap((fixture) => + ["native", "wasm32-wasi"].map((target) => ({ fixture, target }))); +test.for(cases)("published three.js $fixture runs statically through LLVM on $target", async ({ fixture, target }, context) => { + const entry = join(import.meta.dirname, `../fixtures/three/${fixture}.mjs`); if (target === "wasm32-wasi" && !hasZig) context.skip(); const dir = await mkdtemp("/tmp/scriptc-three-"); const previousTarget = process.env["SCRIPTC_TARGET"]; @@ -34,7 +36,7 @@ test.for(["native", "wasm32-wasi"])("published three.js math runs statically thr process.env["SCRIPTC_CC"] = "zigcc"; } const result = await compile(entry, { - outDir: dir, outPath: join(dir, wasm ? "math.wasm" : "math"), + outDir: dir, outPath: join(dir, wasm ? `${fixture}.wasm` : fixture), backend: "llvm", dynamic: false, npmStatic: ["three"], sanitize: !wasm && process.env["SCRIPTC_SAN"] === "1", }); diff --git a/tests/harness/wasi-differential.test.ts b/tests/harness/wasi-differential.test.ts index 01f0f3e7..20e816cc 100644 --- a/tests/harness/wasi-differential.test.ts +++ b/tests/harness/wasi-differential.test.ts @@ -193,17 +193,17 @@ describe.skipIf(!zigOnPath())("wasm32-wasi differential", () => { } }); - test("library mode reports a target diagnostic instead of invoking the WASI toolchain", async () => { + test("sanitized library mode reports a target diagnostic before invoking the WASI toolchain", async () => { const outDir = await mkdtemp("/tmp/scriptc-wasi-library-"); const result = await compileLibrary({ profilePath: join(repoRoot, "tests/library-mode/scalars/profile.json"), - outDir, + outDir, sanitize: true, }); expect(result.ok).toBe(false); if (!result.ok) { expect(result.diagnostics).toHaveLength(1); expect(result.diagnostics[0]?.code).toBe("SC3002"); - expect(result.diagnostics[0]?.message).toMatch(/does not support library-mode archive builds/); + expect(result.diagnostics[0]?.message).toMatch(/does not support sanitized library builds/); } }); diff --git a/tests/harness/wasm-library.test.ts b/tests/harness/wasm-library.test.ts new file mode 100644 index 00000000..18eb652a --- /dev/null +++ b/tests/harness/wasm-library.test.ts @@ -0,0 +1,227 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { WASI } from "node:wasi"; +import { afterAll, beforeAll, describe, expect, test, vi } from "vitest"; +import { compileLibrary as compileNodeLibrary, type CompileLibraryOptions } from "@scriptc/compiler"; +import { NativeCompiler } from "../../packages/compiler/src/native/compiler.js"; +import { nativeCodegenTarget, WASM32_WASI_TARGET } from "../../packages/compiler/src/backend/targets.js"; +import { ts7Executable } from "../../packages/compiler/src/frontend/ts7/rpc-api.js"; + +// Run both compiler drivers against the same real checker transport in this +// harness. The production bootstrap separately exercises the native transport. +vi.mock("../../packages/compiler/src/frontend/pipeline-native.js", async () => { + const { runFrontend } = await import("../../packages/compiler/src/frontend/pipeline.js"); + const { loadProgram } = await import("../../packages/compiler/src/frontend/program-node.js"); + return { runNativeFrontend: (entry: string, _executable: string, npmStatic?: readonly string[] | "auto" | "lib", + externalTypes?: Readonly>, _evaluate?: unknown, libraryNpmStatic?: readonly string[]) => + runFrontend(entry, loadProgram, npmStatic, externalTypes, libraryNpmStatic) }; +}); + +const fixture = join(import.meta.dirname, "../library-mode/wasm"); +const hasZig = spawnSync("zig", ["version"]).status === 0; +type Api = Record number> & { memory: WebAssembly.Memory }; + +describe.skipIf(!hasZig).each(["node", "native"] as const)("Wasm library embedding (%s driver)", (driver) => { + let directory: string; + let module: WebAssembly.Module; + let oldTarget: string | undefined; + let profile: any; + let native: NativeCompiler; + const compileLibrary = (options: CompileLibraryOptions) => driver === "node" + ? compileNodeLibrary(options) : Promise.resolve(native.compileLibrary(options)); + + beforeAll(async () => { + oldTarget = process.env["SCRIPTC_TARGET"]; + if (driver === "native") { + const host = nativeCodegenTarget()!; + const root = join(import.meta.dirname, "../.."); + const helperPackageRoot = join(root, "packages", host.helper.packageName.replace("@scriptc/", "")); + const compilerPackage = JSON.parse(await readFile(join(root, "packages/compiler/package.json"), "utf8")); + native = new NativeCompiler({ + compilerVersion: compilerPackage.version, target: WASM32_WASI_TARGET, helper: host.helper, + helperPackageRoot, helperExecutable: join(helperPackageRoot, "bin", process.platform === "win32" ? "scriptc-llvm-codegen.exe" : "scriptc-llvm-codegen"), + runtimePackRoot: join(root, "packages/runtime-wasm32-wasi"), ts7Executable: ts7Executable(), + linker: "zig", linkerArgs: ["cc"], dsymutil: "unused", + }); + } + process.env["SCRIPTC_TARGET"] = "wasm32-wasi"; + directory = await mkdtemp("/tmp/scriptc-wasm-library-test-"); + profile = JSON.parse(await readFile(join(fixture, "profile.json"), "utf8")); + profile.entry = join(fixture, "lib.ts"); + const result = await compileLibrary({ profilePath: join(fixture, "profile.json"), outDir: directory }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + expect(result.archivePath.endsWith(".wasm")).toBe(true); + module = new WebAssembly.Module(await readFile(result.archivePath)); + }); + + afterAll(async () => { + if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"]; + else process.env["SCRIPTC_TARGET"] = oldTarget; + if (directory) await rm(directory, { recursive: true, force: true }); + }); + + function instantiate(adjust?: (value: number, api: Api) => number) { + const wasi = new WASI({ version: "preview1" }); + let api: Api; + const reports: unknown[] = []; + const panics: string[] = []; + const text = (ptr: number, len: number) => new TextDecoder().decode(new Uint8Array(api.memory.buffer, ptr, len)); + const instance = new WebAssembly.Instance(module, { + ...wasi.getImportObject(), + scriptc: { + adjust: (value: number) => adjust ? adjust(value, api) : value * 2, + report: (p: number, n: number, b: number, size: number, flag: number) => { + reports.push([text(p, n), Array.from(new Uint8Array(api.memory.buffer, b, size)), flag !== 0]); + }, + panic: (p: number, n: number) => { const message = text(p, n); panics.push(message); throw new Error(message); }, + }, + }); + api = instance.exports as Api; + wasi.initialize(instance); + api.app_init(); + return { api, reports, panics, text }; + } + + test("exports only the callable ABI and requires named host imports", () => { + expect(WebAssembly.Module.exports(module).map((e) => e.name).sort()).toEqual([ + "_initialize", "memory", "scriptc_alloc", "scriptc_free", "app_init", "app_collect", "app_reset_results", + "app_step", "app_echo", "app_bytes", "app_fail", + ].sort()); + expect(WebAssembly.Module.imports(module).filter((e) => e.module === "scriptc").map((e) => e.name).sort()).toEqual(["adjust", "panic", "report"]); + expect(() => new WebAssembly.Instance(module, { scriptc: {}, wasi_snapshot_preview1: {} })).toThrow(); + }); + + test("matches Node across stateful calls, strings, buffers, and host callbacks", () => { + const { api, reports, text } = instantiate(); + const values = [api.app_step(3), api.app_step(-2), api.app_step(0.5)]; + const input = new TextEncoder().encode("λ\0hello"); + const ptr = api.scriptc_alloc(input.length + 8); + new Uint8Array(api.memory.buffer, ptr, input.length).set(input); + const out = ptr + input.length; + api.app_echo(ptr, input.length, out, out + 4); + let view = new DataView(api.memory.buffer); + const resultPtr = view.getUint32(out, true), resultLen = view.getUint32(out + 4, true); + const echo = text(resultPtr, resultLen); + new Uint8Array(api.memory.buffer, ptr, 3).set([0, 254, 255]); + api.app_bytes(ptr, 3, out, out + 4); + view = new DataView(api.memory.buffer); + const bytes = Array.from(new Uint8Array(api.memory.buffer, view.getUint32(out, true), view.getUint32(out + 4, true))); + // An explicit reset entry keeps earlier results live across later calls. + expect(text(resultPtr, resultLen)).toBe(echo); + api.scriptc_free(ptr); + api.app_reset_results(); + api.app_collect(); + const reference = execFileSync(process.execPath, ["--no-warnings", "--input-type=module", "-e", ` + const reports=[]; + globalThis.adjust=x=>x*2; + globalThis.report=(s,b,f)=>reports.push([s,Array.from(b),f]); + const api=await import(${JSON.stringify(join(fixture, "lib.ts"))}); + const values=[api.step(3),api.step(-2),api.step(0.5)]; + const echo=api.echo("λ\\0hello"); + const bytes=Array.from(api.bytes(new Uint8Array([0,254,255]))); + console.log(JSON.stringify({values,echo,bytes,reports})); + `], { encoding: "utf8" }); + expect({ values, echo, bytes, reports }).toEqual(JSON.parse(reference)); + }); + + test("instances isolate state and initialization resets the session", () => { + const a = instantiate().api, b = instantiate().api; + expect(a.memory).not.toBe(b.memory); + expect(a.app_step(5)).toBe(17); + expect(b.app_step(1)).toBe(9); + a.app_init(); + expect(a.app_step(0)).toBe(7); + expect(b.app_step(0)).toBe(9); + }); + + test("host allocations survive memory growth and collection", () => { + const { api } = instantiate(); + const p = api.scriptc_alloc(16); + new Uint8Array(api.memory.buffer, p, 16).fill(42); + const original = api.memory.buffer; + const big = api.scriptc_alloc(original.byteLength + 65536); + expect(big).not.toBe(0); + expect(api.memory.buffer).not.toBe(original); + api.app_collect(); + expect(Array.from(new Uint8Array(api.memory.buffer, p, 16))).toEqual(Array(16).fill(42)); + api.scriptc_free(big); + api.scriptc_free(p); + }); + + test("escaped exceptions poison only their instance", () => { + const a = instantiate(), b = instantiate(); + expect(() => a.api.app_fail()).toThrow(/reactor failure/); + expect(a.panics).toHaveLength(1); + expect(a.panics[0]).toContain("app_fail"); + expect(() => a.api.app_step(1)).toThrow(); + expect(() => a.api.scriptc_alloc(8)).toThrow(); + expect(b.api.app_step(1)).toBe(9); + }); + + test.each(["app_init", "app_collect", "app_reset_results", "scriptc_alloc", "scriptc_free", "app_step"])("rejects callback reentry through %s", (entry) => { + const { api, panics } = instantiate((value, api) => api[entry](value)); + expect(() => api.app_step(2)).toThrow(/SC4026/); + expect(panics[0]).toContain(entry); + }); + + test("an escaping host exception prevents reuse of the suspended instance", () => { + const { api } = instantiate(() => { throw new Error("host failure"); }); + expect(() => api.app_step(1)).toThrow("host failure"); + expect(() => api.app_init()).toThrow(/SC4026/); + }); + + test.each(["instance_per_thread", "localize_runtime"])("refuses native-only %s", async (key) => { + const path = join(directory, `${key}.json`); + await writeFile(path, JSON.stringify({ ...profile, abi: { ...profile.abi, [key]: true } })); + const result = await compileLibrary({ profilePath: path, outDir: directory }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.diagnostics[0]?.code).toBe("SC3002"); + }); + + test("refuses collisions with built-in Wasm exports and imports", async () => { + for (const change of [ + { abi: { ...profile.abi, prefix: "scriptc_", init_symbol: "scriptc_alloc", sink_register_symbol: "scriptc_sink", callback_register_symbol: "scriptc_callback", collect_symbol: null, result_reset_symbol: null }, exports: [] }, + { callbacks: [{ name: "panic", params: [], returns: "void" }] }, + ]) { + const path = join(directory, "reserved.json"); + await writeFile(path, JSON.stringify({ ...profile, ...change })); + const result = await compileLibrary({ profilePath: path, outDir: directory }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.diagnostics[0]?.message).toMatch(/reserved/); + } + }); + + test.each(["release", "dev"])("published three.js computes animated camera-space vertices in %s", async (optimization) => { + const threeProfile = JSON.parse(await readFile(join(fixture, "three.json"), "utf8")); + const profilePath = join(directory, `three-${optimization}.json`); + await writeFile(profilePath, JSON.stringify({ ...threeProfile, entry: join(fixture, "three.mjs"), optimization })); + const result = await compileLibrary({ profilePath, outDir: directory }); + if (!result.ok) throw new Error(JSON.stringify(result.diagnostics)); + const wasi = new WASI({ version: "preview1" }); + let api: Api; + const vertices: number[][] = []; + const instance = new WebAssembly.Instance(new WebAssembly.Module(await readFile(result.archivePath)), { + ...wasi.getImportObject(), + scriptc: { + vertex: (...args: number[]) => vertices.push(args.map((x) => Number(x.toFixed(9)))), + panic: (p: number, n: number) => { throw new Error(new TextDecoder().decode(new Uint8Array(api.memory.buffer, p, n))); }, + }, + }); + api = instance.exports as Api; + wasi.initialize(instance); + api.app_init(); + for (const time of [0, 123, 2000]) expect(api.app_frame(time, 1.5)).toBe(24); + const reference = spawnSync(process.execPath, ["--no-warnings", "--input-type=module", "-e", ` + const vertices=[]; + globalThis.vertex=(...args)=>vertices.push(args.map(x=>Number(x.toFixed(9)))); + const {frame}=await import(${JSON.stringify(join(fixture, "three.mjs"))}); + for (const time of [0,123,2000]) frame(time,1.5); + console.log(JSON.stringify(vertices)); + `], { encoding: "utf8" }); + expect({ stdout: JSON.stringify(vertices) + "\n", stderr: "", status: 0 }).toEqual({ stdout: reference.stdout, stderr: reference.stderr, status: reference.status }); + api.app_collect(); + api.app_init(); + expect(api.app_frame(0, 1)).toBe(24); + }); +}); diff --git a/tests/library-mode/wasm/dispatch.mjs b/tests/library-mode/wasm/dispatch.mjs new file mode 100644 index 00000000..602925f2 --- /dev/null +++ b/tests/library-mode/wasm/dispatch.mjs @@ -0,0 +1,13 @@ +/** @returns {number} */ +export function dispatch() { + return consume(1, 2, 3); +} + +function consume(...values) { + const first = values.shift(); + values.push(4); + const object = JSON.parse('{}'); + Object.defineProperties(object, { score: { value: 5, enumerable: true } }); + const add = (left, right) => left + right; + return Number(first) + values.join("").length + Number(object.score) + Number(add.apply(null, [6, 7])); +} diff --git a/tests/library-mode/wasm/lib.ts b/tests/library-mode/wasm/lib.ts new file mode 100644 index 00000000..43a301f2 --- /dev/null +++ b/tests/library-mode/wasm/lib.ts @@ -0,0 +1,24 @@ +declare function adjust(value: number): number; +declare function report(message: string, data: Uint8Array, flag: boolean): void; + +let total = 7; + +export function step(value: number): number { + total += adjust(value); + return total; +} + +export function echo(value: string): string { + return "wasm:" + value; +} + +export function bytes(value: Uint8Array): Uint8Array { + const output = new Uint8Array(value.length); + for (let i = 0; i < value.length; i++) output[i] = value[i] + 1; + report("hello λ", output, true); + return output; +} + +export function fail(): void { + throw new Error("reactor failure"); +} diff --git a/tests/library-mode/wasm/profile.json b/tests/library-mode/wasm/profile.json new file mode 100644 index 00000000..1ed376f4 --- /dev/null +++ b/tests/library-mode/wasm/profile.json @@ -0,0 +1,24 @@ +{ + "profile_format": 1, + "name": "wasm-embedding", + "entry": "lib.ts", + "emission": "llvm", + "abi": { + "prefix": "app_", + "init_symbol": "app_init", + "sink_register_symbol": "app_sink", + "callback_register_symbol": "app_callback", + "collect_symbol": "app_collect", + "result_reset_symbol": "app_reset_results" + }, + "callbacks": [ + { "name": "adjust", "params": ["f64"], "returns": "f64" }, + { "name": "report", "params": ["string", "bytes", "bool"], "returns": "void" } + ], + "exports": [ + { "export": "step", "symbol": "app_step", "params": ["f64"], "returns": "f64" }, + { "export": "echo", "symbol": "app_echo", "params": ["string"], "returns": "string" }, + { "export": "bytes", "symbol": "app_bytes", "params": ["bytes"], "returns": "bytes" }, + { "export": "fail", "symbol": "app_fail", "params": [], "returns": "void" } + ] +} diff --git a/tests/library-mode/wasm/three-host.d.ts b/tests/library-mode/wasm/three-host.d.ts new file mode 100644 index 00000000..fe30b568 --- /dev/null +++ b/tests/library-mode/wasm/three-host.d.ts @@ -0,0 +1 @@ +declare function vertex(index: number, x: number, y: number, z: number): void; diff --git a/tests/library-mode/wasm/three.json b/tests/library-mode/wasm/three.json new file mode 100644 index 00000000..6f0d212f --- /dev/null +++ b/tests/library-mode/wasm/three.json @@ -0,0 +1,38 @@ +{ + "profile_format": 1, + "name": "three-wasm-embedding", + "entry": "three.mjs", + "emission": "llvm", + "npm_static": ["three"], + "abi": { + "prefix": "app_", + "init_symbol": "app_init", + "sink_register_symbol": "app_sink", + "callback_register_symbol": "app_callback", + "collect_symbol": "app_collect", + "result_reset_symbol": "app_reset_results" + }, + "callbacks": [ + { + "name": "vertex", + "params": [ + "u32", + "f64", + "f64", + "f64" + ], + "returns": "void" + } + ], + "exports": [ + { + "export": "frame", + "symbol": "app_frame", + "params": [ + "f64", + "f64" + ], + "returns": "f64" + } + ] +} diff --git a/tests/library-mode/wasm/three.mjs b/tests/library-mode/wasm/three.mjs new file mode 100644 index 00000000..b6f8730c --- /dev/null +++ b/tests/library-mode/wasm/three.mjs @@ -0,0 +1,32 @@ +/// +import { PerspectiveCamera } from 'three/src/cameras/PerspectiveCamera.js'; +import { Scene } from 'three/src/scenes/Scene.js'; +import { BoxGeometry } from 'three/src/geometries/BoxGeometry.js'; +import { Mesh } from 'three/src/objects/Mesh.js'; +import { MeshNormalMaterial } from 'three/src/materials/MeshNormalMaterial.js'; +import { Vector3 } from 'three/src/math/Vector3.js'; + + +const camera = new PerspectiveCamera(70, 1, 0.01, 10); +camera.position.z = 1; +const scene = new Scene(); +const geometry = new BoxGeometry(0.2, 0.2, 0.2); +const mesh = new Mesh(geometry, new MeshNormalMaterial()); +scene.add(mesh); +const point = new Vector3(); + +/** @param {number} time @param {number} aspect @returns {number} */ +export function frame(time, aspect) { + camera.aspect = aspect; + camera.updateProjectionMatrix(); + camera.updateMatrixWorld(); + mesh.rotation.x = time / 2000; + mesh.rotation.y = time / 1000; + scene.updateMatrixWorld(true); + const position = geometry.getAttribute('position'); + for (let i = 0; i < position.count; i++) { + point.fromBufferAttribute(position, i).applyMatrix4(mesh.matrixWorld).project(camera); + vertex(i, point.x, point.y, point.z); + } + return Number(position.count); +} diff --git a/vitest.config.ts b/vitest.config.ts index 406b8a35..c11fc556 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -30,6 +30,11 @@ const workers = process.env["SCRIPTC_TEST_WORKERS"]; export default defineConfig({ resolve: { alias: { + "@scriptc/compiler/cli/command": fileURLToPath(new URL("./packages/compiler/src/cli/command.ts", import.meta.url)), + "@scriptc/compiler/cli/host": fileURLToPath(new URL("./packages/compiler/src/cli/host.ts", import.meta.url)), + "@scriptc/compiler/cli/paths": fileURLToPath(new URL("./packages/compiler/src/cli/paths.ts", import.meta.url)), + "@scriptc/compiler/cli/output-options": fileURLToPath(new URL("./packages/compiler/src/cli/output-options.ts", import.meta.url)), + "@scriptc/compiler/cli/usage": fileURLToPath(new URL("./packages/compiler/src/cli/usage.ts", import.meta.url)), // Tests run against compiler source directly — no build step needed. "@scriptc/compiler": fileURLToPath( new URL("./packages/compiler/src/index.ts", import.meta.url),