diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f2a6d51d0..b296f2ddf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,21 +4,23 @@ on: push: branches: [main] pull_request: - # Nightly full macOS sweep, including on days without a push to main. + # Nightly full macOS sweep. Pushes to main run no macOS jobs, and pull requests run them only when they change + # darwin code (scripts/ci-lanes.mjs). schedule: - cron: "17 7 * * *" - # A manual run; macos_full runs the complete suite on macOS as the nightly sweep does (for example on a - # release candidate commit). + # A manual run, for example on a release candidate commit. It runs every lane; macos_suite selects the complete + # suite on macOS, as the nightly sweep runs it, or the darwin suite that pull requests run. workflow_dispatch: inputs: - macos_full: - description: Run the complete test suite on macOS - type: boolean - default: false + macos_suite: + description: The test suite to run on macOS + type: choice + options: [full, darwin] + default: full concurrency: # A new push to a pull request cancels its previous run. Every push to main runs to completion in a group of - # its own commit, so each merged commit gets a full result (releases require a green run on their commit). The + # its own commit, so each merged commit gets a result (releases require a green run on their commit). The # nightly sweep and manual runs neither cancel nor are cancelled by main pushes. group: ci-${{ github.event_name == 'schedule' && 'schedule' || github.event_name == 'workflow_dispatch' && format('dispatch-{0}', github.run_id) || github.event.pull_request.number || github.sha }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -59,7 +61,8 @@ jobs: NODE # Which expensive lanes a pull request needs (scripts/ci-lanes.mjs, run from the base commit against the pull - # request's diff, so a pull request cannot change its own selection). Pushes and the nightly sweep run every lane. + # request's diff, so a pull request cannot change its own selection). Pushes to main run every lane except macos, + # whose jobs the nightly sweep runs on main. The nightly sweep and manual runs run every lane. lanes: needs: validation_scope if: ${{ needs.validation_scope.outputs.full == 'true' }} @@ -86,7 +89,9 @@ jobs: HEAD: ${{ github.event.pull_request.head.sha }} run: | every() { printf 'linux=true\nmacos=true\neffects=true\nbootstrap=true\n' | tee -a "$GITHUB_OUTPUT"; } - if [ "$EVENT" != pull_request ]; then + if [ "$EVENT" = push ]; then + printf 'linux=true\nmacos=false\neffects=true\nbootstrap=true\n' | tee -a "$GITHUB_OUTPUT" + elif [ "$EVENT" != pull_request ]; then every elif git show "$BASE:scripts/ci-lanes.mjs" > "$RUNNER_TEMP/ci-lanes.mjs" 2> /dev/null; then node "$RUNNER_TEMP/ci-lanes.mjs" --base "$BASE" --head "$HEAD" --github-output @@ -117,6 +122,8 @@ jobs: SCRIPTC_TEST_WORKERS: "1" - name: Verify mixed-source generated AST getters run: pnpm generate:ts7-ast --check + - name: Verify the generated IR field readers + run: pnpm generate:ir-fields --check native_macos: needs: [validation_scope, lanes] @@ -172,7 +179,9 @@ jobs: SCRIPTC_SAN: ${{ matrix.flavor == 'san' && '1' || '' }} # Match macOS, where Apple ASan has no LeakSanitizer: the RC audit owns # leak detection, including intentional abandonment. Plain shards also - # build some sanitized probes, so this applies to both flavors. + # build some sanitized probes, so this applies to both flavors. The + # san shards' case-sharded corpus turns LeakSanitizer on + # (scripts/ci-test-plan.mjs, tests/harness/lsan.supp). ASAN_OPTIONS: detect_leaks=0 # Within-file partition for the large case-driven harnesses # (differential, llvm-differential, npm, server, Test262, coverage, and @@ -209,33 +218,48 @@ jobs: # Start the corpus immediately beside a serial side queue. One shared # worker budget covers both processes; file and case shard axes stay # separate so every fixture still runs exactly once in each lane. + # The tests run in a transient scope capped below the runner's 16 GB, so + # the runner itself keeps memory: past 15 GB of RAM the scope swaps, as + # the whole runner did before, and when RAM and swap run out the kernel + # kills the scope's largest process. A sanitized shard peaks near 16 GB, + # and the sanitized self-hosting native-frontend build alone needs 15 GB + # (its clang compile of the frontend takes 10.5 GB). + # OOMPolicy=continue leaves the rest of the scope running, so vitest + # reports the killed worker and the job log names the test. Node sizes + # its default heap from the memory it may use, which the scope lowers + # below 4 GB; the workers keep the 4 GB heap they had on the bare + # runner. - name: Tests (corpus and integration suites) run: | ulimit -c 0 + if [ -d /run/systemd/system ]; then + exec sudo --preserve-env systemd-run --scope --quiet -p MemoryMax=15G -p OOMPolicy=continue \ + --uid="$(id -u)" --gid="$(id -g)" -- env PATH="$PATH" HOME="$HOME" NODE_OPTIONS="--max-old-space-size=4096" node scripts/ci-test.mjs + fi node scripts/ci-test.mjs tests_macos: needs: [validation_scope, lanes, native_macos] if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.macos == 'true' }} - name: test (macOS, ${{ matrix.flavor }}, ${{ (github.event_name != 'schedule' && !inputs.macos_full) && 'darwin ' || '' }}${{ matrix.shard }}/${{ (github.event_name != 'schedule' && !inputs.macos_full) && '3' || '10' }}) + name: test (macOS, ${{ matrix.flavor }}, ${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && 'darwin ' || '' }}${{ matrix.shard }}/${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && '3' || '10' }}) # arm64 macOS: same clang/libc the differential + number-formatting - # oracles were verified against. Pull requests and pushes to main run the - # darwin suite (scripts/ci-test-plan.mjs) in three slices; the nightly - # schedule and manual runs with macos_full run the complete suite in ten. - # The complete suite on every push needed more macOS runners than the - # repository has. + # oracles were verified against. Pull requests that change darwin code and + # manual runs with macos_suite set to darwin run the darwin suite + # (scripts/ci-test-plan.mjs) in three slices; the nightly schedule and + # other manual runs run the complete suite in ten. The complete suite on + # every push needed more macOS runners than the repository has. runs-on: macos-15 strategy: fail-fast: false matrix: flavor: [plain, san] - shard: ${{ fromJSON((github.event_name != 'schedule' && !inputs.macos_full) && '[1, 2, 3]' || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }} + shard: ${{ fromJSON((github.event_name != 'schedule' && inputs.macos_suite != 'full') && '[1, 2, 3]' || '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') }} env: # See the Linux tests job for the worker cap and case partitioning. SCRIPTC_TEST_WORKERS: "4" SCRIPTC_SAN: ${{ matrix.flavor == 'san' && '1' || '' }} - SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/${{ (github.event_name != 'schedule' && !inputs.macos_full) && '3' || '10' }} - SCRIPTC_CI_SUITE: ${{ (github.event_name != 'schedule' && !inputs.macos_full) && 'darwin' || 'full' }} + SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && '3' || '10' }} + SCRIPTC_CI_SUITE: ${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && 'darwin' || 'full' }} steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -252,7 +276,7 @@ jobs: path: ${{ runner.temp }} - run: tar -xzf "$RUNNER_TEMP/native-darwin-arm64.tar.gz" - run: pnpm build - - name: Tests (${{ (github.event_name != 'schedule' && !inputs.macos_full) && 'darwin suite' || 'corpus and integration suites' }}) + - name: Tests (${{ (github.event_name != 'schedule' && inputs.macos_suite != 'full') && 'darwin suite' || 'corpus and integration suites' }}) run: node scripts/ci-test.mjs native_linux: @@ -324,6 +348,37 @@ jobs: - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" - run: pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts + # The sanitized three.js nodes build alone: beside the san corpus's other + # workers it took 860 to more than 1200 seconds of its shard + # (SCRIPTC_THREE_SANITIZED_NODES_JOB in scripts/ci-test-plan.mjs). It keeps + # the corpus's LeakSanitizer settings. + three_nodes_san: + needs: [validation_scope, lanes, native_linux] + if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} + name: test (Linux, san, three.js nodes) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + env: + SCRIPTC_SAN: "1" + ASAN_OPTIONS: detect_leaks=1 + LSAN_OPTIONS: suppressions=${{ github.workspace }}/tests/harness/lsan.supp:print_suppressions=0 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + name: native-linux-x64-gnu + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz" + - run: pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'" + # Build each instrumented seed once. Downstream contracts retain the same # compiler and source artifacts without repeating seed construction. bootstrap_seed: @@ -357,6 +412,20 @@ jobs: target: wasm32-wasi helper: "false" - run: pnpm build + # The sanitized seed compiles the compiler's 700 MB module with one clang + # process under AddressSanitizer. It peaks near 14 GB while the Node + # process that emitted the module still holds about 6 GB. Together they + # exceed the runner's 16 GB of memory, so back them with swap. + - name: Enlarge swap for the sanitized seed + if: matrix.flavor == 'san' + run: | + sudo swapoff -a + sudo rm -f /mnt/swapfile + sudo fallocate -l 24G /mnt/swapfile + sudo chmod 600 /mnt/swapfile + sudo mkswap /mnt/swapfile + sudo swapon /mnt/swapfile + free -h - name: Prepare bootstrap seed run: node scripts/bootstrap-seed.mjs env: @@ -496,6 +565,32 @@ jobs: - name: TypeScript 7 full parity sweep run: pnpm test:ts7 + # tsc-ts, the default checker, and tsgo (SCRIPTC_CHECKER=tsgo) serve the same TypeScript 7 API. Every corpus + # program must compile to the same LLVM IR, with the same diagnostics, with either checker. + checker_identity: + needs: [validation_scope, lanes] + if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} + name: test (tsgo and tsc-ts give the same IR) + runs-on: ubuntu-24.04 + # The two corpus compiles run back to back and take close to 30 minutes on a + # slow hosted runner, so a 30-minute limit cancels otherwise healthy runs. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build + - name: Corpus LLVM IR with tsgo and with tsc-ts (release) + run: pnpm test:checker-identity --optimization release + - name: Corpus LLVM IR with tsgo and with tsc-ts (dev) + run: pnpm test:checker-identity --optimization dev + llvm_artifacts_linux_x64: needs: [validation_scope, lanes] if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }} @@ -570,11 +665,10 @@ jobs: if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.macos == 'true' }} name: test (macOS arm64 LLVM artifacts, no clang, ${{ matrix.shard }}/5) runs-on: macos-15 - # Shard 1 additionally runs package/install and link-parity contracts - # before its LLVM differential slice. The slice itself can take ~10 - # minutes on a cold hosted runner, so 20 minutes cancels an otherwise - # healthy job before the differential completes. - timeout-minutes: 35 + # Each LLVM differential slice takes 30 to 35 minutes on a hosted runner. + # The package/install and link-parity contracts run in their own job + # (llvm_artifacts_macos_arm64_contracts), so no slice carries them. + timeout-minutes: 50 strategy: fail-fast: false matrix: @@ -584,6 +678,36 @@ jobs: # keeping each macOS job below its timeout while the matrix union still # exercises every LLVM-tier program. SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/5 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - uses: actions/download-artifact@v4 + with: + name: native-darwin-arm64 + path: ${{ runner.temp }} + - run: tar -xzf "$RUNNER_TEMP/native-darwin-arm64.tar.gz" + - run: pnpm build + - name: LLVM release/dev differential (${{ matrix.shard }}/5) + env: + SCRIPTC_TEST_WORKERS: "4" + run: pnpm test tests/harness/llvm-differential.test.ts + + llvm_artifacts_macos_arm64_contracts: + needs: [validation_scope, lanes, native_macos] + if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.macos == 'true' }} + name: test (macOS arm64 LLVM artifacts, no clang, contracts) + runs-on: macos-15 + # The no-clang object, helper parity, packaged helper and packed npm + # installation contracts: about 12 minutes, which used to run in front + # of shard 1's differential slice and took that job to 45-50 minutes. + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -601,10 +725,8 @@ jobs: - run: tar -xzf "$RUNNER_TEMP/native-darwin-arm64.tar.gz" - run: pnpm build - name: No-clang assembly/object contract - if: matrix.shard == 1 run: pnpm test packages/cli/test/native-output.test.ts - name: Helper diagnostics and object/link parity - if: matrix.shard == 1 run: >- pnpm test packages/compiler/test/native-codegen-integration.test.ts @@ -612,17 +734,11 @@ jobs: packages/cli/test/native-link-info.test.ts packages/cli/test/runtime-pack.test.ts tests/harness/native-object-example.test.ts - - name: LLVM release/dev differential (${{ matrix.shard }}/5) - env: - SCRIPTC_TEST_WORKERS: "4" - run: pnpm test tests/harness/llvm-differential.test.ts - name: Packaged helper contract - if: matrix.shard == 1 run: | TARBALL=$(pnpm --dir packages/llvm-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent) node scripts/verify-llvm-package.mjs "$RUNNER_TEMP/$(basename "$TARBALL")" - name: Packed npm installation smoke - if: matrix.shard == 1 env: # Release optimization is covered by the compiler bootstrap jobs. # This lane verifies packaging and installation with a faster seed. @@ -728,17 +844,22 @@ jobs: run: >- pnpm exec vitest run tests/harness/worktree-files.test.ts --testNamePattern "workspace reset keeps Linux Sandbox paths POSIX" + # The Node host checks with tsc-ts, the default, over the named pipe it + # serves on Windows, and with tsgo. - name: Windows scriptc run smoke shell: pwsh run: | zig version - $output = & node packages/cli/dist/main.js run tests/corpus/001-hello.ts --backend llvm - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - $actual = ($output -join "`n").Trim() - if ($actual -ne "hello world") { - throw "unexpected scriptc output: $actual" + foreach ($checker in @("tsc-ts", "tsgo")) { + $env:SCRIPTC_CHECKER = $checker + $output = & node packages/cli/dist/main.js run tests/corpus/001-hello.ts --backend llvm + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + $actual = ($output -join "`n").Trim() + if ($actual -ne "hello world") { + throw "unexpected scriptc output with ${checker}: $actual" + } } - name: Windows PE subsystem selection env: @@ -842,14 +963,25 @@ jobs: strategy: fail-fast: false matrix: - include: - - runtime: runtime-ios-arm64 - runner: macos-15 - - runtime: runtime-ios-simulator-arm64 - runner: macos-15 - - runtime: runtime-android-arm64 - runner: ubuntu-24.04 - runs-on: ${{ matrix.runner }} + runtime: [runtime-android-arm64] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/mobile-runtime-pack + with: + runtime: ${{ matrix.runtime }} + + # The iOS runtime packs build on macOS runners, so they run in the macos lane. + mobile_runtime_packs_ios: + needs: [validation_scope, lanes] + if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.macos == 'true' }} + name: test (${{ matrix.runtime }}) + strategy: + fail-fast: false + matrix: + runtime: [runtime-ios-arm64, runtime-ios-simulator-arm64] + runs-on: macos-15 timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -861,7 +993,31 @@ jobs: # Require this check on the stack base. Upper layers must not pass before # full validation, so untested groups cannot merge. Merge bottom-up. test: - needs: [validation_scope, lanes, code_quality, native_macos, tests, tests_macos, native_linux, effects, bootstrap_seed, bootstrap, linux_host_clang, ts7_parity, llvm_artifacts_linux_x64, llvm_artifacts_macos_arm64, llvm_artifacts_macos_x64, windows_cli, llvm_artifacts_windows_x64, mobile_runtime_packs] + needs: + [ + validation_scope, + lanes, + code_quality, + native_macos, + tests, + tests_macos, + native_linux, + effects, + three_nodes_san, + bootstrap_seed, + bootstrap, + linux_host_clang, + ts7_parity, + checker_identity, + llvm_artifacts_linux_x64, + llvm_artifacts_macos_arm64, + llvm_artifacts_macos_arm64_contracts, + llvm_artifacts_macos_x64, + windows_cli, + llvm_artifacts_windows_x64, + mobile_runtime_packs, + mobile_runtime_packs_ios, + ] # Run even after failed or cancelled lanes: skipped required jobs allow merging. if: ${{ always() }} runs-on: ubuntu-latest @@ -881,16 +1037,20 @@ jobs: lane tests_macos "${{ needs.tests_macos.result }}" "${{ needs.lanes.outputs.macos }}" lane native_linux "${{ needs.native_linux.result }}" "${{ needs.lanes.outputs.linux }}" lane effects "${{ needs.effects.result }}" "${{ needs.lanes.outputs.effects }}" + lane three_nodes_san "${{ needs.three_nodes_san.result }}" "${{ needs.lanes.outputs.linux }}" lane bootstrap_seed "${{ needs.bootstrap_seed.result }}" "${{ needs.lanes.outputs.bootstrap }}" lane bootstrap "${{ needs.bootstrap.result }}" "${{ needs.lanes.outputs.bootstrap }}" lane linux_host_clang "${{ needs.linux_host_clang.result }}" "${{ needs.lanes.outputs.linux }}" lane ts7_parity "${{ needs.ts7_parity.result }}" "${{ needs.lanes.outputs.linux }}" + lane checker_identity "${{ needs.checker_identity.result }}" "${{ needs.lanes.outputs.linux }}" lane llvm_artifacts_linux_x64 "${{ needs.llvm_artifacts_linux_x64.result }}" "${{ needs.lanes.outputs.linux }}" lane llvm_artifacts_macos_arm64 "${{ needs.llvm_artifacts_macos_arm64.result }}" "${{ needs.lanes.outputs.macos }}" + lane llvm_artifacts_macos_arm64_contracts "${{ needs.llvm_artifacts_macos_arm64_contracts.result }}" "${{ needs.lanes.outputs.macos }}" lane llvm_artifacts_macos_x64 "${{ needs.llvm_artifacts_macos_x64.result }}" "${{ needs.lanes.outputs.macos }}" lane windows_cli "${{ needs.windows_cli.result }}" "${{ needs.lanes.outputs.linux }}" lane llvm_artifacts_windows_x64 "${{ needs.llvm_artifacts_windows_x64.result }}" "${{ needs.lanes.outputs.linux }}" lane mobile_runtime_packs "${{ needs.mobile_runtime_packs.result }}" "${{ needs.lanes.outputs.linux }}" + lane mobile_runtime_packs_ios "${{ needs.mobile_runtime_packs_ios.result }}" "${{ needs.lanes.outputs.macos }}" ;; false) echo "::notice::Full CI is deferred until this PR becomes the lowest unmerged layer. Merge the stack from the bottom up." diff --git a/.github/workflows/llvm-dist.yml b/.github/workflows/llvm-dist.yml new file mode 100644 index 0000000000..233e97e98c --- /dev/null +++ b/.github/workflows/llvm-dist.yml @@ -0,0 +1,93 @@ +name: LLVM distribution + +# Builds the LLVM that the Linux release helpers link +# (native/llvm-codegen/llvm-dist.mjs): LLVM from its release sources with +# ThinLTO and native/llvm-codegen/llvm.profdata.gz, for each Linux +# architecture. Each job builds a helper against it and one against LLVM's +# release archive, requires the two to give identical objects on this +# checkout's helper inputs (scripts/check-llvm-dist.mjs), prints both +# helpers' times, and attaches the archive to the LLVM_DIST_TAG release. +# The arm64 build uses the profile collected on x64; its job log reports how +# many functions the profile no longer matches. Run it after a profile refresh +# or an LLVM version change, with a new LLVM_DIST_TAG. +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: ${{ github.workflow }} + +jobs: + build: + name: Build LLVM (${{ matrix.arch }}) + if: github.repository == 'vercel-labs/scriptc' + strategy: + fail-fast: false + matrix: + include: + - arch: x64 + runner: ubuntu-24.04 + llvm_asset: LLVM-22.1.8-Linux-X64 + - arch: arm64 + runner: ubuntu-24.04-arm + llvm_asset: LLVM-22.1.8-Linux-ARM64 + runs-on: ${{ matrix.runner }} + # About an hour for LLVM on 4 cores, and 5 minutes for each helper link. + timeout-minutes: 240 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: pnpm + - name: Install LLVM's release archive and sources + run: | + sudo apt-get update + sudo apt-get install --yes ninja-build zlib1g-dev libzstd-dev + base=https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8 + curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" "$base/${{ matrix.llvm_asset }}.tar.xz" + tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP" + mv "$RUNNER_TEMP/${{ matrix.llvm_asset }}" "$RUNNER_TEMP/llvm-22.1.8" + curl -fL --retry 3 -o "$RUNNER_TEMP/llvm-src.tar.xz" "$base/llvm-project-22.1.8.src.tar.xz" + tar -xJf "$RUNNER_TEMP/llvm-src.tar.xz" -C "$RUNNER_TEMP" + rm "$RUNNER_TEMP/llvm.tar.xz" "$RUNNER_TEMP/llvm-src.tar.xz" + echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" + - run: pnpm install --frozen-lockfile + - run: pnpm --filter @scriptc/compiler --filter scriptc build + - name: Build the distribution + run: >- + node scripts/build-llvm-dist.mjs --source "$RUNNER_TEMP/llvm-project-22.1.8.src" + --prefix "$RUNNER_TEMP/dist/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}" + --archive "$RUNNER_TEMP/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}.tar.xz" + --work "$RUNNER_TEMP/llvm-dist-work" + - name: Build a helper against each LLVM + env: + HELPER: llvm-linux-${{ matrix.arch }}-gnu + run: | + LLVM_DIR="$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" pnpm --filter "@scriptc/$HELPER" build:native + cp "packages/$HELPER/bin/scriptc-llvm-codegen" "$RUNNER_TEMP/helper-release-archive" + LLVM_DIR="$RUNNER_TEMP/dist/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}/lib/cmake/llvm" \ + SCRIPTC_LLVM_THINLTO=1 pnpm --filter "@scriptc/$HELPER" build:native + cp "packages/$HELPER/bin/scriptc-llvm-codegen" "$RUNNER_TEMP/helper-profiled" + - name: Compare the helpers + run: >- + node scripts/check-llvm-dist.mjs --baseline "$RUNNER_TEMP/helper-release-archive" + --candidate "$RUNNER_TEMP/helper-profiled" --rounds 3 --work "$RUNNER_TEMP/llvm-check" + - name: Attach the archive to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ARCHIVE: ${{ runner.temp }}/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}.tar.xz + run: | + tag=$(node --input-type=module -e "import { LLVM_DIST_TAG } from './native/llvm-codegen/llvm-dist.mjs'; console.log(LLVM_DIST_TAG)") + # Both architectures' jobs create the release if it is missing. + gh release view "$tag" > /dev/null 2>&1 || + gh release create "$tag" --target "$GITHUB_SHA" --title "$tag" --latest=false \ + --notes "LLVM 22.1.8 built with ThinLTO and the helper profile, for the Linux LLVM helpers (scripts/build-llvm-dist.mjs)." || + gh release view "$tag" > /dev/null + gh release upload "$tag" "$ARCHIVE" "$ARCHIVE.sha256" --clobber diff --git a/.github/workflows/prune-caches.yml b/.github/workflows/prune-caches.yml new file mode 100644 index 0000000000..1509d56ce7 --- /dev/null +++ b/.github/workflows/prune-caches.yml @@ -0,0 +1,41 @@ +name: Prune caches + +# Pull request caches are scoped to that pull request, so once it is closed no +# run can use them again. Deleting them keeps the repository under its cache +# size limit, so caches that main and open pull requests use are not evicted. +on: + pull_request: + types: [closed] + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + +permissions: + actions: write + pull-requests: read + +jobs: + prune: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Delete caches of closed pull requests + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # Read both lists in full before deleting anything: deleting while the cache + # listing pages shifts later pages, and a failed request must stop the job + # instead of leaving open pull requests out of the list. + open=$(gh api --paginate "repos/$REPO/pulls?state=open&per_page=100" --jq '.[].number') + caches=$(gh api --paginate "repos/$REPO/actions/caches?per_page=100" \ + --jq '.actions_caches[] | select(.ref | startswith("refs/pull/")) | "\(.id) \(.ref)"') + while read -r id ref; do + [ -n "$id" ] || continue + number=$(echo "$ref" | cut -d/ -f3) + if ! grep -qx "$number" <<<"$open"; then + gh api --method DELETE "repos/$REPO/actions/caches/$id" >/dev/null + echo "deleted cache $id ($ref)" + fi + done <<<"$caches" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47329cc28e..a402f912fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,6 +83,7 @@ jobs: spawnSync('git', ['diff', '--quiet', run.head_sha, 'HEAD', '--', '.', ':(exclude).github/workflows/release.yml', ':(exclude)scripts/verify-native-cli.mjs', ':(exclude)scripts/package-native-cli.mjs', ':(exclude)tests/harness/native-cli-packaging.test.ts', + ':(exclude)scripts/wait-npm-available.mjs', ':(exclude)tests/harness/npm-available.test.ts', ]).status !== 0) continue; const artifacts = api(`repos/${repo}/actions/runs/${run.id}/artifacts?per_page=100`).artifacts .filter((artifact) => !artifact.expired && artifact.size_in_bytes > 0); @@ -100,10 +101,71 @@ jobs: if (receiptRun) appendFileSync(process.env.GITHUB_OUTPUT, `receipt_run=${receiptRun}\n`); NODE - build-native-packages: - name: Build native package (${{ matrix.platform }}) + # Collects the native compiler's function layout and profile on this commit + # (scripts/refresh-cli-layout.mjs, scripts/refresh-cli-profile.mjs), so every + # platform's CLI build optimizes and lays out the code it compiles. The + # committed ones fall behind main within a day or two. If this job fails, + # the CLI builds use the committed profile and layout. + cli-profile: + name: Collect the native compiler profile needs: check-release if: needs.check-release.outputs.should_release == 'true' && needs.check-release.outputs.reuse_artifacts != 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 75 + continue-on-error: true + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11.1.3 + - uses: actions/setup-node@v4 + with: + node-version: "24" + cache: pnpm + - uses: ./.github/actions/llvm-linux + - run: pnpm install --frozen-lockfile + - uses: ./.github/actions/zig + - uses: ./.github/actions/native-packages + with: + target: linux-x64-gnu + - run: pnpm --filter @scriptc/compiler --filter scriptc build + - name: Check out tsc-ts + run: | + git clone --quiet --depth 1 https://github.com/vercel-labs/tsc-ts "$RUNNER_TEMP/tsc-ts" + echo "tsc-ts $(git -C "$RUNNER_TEMP/tsc-ts" rev-parse HEAD)" + npm ci --prefix "$RUNNER_TEMP/tsc-ts" + # The layout comes first: the profile matches the functions of builds + # with the layout its instrumented compiler was built with, so the + # profile is collected with a second instrumented compiler built with + # this layout. + - name: Collect the function layout + run: >- + node scripts/refresh-cli-layout.mjs --jobs 4 + --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" --work "$RUNNER_TEMP/cli-layout-work" + - name: Collect the profile + env: + SCRIPTC_NATIVE_LAYOUT: ${{ runner.temp }}/cli-profile/cli.layout.gz + run: >- + node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 + --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work" + - uses: actions/upload-artifact@v4 + with: + name: cli-profile + path: | + ${{ runner.temp }}/cli-profile/cli.profdata.gz + ${{ runner.temp }}/cli-profile/cli.layout.gz + retention-days: 7 + if-no-files-found: error + + build-native-packages: + name: Build native package (${{ matrix.platform }}) + needs: [check-release, cli-profile] + # Runs when the profile job fails; the CLI builds then use the committed profile. + if: >- + !cancelled() && needs.check-release.outputs.should_release == 'true' && + needs.check-release.outputs.reuse_artifacts != 'true' strategy: fail-fast: false matrix: @@ -124,6 +186,7 @@ jobs: runtime: runtime-linux-x64-gnu cli: cli-linux-x64-gnu llvm_asset: LLVM-22.1.8-Linux-X64 + llvm_dist: x64 use_zig: true - platform: linux-arm64 runner: ubuntu-24.04-arm @@ -131,6 +194,7 @@ jobs: runtime: runtime-linux-arm64-gnu cli: cli-linux-arm64-gnu llvm_asset: LLVM-22.1.8-Linux-ARM64 + llvm_dist: arm64 use_zig: true - platform: windows-x64 runner: windows-2022 @@ -146,6 +210,7 @@ jobs: seed_helper: llvm-linux-x64-gnu cli_target: x86_64-linux-musl llvm_asset: LLVM-22.1.8-Linux-X64 + llvm_dist: x64 use_zig: true - platform: linux-arm64-musl runner: ubuntu-24.04-arm @@ -155,12 +220,14 @@ jobs: seed_helper: llvm-linux-arm64-gnu cli_target: aarch64-linux-musl llvm_asset: LLVM-22.1.8-Linux-ARM64 + llvm_dist: arm64 use_zig: true - platform: wasm32-wasi runner: ubuntu-24.04 helper: llvm-linux-x64-gnu runtime: runtime-wasm32-wasi llvm_asset: LLVM-22.1.8-Linux-X64 + llvm_dist: x64 use_zig: true runs-on: ${{ matrix.runner }} # The Intel macOS build took 24 minutes for 0.3.2 and passed 30 for 0.3.3. @@ -191,6 +258,28 @@ jobs: mv "$RUNNER_TEMP/${{ matrix.llvm_asset }}" "$RUNNER_TEMP/llvm-22.1.8" echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV" echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH" + # The Linux helpers link LLVM built with ThinLTO and a profile of the + # helper's code generation (native/llvm-codegen/llvm-dist.mjs), from a + # release of this repository. Without that archive they link LLVM's + # release archive, which gives the same objects more slowly. The + # wasm32-wasi job also uploads the x64 GNU helper, so it selects the same. + - name: Select the profiled LLVM distribution (Linux) + if: matrix.llvm_dist != '' + env: + ARCH: ${{ matrix.llvm_dist }} + run: | + tag=$(node --input-type=module -e "import { LLVM_DIST_TAG } from './native/llvm-codegen/llvm-dist.mjs'; console.log(LLVM_DIST_TAG)") + asset=$(node --input-type=module -e "import { llvmDistAsset } from './native/llvm-codegen/llvm-dist.mjs'; console.log(llvmDistAsset(process.env.ARCH))") + base="https://github.com/$GITHUB_REPOSITORY/releases/download/$tag" + if curl -fsSL --retry 3 -o "$RUNNER_TEMP/$asset" "$base/$asset" && + curl -fsSL --retry 3 -o "$RUNNER_TEMP/$asset.sha256" "$base/$asset.sha256" && + (cd "$RUNNER_TEMP" && sha256sum --check "$asset.sha256"); then + tar -xJf "$RUNNER_TEMP/$asset" -C "$RUNNER_TEMP" + echo "SCRIPTC_HELPER_LLVM_DIR=$RUNNER_TEMP/${asset%.tar.xz}/lib/cmake/llvm" >> "$GITHUB_ENV" + echo "The helper links $asset from release $tag" + else + echo "::warning::$asset is not available from release $tag; the helper links LLVM's release archive" + fi - uses: vercel-labs/setup-zig@v1 if: matrix.use_zig == true with: @@ -219,11 +308,25 @@ jobs: Move-Item "$expanded\clang+llvm-22.1.8-x86_64-pc-windows-msvc" $llvm echo "LLVM_DIR=$llvm\lib\cmake\llvm" >> $env:GITHUB_ENV echo "$llvm\bin" >> $env:GITHUB_PATH + # A musl CLI ships tsc-ts's musl executable, and these jobs run on glibc + # runners, where pnpm installs only the glibc platform packages. + - name: Let pnpm install musl platform packages + if: endsWith(matrix.cli_target, '-musl') + run: | + printf 'supportedArchitectures:\n libc: [current, musl]\n' >> pnpm-workspace.yaml - run: pnpm install --frozen-lockfile - name: Build native packages (POSIX) if: matrix.platform != 'windows-x64' run: | - pnpm --filter @scriptc/${{ matrix.helper }} build:native + if [ -n "${SCRIPTC_HELPER_LLVM_DIR:-}" ]; then + LLVM_DIR="$SCRIPTC_HELPER_LLVM_DIR" SCRIPTC_LLVM_THINLTO=1 pnpm --filter @scriptc/${{ matrix.helper }} build:native || { + echo "::warning::the helper did not build against the profiled LLVM; it links LLVM's release archive" + rm -rf node_modules/.cache/scriptc-${{ matrix.helper }} + pnpm --filter @scriptc/${{ matrix.helper }} build:native + } + else + pnpm --filter @scriptc/${{ matrix.helper }} build:native + fi if [ "${{ matrix.use_zig }}" = true ]; then CC=zig AR=zig pnpm --filter @scriptc/${{ matrix.runtime }} build:native else @@ -237,6 +340,33 @@ jobs: - name: Build host helper for cross-libc bootstrap if: matrix.seed_helper != '' run: pnpm --filter @scriptc/${{ matrix.seed_helper }} build:native + - name: Download the profile collected on this commit + id: cli-profile + if: matrix.cli != '' + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: cli-profile + path: ${{ runner.temp }}/cli-profile + # Without the artifact, SCRIPTC_NATIVE_PROFILE_USE and SCRIPTC_NATIVE_LAYOUT + # stay unset and build-native-cli.mts uses the committed profile and layout. + - name: Select the native compiler profile + if: matrix.cli != '' + shell: bash + env: + COLLECTED: ${{ steps.cli-profile.outcome == 'success' && format('{0}/cli-profile/cli.profdata.gz', runner.temp) || '' }} + run: | + if [ -n "$COLLECTED" ]; then + echo "SCRIPTC_NATIVE_PROFILE_USE=$COLLECTED" >> "$GITHUB_ENV" + echo "The native CLI builds with the profile collected on $GITHUB_SHA" + layout="$(dirname "$COLLECTED")/cli.layout.gz" + if [ -f "$layout" ]; then + echo "SCRIPTC_NATIVE_LAYOUT=$layout" >> "$GITHUB_ENV" + echo "The native CLI builds with the function layout collected on $GITHUB_SHA" + fi + else + echo "::warning::No profile was collected on this commit; the native CLI builds with the committed profile" + fi - name: Build native CLI distribution if: matrix.cli != '' env: @@ -311,12 +441,15 @@ jobs: (needs.check-release.outputs.reuse_artifacts == 'true' || (needs.build-native-packages.result == 'success' && needs.build-mobile-runtime-packs.result == 'success')) runs-on: ubuntu-latest - timeout-minutes: 15 + # Two waits for npm availability of at most 25 minutes each. + timeout-minutes: 70 environment: Release permissions: contents: read actions: read id-token: write + outputs: + uploads: ${{ steps.publish.outputs.uploads }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -336,7 +469,7 @@ jobs: VERSION: ${{ needs.check-release.outputs.version }} RECEIPT_RUN: ${{ needs.check-release.outputs.receipt_run }} run: | - touch "$RUNNER_TEMP/npm-accepted.txt" + touch "$RUNNER_TEMP/npm-accepted.txt" "$RUNNER_TEMP/npm-accepted-times.txt" # Include this run so re-running only failed jobs restores its receipts. for run in "$GITHUB_RUN_ID" "$RECEIPT_RUN"; do if [ -z "$run" ]; then continue; fi @@ -345,6 +478,7 @@ jobs: if [ "$count" -eq 0 ]; then continue; fi gh run download "$run" --repo "$GITHUB_REPOSITORY" --pattern "npm-accepted-$VERSION-*" --dir "$RUNNER_TEMP/npm-receipts/$run" find "$RUNNER_TEMP/npm-receipts/$run" -name npm-accepted.txt -exec cat {} + >> "$RUNNER_TEMP/npm-accepted.txt" + find "$RUNNER_TEMP/npm-receipts/$run" -name npm-accepted-times.txt -exec cat {} + >> "$RUNNER_TEMP/npm-accepted-times.txt" done - name: Setup pnpm @@ -394,6 +528,7 @@ jobs: done - name: Publish to npm + id: publish shell: bash env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -413,10 +548,14 @@ jobs: # Upload acceptance and public availability are separate states. # Save acceptance immediately so retries never depend on registry lag. + # npm-release.txt lists every package of the release, and + # npm-accepted-times.txt the time npm accepted each upload, for the + # npm-available job. publish_dir() { dir="$1" packed="${2:-}" name=$(node -p "require('./$dir/package.json').name") + echo "$name@$VERSION" >> "$RUNNER_TEMP/npm-release.txt" if grep -Fxq "$name@$VERSION" "$RUNNER_TEMP/npm-accepted.txt"; then echo "$name@$VERSION upload already accepted, skipping" return 0 @@ -433,6 +572,7 @@ jobs: publish_log="$RUNNER_TEMP/npm-publish.log" if npm publish "$packed" $PROVENANCE --access public 2>&1 | tee "$publish_log"; then echo "$name@$VERSION" >> "$RUNNER_TEMP/npm-accepted.txt" + echo "$name@$VERSION $(date +%s)" >> "$RUNNER_TEMP/npm-accepted-times.txt" return 0 else publish_status=${PIPESTATUS[0]} @@ -455,8 +595,8 @@ jobs: } # pnpm pack resolves workspace dependencies; npm handles OIDC. - # Publish dependencies first. Registry propagation after an accepted - # upload must not block the remaining packages or the GitHub release. + # Publish dependencies first. The npm-available job waits for the + # packages that nothing in this release depends on. publish_dir packages/runtime publish_dir packages/runtime-darwin-arm64 publish_dir packages/llvm-darwin-arm64 @@ -476,31 +616,87 @@ jobs: publish_dir packages/runtime-wasm32-wasi publish_dir packages/runtime-win32-x64-msvc publish_dir packages/llvm-win32-x64-msvc + # A native CLI package depends on its platform's LLVM helper and + # runtime pack packages, and scriptc on the native CLI packages. + # Each group publishes only once the packages it depends on + # resolve, so an install never finds a package whose dependencies + # npm is still processing. A rerun skips accepted uploads and waits + # again. + wait_available() { + awk 'FILENAME == ARGV[1] { accepted[$1] = $2; next } { print $1, accepted[$1] }' \ + "$RUNNER_TEMP/npm-accepted-times.txt" "$RUNNER_TEMP/npm-release.txt" > "$RUNNER_TEMP/npm-wait.txt" + node scripts/wait-npm-available.mjs "$RUNNER_TEMP/npm-wait.txt" --interval 30 --timeout 1500 + } + wait_available for pkg in packages/cli-*; do publish_dir "$pkg"; done publish_dir packages/compiler publish_dir packages/experimental publish_dir packages/threads publish_dir packages/types + wait_available publish_dir packages/cli echo "All package uploads accepted; npm registry propagation may continue after this job." >> "$GITHUB_STEP_SUMMARY" + # Each package of the release with the time its upload was accepted, + # when a run recorded it. + { + echo "uploads<> "$GITHUB_OUTPUT" + - name: Save accepted upload receipts if: always() uses: actions/upload-artifact@v4 with: name: npm-accepted-${{ needs.check-release.outputs.version }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/npm-accepted.txt + path: | + ${{ runner.temp }}/npm-accepted.txt + ${{ runner.temp }}/npm-accepted-times.txt if-no-files-found: ignore retention-days: 90 - # Publish standalone native distributions and the surface manifest after - # npm publishing succeeds. The release body comes from the marked changelog. - github-release: - name: Create GitHub Release + # Waits until every package of the release resolves on the npm registry, + # and reports each package's time from upload to available. Fails with the + # packages that are still missing after 2 hours. It only reads from the + # registry; the next release run retries it, and publishing skips uploads + # npm already accepted. + npm-available: + name: Wait for npm availability needs: [check-release, publish] if: >- !cancelled() && needs.check-release.outputs.should_release == 'true' && needs.publish.result == 'success' runs-on: ubuntu-latest + timeout-minutes: 135 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + + - name: Wait for every package to resolve + env: + UPLOADS: ${{ needs.publish.outputs.uploads }} + run: | + printf '%s\n' "$UPLOADS" > "$RUNNER_TEMP/npm-uploads.txt" + node scripts/wait-npm-available.mjs "$RUNNER_TEMP/npm-uploads.txt" + + # Publish standalone native distributions and the surface manifest after + # every npm package resolves. The release body comes from the marked + # changelog. + github-release: + name: Create GitHub Release + needs: [check-release, publish, npm-available] + if: >- + !cancelled() && needs.check-release.outputs.should_release == 'true' && needs.npm-available.result == 'success' + runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: write @@ -552,6 +748,21 @@ jobs: - name: Package standalone compilers run: node scripts/package-native-cli.mjs packages /tmp/scriptc-release-assets + # The profile the native CLIs were built with. Runs that reuse native + # builds from before the cli-profile job, or whose profile job failed, + # have none; their CLIs used the committed profile. + - name: Download the native compiler profile + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN: ${{ needs.check-release.outputs.artifact_run }} + VERSION: ${{ needs.check-release.outputs.version }} + run: | + if gh run download "$RUN" --repo "$GITHUB_REPOSITORY" --name cli-profile --dir "$RUNNER_TEMP/cli-profile"; then + mv "$RUNNER_TEMP/cli-profile/cli.profdata.gz" "/tmp/scriptc-release-assets/scriptc-$VERSION-cli.profdata.gz" + else + echo "Run $RUN has no cli-profile artifact" + fi + - name: Create GitHub Release run: | VERSION="${{ needs.check-release.outputs.version }}" diff --git a/.gitignore b/.gitignore index e5b54a5fac..81b92ddd53 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,14 @@ node_modules/ !tests/fixtures/npm/node_modules/ !tests/corpus/*/node_modules/ !tests/fixtures/npm-static/node_modules/ +!tests/coverage-fixtures/wasm/node_modules/ +!tests/coverage-fixtures/node-islands*/node_modules/ +!tests/coverage-fixtures/dynamic-tier/package/node_modules/ !tests/fixtures/npm-static/nested/inner/node_modules/ +!tests/fixtures/npm-static/node_modules/*/node_modules/ !tests/fixtures/npm/store/*/node_modules/ +!tests/fixtures/npm/node_modules/*/node_modules/ +!tests/fixtures/npm/workspace/*/node_modules/ !tests/fixtures/npm/cases/*/node_modules/ !tests/library-mode/*/node_modules/ !tests/fixtures/provenance/node_modules/ @@ -28,10 +34,13 @@ output/ /packages/runtime-*/runtime-pack.json /packages/runtime-*/.runtime-pack-* !tests/fixtures/npm/node_modules/*/dist/ +!tests/fixtures/npm/node_modules/@*/*/dist/ !tests/fixtures/npm/workspace/*/dist/ !tests/fixtures/npm/cases/workspace-copied/members/*/dist/ !tests/fixtures/npm/cases/workspace-copied/node_modules/*/dist/ !tests/fixtures/provenance/node_modules/*/dist/ +!tests/fixtures/npm-static/node_modules/*/dist/ +!tests/corpus/*/node_modules/*/dist/ .scriptc/ *.log .DS_Store @@ -45,3 +54,6 @@ tmp-*/ .env.local .env* .env*.local +# the vendored fs-extra corpus package: its lib/output module and jsonfile's own universalify +!tests/corpus/fs-extra-package/node_modules/fs-extra/lib/output/ +!tests/corpus/fs-extra-package/node_modules/jsonfile/node_modules/ diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index 52cbf50262..55c8d0353f 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -241,6 +241,7 @@ "packages/compiler/src/frontend/comptime-node.ts", "packages/compiler/src/frontend/ts7/world-check.ts", "packages/compiler/src/frontend/ts7/source-parser.test.ts", + "packages/compiler/src/frontend/jsx-source.test.ts", "packages/compiler/src/frontend/npm-static-rewrite.ts", "packages/compiler/src/frontend/npm-static-bundled-cjs.ts", "packages/compiler/src/library/semantic-source.ts", @@ -270,6 +271,7 @@ "packages/compiler/src/backend/llvm/**", "packages/compiler/src/backend/loop-polls.test.ts", "packages/compiler/src/backend/may-throw.test.ts", + "packages/compiler/src/backend/stack-observations.test.ts", "packages/compiler/src/ir/**", "packages/compiler/src/library/*.test.ts", "packages/compiler/test/**", @@ -281,10 +283,22 @@ }, { // Ambient declarations describe an existing constructor and its prototype methods. - "files": ["packages/compiler/ambient/scriptc-node-fallback.d.ts"], + "files": [ + "packages/compiler/ambient/scriptc-node-fallback.d.ts", + "packages/compiler/ambient/node-fallback/*.d.ts", + ], "rules": { "typescript/no-unsafe-declaration-merging": "off", }, }, + { + // Each module's fallback declarations are a script that declares the + // module ambiently; it names the files of the modules it uses by + // reference, since a script cannot import them. + "files": ["packages/compiler/ambient/node-fallback/*.d.ts"], + "rules": { + "typescript/triple-slash-reference": "off", + }, + }, ], } diff --git a/AGENTS.md b/AGENTS.md index 4d66cb42fa..3e631a92f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This is the only agent guide for the repository. Its rules apply everywhere, inc ## Markdown style -Write prose paragraphs and list items as single source lines and let editors and renderers wrap them naturally. Do not hard-wrap Markdown at a fixed column; preserve separate lines only where Markdown syntax or intentional formatting requires them, such as headings, lists, tables, blockquotes, and code blocks. +Write prose paragraphs and list items as single source lines and let editors and renderers wrap them naturally. Do not hard-wrap Markdown at a fixed column; preserve separate lines only where Markdown syntax or intentional formatting requires them, such as headings, lists, tables, blockquotes, and code blocks. The Limitations page partials under `docs/content/limitations/` are the exception: they keep one sentence per line (see Limitations page partials). ## Build and test @@ -25,6 +25,8 @@ their supported native-clang contracts locally; other hosts retain those checks in the Sandboxes. Both lanes green is the bar before shipping any change. +`SCRIPTC_SANDBOX_NATIVE_REFERENCE` names a built checkout inside a prepared `SCRIPTC_SANDBOX_SNAPSHOT` or image: each lane copies that checkout's LLVM helper and runtime pack instead of rebuilding them when their source inputs are identical. Sandbox creation is rate limited per team in vCPUs per minute, so the runner retries a rate-limited creation with backoff. Larger Sandboxes (`SCRIPTC_SANDBOX_VCPUS` up to 32, with `SCRIPTC_TEST_WORKERS` at half the vCPUs) shorten the lanes most. + `VERCEL_OIDC_TOKEN` is the preferred credential. `VERCEL_TOKEN` remains supported with explicit `VERCEL_TEAM_ID` and `VERCEL_PROJECT_ID`. The custom `SCRIPTC_SANDBOX_IMAGE` is optional: without it, the gate starts from @@ -46,8 +48,20 @@ SCRIPTC_TEST_WORKERS=4 SCRIPTC_SAN=1 pnpm test # sanitized lane Corpus programs are differential tests against Node: every program runs under Node and as a compiled native binary, and stdout, stderr, and exit codes must match byte-for-byte. A new feature lands with corpus programs that pin its behavior both ways. +Name a new corpus program after what it tests, without a number: lowercase words joined by hyphens, such as `tests/corpus/intersection-restated-members.ts`, or a directory such as `tests/corpus//main.ts` for a multi-file program. Parallel changes kept taking the same next number; with names, two changes collide only when they add the same name, which git reports as a conflict. Leave existing numbered programs as they are: never rename or renumber them, and name a new program even when it extends a numbered family. + +Record a new or remeasured program's CI cost (used to balance the test shards) in `tests/harness/corpus-costs/.json`, not in `tests/harness/corpus-costs.json`; see the README there. + Before opening a PR that adds or changes corpus, diagnostic, or frontend fixtures, run `pnpm test:ts7 --baselines-only`. This fast completeness check uses the same fixture discovery as the parity suite; it does not type-check or replace behavioral parity. Generate missing preflight/order baselines, review their diagnostics and module order, and run the affected `order-parity.test.ts` batches before pushing. Keep existing baseline entries unchanged unless the corresponding behavior intentionally changes. The full `pnpm test:ts7` command also checks completeness before starting its sweep. +A change that must not cost good code anything, such as a compatibility fix that replaces a refusal, proves it with the zero-cost gate. The gate compiles the benchmark workloads, the self-hosted compiler, and the TypeScript corpus with two built checkouts. It requires byte-identical IR, LLVM, and release executables, and compile time within tolerance. `--allow=` names the change's pattern signature: programs whose sources match may differ, and the PR lists them. Any other difference fails the gate, which names the first differing function. The gate also enforces the two-tier runtime separation: no executable may gain symbols from `DYNAMIC_TIER_PREFIXES` in `scripts/zero-cost-compare.mjs`, and fully typed benchmarks may link none; extend that list as dynamic-tier units land. `--quick` checks only the benchmarks; `--cachegrind` adds per-function instruction counts for executables that differ. Run it in a Sandbox with both checkouts prepared: + +```bash +node scripts/zero-cost-gate.mjs --baseline=/vercel/sandbox/prev --candidate=. --allow='\barguments\b' +``` + +The whole gate takes 30 to 75 minutes on one 8-vCPU Sandbox, so shard it across Sandboxes. `--shard=i/n` splits the bench and corpus programs; the self-hosted compiler runs in shard 1, or alone with `--set=self`, and its candidate-source build overlaps its comparison build on hosts with 40 GB or more. Each shard writes `--json`, and `node scripts/zero-cost-merge.mjs ...` merges them into one verdict, recomputing the compile-time intervals over every shard. `--record-baseline-cache=` builds only the baseline and records each program's artifact hashes; `--baseline-cache=` then skips the baseline build of every program whose candidate artifacts hash the same, which halves the work for a change that leaves most programs identical. A record applies only to the built baseline checkout, toolchain, and paths it was made with, and is ignored otherwise; a program whose entry, relative imports, or optimization level changed since the record builds its baseline. + Test location follows scope: - Co-locate white-box unit tests with implementation files under @@ -58,6 +72,16 @@ Test location follows scope: Keep existing tests in place unless a change already touches their organization; new tests should follow this convention. +## Numbered registries + +Parallel changes must never pick the same number by hand. Each registry has one source of truth and a fast check: + +- Diagnostic codes: claim a code by adding its row, in numeric order, to `DIAGNOSTIC_CODES` in `packages/compiler/src/diagnostics/codes.ts` before any site uses it. A shipped construct's code moves to `RETIRED_DIAGNOSTIC_CODES` and is never reused; codes that open work already uses are held in `RESERVED_DIAGNOSTIC_CODES`. `codes.test.ts` fails on a code without a row, a retired code in use, an unused row, or rows out of order, and two claims of one code are a merge conflict or a duplicate key. +- Dyn handle tags: add a new tag by name only to `SCR_DYNH_NEW_TAGS` in `packages/runtime/src/scr_dyn.h`, in name order; the enum numbers it. Pin a tag to an explicit value in `SCR_DYNH_TAGS` only when the LLVM backend needs its number. The header does not compile when two tags share a value, and `dyn-handle-tags.test.ts` names the problem. +- Runtime enums and flag bits the backend mirrors (`ScrDynKind`, `ScrMessageKind`, `ScrBytesElem`, `SCR_CLOSURE_KIND_*`, the `%ScrArr` spare flags) are checked against their C declarations by `packages/compiler/src/backend/llvm/runtime-registries.test.ts`. A new mirrored enum or flag set gets a check there, and C code names flag bits instead of writing their values. + +Before taking a number in a pinned registry, run `node scripts/registry-claims.mjs`: it lists the diagnostic codes and pinned handle tags that open pull requests claim and reports collisions; `--check` reports only collisions with the current branch and exits 1 when there is one. + ## Where things live - `packages/compiler` — the frontend (tsc API to IR), typed IR, validator, serializer, and LLVM backend. @@ -132,11 +156,11 @@ Never hand-edit these files: - `packages/compiler/surface-manifest.json` - `internal/compatibility/generated/node-v24-internal.json` -- `internal/compatibility/generated/node-v24-backlog.json` - `docs/src/generated/node-v24-compatibility.json` -- `docs/src/generated/node-v24-compatibility-meta.json` -Change compiler decision sources or the implementation-owned compatibility manifests, then regenerate. Use `pnpm manifest` when compiler decision tables change and `pnpm node-compat` whenever the Node pin, compatibility manifests, compiler surface manifest, changes. +Change compiler decision sources or the implementation-owned compatibility manifests, then regenerate. Use `pnpm manifest` when compiler decision tables change and `pnpm node-compat` whenever the Node pin, compatibility manifests, compiler surface manifest, changes. `pnpm node-compat --local` reclassifies the committed ledger without fetching Node's docs; it writes the same files as `pnpm node-compat` while the Node pin is unchanged. + +Never resolve a merge conflict in these files by hand. Resolve the source conflicts, then run `node scripts/regen-generated.mjs`: it takes the merged-in branch's version of each conflicted generated file, regenerates all of them from the merged sources, and stages the ones that conflicted. The public docs artifact must not expose repository test paths, internal source identifiers, engineering evidence bookkeeping, documentation/metadata headings, command-line or configuration-only entries, or rows that are N/A in the native tier. Those remain in the complete internal census under `internal/compatibility/generated/`. @@ -151,7 +175,7 @@ For each API family: 5. Add differential evidence that executes under Node and as a compiled binary. Cover stdout, stderr, exit status, success behavior, and important error shapes. A load-only export check does not prove behavior. 6. Update the implementation-owned source or manifest. Positive `supported` and `partial` mappings must name existing test evidence. 7. Run `pnpm manifest` if compiler surface tables changed, then run `pnpm node-compat`. Never patch generated JSON or headers. -8. Inspect the internal ledger, public artifact, and backlog diff. Confirm the intended API family changed and unrelated rows did not. +8. Inspect the internal ledger and public artifact diff, and compare `pnpm node-compat:backlog` output before and after. Confirm the intended API family changed and unrelated rows did not. 9. Run focused tests, `pnpm node-compat:check`, and the full validation gate required for the change. If docs output changed, also run the docs gate. Compatibility commands: @@ -189,6 +213,7 @@ The docs site is a standalone pnpm workspace under `docs/`. It uses Next.js App - A labeled bullet (`- **Crypto:** ...`) is `docs/content/limitations/
/