trending/scriptc
1
0
Fork 0
mirror of https://github.com/vercel-labs/scriptc.git synced 2026-10-31 07:23:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge perf/shape-overlay-cost

Browse Source
This commit is contained in:
Chris Tate
2026-10-26 11:53:34 -05:00
parent 30668f19ba d163df4674
commit 06f7b8631f
1145 changed files with 73292 additions and 17811 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+35
View File
@@ -130,6 +130,8 @@ jobs:
SCRIPTC_TEST_WORKERS: "1"
- name: Verify mixed-source generated AST getters
run: pnpm generate:ts7-ast --check
- name: Verify the generated IR field readers
run: pnpm generate:ir-fields --check
native_macos:
needs: [validation_scope, lanes]
@@ -354,6 +356,37 @@ jobs:
- run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
- run: pnpm test tests/harness/effect.test.ts tests/harness/effect4.test.ts
# The sanitized three.js nodes build alone: beside the san corpus's other
# workers it took 860 to more than 1200 seconds of its shard
# (SCRIPTC_THREE_SANITIZED_NODES_JOB in scripts/ci-test-plan.mjs). It keeps
# the corpus's LeakSanitizer settings.
three_nodes_san:
needs: [validation_scope, lanes, native_linux]
if: ${{ needs.validation_scope.outputs.full == 'true' && needs.lanes.outputs.linux == 'true' }}
name: test (Linux, san, three.js nodes)
runs-on: ubuntu-24.04
timeout-minutes: 40
env:
SCRIPTC_SAN: "1"
ASAN_OPTIONS: detect_leaks=1
LSAN_OPTIONS: suppressions=${{ github.workspace }}/tests/harness/lsan.supp:print_suppressions=0
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- run: pnpm install --frozen-lockfile
- uses: actions/download-artifact@v4
with:
name: native-linux-x64-gnu
path: ${{ runner.temp }}
- run: tar -xzf "$RUNNER_TEMP/native-linux-x64-gnu.tar.gz"
- run: pnpm test tests/harness/three.test.ts -t "'nodes' runs statically through LLVM on 'native'"
# Shape inference's overlay (opt-in, SCRIPTC_SHAPE_OVERLAY=1) over the
# suites of real packages: nightly and on manual runs only (scope
# shape_overlay runs it alone), outside the `test` roll-up. Every case
@@ -1033,6 +1066,7 @@ jobs:
tests_macos,
native_linux,
effects,
three_nodes_san,
bootstrap_seed,
bootstrap,
linux_host_clang,
@@ -1065,6 +1099,7 @@ jobs:
lane tests_macos "${{ needs.tests_macos.result }}" "${{ needs.lanes.outputs.macos }}"
lane native_linux "${{ needs.native_linux.result }}" "${{ needs.lanes.outputs.linux }}"
lane effects "${{ needs.effects.result }}" "${{ needs.lanes.outputs.effects }}"
lane three_nodes_san "${{ needs.three_nodes_san.result }}" "${{ needs.lanes.outputs.linux }}"
lane bootstrap_seed "${{ needs.bootstrap_seed.result }}" "${{ needs.lanes.outputs.bootstrap }}"
lane bootstrap "${{ needs.bootstrap.result }}" "${{ needs.lanes.outputs.bootstrap }}"
lane linux_host_clang "${{ needs.linux_host_clang.result }}" "${{ needs.lanes.outputs.linux }}"
.github/workflows/llvm-dist.yml
+93
View File
@@ -0,0 +1,93 @@
name: LLVM distribution
# Builds the LLVM that the Linux release helpers link
# (native/llvm-codegen/llvm-dist.mjs): LLVM from its release sources with
# ThinLTO and native/llvm-codegen/llvm.profdata.gz, for each Linux
# architecture. Each job builds a helper against it and one against LLVM's
# release archive, requires the two to give identical objects on this
# checkout's helper inputs (scripts/check-llvm-dist.mjs), prints both
# helpers' times, and attaches the archive to the LLVM_DIST_TAG release.
# The arm64 build uses the profile collected on x64; its job log reports how
# many functions the profile no longer matches. Run it after a profile refresh
# or an LLVM version change, with a new LLVM_DIST_TAG.
on:
workflow_dispatch:
permissions:
contents: read
concurrency: ${{ github.workflow }}
jobs:
build:
name: Build LLVM (${{ matrix.arch }})
if: github.repository == 'vercel-labs/scriptc'
strategy:
fail-fast: false
matrix:
include:
- arch: x64
runner: ubuntu-24.04
llvm_asset: LLVM-22.1.8-Linux-X64
- arch: arm64
runner: ubuntu-24.04-arm
llvm_asset: LLVM-22.1.8-Linux-ARM64
runs-on: ${{ matrix.runner }}
# About an hour for LLVM on 4 cores, and 5 minutes for each helper link.
timeout-minutes: 240
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version: "24"
cache: pnpm
- name: Install LLVM's release archive and sources
run: |
sudo apt-get update
sudo apt-get install --yes ninja-build zlib1g-dev libzstd-dev
base=https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8
curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" "$base/${{ matrix.llvm_asset }}.tar.xz"
tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP"
mv "$RUNNER_TEMP/${{ matrix.llvm_asset }}" "$RUNNER_TEMP/llvm-22.1.8"
curl -fL --retry 3 -o "$RUNNER_TEMP/llvm-src.tar.xz" "$base/llvm-project-22.1.8.src.tar.xz"
tar -xJf "$RUNNER_TEMP/llvm-src.tar.xz" -C "$RUNNER_TEMP"
rm "$RUNNER_TEMP/llvm.tar.xz" "$RUNNER_TEMP/llvm-src.tar.xz"
echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH"
- run: pnpm install --frozen-lockfile
- run: pnpm --filter @scriptc/compiler --filter scriptc build
- name: Build the distribution
run: >-
node scripts/build-llvm-dist.mjs --source "$RUNNER_TEMP/llvm-project-22.1.8.src"
--prefix "$RUNNER_TEMP/dist/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}"
--archive "$RUNNER_TEMP/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}.tar.xz"
--work "$RUNNER_TEMP/llvm-dist-work"
- name: Build a helper against each LLVM
env:
HELPER: llvm-linux-${{ matrix.arch }}-gnu
run: |
LLVM_DIR="$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" pnpm --filter "@scriptc/$HELPER" build:native
cp "packages/$HELPER/bin/scriptc-llvm-codegen" "$RUNNER_TEMP/helper-release-archive"
LLVM_DIR="$RUNNER_TEMP/dist/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}/lib/cmake/llvm" \
SCRIPTC_LLVM_THINLTO=1 pnpm --filter "@scriptc/$HELPER" build:native
cp "packages/$HELPER/bin/scriptc-llvm-codegen" "$RUNNER_TEMP/helper-profiled"
- name: Compare the helpers
run: >-
node scripts/check-llvm-dist.mjs --baseline "$RUNNER_TEMP/helper-release-archive"
--candidate "$RUNNER_TEMP/helper-profiled" --rounds 3 --work "$RUNNER_TEMP/llvm-check"
- name: Attach the archive to the release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARCHIVE: ${{ runner.temp }}/scriptc-llvm-22.1.8-linux-${{ matrix.arch }}.tar.xz
run: |
tag=$(node --input-type=module -e "import { LLVM_DIST_TAG } from './native/llvm-codegen/llvm-dist.mjs'; console.log(LLVM_DIST_TAG)")
# Both architectures' jobs create the release if it is missing.
gh release view "$tag" > /dev/null 2>&1 ||
gh release create "$tag" --target "$GITHUB_SHA" --title "$tag" --latest=false \
--notes "LLVM 22.1.8 built with ThinLTO and the helper profile, for the Linux LLVM helpers (scripts/build-llvm-dist.mjs)." ||
gh release view "$tag" > /dev/null
gh release upload "$tag" "$ARCHIVE" "$ARCHIVE.sha256" --clobber
.github/workflows/release.yml
+117 -17
View File
@@ -83,6 +83,7 @@ jobs:
spawnSync('git', ['diff', '--quiet', run.head_sha, 'HEAD', '--', '.',
':(exclude).github/workflows/release.yml', ':(exclude)scripts/verify-native-cli.mjs',
':(exclude)scripts/package-native-cli.mjs', ':(exclude)tests/harness/native-cli-packaging.test.ts',
':(exclude)scripts/wait-npm-available.mjs', ':(exclude)tests/harness/npm-available.test.ts',
]).status !== 0) continue;
const artifacts = api(`repos/${repo}/actions/runs/${run.id}/artifacts?per_page=100`).artifacts
.filter((artifact) => !artifact.expired && artifact.size_in_bytes > 0);
@@ -100,11 +101,11 @@ jobs:
if (receiptRun) appendFileSync(process.env.GITHUB_OUTPUT, `receipt_run=${receiptRun}\n`);
NODE
# Collects the native compiler's profile on this commit
# (scripts/refresh-cli-profile.mjs), so every platform's CLI build optimizes
# with a profile of the code it compiles. The committed profile falls behind
# main within a day or two. If this job fails, the CLI builds use the
# committed profile.
# Collects the native compiler's profile and function layout on this commit
# (scripts/refresh-cli-profile.mjs, scripts/refresh-cli-layout.mjs), so every
# platform's CLI build optimizes and lays out the code it compiles. The
# committed ones fall behind main within a day or two. If this job fails,
# the CLI builds use the committed profile and layout.
cli-profile:
name: Collect the native compiler profile
needs: check-release
@@ -138,11 +139,17 @@ jobs:
- name: Collect the profile
run: >-
node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4
--output "$RUNNER_TEMP/cli-profile/cli.profdata.gz"
--output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work"
- name: Collect the function layout
run: >-
node scripts/refresh-cli-layout.mjs --jobs 4 --instrumented "$RUNNER_TEMP/cli-profile-work/cli"
--output "$RUNNER_TEMP/cli-profile/cli.layout.gz"
- uses: actions/upload-artifact@v4
with:
name: cli-profile
path: ${{ runner.temp }}/cli-profile/cli.profdata.gz
path: |
${{ runner.temp }}/cli-profile/cli.profdata.gz
${{ runner.temp }}/cli-profile/cli.layout.gz
retention-days: 7
if-no-files-found: error
@@ -173,6 +180,7 @@ jobs:
runtime: runtime-linux-x64-gnu
cli: cli-linux-x64-gnu
llvm_asset: LLVM-22.1.8-Linux-X64
llvm_dist: x64
use_zig: true
- platform: linux-arm64
runner: ubuntu-24.04-arm
@@ -180,6 +188,7 @@ jobs:
runtime: runtime-linux-arm64-gnu
cli: cli-linux-arm64-gnu
llvm_asset: LLVM-22.1.8-Linux-ARM64
llvm_dist: arm64
use_zig: true
- platform: windows-x64
runner: windows-2022
@@ -195,6 +204,7 @@ jobs:
seed_helper: llvm-linux-x64-gnu
cli_target: x86_64-linux-musl
llvm_asset: LLVM-22.1.8-Linux-X64
llvm_dist: x64
use_zig: true
- platform: linux-arm64-musl
runner: ubuntu-24.04-arm
@@ -204,12 +214,14 @@ jobs:
seed_helper: llvm-linux-arm64-gnu
cli_target: aarch64-linux-musl
llvm_asset: LLVM-22.1.8-Linux-ARM64
llvm_dist: arm64
use_zig: true
- platform: wasm32-wasi
runner: ubuntu-24.04
helper: llvm-linux-x64-gnu
runtime: runtime-wasm32-wasi
llvm_asset: LLVM-22.1.8-Linux-X64
llvm_dist: x64
use_zig: true
runs-on: ${{ matrix.runner }}
# The Intel macOS build took 24 minutes for 0.3.2 and passed 30 for 0.3.3.
@@ -240,6 +252,28 @@ jobs:
mv "$RUNNER_TEMP/${{ matrix.llvm_asset }}" "$RUNNER_TEMP/llvm-22.1.8"
echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV"
echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH"
# The Linux helpers link LLVM built with ThinLTO and a profile of the
# helper's code generation (native/llvm-codegen/llvm-dist.mjs), from a
# release of this repository. Without that archive they link LLVM's
# release archive, which gives the same objects more slowly. The
# wasm32-wasi job also uploads the x64 GNU helper, so it selects the same.
- name: Select the profiled LLVM distribution (Linux)
if: matrix.llvm_dist != ''
env:
ARCH: ${{ matrix.llvm_dist }}
run: |
tag=$(node --input-type=module -e "import { LLVM_DIST_TAG } from './native/llvm-codegen/llvm-dist.mjs'; console.log(LLVM_DIST_TAG)")
asset=$(node --input-type=module -e "import { llvmDistAsset } from './native/llvm-codegen/llvm-dist.mjs'; console.log(llvmDistAsset(process.env.ARCH))")
base="https://github.com/$GITHUB_REPOSITORY/releases/download/$tag"
if curl -fsSL --retry 3 -o "$RUNNER_TEMP/$asset" "$base/$asset" &&
curl -fsSL --retry 3 -o "$RUNNER_TEMP/$asset.sha256" "$base/$asset.sha256" &&
(cd "$RUNNER_TEMP" && sha256sum --check "$asset.sha256"); then
tar -xJf "$RUNNER_TEMP/$asset" -C "$RUNNER_TEMP"
echo "SCRIPTC_HELPER_LLVM_DIR=$RUNNER_TEMP/${asset%.tar.xz}/lib/cmake/llvm" >> "$GITHUB_ENV"
echo "The helper links $asset from release $tag"
else
echo "::warning::$asset is not available from release $tag; the helper links LLVM's release archive"
fi
- uses: vercel-labs/setup-zig@v1
if: matrix.use_zig == true
with:
@@ -278,7 +312,15 @@ jobs:
- name: Build native packages (POSIX)
if: matrix.platform != 'windows-x64'
run: |
pnpm --filter @scriptc/${{ matrix.helper }} build:native
if [ -n "${SCRIPTC_HELPER_LLVM_DIR:-}" ]; then
LLVM_DIR="$SCRIPTC_HELPER_LLVM_DIR" SCRIPTC_LLVM_THINLTO=1 pnpm --filter @scriptc/${{ matrix.helper }} build:native || {
echo "::warning::the helper did not build against the profiled LLVM; it links LLVM's release archive"
rm -rf node_modules/.cache/scriptc-${{ matrix.helper }}
pnpm --filter @scriptc/${{ matrix.helper }} build:native
}
else
pnpm --filter @scriptc/${{ matrix.helper }} build:native
fi
if [ "${{ matrix.use_zig }}" = true ]; then
CC=zig AR=zig pnpm --filter @scriptc/${{ matrix.runtime }} build:native
else
@@ -300,8 +342,8 @@ jobs:
with:
name: cli-profile
path: ${{ runner.temp }}/cli-profile
# Without the artifact, SCRIPTC_NATIVE_PROFILE_USE stays unset and
# build-native-cli.mts uses the committed profile.
# Without the artifact, SCRIPTC_NATIVE_PROFILE_USE and SCRIPTC_NATIVE_LAYOUT
# stay unset and build-native-cli.mts uses the committed profile and layout.
- name: Select the native compiler profile
if: matrix.cli != ''
shell: bash
@@ -311,6 +353,11 @@ jobs:
if [ -n "$COLLECTED" ]; then
echo "SCRIPTC_NATIVE_PROFILE_USE=$COLLECTED" >> "$GITHUB_ENV"
echo "The native CLI builds with the profile collected on $GITHUB_SHA"
layout="$(dirname "$COLLECTED")/cli.layout.gz"
if [ -f "$layout" ]; then
echo "SCRIPTC_NATIVE_LAYOUT=$layout" >> "$GITHUB_ENV"
echo "The native CLI builds with the function layout collected on $GITHUB_SHA"
fi
else
echo "::warning::No profile was collected on this commit; the native CLI builds with the committed profile"
fi
@@ -394,6 +441,8 @@ jobs:
contents: read
actions: read
id-token: write
outputs:
uploads: ${{ steps.publish.outputs.uploads }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
@@ -413,7 +462,7 @@ jobs:
VERSION: ${{ needs.check-release.outputs.version }}
RECEIPT_RUN: ${{ needs.check-release.outputs.receipt_run }}
run: |
touch "$RUNNER_TEMP/npm-accepted.txt"
touch "$RUNNER_TEMP/npm-accepted.txt" "$RUNNER_TEMP/npm-accepted-times.txt"
# Include this run so re-running only failed jobs restores its receipts.
for run in "$GITHUB_RUN_ID" "$RECEIPT_RUN"; do
if [ -z "$run" ]; then continue; fi
@@ -422,6 +471,7 @@ jobs:
if [ "$count" -eq 0 ]; then continue; fi
gh run download "$run" --repo "$GITHUB_REPOSITORY" --pattern "npm-accepted-$VERSION-*" --dir "$RUNNER_TEMP/npm-receipts/$run"
find "$RUNNER_TEMP/npm-receipts/$run" -name npm-accepted.txt -exec cat {} + >> "$RUNNER_TEMP/npm-accepted.txt"
find "$RUNNER_TEMP/npm-receipts/$run" -name npm-accepted-times.txt -exec cat {} + >> "$RUNNER_TEMP/npm-accepted-times.txt"
done
- name: Setup pnpm
@@ -471,6 +521,7 @@ jobs:
done
- name: Publish to npm
id: publish
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -490,10 +541,14 @@ jobs:
# Upload acceptance and public availability are separate states.
# Save acceptance immediately so retries never depend on registry lag.
# npm-release.txt lists every package of the release, and
# npm-accepted-times.txt the time npm accepted each upload, for the
# npm-available job.
publish_dir() {
dir="$1"
packed="${2:-}"
name=$(node -p "require('./$dir/package.json').name")
echo "$name@$VERSION" >> "$RUNNER_TEMP/npm-release.txt"
if grep -Fxq "$name@$VERSION" "$RUNNER_TEMP/npm-accepted.txt"; then
echo "$name@$VERSION upload already accepted, skipping"
return 0
@@ -510,6 +565,7 @@ jobs:
publish_log="$RUNNER_TEMP/npm-publish.log"
if npm publish "$packed" $PROVENANCE --access public 2>&1 | tee "$publish_log"; then
echo "$name@$VERSION" >> "$RUNNER_TEMP/npm-accepted.txt"
echo "$name@$VERSION $(date +%s)" >> "$RUNNER_TEMP/npm-accepted-times.txt"
return 0
else
publish_status=${PIPESTATUS[0]}
@@ -533,7 +589,8 @@ jobs:
# pnpm pack resolves workspace dependencies; npm handles OIDC.
# Publish dependencies first. Registry propagation after an accepted
# upload must not block the remaining packages or the GitHub release.
# upload must not block the remaining packages; the npm-available
# job waits for it.
publish_dir packages/runtime
publish_dir packages/runtime-darwin-arm64
publish_dir packages/llvm-darwin-arm64
@@ -561,23 +618,66 @@ jobs:
publish_dir packages/cli
echo "All package uploads accepted; npm registry propagation may continue after this job." >> "$GITHUB_STEP_SUMMARY"
# Each package of the release with the time its upload was accepted,
# when a run recorded it.
{
echo "uploads<<NPM_UPLOADS"
awk 'FILENAME == ARGV[1] { accepted[$1] = $2; next } { print $1, accepted[$1] }' \
"$RUNNER_TEMP/npm-accepted-times.txt" "$RUNNER_TEMP/npm-release.txt"
echo "NPM_UPLOADS"
} >> "$GITHUB_OUTPUT"
- name: Save accepted upload receipts
if: always()
uses: actions/upload-artifact@v4
with:
name: npm-accepted-${{ needs.check-release.outputs.version }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/npm-accepted.txt
path: |
${{ runner.temp }}/npm-accepted.txt
${{ runner.temp }}/npm-accepted-times.txt
if-no-files-found: ignore
retention-days: 90
# Publish standalone native distributions and the surface manifest after
# npm publishing succeeds. The release body comes from the marked changelog.
github-release:
name: Create GitHub Release
# Waits until every package of the release resolves on the npm registry,
# and reports each package's time from upload to available. Fails with the
# packages that are still missing after 2 hours. It only reads from the
# registry; the next release run retries it, and publishing skips uploads
# npm already accepted.
npm-available:
name: Wait for npm availability
needs: [check-release, publish]
if: >-
!cancelled() && needs.check-release.outputs.should_release == 'true' && needs.publish.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 135
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "24"
registry-url: "https://registry.npmjs.org"
- name: Wait for every package to resolve
env:
UPLOADS: ${{ needs.publish.outputs.uploads }}
run: |
printf '%s\n' "$UPLOADS" > "$RUNNER_TEMP/npm-uploads.txt"
node scripts/wait-npm-available.mjs "$RUNNER_TEMP/npm-uploads.txt"
# Publish standalone native distributions and the surface manifest after
# every npm package resolves. The release body comes from the marked
# changelog.
github-release:
name: Create GitHub Release
needs: [check-release, publish, npm-available]
if: >-
!cancelled() && needs.check-release.outputs.should_release == 'true' && needs.npm-available.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: write