diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 677e51775..a402f912f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -101,8 +101,8 @@ jobs: if (receiptRun) appendFileSync(process.env.GITHUB_OUTPUT, `receipt_run=${receiptRun}\n`); NODE - # Collects the native compiler's profile and function layout on this commit - # (scripts/refresh-cli-profile.mjs, scripts/refresh-cli-layout.mjs), so every + # Collects the native compiler's function layout and profile on this commit + # (scripts/refresh-cli-layout.mjs, scripts/refresh-cli-profile.mjs), so every # platform's CLI build optimizes and lays out the code it compiles. The # committed ones fall behind main within a day or two. If this job fails, # the CLI builds use the committed profile and layout. @@ -111,7 +111,7 @@ jobs: needs: check-release if: needs.check-release.outputs.should_release == 'true' && needs.check-release.outputs.reuse_artifacts != 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 75 continue-on-error: true permissions: contents: read @@ -136,14 +136,20 @@ jobs: git clone --quiet --depth 1 https://github.com/vercel-labs/tsc-ts "$RUNNER_TEMP/tsc-ts" echo "tsc-ts $(git -C "$RUNNER_TEMP/tsc-ts" rev-parse HEAD)" npm ci --prefix "$RUNNER_TEMP/tsc-ts" + # The layout comes first: the profile matches the functions of builds + # with the layout its instrumented compiler was built with, so the + # profile is collected with a second instrumented compiler built with + # this layout. + - name: Collect the function layout + run: >- + node scripts/refresh-cli-layout.mjs --jobs 4 + --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" --work "$RUNNER_TEMP/cli-layout-work" - name: Collect the profile + env: + SCRIPTC_NATIVE_LAYOUT: ${{ runner.temp }}/cli-profile/cli.layout.gz run: >- node scripts/refresh-cli-profile.mjs --tsc-ts "$RUNNER_TEMP/tsc-ts" --jobs 4 --output "$RUNNER_TEMP/cli-profile/cli.profdata.gz" --work "$RUNNER_TEMP/cli-profile-work" - - name: Collect the function layout - run: >- - node scripts/refresh-cli-layout.mjs --jobs 4 --instrumented "$RUNNER_TEMP/cli-profile-work/cli" - --output "$RUNNER_TEMP/cli-profile/cli.layout.gz" - uses: actions/upload-artifact@v4 with: name: cli-profile diff --git a/RELEASING.md b/RELEASING.md index 4480e3fbc..a33da97f5 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -16,9 +16,9 @@ CI (`.github/workflows/release.yml`) compares the version in `packages/cli/packa The release job builds each pinned LLVM helper, precompiled runtime pack, and native CLI on its matching host. The native CLI package bundles its TypeScript checker, declarations, and comptime evaluator, and depends on its platform's LLVM helper and runtime pack packages at the same version; the compiler resolves them beside itself. The standalone archives bundle the helper and runtime pack too. npm postinstall installs the matching native executable; Node is needed for npm installation but not native compilation or execution. The user's toolchain performs the final platform link. Sanitized builds also use it to instrument program LLVM and compile the C runtime. Platform packages publish ahead of `@scriptc/compiler` and `scriptc`. The GitHub release attaches standalone compiler archives, their SHA-256 checksums, and the surface manifest after npm publication succeeds. -Release builds of the native CLI optimize the compiler with a profile collected on the release commit. The `cli-profile` job runs `node scripts/refresh-cli-profile.mjs` on Linux x64, which builds an instrumented compiler, runs it on tsc-ts, a slice of the corpus, and the runtime benchmark workloads, and writes the merged sparse profile. Every platform's CLI build uses that profile (`SCRIPTC_NATIVE_PROFILE_USE`), and the GitHub release attaches it as `scriptc--cli.profdata.gz`. Code the profile does not describe, or describes from older compiler sources, is optimized as without a profile. When the profile job fails, the CLI builds use the committed profile `packages/compiler/native/cli.profdata.gz`, which local and CI builds of the native compiler also use. Refresh the committed profile when compiler code has changed substantially since the last refresh: run `node scripts/refresh-cli-profile.mjs --tsc-ts ` on a quiet Linux x64 host and commit the new profile on its own. +Release builds of the native CLI optimize the compiler with a profile collected on the release commit. The `cli-profile` job runs `node scripts/refresh-cli-profile.mjs` on Linux x64, which builds an instrumented compiler, runs it on tsc-ts, a slice of the corpus, and the runtime benchmark workloads, and writes the merged profile. The profile keeps the functions that training never ran, with zero counts, so the CLI builds optimize them as cold code and place them after the code that ran. Every platform's CLI build uses that profile (`SCRIPTC_NATIVE_PROFILE_USE`), and the GitHub release attaches it as `scriptc--cli.profdata.gz`. Code the profile does not describe, or describes from older compiler sources, is optimized as without a profile. When the profile job fails, the CLI builds use the committed profile `packages/compiler/native/cli.profdata.gz`, which local and CI builds of the native compiler also use. Refresh the committed profile when compiler code has changed substantially since the last refresh: run `node scripts/refresh-cli-profile.mjs --tsc-ts ` on a quiet Linux x64 host and commit the new profile on its own. -Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`, reusing the job's instrumented compiler), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Refresh the committed layout the same way as the profile, with `node scripts/refresh-cli-layout.mjs`, and commit it on its own. +Builds of the native compiler also order its functions by the committed layout `packages/compiler/native/cli.layout.gz`: the functions that `--version` and rebuilds of unchanged programs run come first, then the ones other builds run, ordered by the share of the layout's training builds that run each, so a run maps fewer pages of the executable. Functions the layout does not list keep their usual order. Function names of closures and generated helpers change with nearby edits, so the `cli-profile` job also collects a layout on the release commit (`scripts/refresh-cli-layout.mjs`), and the CLI builds use it (`SCRIPTC_NATIVE_LAYOUT`). Instrumented builds use the same layout as optimized builds, because a function laid out in a different partition than in the instrumented build no longer matches its profile record. The job therefore collects the layout first and then builds the instrumented compiler for the profile with that layout. Refresh the committed layout the same way, with `node scripts/refresh-cli-layout.mjs`, before the profile, and commit each on its own. Publishing uses npm trusted publishing (OIDC) — there is no npm token secret. Each published package must have a GitHub Actions trusted publisher for diff --git a/native/llvm-codegen/src/emit.cpp b/native/llvm-codegen/src/emit.cpp index c7efef13f..a263ee601 100644 --- a/native/llvm-codegen/src/emit.cpp +++ b/native/llvm-codegen/src/emit.cpp @@ -212,8 +212,8 @@ struct UnnameStructTypes : PassInfoMixin { // How the --profile-use profile matched the functions it annotated, summed // over partitions: functions that received its counts, functions whose // control flow changed since it was collected (hash mismatch), and functions -// it has no record of (new, or never run in training: the merged profile is -// sparse). A partition reused from the cache without its counts is counted +// it has no record of (new since it was collected, or never run in training +// when the profile was merged with --sparse). A partition reused from the cache without its counts is counted // in Unknown. struct ProfileMatch { std::atomic Annotated{0}; diff --git a/packages/compiler/native/cli.layout.gz b/packages/compiler/native/cli.layout.gz index 4ede69568..84a2558b5 100644 Binary files a/packages/compiler/native/cli.layout.gz and b/packages/compiler/native/cli.layout.gz differ diff --git a/packages/compiler/native/cli.profdata.gz b/packages/compiler/native/cli.profdata.gz index aa9398b5f..4480e4ecb 100644 Binary files a/packages/compiler/native/cli.profdata.gz and b/packages/compiler/native/cli.profdata.gz differ diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 601be16c4..207fe7f0b 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -21,7 +21,7 @@ import { import { isDeclBoundaryFile } from "../decl-projection.js"; import { relieveHeapPressure } from "../../heap-pressure.js"; import { moduleNamespaceObject, moduleNamespaceValue } from "./module-namespace-elements.js"; -import { keyedHelperName } from "./stable-names.js"; +import { keyedHelperName, moduleTagNumber, portableRelativePath } from "./stable-names.js"; import { positionTag, StructuralPositions } from "./structural-positions.js"; import { bindingSymbolOfRead, onlyForwarded, stripTypeWrappers } from "./typed-arguments.js"; import { @@ -217,7 +217,7 @@ export { newFnCtx, type FnCtx } from "./function-context.js"; * - Lexical scoping is resolved here: locals get function-unique ids * ("x.0", "x.1" for shadowing); the IR is scope-flat. */ -import { dirname, relative, resolve } from "node:path"; +import { dirname, resolve } from "node:path"; import { bindingSource } from "../binding-source.js"; import { compilerOwnedFileName, tsgoPath } from "../dts-paths.js"; import * as ts from "../ts7/adapter.js"; @@ -3616,12 +3616,17 @@ export class Lowerer { * names do not depend on where the program sits, default library files * by their base name (two checkers install them in different * directories), and the files scriptc ships by their name inside scriptc, - * so names do not depend on where scriptc is installed either. */ - readonly positions = new StructuralPositions((sf) => - this.program.isSourceFileDefaultLibrary(sf) + * so names do not depend on where scriptc is installed either. Relative + * paths use `/` on every platform. */ + readonly positions = new StructuralPositions((sf) => this.stableFileName(sf)); + + /** The name of `sf` in positions and module tags. */ + stableFileName(sf: ts.SourceFile): string { + return this.program.isSourceFileDefaultLibrary(sf) ? `lib:${sf.fileName.slice(sf.fileName.lastIndexOf("/") + 1)}` - : (compilerOwnedFileName(sf.fileName) ?? relative(dirname(this.entry.fileName), sf.fileName)), - ); + : (compilerOwnedFileName(sf.fileName) ?? + portableRelativePath(dirname(this.entry.fileName), sf.fileName)); + } /** The outermost node whose token span is exactly `loc`, or null. */ nodeAtLoc(loc: SrcLoc): ts.Node | null { @@ -4320,9 +4325,20 @@ export class Lowerer { // moduleOrder BEFORE any pass constructed — lowerToIr runs // appendDynamicImportModules once on the shared array (a per-pass run // here would repeatedly extend the graph and duplicate cycle reports). - this.moduleOrder.forEach((sf, i) => { - this.fileTag.set(sf, sf === entry ? "" : `%m${i}.`); - }); + // A module's tag hashes its stable file name rather than numbering its + // place in moduleOrder, so adding or reordering modules leaves the other + // modules' function names, and the closure names keyed by them, + // unchanged. The native compiler's profile matches functions by name. + const tagNumbers = new Set(); + for (const sf of this.moduleOrder) { + if (sf === entry) { + this.fileTag.set(sf, ""); + continue; + } + const n = moduleTagNumber(this.stableFileName(sf), tagNumbers); + tagNumbers.add(n); + this.fileTag.set(sf, `%m${n}.`); + } if (this.moduleOrder.length === 0) this.fileTag.set(entry, ""); const jsDynamic = jsDynamicDeclarations(this.moduleOrder, this.checker); if (jsDynamic.size > 0) this.typeCtx.jsDynamicDeclaration = (decl) => jsDynamic.has(decl); diff --git a/packages/compiler/src/frontend/lowering/stable-names.test.ts b/packages/compiler/src/frontend/lowering/stable-names.test.ts new file mode 100644 index 000000000..dbef9b29e --- /dev/null +++ b/packages/compiler/src/frontend/lowering/stable-names.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from "vitest"; +import { moduleTagNumber, portableRelativePath } from "./stable-names.js"; + +test("module tag numbers depend only on the file name and stay distinct", () => { + const first = moduleTagNumber("lib/a.ts", new Set()); + expect(moduleTagNumber("lib/a.ts", new Set([7, 8]))).toBe(first); + expect(moduleTagNumber("lib/b.ts", new Set())).not.toBe(first); + expect(moduleTagNumber("lib/a.ts", new Set([first]))).toBe((first + 1) >>> 0); + expect(moduleTagNumber("lib/a.ts", new Set([first, (first + 1) >>> 0]))).toBe((first + 2) >>> 0); +}); + +test("relative module paths use / on every platform", () => { + expect(portableRelativePath("/repo/src", "/repo/src/lib/a.ts", "linux")).toBe("lib/a.ts"); + expect(portableRelativePath("C:/repo/src", "C:/repo/src/lib/a.ts", "win32")).toBe("lib/a.ts"); + expect(portableRelativePath("C:\\repo\\src", "C:/repo/node_modules/x/index.d.ts", "win32")).toBe( + "../node_modules/x/index.d.ts", + ); + expect( + moduleTagNumber(portableRelativePath("C:/repo", "C:/repo/lib/a.ts", "win32"), new Set()), + ).toBe(moduleTagNumber(portableRelativePath("/repo", "/repo/lib/a.ts", "linux"), new Set())); +}); diff --git a/packages/compiler/src/frontend/lowering/stable-names.ts b/packages/compiler/src/frontend/lowering/stable-names.ts index 8f7d0967e..0d57bd2c9 100644 --- a/packages/compiler/src/frontend/lowering/stable-names.ts +++ b/packages/compiler/src/frontend/lowering/stable-names.ts @@ -1,4 +1,7 @@ +import * as posix from "node:path/posix"; +import * as win32 from "node:path/win32"; import { InternalCompilerError } from "../../errors.js"; +import { tsgoPath } from "../ts7/session-path.js"; /** The owner of the names keyedHelperName hands out. */ export interface KeyedNameOwner { @@ -46,3 +49,26 @@ export function keyedHelperName( throw new InternalCompilerError(`helper name ${name} is shared by two helper keys`); return name; } + +/** The number in the `%m.` tag of the module named `fileName`: a + * 32-bit FNV-1a hash of the name, so a module keeps its tag when other + * modules are added, removed or reordered. A number in `taken` moves to the + * next free one, which keeps tags distinct. */ +export function moduleTagNumber(fileName: string, taken: ReadonlySet): number { + let h = 0x811c9dc5; + for (let i = 0; i < fileName.length; i++) h = Math.imul(h ^ fileName.charCodeAt(i), 0x01000193); + let n = h >>> 0; + while (taken.has(n)) n = (n + 1) >>> 0; + return n; +} + +/** The path of `file` relative to the directory `from`, with `/` separators + * on every platform, so names built from it match across hosts. */ +export function portableRelativePath( + from: string, + file: string, + platform: NodeJS.Platform = process.platform, +): string { + const path = platform === "win32" ? win32.relative(from, file) : posix.relative(from, file); + return tsgoPath(path, platform); +} diff --git a/packages/compiler/test/stable-names.test.ts b/packages/compiler/test/stable-names.test.ts index c82db039d..13ed000df 100644 --- a/packages/compiler/test/stable-names.test.ts +++ b/packages/compiler/test/stable-names.test.ts @@ -117,3 +117,44 @@ test("names do not depend on the directory the program sits in", async () => { const [second] = await lower(recursive); expect(names(second!)).toEqual(names(first!)); }); + +/** Lowers a program of several files; `main.ts` is the entry. */ +async function lowerFiles(files: Record): Promise { + const dir = await mkdtemp(join(tmpdir(), "scriptc-stable-names-")); + try { + for (const [name, source] of Object.entries(files)) await writeFile(join(dir, name), source); + const outPath = join(dir, "main.ir.json"); + const result = await compile(join(dir, "main.ts"), { outDir: dir, outPath, outputKind: "ir" }); + if (!result.ok) + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + return deserializeModule(await readFile(outPath, "utf8")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +test("a module's names do not depend on the modules loaded before it", async () => { + const shapes = ` +export class Box { + constructor(private value: number) {} + scaled(by: number): number { return this.value * by; } +} +export function boxes(count: number): number[] { + return Array.from({ length: count }, (_, i) => new Box(i).scaled(2)); +} +`; + const before = await lowerFiles({ + "shapes.ts": shapes, + "main.ts": `import { boxes } from "./shapes";\nconsole.log(boxes(3).join(","));\n`, + }); + const after = await lowerFiles({ + "offsets.ts": `export function offset(x: number): number { return x + 1; }\n`, + "shapes.ts": shapes, + "main.ts": `import { offset } from "./offsets";\nimport { boxes } from "./shapes";\nconsole.log(offset(1), boxes(3).join(","));\n`, + }); + // The class, its methods, the function and the closure it lifts. + const ofShapes = (mod: IrModule) => + names(mod).filter((name) => /Box|boxes/.test(name) || name.startsWith("%fn.")); + expect(ofShapes(before).length).toBeGreaterThanOrEqual(4); + expect(ofShapes(after)).toEqual(ofShapes(before)); +}); diff --git a/scripts/build-native-cli.mts b/scripts/build-native-cli.mts index 3dbe22bf3..22b9967ba 100644 --- a/scripts/build-native-cli.mts +++ b/scripts/build-native-cli.mts @@ -31,10 +31,10 @@ const sanitize = process.env["SCRIPTC_SAN"] === "1"; // SCRIPTC_NATIVE_PROFILE_USE builds without one. const profileGenerate = process.env["SCRIPTC_NATIVE_PROFILE_GENERATE"] || undefined; // Below this share of matching functions, the build warns that the profile -// has lost most of its gain. A profile refreshed on the same commit matched -// 92.7% of the native compiler's profiled functions, one refreshed 1.5 days -// and about 200 commits earlier 87.2%, and that one kept about half of the -// fresh profile's gain on a cold tsc-ts build. +// has lost most of its gain. A profile refreshed on the same commit, with +// the same layout, matches all of the native compiler's profiled functions. +// One refreshed 1.5 days and about 200 commits earlier matched 87.2%, and +// kept about half of the fresh profile's gain on a cold tsc-ts build. const PROFILE_MATCH_WARNING = 0.9; const committedProfile = join(root, "packages/compiler/native/cli.profdata.gz"); const requestedProfile = process.env["SCRIPTC_NATIVE_PROFILE_USE"]; @@ -46,11 +46,15 @@ const useProfile = (requestedProfile !== undefined || existsSync(committedProfile)); // Builds lay out the compiler's functions with the committed layout // (scripts/refresh-cli-layout.mjs), or the one SCRIPTC_NATIVE_LAYOUT names; -// an empty SCRIPTC_NATIVE_LAYOUT, and instrumented builds, use none. +// an empty SCRIPTC_NATIVE_LAYOUT uses none. Instrumented builds use the same +// layout as the builds their profile is for. The layout decides which +// functions share a partition, and a function whose partition differs from +// the instrumented build's is optimized differently before its control flow +// is hashed, so the profile's record for it no longer matches. const committedLayout = join(root, "packages/compiler/native/cli.layout.gz"); const requestedLayout = process.env["SCRIPTC_NATIVE_LAYOUT"]; const functionLayout = - profileGenerate !== undefined || requestedLayout === "" + requestedLayout === "" ? undefined : (requestedLayout ?? (existsSync(committedLayout) ? committedLayout : undefined)); let target = nativeCodegenTarget(); diff --git a/scripts/refresh-cli-profile.mjs b/scripts/refresh-cli-profile.mjs index dde4e21b9..a9436b505 100644 --- a/scripts/refresh-cli-profile.mjs +++ b/scripts/refresh-cli-profile.mjs @@ -6,7 +6,7 @@ * * Builds an instrumented native compiler for this Linux host, runs it on the * training builds below, and merges the raw profiles it writes into one - * sparse indexed profile. The training builds are tsc-ts (a cold `speed` + * indexed profile. The training builds are tsc-ts (a cold `speed` * build, then a rebuild after an edit), every seventh top-level corpus * program (by number, or by a hash of the name of a program without one) at * the default optimization and a further slice in `dev`, and the runtime @@ -14,6 +14,11 @@ * move much compiler code; a stale profile only loses part of its gain. A * refresh takes about 11 minutes on 16 cores. * + * The instrumented compiler is built with the committed function layout, or + * the one SCRIPTC_NATIVE_LAYOUT names, and the profile matches builds that + * use the same layout. When refreshing both, refresh the layout first + * (scripts/refresh-cli-layout.mjs). + * * node scripts/refresh-cli-profile.mjs --tsc-ts [--jobs 8] */ import { spawn, spawnSync } from "node:child_process"; @@ -210,12 +215,15 @@ const profiles = readdirSync(raw) .map((name) => join(raw, name)); rmSync(builds, { recursive: true, force: true }); const indexed = join(work, "cli.profdata"); -const merged = spawnSync(scriptc, ["merge-profiles", "--sparse", "-o", indexed, ...profiles], { +// The profile keeps the functions that never ran in training, with zero +// counts. Builds optimize those as cold and place them after the trained code, +// which keeps the compiler's .text smaller than a sparse profile does. +const merged = spawnSync(scriptc, ["merge-profiles", "-o", indexed, ...profiles], { stdio: "inherit", env: { ...trainingEnv, LLVM_PROFILE_FILE: join(work, "merge.profraw") }, }); if (merged.status !== 0) throw new Error("merge-profiles failed"); -// The indexed profile compresses to about an eighth of its size. +// The indexed profile compresses to about a sixth of its size. writeFileSync( profileOutput, gzipSync(readFileSync(indexed), { level: constants.Z_BEST_COMPRESSION }),