From 94173bebdc4480d661d5316e3ac96a6eb8374a33 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Thu, 22 Oct 2026 19:12:16 -0700 Subject: [PATCH] Name the compiler's own shim files independently of the install path Programs that compile against a package shim (fs-extra, chalk, undici, hot-shots) produced IR and LLVM that depended on where the compiler was installed: lifted helper and closure names hashed the shim file's path relative to the entry, the shared-helper location pass ordered candidate sites by absolute path, and IR artifacts spelled every shim location as an absolute path. Two checkouts of the same sources therefore emitted different artifacts, and the zero-cost gate always reported fs-extra-package as differing. Files scriptc ships are now named by their place inside scriptc (scriptc:shims/...) in structural positions, in the shared-helper ordering, and in serialized IR artifacts. In-memory locations keep the real path, so diagnostics and source lookups are unchanged. Co-Authored-By: Claude Opus 5.5 --- packages/compiler/src/frontend/dts-paths.ts | 25 +++++++++++++ .../compiler/src/frontend/lowering/lowerer.ts | 30 +++++++++++----- packages/compiler/src/index.ts | 9 ++--- packages/compiler/src/ir/serialize.test.ts | 21 +++++++++++ packages/compiler/src/ir/serialize.ts | 36 +++++++++++++++++-- packages/compiler/src/native/prepare.ts | 7 ++-- 6 files changed, 111 insertions(+), 17 deletions(-) diff --git a/packages/compiler/src/frontend/dts-paths.ts b/packages/compiler/src/frontend/dts-paths.ts index 3b2f32413..009d1c333 100644 --- a/packages/compiler/src/frontend/dts-paths.ts +++ b/packages/compiler/src/frontend/dts-paths.ts @@ -34,6 +34,31 @@ export function packageShimRoot(): string { : join(installedDeclarationRoot, "shims"); } +/** The directories of files scriptc itself ships, each with the prefix + * that names its files independently of where scriptc is installed + * (compilerOwnedFileName). IR artifacts spell source locations in them this + * way (serializeModuleParts). */ +export function compilerOwnedFileRoots(): readonly { from: string; to: string }[] { + return [{ from: tsgoPath(packageShimRoot()) + "/", to: "scriptc:shims/" }]; +} + +/** The name of a file scriptc itself ships (a package shim or one of the + * shipped declaration files) that does not depend on where scriptc is + * installed: `scriptc:shims/` or + * `scriptc:`. Null for every other file. Names and + * artifacts that would otherwise spell the installation's absolute path use + * it, so two installations of one compiler emit the same output. */ +export function compilerOwnedFileName(file: string): string | null { + const path = tsgoPath(file); + for (const { from, to } of compilerOwnedFileRoots()) { + if (path.startsWith(from)) return to + path.slice(from.length); + } + for (const declaration of [ambientDtsPath(), overridesDtsPath(), fallbackDtsPath()]) { + if (path === declaration) return `scriptc:${path.slice(path.lastIndexOf("/") + 1)}`; + } + return null; +} + /** Path of the shipped ambient declarations — the always-shipped CORE * (comptime, setTimeout). Part of EVERY program scriptc * builds, the project-world preflight program included. */ diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index 3ba81dd6b..60eba3107 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -184,7 +184,7 @@ export { newFnCtx, type FnCtx } from "./function-context.js"; */ import { dirname, relative, resolve } from "node:path"; import { bindingSource } from "../binding-source.js"; -import { tsgoPath } from "../dts-paths.js"; +import { compilerOwnedFileName, tsgoPath } from "../dts-paths.js"; import * as ts from "../ts7/adapter.js"; import { isBundledShimForwarder } from "../package-shims.js"; import { erasedTypeOnlyImport } from "../type-only-imports.js"; @@ -3200,13 +3200,14 @@ export class Lowerer { /** Structural source positions (structural-positions.ts) for names that * stay the same when unrelated code moves. */ /** Files are named by their path relative to the entry's directory, so - * names do not depend on where the program sits, and default library - * files by their base name: two checkers install them in different - * directories. */ + * names do not depend on where the program sits, default library files + * by their base name (two checkers install them in different + * directories), and the files scriptc ships by their name inside scriptc, + * so names do not depend on where scriptc is installed either. */ readonly positions = new StructuralPositions((sf) => this.program.isSourceFileDefaultLibrary(sf) ? `lib:${sf.fileName.slice(sf.fileName.lastIndexOf("/") + 1)}` - : relative(dirname(this.entry.fileName), sf.fileName), + : (compilerOwnedFileName(sf.fileName) ?? relative(dirname(this.entry.fileName), sf.fileName)), ); /** The outermost node whose token span is exactly `loc`, or null. */ @@ -3238,7 +3239,10 @@ export class Lowerer { * before it. */ closureName(loc: SrcLoc, suffix = "", prefix = "%fn"): string { const node = this.nodeAtLoc(loc); - const where = node === null ? `${loc.file}:${loc.start}:${loc.end}` : this.positions.of(node); + const where = + node === null + ? `${compilerOwnedFileName(loc.file) ?? loc.file}:${loc.start}:${loc.end}` + : this.positions.of(node); const key = `${prefix}|${this.closureRoot}|${where}|${suffix}`; const first = keyedHelperName(this, prefix, key); const seen = this.closureNameCounts.get(first) ?? 0; @@ -3379,8 +3383,18 @@ export class Lowerer { if (interned.has(fn.name)) this.sharedHelpers.add(fn); if (this.sharedHelpers.size === 0) return; const shared = new Map([...this.sharedHelpers].map((fn) => [fn.name, fn])); - const earlier = (left: SrcLoc, right: SrcLoc): boolean => - left.file < right.file || (left.file === right.file && left.start < right.start); + // Files scriptc ships compare by their name inside scriptc, so the + // choice does not depend on where scriptc is installed. + const fileKeys = new Map(); + const fileKey = (file: string): string => { + let key = fileKeys.get(file); + if (key === undefined) fileKeys.set(file, (key = compilerOwnedFileName(file) ?? file)); + return key; + }; + const earlier = (left: SrcLoc, right: SrcLoc): boolean => { + if (left.file === right.file) return left.start < right.start; + return fileKey(left.file) < fileKey(right.file); + }; // The earliest use of each shared helper outside the shared helpers, and // which shared helpers each shared helper uses. const uses = new Map(); diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index f0f19a408..daa5af366 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -126,6 +126,7 @@ import { moduleRuntimeFeatures } from "./ir/runtime-features.js"; import { serializeModuleParts } from "./ir/serialize.js"; import { readTextParts, writeTextParts } from "./text-parts.js"; import { validateModule } from "./ir/validate.js"; +import { compilerOwnedFileRoots } from "./frontend/dts-paths.js"; import { loadProgram } from "./frontend/program-node.js"; import { runFrontend, type FrontendFactory } from "./frontend/pipeline.js"; import { @@ -785,7 +786,7 @@ async function prepareExecutableInput( if (outputKind === "ir") { await mkdir(dirname(opts.outPath), { recursive: true }); - await writeTextParts(opts.outPath, serializeModuleParts(mod)); + await writeTextParts(opts.outPath, serializeModuleParts(mod, compilerOwnedFileRoots())); return { ok: true, artifact: { kind: "ir", path: opts.outPath } }; } @@ -885,7 +886,7 @@ async function prepareExecutableInput( if (opts.emitIr) { irPath = defaultSourcePaths.ir; irSourcePath = privateSiblingPath(irPath, "ir"); - await writeTextParts(irSourcePath, serializeModuleParts(mod)); + await writeTextParts(irSourcePath, serializeModuleParts(mod, compilerOwnedFileRoots())); } const nativeFeatures = executableNativeFeatures(mod, backend, opts.optimization ?? "release"); @@ -1986,7 +1987,7 @@ async function emitSemanticLibraryHit( let irPath: string | undefined; if (opts.emitIr) { irPath = join(opts.outDir, `${stem}.lib.ir.json`); - await writeTextParts(irPath, serializeModuleParts(mod)); + await writeTextParts(irPath, serializeModuleParts(mod, compilerOwnedFileRoots())); } await compileLibraryNative(profile, llvmPath, archivePath, opts.sanitize ?? false, hit.native); timing("native-archive"); @@ -2272,7 +2273,7 @@ async function compileLibraryTracked( let irPath: string | undefined; if (opts.emitIr) { irPath = join(opts.outDir, `${stem}.lib.ir.json`); - await writeTextParts(irPath, serializeModuleParts(mod)); + await writeTextParts(irPath, serializeModuleParts(mod, compilerOwnedFileRoots())); } const nativeFeatures = libraryNativeFeatures(mod, profile.emission); diff --git a/packages/compiler/src/ir/serialize.test.ts b/packages/compiler/src/ir/serialize.test.ts index 8345ab10a..c24e1f198 100644 --- a/packages/compiler/src/ir/serialize.test.ts +++ b/packages/compiler/src/ir/serialize.test.ts @@ -198,3 +198,24 @@ test("byte decoding keeps the whole-document errors and version refusal", () => const sentinel = text.replace('"nan"', '"future"'); expect(() => deserializeModuleBytes(Buffer.from(sentinel))).toThrow("Invalid IR number sentinel"); }); + +test("file roots respell the source locations under them and nothing else", () => { + const mod = numbers([1, 2]); + const shim = { file: '/opt/scriptc "a"/shims/pkg/index.ts', start: 3, end: 4 }; + const other = { file: "/opt/elsewhere/shims/pkg/index.ts", start: 5, end: 6 }; + const body = mod.functions[0]!.body as IrStmt[]; + body.push( + { kind: "exprStmt", expr: { kind: "numLit", value: 3, type: F64, loc: shim }, loc: shim }, + { kind: "exprStmt", expr: { kind: "numLit", value: 4, type: F64, loc: other }, loc: other }, + ); + const roots = [{ from: '/opt/scriptc "a"/shims/', to: "scriptc:shims/" }]; + const relocated = deserializeModule(serializeModuleParts(mod, roots).join("")); + const files = (relocated.functions[0]!.body as IrStmt[]).map((stmt) => stmt.loc.file); + expect(files).toEqual([ + "numbers.ts", + "numbers.ts", + "scriptc:shims/pkg/index.ts", + "/opt/elsewhere/shims/pkg/index.ts", + ]); + expect(serializeModuleParts(mod).join("")).toBe(serializeModule(mod, true)); +}); diff --git a/packages/compiler/src/ir/serialize.ts b/packages/compiler/src/ir/serialize.ts index ca5eb955e..314f258d6 100644 --- a/packages/compiler/src/ir/serialize.ts +++ b/packages/compiler/src/ir/serialize.ts @@ -46,11 +46,41 @@ function encodeFunction(fn: IrFunction): string | undefined { return ordinaryNumbers(fn) ? JSON.stringify(fn) : JSON.stringify(fn, encodeNumber); } +/** A directory whose source location files an IR artifact names by a + * prefix of its own: a location file `from + rest` serializes as + * `to + rest`. */ +export interface SerializedFileRoot { + from: string; + to: string; +} + +/** Respells the source location files under `roots` in serialized parts. + * A location's file is the first member of its JSON object, and the + * escaped spelling of a path prefix is the prefix of the escaped path. */ +function relocateFiles(parts: string[], roots: readonly SerializedFileRoot[]): string[] { + const needles = roots.map(({ from, to }) => ({ + from: `"file":${JSON.stringify(from).slice(0, -1)}`, + to: `"file":${JSON.stringify(to).slice(0, -1)}`, + })); + return parts.map((part) => + needles.reduce( + (text, { from, to }) => (text.includes(from) ? text.replaceAll(from, to) : text), + part, + ), + ); +} + /** The compact encoding as ordered parts: the module header up to the * function array, one part per function (with separate comma parts), and the * header's remainder. Concatenating the parts yields exactly - * `serializeModule(mod, true)`; writers stream them to disk in batches. */ -export function serializeModuleParts(mod: IrModule): string[] { + * `serializeModule(mod, true)`; writers stream them to disk in batches. + * `fileRoots` respells the source locations in those directories, so + * artifacts do not depend on where the compiler that wrote them is + * installed. */ +export function serializeModuleParts( + mod: IrModule, + fileRoots: readonly SerializedFileRoot[] = [], +): string[] { // A replacer reads through live native views. Serializing the complete // function array through one view refreshes every function capsule for // each element, making large compiler artifacts quadratic. The sentinel @@ -65,7 +95,7 @@ export function serializeModuleParts(mod: IrModule): string[] { parts.push(encodeFunction(fn) ?? "null"); } parts.push("]" + header.slice(offset + slot.length)); - return parts; + return fileRoots.length === 0 ? parts : relocateFiles(parts, fileRoots); } /** Compiler artifacts use compact JSON; API consumers can retain the diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index a2c627edf..da90dc2ce 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -26,6 +26,7 @@ import { type FrontendInputExclusions, trackedReadFile, } from "../frontend/input-tracker.js"; +import { compilerOwnedFileRoots } from "../frontend/dts-paths.js"; import { provenanceSources } from "../frontend/provenance-registry.js"; import { processFork } from "../frontend/process-fork.js"; import { prepareLibrary } from "../library/prepare.js"; @@ -312,7 +313,7 @@ export function prepareNativeExecutable( if (irInput !== "") timing("ir-input"); const ir = outputKind === "ir" || options.emitIr - ? batchTextParts(serializeModuleParts(prepared.mod)) + ? batchTextParts(serializeModuleParts(prepared.mod, compilerOwnedFileRoots())) : null; timing("ir-serialize"); const llvm = @@ -445,7 +446,9 @@ export function prepareNativeLibrary( const input: NativeExecutableInput = { ok: true, llvm, - ir: options.emitIr ? batchTextParts(serializeModuleParts(prepared.mod)) : null, + ir: options.emitIr + ? batchTextParts(serializeModuleParts(prepared.mod, compilerOwnedFileRoots())) + : null, sidecarJson: prepared.sidecarJson, features: executableLinkFeatures(prepared.mod), sources: [...prepared.sourceTexts],