From 3b374f3201c045c5805faaf0b150ded3d09c4068 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Tue, 27 Oct 2026 08:21:48 -0500 Subject: [PATCH] Check with the shared tsc-ts server by default on Linux and macOS Builds on Linux and macOS now connect to a shared `tsc-ts --api --listen` server by default, which keeps the files it parsed and bound, so a later build in the same working directory does not parse the TypeScript libs and scriptc's declaration files again. SCRIPTC_CHECKER_SERVER=0 turns it off. When CI is set to a value other than 0 or false, builds use their own tsc-ts process unless SCRIPTC_CHECKER_SERVER=1. - The server exits after 600 seconds without a build (was 300). - Its name also digests the compiler executable's identity, so each scriptc installation and build connects only to servers it started. - The server process runs in the root directory, so it does not keep the first build's directory in use; its --cwd argument is unchanged. - When the connection to the server fails after it answered, through a failed transfer or an ended stream, the native CLI runs the command again in a new process with SCRIPTC_CHECKER_SERVER=0 and exits with its status. - SCRIPTC_CHECKER_SERVER no longer changes the frontend cache key: the server's answers are the build's own checker's. The CLI docs describe the background process, its socket directory, when it exits and how to stop it. --- docs/content/docs/cli.mdx | 2 +- .../compiler/native/ts7-process-child.test.c | 12 +++--- packages/compiler/native/ts7-process.c | 8 ++-- packages/compiler/native/ts7-process.test.c | 7 +-- .../src/frontend/ts7/checker-server.test.ts | 11 +++-- .../src/frontend/ts7/checker-server.ts | 38 +++++++++++----- .../compiler/src/frontend/ts7/native-api.ts | 43 ++++++++++++++++++- packages/compiler/src/native/cli.ts | 35 +++++++++++++-- packages/compiler/src/native/prepare.ts | 2 + .../self-hosting/ts7-native-process.ts | 6 ++- 10 files changed, 130 insertions(+), 34 deletions(-) diff --git a/docs/content/docs/cli.mdx b/docs/content/docs/cli.mdx index b1583a617..fd15a000d 100644 --- a/docs/content/docs/cli.mdx +++ b/docs/content/docs/cli.mdx @@ -246,7 +246,7 @@ With `--sparse`, the profile leaves out the functions the runs never reached. Th
The TypeScript 7 API server that checks the program: tsc-ts (the default), which scriptc includes, or tsgo, from the typescript package scriptc also includes. Both give the same diagnostics and generated code.
SCRIPTC_CHECKER_SERVER
-
Set to 1 on Linux and macOS to check with a tsc-ts server that later builds share. It takes effect with a tsc-ts release that serves tsc-ts --api --listen. The tsc-ts 0.0.9 that this version of scriptc includes does not, so each build checks with its own tsc-ts process, as without the variable. A build connects to the server for its working directory, starting it when none answers, and the server keeps the files it parsed, so the next build of the same program loads it faster. A build that finds the server in use by another build, or cannot start or reach it, checks with its own tsc-ts process. The server exits after 5 minutes without a build, or after a build when its resident memory is above 2048 MB (TSC_TS_LISTEN_MEMORY sets another limit in megabytes). Diagnostics and generated code are the same with and without it. It applies to the default checker, tsc-ts.
+
On Linux and macOS, builds check with a tsc-ts server that later builds share. The first build in a working directory starts it: a tsc-ts --api --listen process that keeps running in the background with no terminal and keeps the files it parsed, so the next build in that directory loads its program faster. Each user, working directory and scriptc installation gets its own server, and a build connects only to the server of its own scriptc and tsc-ts. The socket is in $XDG_RUNTIME_DIR/scriptc, or /tmp/scriptc-<uid>, a directory only that user can access. A build that finds the server in use by another build, cannot start or reach it, or loses its connection to it, checks with its own tsc-ts process. The server exits after 10 minutes without a build, or after a build when its resident memory is above 2048 MB (TSC_TS_LISTEN_MEMORY sets another limit in megabytes). To stop it sooner, end that process, for example with pkill -f 'tsc-ts --api --listen'. Set to 0 to check with a tsc-ts process of each build's own. When CI is set to a value other than 0 or false, builds use their own process unless this variable is 1. Diagnostics and generated code are the same with and without the server. It applies to the default checker, tsc-ts.
```console diff --git a/packages/compiler/native/ts7-process-child.test.c b/packages/compiler/native/ts7-process-child.test.c index 4d1cbc081..d387f735a 100644 --- a/packages/compiler/native/ts7-process-child.test.c +++ b/packages/compiler/native/ts7-process-child.test.c @@ -20,8 +20,9 @@ /* `--listen PATH --idle-exit SECONDS`: a stand-in for a shared server. In * mode "listen-exit" it exits as an executable without --listen does. In * mode "listen" it answers each client, in order, with its client number, - * its listen arguments and TSC_TS_LISTEN_MEMORY, then echoes until the - * client closes, and exits after SECONDS (at most 5) without a client. */ + * its listen arguments, TSC_TS_LISTEN_MEMORY and its working directory, then + * echoes until the client closes, and exits after SECONDS (at most 5) without + * a client. */ static int listen_main(const char *mode, char **argv) { if (strcmp(mode, "listen-exit") == 0) return 2; if (strcmp(mode, "listen") != 0) return 73; @@ -42,9 +43,10 @@ static int listen_main(const char *mode, char **argv) { if (poll(&entry, 1, idle * 1000) <= 0) break; int fd = accept(listener, NULL, NULL); if (fd < 0) return 79; - char line[512]; - int size = snprintf(line, sizeof(line), "client %d\n%s %s\n%s\n", client, argv[0], argv[2], - memory == NULL ? "unset" : memory); + char line[1024], cwd[256]; + if (getcwd(cwd, sizeof(cwd)) == NULL) return 81; + int size = snprintf(line, sizeof(line), "client %d\n%s %s\n%s\n%s\n", client, argv[0], argv[2], + memory == NULL ? "unset" : memory, cwd); if (size <= 0 || write(fd, line, (size_t)size) != size) return 80; uint8_t bytes[17]; ssize_t count; diff --git a/packages/compiler/native/ts7-process.c b/packages/compiler/native/ts7-process.c index 8261c341b..bf486c87d 100644 --- a/packages/compiler/native/ts7-process.c +++ b/packages/compiler/native/ts7-process.c @@ -414,8 +414,10 @@ static int connect_server(const char *path) { /* Starts the server in its own process group, so that a terminal's signals * to the build do not reach it, with its standard streams on /dev/null and * no other descriptor of this process, so that it holds no pipe a caller of - * the build waits on. `limit` (TSC_TS_LISTEN_MEMORY=...) is added to the - * environment unless it sets TSC_TS_LISTEN_MEMORY. */ + * the build waits on. It runs in the root directory, so that it does not + * keep the build's directory in use; its working directory for the build's + * files is the --cwd argument. `limit` (TSC_TS_LISTEN_MEMORY=...) is added + * to the environment unless it sets TSC_TS_LISTEN_MEMORY. */ static pid_t start_server(char **args, char *limit) { #ifdef __APPLE__ char **inherited = *_NSGetEnviron(); @@ -440,7 +442,7 @@ static pid_t start_server(char **args, char *limit) { // Only async-signal-safe calls until exec. int null = open("/dev/null", O_RDWR); if (setpgid(0, 0) != 0 || null < 0 || dup2(null, STDIN_FILENO) < 0 || dup2(null, STDOUT_FILENO) < 0 || - dup2(null, STDERR_FILENO) < 0) + dup2(null, STDERR_FILENO) < 0 || chdir("/") != 0) _exit(127); #if defined(__linux__) && defined(SYS_close_range) if (syscall(SYS_close_range, 3U, ~0U, 0U) != 0) diff --git a/packages/compiler/native/ts7-process.test.c b/packages/compiler/native/ts7-process.test.c index 85de63db1..e07a7d10c 100644 --- a/packages/compiler/native/ts7-process.test.c +++ b/packages/compiler/native/ts7-process.test.c @@ -208,9 +208,9 @@ static void attach_cases(void) { uint8_t bytes[4]; snprintf(directory, sizeof(directory), "%s/scriptc", runtime); snprintf(socket_path, sizeof(socket_path), "%s/server.sock", directory); - // The first build starts the server with its idle time and memory limit. - // The server holds none of the build's descriptors, such as an output - // pipe its caller reads to the end. + // The first build starts the server with its idle time and memory limit, + // in the root directory. The server holds none of the build's descriptors, + // such as an output pipe its caller reads to the end. int output[2]; assert(pipe(output) == 0); uint32_t first = attach("server", "listen", 123); @@ -224,6 +224,7 @@ static void attach_cases(void) { snprintf(expected, sizeof(expected), "%s 1\n", socket_path); expect_line(first, expected); expect_line(first, "123\n"); + expect_line(first, "/\n"); write_exact(first, (const uint8_t *)"ping", 4, 0, 4); read_exact(first, bytes, 4); assert(memcmp(bytes, "ping", 4) == 0); diff --git a/packages/compiler/src/frontend/ts7/checker-server.test.ts b/packages/compiler/src/frontend/ts7/checker-server.test.ts index 9a0371fe6..e6cc5d29a 100644 --- a/packages/compiler/src/frontend/ts7/checker-server.test.ts +++ b/packages/compiler/src/frontend/ts7/checker-server.test.ts @@ -12,13 +12,18 @@ const posix = process.platform === "linux" || process.platform === "darwin"; afterAll(() => rmSync(dir, { recursive: true, force: true })); -test("builds use the shared checker server only when they opt in, on Linux and macOS, with tsc-ts", () => { - expect(checkerServerName(executable, dir, false, {})).toBeNull(); +test("builds use the shared checker server by default outside CI, on Linux and macOS, with tsc-ts", () => { expect(checkerServerName(executable, dir, false, { SCRIPTC_CHECKER_SERVER: "0" })).toBeNull(); + expect(checkerServerName(executable, dir, false, { SCRIPTC_CHECKER: "tsgo" })).toBeNull(); expect( checkerServerName(executable, dir, false, { ...enabled, SCRIPTC_CHECKER: "tsgo" }), ).toBeNull(); - const name = checkerServerName(executable, dir, false, enabled); + // CI turns the default off; SCRIPTC_CHECKER_SERVER=1 turns it back on. + expect(checkerServerName(executable, dir, false, { CI: "true" })).toBeNull(); + expect(checkerServerName(executable, dir, false, { CI: "1" })).toBeNull(); + const name = checkerServerName(executable, dir, false, {}); + for (const env of [enabled, { CI: "false" }, { CI: "0" }, { CI: "" }, { ...enabled, CI: "true" }]) + expect(checkerServerName(executable, dir, false, env)).toBe(name); if (!posix) expect(name).toBeNull(); else expect(name).toMatch(/^[0-9a-f]{32}$/); }); diff --git a/packages/compiler/src/frontend/ts7/checker-server.ts b/packages/compiler/src/frontend/ts7/checker-server.ts index 94c80f8db..36a951e4c 100644 --- a/packages/compiler/src/frontend/ts7/checker-server.ts +++ b/packages/compiler/src/frontend/ts7/checker-server.ts @@ -5,23 +5,37 @@ import { TS7_FILE_SYSTEM_CALLBACKS } from "./rpc-filesystem.js"; /** How long a shared checker server waits for the next build before it * exits, in seconds. */ -export const CHECKER_SERVER_IDLE_EXIT = 300; +export const CHECKER_SERVER_IDLE_EXIT = 600; /** The resident memory, in megabytes, above which a shared checker server * exits after a build. tsc-ts never frees a program's syntax trees, types and * symbols, so its memory grows with each build it serves. */ export const CHECKER_SERVER_MEMORY = 2048; +/** Whether builds use a shared checker server: by default, except in CI + * (`CI` set to anything but `0` or `false`), where a job usually builds once + * per directory and would only leave the server behind. + * `SCRIPTC_CHECKER_SERVER=0` turns it off and `SCRIPTC_CHECKER_SERVER=1` turns + * it on in CI. */ +function checkerServerEnabled(env: NodeJS.ProcessEnv): boolean { + const setting = env["SCRIPTC_CHECKER_SERVER"]; + if (setting === "1") return true; + if (setting === "0") return false; + const ci = env["CI"]; + return ci === undefined || ci === "" || ci === "0" || ci === "false"; +} + /** - * With `SCRIPTC_CHECKER_SERVER=1` on Linux and macOS, the native compiler - * checks with a tsc-ts server shared by the builds that follow - * (`tsc-ts --api --listen`): a build connects to it, starting it when none - * answers, and the server keeps the files it parsed for the next build. - * Returns the server's name, a digest of everything a server's answers - * depend on besides the files it reads through the build: the tsc-ts - * executable, the working directory, the file system callbacks, timing, - * whether it reads its library files from the disk, and the `TSC_TS_*` - * environment. Returns null when the shared server is not used. + * On Linux and macOS, the native compiler checks with a tsc-ts server shared + * by the builds that follow (`tsc-ts --api --listen`): a build connects to + * it, starting it when none answers, and the server keeps the files it + * parsed for the next build (see checkerServerEnabled for when). Returns the + * server's name, a digest of everything a server's answers depend on + * besides the files it reads through the build: the tsc-ts executable, the + * working directory, the file system callbacks, timing, whether it reads its + * library files from the disk, and the `TSC_TS_*` environment; and of the + * compiler executable, so that each scriptc build has servers of its own. + * Returns null when the shared server is not used. */ export function checkerServerName( executable: string, @@ -29,8 +43,9 @@ export function checkerServerName( timing: boolean, env: NodeJS.ProcessEnv = process.env, librariesFromDisk = false, + compiler: string = process.execPath, ): string | null { - if (env["SCRIPTC_CHECKER_SERVER"] !== "1") return null; + if (!checkerServerEnabled(env)) return null; if (process.platform !== "linux" && process.platform !== "darwin") return null; if (checkerBackend(env) !== "tsc-ts") return null; const tscTsEnv = Object.keys(env) @@ -41,6 +56,7 @@ export function checkerServerName( .update( JSON.stringify([ checkerExecutableIdentity(executable), + checkerExecutableIdentity(compiler), cwd, TS7_FILE_SYSTEM_CALLBACKS, timing, diff --git a/packages/compiler/src/frontend/ts7/native-api.ts b/packages/compiler/src/frontend/ts7/native-api.ts index fa0690e69..03be8974c 100644 --- a/packages/compiler/src/frontend/ts7/native-api.ts +++ b/packages/compiler/src/frontend/ts7/native-api.ts @@ -4,7 +4,41 @@ import { checkerServerName } from "./checker-server.js"; import { attachNativeTs7Process, spawnNativeTs7Wire } from "./native-process.js"; import { Ts7RpcClient } from "./rpc-client.js"; import type { Ts7FileSystem } from "./rpc-filesystem.js"; -import { Ts7Wire } from "./rpc-wire.js"; +import { Ts7Wire, type Ts7WireIo } from "./rpc-wire.js"; + +let sharedServerLost = false; + +/** Whether a connection to the shared checker server failed in this process + * after the server answered: a transfer failed or the server ended the + * connection. The build then runs again with its own checker (cli.ts). */ +export function checkerServerLost(): boolean { + return sharedServerLost; +} + +/** `io`, recording a failed transfer or an ended stream in sharedServerLost. */ +function watched(io: Ts7WireIo): Ts7WireIo { + return { + read: (buffer, offset, length) => { + try { + const count = io.read(buffer, offset, length); + if (count === 0) sharedServerLost = true; + return count; + } catch (error) { + sharedServerLost = true; + throw error; + } + }, + write: (buffer, offset, length) => { + try { + return io.write(buffer, offset, length); + } catch (error) { + sharedServerLost = true; + throw error; + } + }, + close: () => io.close(), + }; +} /** The native driver supplies its installed TypeScript executable path. * Package discovery is separate from starting and owning a server. */ @@ -32,10 +66,15 @@ export function createNativeTs7Api(options: { try { // A server that ended the connection leaves this build to start its own. rpc.requestText("ping", "null"); - return new Ts7Api(rpc, options.fs, timing); } catch { rpc.close(); + return new Ts7Api( + new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)), + options.fs, + timing, + ); } + return new Ts7Api(new Ts7RpcClient(new Ts7Wire(watched(shared))), options.fs, timing); } return new Ts7Api( new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)), diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index 4209ecf0e..961fd02a9 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -13,7 +13,7 @@ import { setDeclarationRoot } from "../frontend/dts-paths.js"; import { setFrontendDisposedHook } from "../frontend/dispose-hook.js"; import { FrontendServices } from "../frontend/services.js"; import { runTypePatchCommand } from "../frontend/type-patches/command.js"; -import { createNativeTs7Api } from "../frontend/ts7/native-api.js"; +import { checkerServerLost, createNativeTs7Api } from "../frontend/ts7/native-api.js"; import { resolveProvenanceSourcesWithParser } from "../frontend/provenance-core.js"; import { setCompilerReleaseVersion } from "../library/sidecar.js"; import { NativeCompiler } from "./compiler.js"; @@ -41,6 +41,30 @@ declare function compilerNativeLock(): void; declare function compilerNativeUnlock(): void; declare function compilerNativeBoundCycleGrowth(): void; +/** Runs a build. When this process's connection to the shared checker server + * failed during it, the command runs again in a new process that checks with + * its own tsc-ts process (SCRIPTC_CHECKER_SERVER=0), and this process exits + * with that one's status: the new process prints the build's output. */ +async function withCheckerFallback(work: () => Promise): Promise { + let result: T; + try { + result = await work(); + } catch (error) { + if (!checkerServerLost()) throw error; + return runWithOwnChecker(); + } + return checkerServerLost() ? runWithOwnChecker() : result; +} + +function runWithOwnChecker(): never { + const run = spawnSync(process.execPath, process.argv.slice(2), { + stdio: "inherit", + env: { ...process.env, SCRIPTC_CHECKER_SERVER: "0" }, + }); + if (run.error) throw new Error(run.error.message); + process.exit(run.status ?? 1); +} + async function main(): Promise { compilerNativeBoundCycleGrowth(); setFrontendDisposedHook(() => compilerNativeCollectCycles()); @@ -100,9 +124,12 @@ async function main(): Promise { return runCli(process.argv.slice(2), { version: () => loadNativeToolchain(manifestPath).compilerVersion, sourceTargetPlatform: () => getCompiler().toolchain.target.platform, - analyze: (entry, options) => runCompilerTask(() => getCompiler().analyzeCached(entry, options)), - compile: (entry, options) => runCompilerTask(() => getCompiler().compile(entry, options)), - compileLibrary: (options) => runCompilerTask(() => getCompiler().compileLibrary(options)), + analyze: (entry, options) => + withCheckerFallback(() => runCompilerTask(() => getCompiler().analyzeCached(entry, options))), + compile: (entry, options) => + withCheckerFallback(() => runCompilerTask(() => getCompiler().compile(entry, options))), + compileLibrary: (options) => + withCheckerFallback(() => runCompilerTask(() => getCompiler().compileLibrary(options))), resolveProvenanceSources: async (entry) => { const toolchain = getCompiler().toolchain; const services = new FrontendServices((options) => diff --git a/packages/compiler/src/native/prepare.ts b/packages/compiler/src/native/prepare.ts index 339a50faa..33c239c64 100644 --- a/packages/compiler/src/native/prepare.ts +++ b/packages/compiler/src/native/prepare.ts @@ -217,6 +217,8 @@ function implementationIdentity(toolchain: NativeToolchain): string { const FRONTEND_INDEPENDENT_VARIABLES = new Set([ "SCRIPTC_CACHE_DIR", "SCRIPTC_CACHE_MAX_MB", + // The shared checker server answers as the build's own checker does. + "SCRIPTC_CHECKER_SERVER", "SCRIPTC_EMIT_WORKERS", "SCRIPTC_EMIT_WORKER_FUNCTIONS", "SCRIPTC_TIMING", diff --git a/tests/fixtures/self-hosting/ts7-native-process.ts b/tests/fixtures/self-hosting/ts7-native-process.ts index 3b04a784c..8ca9ec8e3 100644 --- a/tests/fixtures/self-hosting/ts7-native-process.ts +++ b/tests/fixtures/self-hosting/ts7-native-process.ts @@ -109,13 +109,15 @@ function readLine(connection: NativeTs7Process): string { } // The stand-in server (ts7-process-child.test.c, mode "listen") answers each -// client with its number, its socket and idle time, and its memory limit. +// client with its number, its socket and idle time, its memory limit and its +// working directory. function checkShared(executable: string): void { const first = attachNativeTs7Process("server", executable, "listen"); check(first !== null, "shared server started"); check(readLine(first!) === "client 1", "first client"); - check(readLine(first!).endsWith("/scriptc/server.sock 300"), "listen arguments"); + check(readLine(first!).endsWith("/scriptc/server.sock 600"), "listen arguments"); check(readLine(first!) === "2048", "memory limit"); + check(readLine(first!) === "/", "server in the root directory"); check(attachNativeTs7Process("server", executable, "listen") === null, "shared server in use"); first!.close(); first!.close();