diff --git a/packages/compiler/native/ts7-process-child.test.c b/packages/compiler/native/ts7-process-child.test.c index 386a0a7c3..4d1cbc081 100644 --- a/packages/compiler/native/ts7-process-child.test.c +++ b/packages/compiler/native/ts7-process-child.test.c @@ -64,8 +64,11 @@ static int child_main(int argc, char **argv) { #endif int listening = argc >= 9 && strcmp(argv[5], "--listen") == 0 && strcmp(argv[7], "--idle-exit") == 0; int fixed = listening ? 9 : 5; - if (argc < fixed || argc > fixed + 1 || strcmp(argv[1], "--api") != 0 || strcmp(argv[2], "--cwd") != 0 || - strncmp(argv[4], "--callbacks=", 12) != 0 || (argc == fixed + 1 && strcmp(argv[fixed], "--timing") != 0)) + int at = fixed, timing = 0, libraries = 0; + if (at < argc && strcmp(argv[at], "--timing") == 0) { timing = 1; at++; } + if (at < argc && strcmp(argv[at], "--default-libraries-from-disk") == 0) { libraries = 1; at++; } + if (argc < fixed || at != argc || strcmp(argv[1], "--api") != 0 || strcmp(argv[2], "--cwd") != 0 || + strncmp(argv[4], "--callbacks=", 12) != 0) return 71; const char *mode = argv[3]; #ifndef _WIN32 @@ -76,7 +79,8 @@ static int child_main(int argc, char **argv) { fputc('\n', stdout); fputs(argv[4], stdout); fputc('\n', stdout); - fputs(argc == 6 ? "timing\n" : "plain\n", stdout); + fputs(timing ? "timing\n" : "plain\n", stdout); + if (libraries) fputs("libraries from disk\n", stdout); return ferror(stdout) ? 72 : 0; } if (strcmp(mode, "exit") == 0) return 0; diff --git a/packages/compiler/native/ts7-process.c b/packages/compiler/native/ts7-process.c index 3fd38a423..8261c341b 100644 --- a/packages/compiler/native/ts7-process.c +++ b/packages/compiler/native/ts7-process.c @@ -161,7 +161,7 @@ static wchar_t *command_line(wchar_t **arguments, size_t count) { } static int start_process(Ts7Process *p, char **args, size_t argc) { - wchar_t *wide[6] = {0}; + wchar_t *wide[7] = {0}; wchar_t *line = NULL; HANDLE child_input = NULL, child_output = NULL, child_error = NULL; LPPROC_THREAD_ATTRIBUTE_LIST attributes = NULL; @@ -330,7 +330,7 @@ static void stop_process(Ts7Process *p) { uint32_t scriptc_ts7_open(const uint8_t *executable, size_t executable_size, const uint8_t *cwd, size_t cwd_size, const uint8_t *callbacks, size_t callbacks_size, - uint8_t timing) { + uint8_t timing, uint8_t libraries_from_disk) { if (next_id == 0) { error_text("server handle IDs are exhausted"); return 0; } char *exe = copy_argument(executable, executable_size, ""); if (exe == NULL) return 0; @@ -346,8 +346,12 @@ uint32_t scriptc_ts7_open(const uint8_t *executable, size_t executable_size, } exit_registered = 1; } - char *args[] = {exe, "--api", "--cwd", directory, callback_flag, timing ? "--timing" : NULL, NULL}; - int started = start_process(p, args, timing ? 6 : 5); + char *args[8] = {exe, "--api", "--cwd", directory, callback_flag}; + size_t argc = 5; + if (timing) args[argc++] = "--timing"; + if (libraries_from_disk) args[argc++] = "--default-libraries-from-disk"; + args[argc] = NULL; + int started = start_process(p, args, argc); free(exe); free(directory); free(callback_flag); if (!started) { free(p); return 0; } p->id = next_id++; @@ -361,9 +365,10 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size, const uint8_t *executable, size_t executable_size, const uint8_t *cwd, size_t cwd_size, const uint8_t *callbacks, size_t callbacks_size, - uint8_t timing, uint32_t idle_exit, uint32_t memory) { + uint8_t timing, uint8_t libraries_from_disk, uint32_t idle_exit, + uint32_t memory) { (void)name; (void)name_size; (void)executable; (void)executable_size; (void)cwd; (void)cwd_size; - (void)callbacks; (void)callbacks_size; (void)timing; (void)idle_exit; (void)memory; + (void)callbacks; (void)callbacks_size; (void)timing; (void)libraries_from_disk; (void)idle_exit; (void)memory; error_text("a shared checker server needs Unix sockets"); return 0; } @@ -455,7 +460,8 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size, const uint8_t *executable, size_t executable_size, const uint8_t *cwd, size_t cwd_size, const uint8_t *callbacks, size_t callbacks_size, - uint8_t timing, uint32_t idle_exit, uint32_t memory) { + uint8_t timing, uint8_t libraries_from_disk, uint32_t idle_exit, + uint32_t memory) { if (next_id == 0) { error_text("server handle IDs are exhausted"); return 0; } char *server = copy_argument(name, name_size, ""); if (server == NULL) return 0; @@ -491,8 +497,11 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size, (void)snprintf(limit, sizeof(limit), "TSC_TS_LISTEN_MEMORY=%lu", (unsigned long)memory); pid_t pid = -1; if (callback_flag != NULL) { - char *args[] = {exe, "--api", "--cwd", dir, callback_flag, "--listen", socket_path, "--idle-exit", idle, - timing ? "--timing" : NULL, NULL}; + char *args[12] = {exe, "--api", "--cwd", dir, callback_flag, "--listen", socket_path, "--idle-exit", idle}; + size_t argc = 9; + if (timing) args[argc++] = "--timing"; + if (libraries_from_disk) args[argc++] = "--default-libraries-from-disk"; + args[argc] = NULL; pid = start_server(args, limit); } free(exe); free(dir); free(callback_flag); diff --git a/packages/compiler/native/ts7-process.ffi.json b/packages/compiler/native/ts7-process.ffi.json index c503e3cae..963c81f15 100644 --- a/packages/compiler/native/ts7-process.ffi.json +++ b/packages/compiler/native/ts7-process.ffi.json @@ -1,8 +1,8 @@ { "ffi_format": 6, "functions": [ - { "name": "ts7NativeOpen", "symbol": "scriptc_ts7_open", "params": ["string", "string", "string", "bool"], "returns": "u32" }, - { "name": "ts7NativeAttach", "symbol": "scriptc_ts7_attach", "params": ["string", "string", "string", "string", "bool", "u32", "u32"], "returns": "u32" }, + { "name": "ts7NativeOpen", "symbol": "scriptc_ts7_open", "params": ["string", "string", "string", "bool", "bool"], "returns": "u32" }, + { "name": "ts7NativeAttach", "symbol": "scriptc_ts7_attach", "params": ["string", "string", "string", "string", "bool", "bool", "u32", "u32"], "returns": "u32" }, { "name": "ts7NativeRead", "symbol": "scriptc_ts7_read", "params": ["u32", "mutable-bytes", "u32", "u32"], "returns": "i32" }, { "name": "ts7NativeWrite", "symbol": "scriptc_ts7_write", "params": ["u32", "bytes", "u32", "u32"], "returns": "i32" }, { "name": "ts7NativePid", "symbol": "scriptc_ts7_pid", "params": ["u32"], "returns": "u32" }, diff --git a/packages/compiler/native/ts7-process.h b/packages/compiler/native/ts7-process.h index 4199e8795..d51f939a9 100644 --- a/packages/compiler/native/ts7-process.h +++ b/packages/compiler/native/ts7-process.h @@ -12,7 +12,7 @@ uint32_t scriptc_ts7_open(const uint8_t *executable, size_t executable_size, const uint8_t *cwd, size_t cwd_size, const uint8_t *callbacks, size_t callbacks_size, - uint8_t timing); + uint8_t timing, uint8_t libraries_from_disk); /* Connects to the shared checker server `name` (`--api --listen`) in this * user's private server directory, starting it in its own process group * with `--idle-exit idle_exit` and a limit of `memory` megabytes @@ -25,7 +25,8 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size, const uint8_t *executable, size_t executable_size, const uint8_t *cwd, size_t cwd_size, const uint8_t *callbacks, size_t callbacks_size, - uint8_t timing, uint32_t idle_exit, uint32_t memory); + uint8_t timing, uint8_t libraries_from_disk, uint32_t idle_exit, + uint32_t memory); int32_t scriptc_ts7_read(uint32_t handle, uint8_t *bytes, size_t size, uint32_t offset, uint32_t length); int32_t scriptc_ts7_write(uint32_t handle, const uint8_t *bytes, size_t size, diff --git a/packages/compiler/native/ts7-process.test.c b/packages/compiler/native/ts7-process.test.c index dc0c67ced..b6d760abd 100644 --- a/packages/compiler/native/ts7-process.test.c +++ b/packages/compiler/native/ts7-process.test.c @@ -21,12 +21,14 @@ static const char *executable; static const char *callbacks = "readFile,fileExists"; -static uint32_t start(const char *mode, int timing) { +static uint32_t start_with(const char *mode, int timing, int libraries) { return scriptc_ts7_open((const uint8_t *)executable, strlen(executable), (const uint8_t *)mode, strlen(mode), - (const uint8_t *)callbacks, strlen(callbacks), (uint8_t)timing); + (const uint8_t *)callbacks, strlen(callbacks), (uint8_t)timing, (uint8_t)libraries); } +static uint32_t start(const char *mode, int timing) { return start_with(mode, timing, 0); } + static void fail_message(void) { uint8_t message[1024]; int32_t size = scriptc_ts7_error(message, sizeof(message)); @@ -83,11 +85,13 @@ static void argument_cases(void) { "", "a b", "double\"quote", "end\\", "many\\\\\"quotes\\\\", "a;&$()`'b", "h\xc3\xa9llo \xf0\x9f\x8c\x8d", }; for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { - for (int timing = 0; timing <= 1; timing++) { + for (int flags = 0; flags < 4; flags++) { + int timing = flags & 1, libraries = flags >> 1; char mode[256], expected[512]; snprintf(mode, sizeof(mode), "argv:%s", values[i]); - snprintf(expected, sizeof(expected), "%s\n--callbacks=%s\n%s\n", values[i], callbacks, timing ? "timing" : "plain"); - uint32_t id = start(mode, timing); + snprintf(expected, sizeof(expected), "%s\n--callbacks=%s\n%s\n%s", values[i], callbacks, + timing ? "timing" : "plain", libraries ? "libraries from disk\n" : ""); + uint32_t id = start_with(mode, timing, libraries); assert(id != 0); uint8_t actual[512]; read_exact(id, actual, strlen(expected)); @@ -147,10 +151,10 @@ static void failure_cases(void) { executable = "/scriptc/does-not-exist/tsgo"; for (int i = 0; i < 30; i++) { assert(start("echo", 0) == 0); fail_message(); } executable = saved; - assert(scriptc_ts7_open(NULL, 0, bytes, 1, bytes, 1, 0) == 0); - assert(scriptc_ts7_open((const uint8_t *)"abc\0def", 7, bytes, 1, bytes, 1, 0) == 0); - assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), NULL, 0, bytes, 1, 0) == 0); - assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), bytes, 1, NULL, 0, 0) == 0); + assert(scriptc_ts7_open(NULL, 0, bytes, 1, bytes, 1, 0, 0) == 0); + assert(scriptc_ts7_open((const uint8_t *)"abc\0def", 7, bytes, 1, bytes, 1, 0, 0) == 0); + assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), NULL, 0, bytes, 1, 0, 0) == 0); + assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), bytes, 1, NULL, 0, 0, 0) == 0); for (int i = 0; i < 30; i++) { uint32_t id = start("exit", 0); assert(id != 0); @@ -177,7 +181,7 @@ static void failure_cases(void) { static uint32_t attach(const char *name, const char *mode, uint32_t memory) { return scriptc_ts7_attach((const uint8_t *)name, strlen(name), (const uint8_t *)executable, strlen(executable), (const uint8_t *)mode, strlen(mode), (const uint8_t *)callbacks, strlen(callbacks), - 0, 1, memory); + 0, 0, 1, memory); } static void expect_line(uint32_t id, const char *expected) { diff --git a/packages/compiler/src/frontend/ts7/checker-features.test.ts b/packages/compiler/src/frontend/ts7/checker-features.test.ts new file mode 100644 index 000000000..fcd3576cb --- /dev/null +++ b/packages/compiler/src/frontend/ts7/checker-features.test.ts @@ -0,0 +1,45 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, afterEach, expect, test } from "vitest"; +import { checkerReadsLibrariesFromDisk, setCheckerFeatureStore } from "./checker-features.js"; + +const dir = mkdtempSync(join(tmpdir(), "scriptc-checker-features-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +afterEach(() => setCheckerFeatureStore(null)); + +/** A stand-in checker that records its arguments and exits with `status`. */ +function checker(name: string, status: number): { path: string; log: string } { + const path = join(dir, name); + const log = path + ".log"; + writeFileSync(path, `#!/bin/sh\necho "$@" >> '${log}'\nexit ${status}\n`); + chmodSync(path, 0o755); + return { path, log }; +} + +test.skipIf(process.platform === "win32")( + "the checker is asked once whether it reads its library files from the disk, and the answer is stored", + async () => { + const { readFileSync, existsSync } = await import("node:fs"); + const entries = new Map(); + setCheckerFeatureStore({ + read: (key) => entries.get(key) ?? null, + write: (key, text) => void entries.set(key, text), + }); + const current = checker("current", 0); + const released = checker("released", 2); + expect(checkerReadsLibrariesFromDisk(current.path, {})).toBe(true); + expect(checkerReadsLibrariesFromDisk(released.path, {})).toBe(false); + expect(readFileSync(current.log, "utf8")).toBe("--api --default-libraries-from-disk\n"); + expect([...entries.values()].sort()).toEqual(["0", "1"]); + // Later asks use the answers: the checkers do not run again. + expect(checkerReadsLibrariesFromDisk(current.path, {})).toBe(true); + expect(checkerReadsLibrariesFromDisk(released.path, {})).toBe(false); + expect(readFileSync(current.log, "utf8").split("\n").length).toBe(2); + expect(readFileSync(released.log, "utf8").split("\n").length).toBe(2); + // tsgo is never asked. + const tsgo = checker("tsgo", 0); + expect(checkerReadsLibrariesFromDisk(tsgo.path, { SCRIPTC_CHECKER: "tsgo" })).toBe(false); + expect(existsSync(tsgo.log)).toBe(false); + }, +); diff --git a/packages/compiler/src/frontend/ts7/checker-features.ts b/packages/compiler/src/frontend/ts7/checker-features.ts new file mode 100644 index 000000000..b80a54994 --- /dev/null +++ b/packages/compiler/src/frontend/ts7/checker-features.ts @@ -0,0 +1,53 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { checkerBackend } from "./checker-backend.js"; +import { checkerExecutableIdentity } from "./lib-diagnostics.js"; + +/** Text entries kept between builds (the native compiler's cache). */ +export interface CheckerFeatureStore { + read(key: string): string | null; + write(key: string, text: string): void; +} + +let featureStore: CheckerFeatureStore | null = null; +const answers = new Map(); + +/** Keeps the answers of checkerReadsLibrariesFromDisk between builds. */ +export function setCheckerFeatureStore(store: CheckerFeatureStore | null): void { + featureStore = store; +} + +/** + * Whether the tsc-ts at `executable` takes `--api --default-libraries-from-disk`, which has it read the files + * of its default library directory from the disk instead of through the file system callbacks. tsc-ts releases + * before it reject the flag, so it is asked once per executable identity: the stored answer, or else + * `tsc-ts --api --default-libraries-from-disk` with an empty input, which exits with 0 when the flag is known + * and with a usage error when it is not. tsgo is never asked. + */ +export function checkerReadsLibrariesFromDisk( + executable: string, + env: NodeJS.ProcessEnv = process.env, +): boolean { + if (checkerBackend(env) !== "tsc-ts") return false; + const key = createHash("sha256") + .update(JSON.stringify(["default-libraries-from-disk", checkerExecutableIdentity(executable)])) + .digest("hex"); + const known = answers.get(key); + if (known !== undefined) return known; + const stored = featureStore?.read(key) ?? null; + let answer: boolean; + if (stored === "1" || stored === "0") answer = stored === "1"; + else { + const probe = spawnSync(executable, ["--api", "--default-libraries-from-disk"], { + encoding: "utf8", + stdio: "pipe", + timeout: 10_000, + }); + answer = probe.error === undefined && probe.status === 0; + // A probe that could not run says nothing about the flag: ask again next time. + if (probe.error === undefined && probe.status !== null) + featureStore?.write(key, answer ? "1" : "0"); + } + answers.set(key, answer); + return answer; +} diff --git a/packages/compiler/src/frontend/ts7/checker-server.ts b/packages/compiler/src/frontend/ts7/checker-server.ts index 6d80e2599..94c80f8db 100644 --- a/packages/compiler/src/frontend/ts7/checker-server.ts +++ b/packages/compiler/src/frontend/ts7/checker-server.ts @@ -19,14 +19,16 @@ export const CHECKER_SERVER_MEMORY = 2048; * answers, and the server keeps the files it parsed for the next build. * Returns the server's name, a digest of everything a server's answers * depend on besides the files it reads through the build: the tsc-ts - * executable, the working directory, the file system callbacks, timing, and - * the `TSC_TS_*` environment. Returns null when the shared server is not used. + * executable, the working directory, the file system callbacks, timing, + * whether it reads its library files from the disk, and the `TSC_TS_*` + * environment. Returns null when the shared server is not used. */ export function checkerServerName( executable: string, cwd: string, timing: boolean, env: NodeJS.ProcessEnv = process.env, + librariesFromDisk = false, ): string | null { if (env["SCRIPTC_CHECKER_SERVER"] !== "1") return null; if (process.platform !== "linux" && process.platform !== "darwin") return null; @@ -42,6 +44,7 @@ export function checkerServerName( cwd, TS7_FILE_SYSTEM_CALLBACKS, timing, + librariesFromDisk, tscTsEnv, ]), ) diff --git a/packages/compiler/src/frontend/ts7/native-api.ts b/packages/compiler/src/frontend/ts7/native-api.ts index f8e10efc5..fa0690e69 100644 --- a/packages/compiler/src/frontend/ts7/native-api.ts +++ b/packages/compiler/src/frontend/ts7/native-api.ts @@ -1,4 +1,5 @@ import { Ts7Api } from "./api.js"; +import { checkerReadsLibrariesFromDisk } from "./checker-features.js"; import { checkerServerName } from "./checker-server.js"; import { attachNativeTs7Process, spawnNativeTs7Wire } from "./native-process.js"; import { Ts7RpcClient } from "./rpc-client.js"; @@ -15,12 +16,17 @@ export function createNativeTs7Api(options: { shared?: boolean; }): Ts7Api { const timing = options.collectTiming ?? false; + // The checker reads its own library files when it can: they are part of the + // installed tsc-ts, so no build serves them differently. + const libraries = checkerReadsLibrariesFromDisk(options.executable); const server = - options.shared === false ? null : checkerServerName(options.executable, options.cwd, timing); + options.shared === false + ? null + : checkerServerName(options.executable, options.cwd, timing, process.env, libraries); const shared = server === null ? null - : attachNativeTs7Process(server, options.executable, options.cwd, timing); + : attachNativeTs7Process(server, options.executable, options.cwd, timing, libraries); if (shared !== null) { const rpc = new Ts7RpcClient(new Ts7Wire(shared)); try { @@ -32,7 +38,7 @@ export function createNativeTs7Api(options: { } } return new Ts7Api( - new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing)), + new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)), options.fs, timing, ); diff --git a/packages/compiler/src/frontend/ts7/native-process.ts b/packages/compiler/src/frontend/ts7/native-process.ts index 0582ae673..d59543f32 100644 --- a/packages/compiler/src/frontend/ts7/native-process.ts +++ b/packages/compiler/src/frontend/ts7/native-process.ts @@ -10,6 +10,7 @@ declare function ts7NativeOpen( cwd: string, callbacks: string, timing: boolean, + librariesFromDisk: boolean, ): number; declare function ts7NativeAttach( name: string, @@ -17,6 +18,7 @@ declare function ts7NativeAttach( cwd: string, callbacks: string, timing: boolean, + librariesFromDisk: boolean, idleExit: number, memory: number, ): number; @@ -63,12 +65,26 @@ export class NativeTs7Process implements Ts7WireIo { private handle: number; readonly pid: number; - /** With `handle`, a connection to a shared checker server (attachNativeTs7Process). */ - constructor(executable: string, cwd: string, collectTiming = false, handle = 0) { + /** With `handle`, a connection to a shared checker server (attachNativeTs7Process). With + * `librariesFromDisk`, the server reads its default library files from the disk + * (checkerReadsLibrariesFromDisk). */ + constructor( + executable: string, + cwd: string, + collectTiming = false, + handle = 0, + librariesFromDisk = false, + ) { this.handle = handle !== 0 ? handle - : ts7NativeOpen(executable, cwd, TS7_FILE_SYSTEM_CALLBACKS, collectTiming); + : ts7NativeOpen( + executable, + cwd, + TS7_FILE_SYSTEM_CALLBACKS, + collectTiming, + librariesFromDisk, + ); if (this.handle === 0) throw nativeError("startup"); this.pid = ts7NativePid(this.handle); } @@ -125,6 +141,7 @@ export function attachNativeTs7Process( executable: string, cwd: string, collectTiming = false, + librariesFromDisk = false, ): NativeTs7Process | null { const handle = ts7NativeAttach( name, @@ -132,6 +149,7 @@ export function attachNativeTs7Process( cwd, TS7_FILE_SYSTEM_CALLBACKS, collectTiming, + librariesFromDisk, CHECKER_SERVER_IDLE_EXIT, CHECKER_SERVER_MEMORY, ); @@ -142,6 +160,7 @@ export function spawnNativeTs7Wire( executable: string, cwd: string, collectTiming = false, + librariesFromDisk = false, ): Ts7Wire { - return new Ts7Wire(new NativeTs7Process(executable, cwd, collectTiming)); + return new Ts7Wire(new NativeTs7Process(executable, cwd, collectTiming, 0, librariesFromDisk)); } diff --git a/packages/compiler/src/native/cli.ts b/packages/compiler/src/native/cli.ts index b729b2557..b3de05846 100644 --- a/packages/compiler/src/native/cli.ts +++ b/packages/compiler/src/native/cli.ts @@ -21,6 +21,7 @@ import { checkerExecutableIdentity, setLibDiagnosticsStore, } from "../frontend/ts7/lib-diagnostics.js"; +import { setCheckerFeatureStore } from "../frontend/ts7/checker-features.js"; import { loadNativeToolchain } from "./toolchain.js"; import { buildSanitizedRuntime } from "./sanitizer.js"; import { runCompilerTask } from "./task.js"; @@ -67,6 +68,11 @@ async function main(): Promise { }, checkerExecutableIdentity(toolchain.ts7Executable), ); + setCheckerFeatureStore({ + read: (key) => + store.read("checker-features", key, NATIVE_STRING_BYTES)?.toString("utf8") ?? null, + write: (key, text) => store.write("checker-features", key, text), + }); } compiler = new NativeCompiler(toolchain, cache); return compiler;