Have tsc-ts read its library files from disk when it can (#2208)

packages/compiler/native/ts7-process-child.test.c

@@ -64,8 +64,11 @@ static int child_main(int argc, char **argv) {
 #endif
   int listening = argc >= 9 && strcmp(argv[5], "--listen") == 0 && strcmp(argv[7], "--idle-exit") == 0;
   int fixed = listening ? 9 : 5;
-  if (argc < fixed || argc > fixed + 1 || strcmp(argv[1], "--api") != 0 || strcmp(argv[2], "--cwd") != 0 ||
-      strncmp(argv[4], "--callbacks=", 12) != 0 || (argc == fixed + 1 && strcmp(argv[fixed], "--timing") != 0))
+  int at = fixed, timing = 0, libraries = 0;
+  if (at < argc && strcmp(argv[at], "--timing") == 0) { timing = 1; at++; }
+  if (at < argc && strcmp(argv[at], "--default-libraries-from-disk") == 0) { libraries = 1; at++; }
+  if (argc < fixed || at != argc || strcmp(argv[1], "--api") != 0 || strcmp(argv[2], "--cwd") != 0 ||
+      strncmp(argv[4], "--callbacks=", 12) != 0)
     return 71;
   const char *mode = argv[3];
 #ifndef _WIN32
@@ -76,7 +79,8 @@ static int child_main(int argc, char **argv) {
     fputc('\n', stdout);
     fputs(argv[4], stdout);
     fputc('\n', stdout);
-    fputs(argc == 6 ? "timing\n" : "plain\n", stdout);
+    fputs(timing ? "timing\n" : "plain\n", stdout);
+    if (libraries) fputs("libraries from disk\n", stdout);
     return ferror(stdout) ? 72 : 0;
   }
   if (strcmp(mode, "exit") == 0) return 0;

packages/compiler/native/ts7-process.c

@@ -161,7 +161,7 @@ static wchar_t *command_line(wchar_t **arguments, size_t count) {
 }
 
 static int start_process(Ts7Process *p, char **args, size_t argc) {
-  wchar_t *wide[6] = {0};
+  wchar_t *wide[7] = {0};
   wchar_t *line = NULL;
   HANDLE child_input = NULL, child_output = NULL, child_error = NULL;
   LPPROC_THREAD_ATTRIBUTE_LIST attributes = NULL;
@@ -330,7 +330,7 @@ static void stop_process(Ts7Process *p) {
 uint32_t scriptc_ts7_open(const uint8_t *executable, size_t executable_size,
                          const uint8_t *cwd, size_t cwd_size,
                          const uint8_t *callbacks, size_t callbacks_size,
-                         uint8_t timing) {
+                         uint8_t timing, uint8_t libraries_from_disk) {
   if (next_id == 0) { error_text("server handle IDs are exhausted"); return 0; }
   char *exe = copy_argument(executable, executable_size, "");
   if (exe == NULL) return 0;
@@ -346,8 +346,12 @@ uint32_t scriptc_ts7_open(const uint8_t *executable, size_t executable_size,
     }
     exit_registered = 1;
   }
-  char *args[] = {exe, "--api", "--cwd", directory, callback_flag, timing ? "--timing" : NULL, NULL};
-  int started = start_process(p, args, timing ? 6 : 5);
+  char *args[8] = {exe, "--api", "--cwd", directory, callback_flag};
+  size_t argc = 5;
+  if (timing) args[argc++] = "--timing";
+  if (libraries_from_disk) args[argc++] = "--default-libraries-from-disk";
+  args[argc] = NULL;
+  int started = start_process(p, args, argc);
   free(exe); free(directory); free(callback_flag);
   if (!started) { free(p); return 0; }
   p->id = next_id++;
@@ -361,9 +365,10 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size,
                            const uint8_t *executable, size_t executable_size,
                            const uint8_t *cwd, size_t cwd_size,
                            const uint8_t *callbacks, size_t callbacks_size,
-                           uint8_t timing, uint32_t idle_exit, uint32_t memory) {
+                           uint8_t timing, uint8_t libraries_from_disk, uint32_t idle_exit,
+                           uint32_t memory) {
   (void)name; (void)name_size; (void)executable; (void)executable_size; (void)cwd; (void)cwd_size;
-  (void)callbacks; (void)callbacks_size; (void)timing; (void)idle_exit; (void)memory;
+  (void)callbacks; (void)callbacks_size; (void)timing; (void)libraries_from_disk; (void)idle_exit; (void)memory;
   error_text("a shared checker server needs Unix sockets");
   return 0;
 }
@@ -455,7 +460,8 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size,
                            const uint8_t *executable, size_t executable_size,
                            const uint8_t *cwd, size_t cwd_size,
                            const uint8_t *callbacks, size_t callbacks_size,
-                           uint8_t timing, uint32_t idle_exit, uint32_t memory) {
+                           uint8_t timing, uint8_t libraries_from_disk, uint32_t idle_exit,
+                           uint32_t memory) {
   if (next_id == 0) { error_text("server handle IDs are exhausted"); return 0; }
   char *server = copy_argument(name, name_size, "");
   if (server == NULL) return 0;
@@ -491,8 +497,11 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size,
     (void)snprintf(limit, sizeof(limit), "TSC_TS_LISTEN_MEMORY=%lu", (unsigned long)memory);
     pid_t pid = -1;
     if (callback_flag != NULL) {
-      char *args[] = {exe, "--api", "--cwd", dir, callback_flag, "--listen", socket_path, "--idle-exit", idle,
-                      timing ? "--timing" : NULL, NULL};
+      char *args[12] = {exe, "--api", "--cwd", dir, callback_flag, "--listen", socket_path, "--idle-exit", idle};
+      size_t argc = 9;
+      if (timing) args[argc++] = "--timing";
+      if (libraries_from_disk) args[argc++] = "--default-libraries-from-disk";
+      args[argc] = NULL;
       pid = start_server(args, limit);
     }
     free(exe); free(dir); free(callback_flag);

packages/compiler/native/ts7-process.ffi.json

@@ -1,8 +1,8 @@
 {
   "ffi_format": 6,
   "functions": [
-    { "name": "ts7NativeOpen", "symbol": "scriptc_ts7_open", "params": ["string", "string", "string", "bool"], "returns": "u32" },
-    { "name": "ts7NativeAttach", "symbol": "scriptc_ts7_attach", "params": ["string", "string", "string", "string", "bool", "u32", "u32"], "returns": "u32" },
+    { "name": "ts7NativeOpen", "symbol": "scriptc_ts7_open", "params": ["string", "string", "string", "bool", "bool"], "returns": "u32" },
+    { "name": "ts7NativeAttach", "symbol": "scriptc_ts7_attach", "params": ["string", "string", "string", "string", "bool", "bool", "u32", "u32"], "returns": "u32" },
     { "name": "ts7NativeRead", "symbol": "scriptc_ts7_read", "params": ["u32", "mutable-bytes", "u32", "u32"], "returns": "i32" },
     { "name": "ts7NativeWrite", "symbol": "scriptc_ts7_write", "params": ["u32", "bytes", "u32", "u32"], "returns": "i32" },
     { "name": "ts7NativePid", "symbol": "scriptc_ts7_pid", "params": ["u32"], "returns": "u32" },

packages/compiler/native/ts7-process.h

@@ -12,7 +12,7 @@
 uint32_t scriptc_ts7_open(const uint8_t *executable, size_t executable_size,
                          const uint8_t *cwd, size_t cwd_size,
                          const uint8_t *callbacks, size_t callbacks_size,
-                         uint8_t timing);
+                         uint8_t timing, uint8_t libraries_from_disk);
 /* Connects to the shared checker server `name` (`--api --listen`) in this
  * user's private server directory, starting it in its own process group
  * with `--idle-exit idle_exit` and a limit of `memory` megabytes
@@ -25,7 +25,8 @@ uint32_t scriptc_ts7_attach(const uint8_t *name, size_t name_size,
                            const uint8_t *executable, size_t executable_size,
                            const uint8_t *cwd, size_t cwd_size,
                            const uint8_t *callbacks, size_t callbacks_size,
-                           uint8_t timing, uint32_t idle_exit, uint32_t memory);
+                           uint8_t timing, uint8_t libraries_from_disk, uint32_t idle_exit,
+                           uint32_t memory);
 int32_t scriptc_ts7_read(uint32_t handle, uint8_t *bytes, size_t size,
                         uint32_t offset, uint32_t length);
 int32_t scriptc_ts7_write(uint32_t handle, const uint8_t *bytes, size_t size,

packages/compiler/native/ts7-process.test.c

@@ -21,12 +21,14 @@
 static const char *executable;
 static const char *callbacks = "readFile,fileExists";
 
-static uint32_t start(const char *mode, int timing) {
+static uint32_t start_with(const char *mode, int timing, int libraries) {
   return scriptc_ts7_open((const uint8_t *)executable, strlen(executable),
                           (const uint8_t *)mode, strlen(mode),
-                          (const uint8_t *)callbacks, strlen(callbacks), (uint8_t)timing);
+                          (const uint8_t *)callbacks, strlen(callbacks), (uint8_t)timing, (uint8_t)libraries);
 }
 
+static uint32_t start(const char *mode, int timing) { return start_with(mode, timing, 0); }
+
 static void fail_message(void) {
   uint8_t message[1024];
   int32_t size = scriptc_ts7_error(message, sizeof(message));
@@ -83,11 +85,13 @@ static void argument_cases(void) {
     "", "a b", "double\"quote", "end\\", "many\\\\\"quotes\\\\", "a;&$()`'b", "h\xc3\xa9llo \xf0\x9f\x8c\x8d",
   };
   for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
-    for (int timing = 0; timing <= 1; timing++) {
+    for (int flags = 0; flags < 4; flags++) {
+      int timing = flags & 1, libraries = flags >> 1;
       char mode[256], expected[512];
       snprintf(mode, sizeof(mode), "argv:%s", values[i]);
-      snprintf(expected, sizeof(expected), "%s\n--callbacks=%s\n%s\n", values[i], callbacks, timing ? "timing" : "plain");
-      uint32_t id = start(mode, timing);
+      snprintf(expected, sizeof(expected), "%s\n--callbacks=%s\n%s\n%s", values[i], callbacks,
+               timing ? "timing" : "plain", libraries ? "libraries from disk\n" : "");
+      uint32_t id = start_with(mode, timing, libraries);
       assert(id != 0);
       uint8_t actual[512];
       read_exact(id, actual, strlen(expected));
@@ -147,10 +151,10 @@ static void failure_cases(void) {
   executable = "/scriptc/does-not-exist/tsgo";
   for (int i = 0; i < 30; i++) { assert(start("echo", 0) == 0); fail_message(); }
   executable = saved;
-  assert(scriptc_ts7_open(NULL, 0, bytes, 1, bytes, 1, 0) == 0);
-  assert(scriptc_ts7_open((const uint8_t *)"abc\0def", 7, bytes, 1, bytes, 1, 0) == 0);
-  assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), NULL, 0, bytes, 1, 0) == 0);
-  assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), bytes, 1, NULL, 0, 0) == 0);
+  assert(scriptc_ts7_open(NULL, 0, bytes, 1, bytes, 1, 0, 0) == 0);
+  assert(scriptc_ts7_open((const uint8_t *)"abc\0def", 7, bytes, 1, bytes, 1, 0, 0) == 0);
+  assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), NULL, 0, bytes, 1, 0, 0) == 0);
+  assert(scriptc_ts7_open((const uint8_t *)executable, strlen(executable), bytes, 1, NULL, 0, 0, 0) == 0);
   for (int i = 0; i < 30; i++) {
     uint32_t id = start("exit", 0);
     assert(id != 0);
@@ -177,7 +181,7 @@ static void failure_cases(void) {
 static uint32_t attach(const char *name, const char *mode, uint32_t memory) {
   return scriptc_ts7_attach((const uint8_t *)name, strlen(name), (const uint8_t *)executable, strlen(executable),
                             (const uint8_t *)mode, strlen(mode), (const uint8_t *)callbacks, strlen(callbacks),
-                            0, 1, memory);
+                            0, 0, 1, memory);
 }
 
 static void expect_line(uint32_t id, const char *expected) {

packages/compiler/src/frontend/ts7/checker-features.test.ts

@@ -0,0 +1,45 @@
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, afterEach, expect, test } from "vitest";
+import { checkerReadsLibrariesFromDisk, setCheckerFeatureStore } from "./checker-features.js";
+
+const dir = mkdtempSync(join(tmpdir(), "scriptc-checker-features-"));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+afterEach(() => setCheckerFeatureStore(null));
+
+/** A stand-in checker that records its arguments and exits with `status`. */
+function checker(name: string, status: number): { path: string; log: string } {
+  const path = join(dir, name);
+  const log = path + ".log";
+  writeFileSync(path, `#!/bin/sh\necho "$@" >> '${log}'\nexit ${status}\n`);
+  chmodSync(path, 0o755);
+  return { path, log };
+}
+
+test.skipIf(process.platform === "win32")(
+  "the checker is asked once whether it reads its library files from the disk, and the answer is stored",
+  async () => {
+    const { readFileSync, existsSync } = await import("node:fs");
+    const entries = new Map<string, string>();
+    setCheckerFeatureStore({
+      read: (key) => entries.get(key) ?? null,
+      write: (key, text) => void entries.set(key, text),
+    });
+    const current = checker("current", 0);
+    const released = checker("released", 2);
+    expect(checkerReadsLibrariesFromDisk(current.path, {})).toBe(true);
+    expect(checkerReadsLibrariesFromDisk(released.path, {})).toBe(false);
+    expect(readFileSync(current.log, "utf8")).toBe("--api --default-libraries-from-disk\n");
+    expect([...entries.values()].sort()).toEqual(["0", "1"]);
+    // Later asks use the answers: the checkers do not run again.
+    expect(checkerReadsLibrariesFromDisk(current.path, {})).toBe(true);
+    expect(checkerReadsLibrariesFromDisk(released.path, {})).toBe(false);
+    expect(readFileSync(current.log, "utf8").split("\n").length).toBe(2);
+    expect(readFileSync(released.log, "utf8").split("\n").length).toBe(2);
+    // tsgo is never asked.
+    const tsgo = checker("tsgo", 0);
+    expect(checkerReadsLibrariesFromDisk(tsgo.path, { SCRIPTC_CHECKER: "tsgo" })).toBe(false);
+    expect(existsSync(tsgo.log)).toBe(false);
+  },
+);

packages/compiler/src/frontend/ts7/checker-features.ts

@@ -0,0 +1,53 @@
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { checkerBackend } from "./checker-backend.js";
+import { checkerExecutableIdentity } from "./lib-diagnostics.js";
+
+/** Text entries kept between builds (the native compiler's cache). */
+export interface CheckerFeatureStore {
+  read(key: string): string | null;
+  write(key: string, text: string): void;
+}
+
+let featureStore: CheckerFeatureStore | null = null;
+const answers = new Map<string, boolean>();
+
+/** Keeps the answers of checkerReadsLibrariesFromDisk between builds. */
+export function setCheckerFeatureStore(store: CheckerFeatureStore | null): void {
+  featureStore = store;
+}
+
+/**
+ * Whether the tsc-ts at `executable` takes `--api --default-libraries-from-disk`, which has it read the files
+ * of its default library directory from the disk instead of through the file system callbacks. tsc-ts releases
+ * before it reject the flag, so it is asked once per executable identity: the stored answer, or else
+ * `tsc-ts --api --default-libraries-from-disk` with an empty input, which exits with 0 when the flag is known
+ * and with a usage error when it is not. tsgo is never asked.
+ */
+export function checkerReadsLibrariesFromDisk(
+  executable: string,
+  env: NodeJS.ProcessEnv = process.env,
+): boolean {
+  if (checkerBackend(env) !== "tsc-ts") return false;
+  const key = createHash("sha256")
+    .update(JSON.stringify(["default-libraries-from-disk", checkerExecutableIdentity(executable)]))
+    .digest("hex");
+  const known = answers.get(key);
+  if (known !== undefined) return known;
+  const stored = featureStore?.read(key) ?? null;
+  let answer: boolean;
+  if (stored === "1" || stored === "0") answer = stored === "1";
+  else {
+    const probe = spawnSync(executable, ["--api", "--default-libraries-from-disk"], {
+      encoding: "utf8",
+      stdio: "pipe",
+      timeout: 10_000,
+    });
+    answer = probe.error === undefined && probe.status === 0;
+    // A probe that could not run says nothing about the flag: ask again next time.
+    if (probe.error === undefined && probe.status !== null)
+      featureStore?.write(key, answer ? "1" : "0");
+  }
+  answers.set(key, answer);
+  return answer;
+}

packages/compiler/src/frontend/ts7/checker-server.ts

@@ -19,14 +19,16 @@ export const CHECKER_SERVER_MEMORY = 2048;
  * answers, and the server keeps the files it parsed for the next build.
  * Returns the server's name, a digest of everything a server's answers
  * depend on besides the files it reads through the build: the tsc-ts
- * executable, the working directory, the file system callbacks, timing, and
- * the `TSC_TS_*` environment. Returns null when the shared server is not used.
+ * executable, the working directory, the file system callbacks, timing,
+ * whether it reads its library files from the disk, and the `TSC_TS_*`
+ * environment. Returns null when the shared server is not used.
  */
 export function checkerServerName(
   executable: string,
   cwd: string,
   timing: boolean,
   env: NodeJS.ProcessEnv = process.env,
+  librariesFromDisk = false,
 ): string | null {
   if (env["SCRIPTC_CHECKER_SERVER"] !== "1") return null;
   if (process.platform !== "linux" && process.platform !== "darwin") return null;
@@ -42,6 +44,7 @@ export function checkerServerName(
         cwd,
         TS7_FILE_SYSTEM_CALLBACKS,
         timing,
+        librariesFromDisk,
         tscTsEnv,
       ]),
     )

packages/compiler/src/frontend/ts7/native-api.ts

@@ -1,4 +1,5 @@
 import { Ts7Api } from "./api.js";
+import { checkerReadsLibrariesFromDisk } from "./checker-features.js";
 import { checkerServerName } from "./checker-server.js";
 import { attachNativeTs7Process, spawnNativeTs7Wire } from "./native-process.js";
 import { Ts7RpcClient } from "./rpc-client.js";
@@ -15,12 +16,17 @@ export function createNativeTs7Api(options: {
   shared?: boolean;
 }): Ts7Api {
   const timing = options.collectTiming ?? false;
+  // The checker reads its own library files when it can: they are part of the
+  // installed tsc-ts, so no build serves them differently.
+  const libraries = checkerReadsLibrariesFromDisk(options.executable);
   const server =
-    options.shared === false ? null : checkerServerName(options.executable, options.cwd, timing);
+    options.shared === false
+      ? null
+      : checkerServerName(options.executable, options.cwd, timing, process.env, libraries);
   const shared =
     server === null
       ? null
-      : attachNativeTs7Process(server, options.executable, options.cwd, timing);
+      : attachNativeTs7Process(server, options.executable, options.cwd, timing, libraries);
   if (shared !== null) {
     const rpc = new Ts7RpcClient(new Ts7Wire(shared));
     try {
@@ -32,7 +38,7 @@ export function createNativeTs7Api(options: {
     }
   }
   return new Ts7Api(
-    new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing)),
+    new Ts7RpcClient(spawnNativeTs7Wire(options.executable, options.cwd, timing, libraries)),
     options.fs,
     timing,
   );

packages/compiler/src/frontend/ts7/native-process.ts

@@ -10,6 +10,7 @@ declare function ts7NativeOpen(
   cwd: string,
   callbacks: string,
   timing: boolean,
+  librariesFromDisk: boolean,
 ): number;
 declare function ts7NativeAttach(
   name: string,
@@ -17,6 +18,7 @@ declare function ts7NativeAttach(
   cwd: string,
   callbacks: string,
   timing: boolean,
+  librariesFromDisk: boolean,
   idleExit: number,
   memory: number,
 ): number;
@@ -63,12 +65,26 @@ export class NativeTs7Process implements Ts7WireIo {
   private handle: number;
   readonly pid: number;
 
-  /** With `handle`, a connection to a shared checker server (attachNativeTs7Process). */
-  constructor(executable: string, cwd: string, collectTiming = false, handle = 0) {
+  /** With `handle`, a connection to a shared checker server (attachNativeTs7Process). With
+   * `librariesFromDisk`, the server reads its default library files from the disk
+   * (checkerReadsLibrariesFromDisk). */
+  constructor(
+    executable: string,
+    cwd: string,
+    collectTiming = false,
+    handle = 0,
+    librariesFromDisk = false,
+  ) {
     this.handle =
       handle !== 0
         ? handle
-        : ts7NativeOpen(executable, cwd, TS7_FILE_SYSTEM_CALLBACKS, collectTiming);
+        : ts7NativeOpen(
+            executable,
+            cwd,
+            TS7_FILE_SYSTEM_CALLBACKS,
+            collectTiming,
+            librariesFromDisk,
+          );
     if (this.handle === 0) throw nativeError("startup");
     this.pid = ts7NativePid(this.handle);
   }
@@ -125,6 +141,7 @@ export function attachNativeTs7Process(
   executable: string,
   cwd: string,
   collectTiming = false,
+  librariesFromDisk = false,
 ): NativeTs7Process | null {
   const handle = ts7NativeAttach(
     name,
@@ -132,6 +149,7 @@ export function attachNativeTs7Process(
     cwd,
     TS7_FILE_SYSTEM_CALLBACKS,
     collectTiming,
+    librariesFromDisk,
     CHECKER_SERVER_IDLE_EXIT,
     CHECKER_SERVER_MEMORY,
   );
@@ -142,6 +160,7 @@ export function spawnNativeTs7Wire(
   executable: string,
   cwd: string,
   collectTiming = false,
+  librariesFromDisk = false,
 ): Ts7Wire {
-  return new Ts7Wire(new NativeTs7Process(executable, cwd, collectTiming));
+  return new Ts7Wire(new NativeTs7Process(executable, cwd, collectTiming, 0, librariesFromDisk));
 }

packages/compiler/src/native/cli.ts

@@ -21,6 +21,7 @@ import {
   checkerExecutableIdentity,
   setLibDiagnosticsStore,
 } from "../frontend/ts7/lib-diagnostics.js";
+import { setCheckerFeatureStore } from "../frontend/ts7/checker-features.js";
 import { loadNativeToolchain } from "./toolchain.js";
 import { buildSanitizedRuntime } from "./sanitizer.js";
 import { runCompilerTask } from "./task.js";
@@ -67,6 +68,11 @@ async function main(): Promise<number> {
         },
         checkerExecutableIdentity(toolchain.ts7Executable),
       );
+      setCheckerFeatureStore({
+        read: (key) =>
+          store.read("checker-features", key, NATIVE_STRING_BYTES)?.toString("utf8") ?? null,
+        write: (key, text) => store.write("checker-features", key, text),
+      });
     }
     compiler = new NativeCompiler(toolchain, cache);
     return compiler;