Merge pull request #859 from vercel-labs/perf/worker-fd-table

Grow the descriptor table before worker threads share it

packages/runtime/src/scr_worker.c

@@ -11,6 +11,33 @@
 #include <pthread.h>
 #endif
 
+#if defined(__linux__)
+#include <fcntl.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+/* Every worker's mailbox holds descriptors. Once a process passes 64 of
+ * them, the kernel grows its descriptor table, and while other threads share
+ * the table it first waits a full RCU grace period (synchronize_rcu in
+ * expand_fdtable): measured 11-25 ms on a 16-vCPU host, on the thread that
+ * starts the next worker. Growing the table at load, while the process still
+ * has one thread, never waits; the table never shrinks. 256 entries leave
+ * room for the most workers the allocator gives arenas to. */
+__attribute__((constructor)) static void scr_worker_reserve_fd_table(void) {
+  const int top = 255;
+  struct rlimit limit;
+  if (getrlimit(RLIMIT_NOFILE, &limit) == 0 && limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur <= (rlim_t)top)
+    return;
+  for (int fd = 0; fd <= 2; fd++) {
+    int copy = fcntl(fd, F_DUPFD_CLOEXEC, top);
+    if (copy >= 0) {
+      (void)close(copy);
+      return;
+    }
+  }
+}
+#endif
+
 struct ScrWorkerThread {
   atomic_size_t references;
   atomic_bool cancel;

packages/runtime/src/scr_worker.test.c

@@ -54,7 +54,32 @@ static int entry(uint32_t root_id) {
   return (int)root_id;
 }
 
+#if defined(__linux__)
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+
+/* Worker builds grow the descriptor table at load, while the process has one
+ * thread: growing it later, with threads sharing it, waits a full RCU grace
+ * period on the thread that opens the next worker's mailbox. */
+static void check_fd_table_reserved(void) {
+  struct rlimit limit;
+  if (getrlimit(RLIMIT_NOFILE, &limit) != 0 || (limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur <= 255)) return;
+  FILE *status = fopen("/proc/self/status", "r");
+  assert(status);
+  char line[256];
+  long size = -1;
+  while (fgets(line, sizeof line, status))
+    if (strncmp(line, "FDSize:", 7) == 0) size = strtol(line + 7, NULL, 10);
+  fclose(status);
+  assert(size >= 256);
+}
+#endif
+
 int main(void) {
+#if defined(__linux__)
+  check_fd_table_reserved();
+#endif
   scr_init();
   scr_lib_init(0, NULL);
   ScrMailbox *mailbox = scr_mailbox_new();