Return the pages faulted in past a parked arena's frontiers

When a script thread parks its allocator arena, each class slice still
holds the pages past its bump pointer that were faulted in but never
carved: the rest of the huge page the bump pointer is in, and the pages
the pre-fault helper faulted in ahead of it. The same applies to region
chunks when the thread exits. Until another thread claims the arena,
nothing carves there, so a program that continues on threads that never
claim it (parse threads ahead of a check on the main thread) kept them.

- scr_sa_park_arena returns those pages with MADV_DONTNEED. The next
  thread to claim the arena faults them in again as it carves, and they
  read as zero, as fresh pages do.
- scr_rgn_thread_exit returns each chunk's pages past its bump on worker
  threads; nothing carves in an exited thread's regions.
- scr_mem_release_ahead drops pre-fault requests that overlap the range
  and waits for one in progress before returning it, so the helper does
  not fault the pages in again.
- scr_alloc.test.c checks that a worker that shares and carves past the
  huge-page switch leaves no resident pages past its frontier, keeps its
  blocks, and that the next thread carves on from the frontier, zeroed.

packages/runtime/src/scr_alloc.c

@@ -226,6 +226,11 @@ static pthread_cond_t scr_sa_pf_cond = PTHREAD_COND_INITIALIZER;
 static ScrSaPfRange scr_sa_pf_ring[SCR_SA_PF_RING];
 static unsigned scr_sa_pf_head, scr_sa_pf_tail; /* under the lock */
 static bool scr_sa_pf_started, scr_sa_pf_waiting;
+/* The range the helper is faulting in, and the threads that wait for it to
+ * finish (scr_sa_pf_forget); under the lock. */
+static ScrSaPfRange scr_sa_pf_busy;
+static unsigned scr_sa_pf_forgetting;
+static pthread_cond_t scr_sa_pf_done = PTHREAD_COND_INITIALIZER;
 /* 0 = not decided yet, 1 = on, -1 = off (environment, no thread, or a
  * kernel without MADV_POPULATE_WRITE). */
 static _Atomic(int) scr_sa_pf_state;
@@ -323,8 +328,9 @@ static void *scr_sa_pf_main(void *arg) {
       scr_sa_pf_waiting = false;
     }
     ScrSaPfRange r = scr_sa_pf_ring[scr_sa_pf_tail++ % SCR_SA_PF_RING];
+    scr_sa_pf_busy = r;
     pthread_mutex_unlock(&scr_sa_pf_lock);
-    scr_sa_pf_populate(r.lo, r.hi);
+    if (r.hi > r.lo) scr_sa_pf_populate(r.lo, r.hi);
 #ifdef SCR_SA_PF_HEAP
     if (scr_sa_pf_heap_mode) {
       (void)scr_sa_pf_heap();
@@ -332,10 +338,26 @@ static void *scr_sa_pf_main(void *arg) {
     }
 #endif
     pthread_mutex_lock(&scr_sa_pf_lock);
+    scr_sa_pf_busy = (ScrSaPfRange){NULL, NULL};
+    if (scr_sa_pf_forgetting) pthread_cond_broadcast(&scr_sa_pf_done);
   }
   return NULL;
 }
 
+/* Drops the requests that overlap [lo, hi) and waits for the one the helper
+ * is faulting in if it overlaps, so pages returned there stay returned. */
+static void scr_sa_pf_forget(char *lo, char *hi) {
+  pthread_mutex_lock(&scr_sa_pf_lock);
+  for (unsigned k = scr_sa_pf_tail; k != scr_sa_pf_head; k++) {
+    ScrSaPfRange *r = &scr_sa_pf_ring[k % SCR_SA_PF_RING];
+    if (r->lo < hi && r->hi > lo) r->hi = r->lo;
+  }
+  scr_sa_pf_forgetting++;
+  while (scr_sa_pf_busy.lo < hi && scr_sa_pf_busy.hi > lo) pthread_cond_wait(&scr_sa_pf_done, &scr_sa_pf_lock);
+  scr_sa_pf_forgetting--;
+  pthread_mutex_unlock(&scr_sa_pf_lock);
+}
+
 static bool scr_sa_pf_on(void) {
   int s = atomic_load_explicit(&scr_sa_pf_state, memory_order_relaxed);
   if (s == 0) {
@@ -1482,6 +1504,38 @@ void scr_sa_thread_park(void) {
 #endif
 }
 
+#if defined(SCR_SA_TRIM) && defined(MADV_DONTNEED)
+/* Returns the pages past each slice's frontier that a parking thread leaves
+ * faulted in but never carved: the rest of the huge page the frontier is in
+ * (huge pages fault in whole) and the steps faulted ahead of it. The
+ * thread's objects stay where they are; the next thread to claim the arena
+ * faults the pages in again as it carves, and they read as zero, as fresh
+ * pages do. Until then, a program that does its remaining work on threads
+ * that never claim the arena (parse threads ahead of a check on the main
+ * thread) does not keep them. */
+static void scr_sa_release_frontiers(void) {
+  uintptr_t page = (uintptr_t)sysconf(_SC_PAGESIZE);
+  if (page == 0 || page == (uintptr_t)-1) page = 4096;
+#ifdef SCR_SA_THP
+  const uintptr_t ahead = SCR_SA_HUGE_PAGE;
+#else
+  const uintptr_t ahead = page;
+#endif
+  for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
+    if (!scr_sa.lim[c]) continue;
+    const ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
+    char *bump = h->lim ? h->bump : scr_sa.bump[c];
+    char *lim = h->lim ? h->lim : scr_sa.lim[c];
+    char *end = (char *)scr_sa.base + ((size_t)(c + 1) << scr_sa.shift);
+    char *from = (char *)(((uintptr_t)bump + page - 1) & ~(page - 1));
+    /* At most one step of `ahead` is faulted in past the limit. */
+    char *to = (char *)((((uintptr_t)lim + ahead - 1) & ~(ahead - 1)) + ahead);
+    if (to > end) to = end;
+    if (to > from) scr_mem_release_ahead(from, to);
+  }
+}
+#endif
+
 static void scr_sa_park_arena(void) {
   if (scr_sa_arena < 0) return;
   unsigned i = (unsigned)scr_sa_arena;
@@ -1503,6 +1557,9 @@ static void scr_sa_park_arena(void) {
     return;
   }
   if (k) scr_sa_keep_merge(k);
+#ifdef MADV_DONTNEED
+  scr_sa_release_frontiers();
+#endif
   if (scr_sa_trim_worth()) {
     if (scr_sa_start_helper(i)) {
       /* Later frees on this thread take the foreign path into the arena's
@@ -1748,6 +1805,18 @@ void scr_mem_prefault(void *lo, void *hi) {
 #endif
 }
 
+void scr_mem_release_ahead(void *lo, void *hi) {
+#if SCR_SMALL_ALLOC && defined(SCR_SA_PREFAULT)
+  if (atomic_load_explicit(&scr_sa_pf_state, memory_order_relaxed) > 0) scr_sa_pf_forget(lo, hi);
+#endif
+#if defined(__linux__) && defined(MADV_DONTNEED)
+  if ((char *)hi > (char *)lo) (void)madvise(lo, (size_t)((char *)hi - (char *)lo), MADV_DONTNEED);
+#else
+  (void)lo;
+  (void)hi;
+#endif
+}
+
 /* Free order. Free lists are LIFO, so blocks freed in descending address
  * order come back in ascending order, the order the bump pointers carve
  * fresh memory in, and the order the program then touches them in. The

packages/runtime/src/scr_alloc.test.c

@@ -442,6 +442,68 @@ static void check_trim(void) {
   }
   free(trim_blocks);
 }
+
+/* A thread that parks returns the pages faulted in past each slice's
+ * frontier (the rest of the frontier's huge page and the pages faulted in
+ * ahead), and its blocks stay; the next thread to claim the arena carves on
+ * from the frontier, zeroed. FRONTIER_SIZE is a class the other checks leave
+ * unused, and FRONTIER_BYTES carves past SCR_SA_HUGE_AFTER. */
+enum { FRONTIER_SIZE = 120, FRONTIER_BYTES = 12 << 20, FRONTIER_MORE = 4096 };
+static unsigned char **frontier_blocks;
+static size_t frontier_n;
+static uintptr_t frontier_hi, frontier_end;
+static size_t frontier_past;
+
+static void *frontier_allocate(void *arg) {
+  (void)arg;
+  scr_sa_thread_starting();
+  for (size_t i = 0; i < frontier_n; i++) {
+    unsigned char *p = scr_mem_alloc(FRONTIER_SIZE);
+    assert(p && owned(p));
+    fill(p, FRONTIER_SIZE, (unsigned char)i);
+    frontier_blocks[i] = p;
+    if ((uintptr_t)p + FRONTIER_SIZE > frontier_hi) frontier_hi = (uintptr_t)p + FRONTIER_SIZE;
+  }
+  uintptr_t c = ((uintptr_t)frontier_blocks[0] - scr_sa.base) >> scr_sa.shift;
+  frontier_end = scr_sa.base + ((c + 1) << scr_sa.shift);
+  scr_sa_thread_shares();
+  scr_sa_thread_park();
+  return NULL;
+}
+
+static void *frontier_continue(void *arg) {
+  (void)arg;
+  for (size_t i = 0; i < FRONTIER_MORE; i++) {
+    unsigned char *p = scr_mem_calloc(FRONTIER_SIZE);
+    assert(p && owned(p));
+    for (size_t k = 0; k < FRONTIER_SIZE; k++) assert(p[k] == 0);
+    if ((uintptr_t)p >= frontier_hi) frontier_past++;
+    scr_mem_free(p);
+  }
+  scr_sa_thread_park();
+  return NULL;
+}
+
+static void check_release_frontiers(void) {
+  frontier_n = FRONTIER_BYTES / FRONTIER_SIZE;
+  frontier_blocks = malloc(frontier_n * sizeof *frontier_blocks);
+  assert(frontier_blocks);
+  pthread_t t;
+  assert(pthread_create(&t, NULL, frontier_allocate, NULL) == 0);
+  assert(pthread_join(t, NULL) == 0);
+  scr_sa_wait_trims();
+  size_t page = (size_t)sysconf(_SC_PAGESIZE);
+  uintptr_t from = (frontier_hi + page - 1) & ~(uintptr_t)(page - 1);
+  uintptr_t to = from + ((size_t)4 << 20) < frontier_end ? from + ((size_t)4 << 20) : frontier_end;
+  if (to > from) assert(resident_pages(from, to) == 0);
+  for (size_t i = 0; i < frontier_n; i++) verify(frontier_blocks[i], FRONTIER_SIZE, (unsigned char)i);
+  assert(pthread_create(&t, NULL, frontier_continue, NULL) == 0);
+  assert(pthread_join(t, NULL) == 0);
+  /* The freed block comes back each time: one block past the frontier. */
+  assert(frontier_past >= 1);
+  for (size_t i = 0; i < frontier_n; i++) scr_mem_free(frontier_blocks[i]); /* remote frees */
+  free(frontier_blocks);
+}
 #endif
 
 #if defined(SCR_WORKERS) && SCR_SMALL_ALLOC && defined(__linux__)
@@ -920,6 +982,7 @@ int main(void) {
   check_threads();
 #ifdef __linux__
   check_trim();
+  check_release_frontiers();
   check_discard();
   check_discard_delay_grows();
   check_discard_waits_for_loop();

packages/runtime/src/scr_rc.h

@@ -273,6 +273,10 @@ void scr_rt_free(void *p);
  * of [lo, hi), aligned to 256 KiB, in ahead of use; builds without the
  * helper ignore it. */
 void scr_mem_prefault(void *lo, void *hi);
+/* Returns the pages of [lo, hi) (page aligned), which hold no data, to the
+ * system (Linux): pre-fault requests for them are dropped first, so they
+ * stay returned. Other builds ignore it. */
+void scr_mem_release_ahead(void *lo, void *hi);
 /* Reorders ptrs, distinct pointers `offset` bytes into blocks that are
  * about to be freed in array order, so that the free lists hand the blocks
  * back in ascending address order (scr_alloc.c). Builds without the

packages/runtime/src/scr_region.c

@@ -1362,6 +1362,18 @@ static void scr_rgn_thread_exit(void) {
     ScrRegion *r = i < t->nregions ? t->regions[i] : all[0];
     if (!r) continue;
     scr_rgn_spare_drop(r);
+#if defined(SCR_WORKERS) && SCR_RGN_MMAP && !defined(SCR_RGN_POISON)
+    /* Nothing is carved here after a script thread exits: the pages faulted
+     * in ahead of each chunk's bump go back while the program goes on, and
+     * its objects stay. */
+    if (!scr_context_is_main())
+      for (ScrRgnChunk *c = r->chunks; c; c = c->next) {
+        /* 64 KiB steps cover every page size, as in scr_rgn_chunk_retire. */
+        char *from = (char *)(((uintptr_t)c->bump + 0xffff) & ~(uintptr_t)0xffff);
+        if (c->touched > from) scr_mem_release_ahead(from, c->touched);
+        c->touched = c->bump;
+      }
+#endif
     for (ScrRgnSeg *s = r->segs, *next; s; s = next) {
       next = s->next;
       free(s);