diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index 9f8f4c6c1..8cf14fb7f 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -226,6 +226,11 @@ static pthread_cond_t scr_sa_pf_cond = PTHREAD_COND_INITIALIZER; static ScrSaPfRange scr_sa_pf_ring[SCR_SA_PF_RING]; static unsigned scr_sa_pf_head, scr_sa_pf_tail; /* under the lock */ static bool scr_sa_pf_started, scr_sa_pf_waiting; +/* The range the helper is faulting in, and the threads that wait for it to + * finish (scr_sa_pf_forget); under the lock. */ +static ScrSaPfRange scr_sa_pf_busy; +static unsigned scr_sa_pf_forgetting; +static pthread_cond_t scr_sa_pf_done = PTHREAD_COND_INITIALIZER; /* 0 = not decided yet, 1 = on, -1 = off (environment, no thread, or a * kernel without MADV_POPULATE_WRITE). */ static _Atomic(int) scr_sa_pf_state; @@ -323,8 +328,9 @@ static void *scr_sa_pf_main(void *arg) { scr_sa_pf_waiting = false; } ScrSaPfRange r = scr_sa_pf_ring[scr_sa_pf_tail++ % SCR_SA_PF_RING]; + scr_sa_pf_busy = r; pthread_mutex_unlock(&scr_sa_pf_lock); - scr_sa_pf_populate(r.lo, r.hi); + if (r.hi > r.lo) scr_sa_pf_populate(r.lo, r.hi); #ifdef SCR_SA_PF_HEAP if (scr_sa_pf_heap_mode) { (void)scr_sa_pf_heap(); @@ -332,10 +338,26 @@ static void *scr_sa_pf_main(void *arg) { } #endif pthread_mutex_lock(&scr_sa_pf_lock); + scr_sa_pf_busy = (ScrSaPfRange){NULL, NULL}; + if (scr_sa_pf_forgetting) pthread_cond_broadcast(&scr_sa_pf_done); } return NULL; } +/* Drops the requests that overlap [lo, hi) and waits for the one the helper + * is faulting in if it overlaps, so pages returned there stay returned. */ +static void scr_sa_pf_forget(char *lo, char *hi) { + pthread_mutex_lock(&scr_sa_pf_lock); + for (unsigned k = scr_sa_pf_tail; k != scr_sa_pf_head; k++) { + ScrSaPfRange *r = &scr_sa_pf_ring[k % SCR_SA_PF_RING]; + if (r->lo < hi && r->hi > lo) r->hi = r->lo; + } + scr_sa_pf_forgetting++; + while (scr_sa_pf_busy.lo < hi && scr_sa_pf_busy.hi > lo) pthread_cond_wait(&scr_sa_pf_done, &scr_sa_pf_lock); + scr_sa_pf_forgetting--; + pthread_mutex_unlock(&scr_sa_pf_lock); +} + static bool scr_sa_pf_on(void) { int s = atomic_load_explicit(&scr_sa_pf_state, memory_order_relaxed); if (s == 0) { @@ -1482,6 +1504,38 @@ void scr_sa_thread_park(void) { #endif } +#if defined(SCR_SA_TRIM) && defined(MADV_DONTNEED) +/* Returns the pages past each slice's frontier that a parking thread leaves + * faulted in but never carved: the rest of the huge page the frontier is in + * (huge pages fault in whole) and the steps faulted ahead of it. The + * thread's objects stay where they are; the next thread to claim the arena + * faults the pages in again as it carves, and they read as zero, as fresh + * pages do. Until then, a program that does its remaining work on threads + * that never claim the arena (parse threads ahead of a check on the main + * thread) does not keep them. */ +static void scr_sa_release_frontiers(void) { + uintptr_t page = (uintptr_t)sysconf(_SC_PAGESIZE); + if (page == 0 || page == (uintptr_t)-1) page = 4096; +#ifdef SCR_SA_THP + const uintptr_t ahead = SCR_SA_HUGE_PAGE; +#else + const uintptr_t ahead = page; +#endif + for (unsigned c = 0; c < SCR_SA_NCLASS; c++) { + if (!scr_sa.lim[c]) continue; + const ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c]; + char *bump = h->lim ? h->bump : scr_sa.bump[c]; + char *lim = h->lim ? h->lim : scr_sa.lim[c]; + char *end = (char *)scr_sa.base + ((size_t)(c + 1) << scr_sa.shift); + char *from = (char *)(((uintptr_t)bump + page - 1) & ~(page - 1)); + /* At most one step of `ahead` is faulted in past the limit. */ + char *to = (char *)((((uintptr_t)lim + ahead - 1) & ~(ahead - 1)) + ahead); + if (to > end) to = end; + if (to > from) scr_mem_release_ahead(from, to); + } +} +#endif + static void scr_sa_park_arena(void) { if (scr_sa_arena < 0) return; unsigned i = (unsigned)scr_sa_arena; @@ -1503,6 +1557,9 @@ static void scr_sa_park_arena(void) { return; } if (k) scr_sa_keep_merge(k); +#ifdef MADV_DONTNEED + scr_sa_release_frontiers(); +#endif if (scr_sa_trim_worth()) { if (scr_sa_start_helper(i)) { /* Later frees on this thread take the foreign path into the arena's @@ -1748,6 +1805,18 @@ void scr_mem_prefault(void *lo, void *hi) { #endif } +void scr_mem_release_ahead(void *lo, void *hi) { +#if SCR_SMALL_ALLOC && defined(SCR_SA_PREFAULT) + if (atomic_load_explicit(&scr_sa_pf_state, memory_order_relaxed) > 0) scr_sa_pf_forget(lo, hi); +#endif +#if defined(__linux__) && defined(MADV_DONTNEED) + if ((char *)hi > (char *)lo) (void)madvise(lo, (size_t)((char *)hi - (char *)lo), MADV_DONTNEED); +#else + (void)lo; + (void)hi; +#endif +} + /* Free order. Free lists are LIFO, so blocks freed in descending address * order come back in ascending order, the order the bump pointers carve * fresh memory in, and the order the program then touches them in. The diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index 042c04c65..2d8024c6d 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -442,6 +442,68 @@ static void check_trim(void) { } free(trim_blocks); } + +/* A thread that parks returns the pages faulted in past each slice's + * frontier (the rest of the frontier's huge page and the pages faulted in + * ahead), and its blocks stay; the next thread to claim the arena carves on + * from the frontier, zeroed. FRONTIER_SIZE is a class the other checks leave + * unused, and FRONTIER_BYTES carves past SCR_SA_HUGE_AFTER. */ +enum { FRONTIER_SIZE = 120, FRONTIER_BYTES = 12 << 20, FRONTIER_MORE = 4096 }; +static unsigned char **frontier_blocks; +static size_t frontier_n; +static uintptr_t frontier_hi, frontier_end; +static size_t frontier_past; + +static void *frontier_allocate(void *arg) { + (void)arg; + scr_sa_thread_starting(); + for (size_t i = 0; i < frontier_n; i++) { + unsigned char *p = scr_mem_alloc(FRONTIER_SIZE); + assert(p && owned(p)); + fill(p, FRONTIER_SIZE, (unsigned char)i); + frontier_blocks[i] = p; + if ((uintptr_t)p + FRONTIER_SIZE > frontier_hi) frontier_hi = (uintptr_t)p + FRONTIER_SIZE; + } + uintptr_t c = ((uintptr_t)frontier_blocks[0] - scr_sa.base) >> scr_sa.shift; + frontier_end = scr_sa.base + ((c + 1) << scr_sa.shift); + scr_sa_thread_shares(); + scr_sa_thread_park(); + return NULL; +} + +static void *frontier_continue(void *arg) { + (void)arg; + for (size_t i = 0; i < FRONTIER_MORE; i++) { + unsigned char *p = scr_mem_calloc(FRONTIER_SIZE); + assert(p && owned(p)); + for (size_t k = 0; k < FRONTIER_SIZE; k++) assert(p[k] == 0); + if ((uintptr_t)p >= frontier_hi) frontier_past++; + scr_mem_free(p); + } + scr_sa_thread_park(); + return NULL; +} + +static void check_release_frontiers(void) { + frontier_n = FRONTIER_BYTES / FRONTIER_SIZE; + frontier_blocks = malloc(frontier_n * sizeof *frontier_blocks); + assert(frontier_blocks); + pthread_t t; + assert(pthread_create(&t, NULL, frontier_allocate, NULL) == 0); + assert(pthread_join(t, NULL) == 0); + scr_sa_wait_trims(); + size_t page = (size_t)sysconf(_SC_PAGESIZE); + uintptr_t from = (frontier_hi + page - 1) & ~(uintptr_t)(page - 1); + uintptr_t to = from + ((size_t)4 << 20) < frontier_end ? from + ((size_t)4 << 20) : frontier_end; + if (to > from) assert(resident_pages(from, to) == 0); + for (size_t i = 0; i < frontier_n; i++) verify(frontier_blocks[i], FRONTIER_SIZE, (unsigned char)i); + assert(pthread_create(&t, NULL, frontier_continue, NULL) == 0); + assert(pthread_join(t, NULL) == 0); + /* The freed block comes back each time: one block past the frontier. */ + assert(frontier_past >= 1); + for (size_t i = 0; i < frontier_n; i++) scr_mem_free(frontier_blocks[i]); /* remote frees */ + free(frontier_blocks); +} #endif #if defined(SCR_WORKERS) && SCR_SMALL_ALLOC && defined(__linux__) @@ -920,6 +982,7 @@ int main(void) { check_threads(); #ifdef __linux__ check_trim(); + check_release_frontiers(); check_discard(); check_discard_delay_grows(); check_discard_waits_for_loop(); diff --git a/packages/runtime/src/scr_rc.h b/packages/runtime/src/scr_rc.h index 889202975..8fed602c6 100644 --- a/packages/runtime/src/scr_rc.h +++ b/packages/runtime/src/scr_rc.h @@ -273,6 +273,10 @@ void scr_rt_free(void *p); * of [lo, hi), aligned to 256 KiB, in ahead of use; builds without the * helper ignore it. */ void scr_mem_prefault(void *lo, void *hi); +/* Returns the pages of [lo, hi) (page aligned), which hold no data, to the + * system (Linux): pre-fault requests for them are dropped first, so they + * stay returned. Other builds ignore it. */ +void scr_mem_release_ahead(void *lo, void *hi); /* Reorders ptrs, distinct pointers `offset` bytes into blocks that are * about to be freed in array order, so that the free lists hand the blocks * back in ascending address order (scr_alloc.c). Builds without the diff --git a/packages/runtime/src/scr_region.c b/packages/runtime/src/scr_region.c index c9d6c5511..3a7dad541 100644 --- a/packages/runtime/src/scr_region.c +++ b/packages/runtime/src/scr_region.c @@ -1362,6 +1362,18 @@ static void scr_rgn_thread_exit(void) { ScrRegion *r = i < t->nregions ? t->regions[i] : all[0]; if (!r) continue; scr_rgn_spare_drop(r); +#if defined(SCR_WORKERS) && SCR_RGN_MMAP && !defined(SCR_RGN_POISON) + /* Nothing is carved here after a script thread exits: the pages faulted + * in ahead of each chunk's bump go back while the program goes on, and + * its objects stay. */ + if (!scr_context_is_main()) + for (ScrRgnChunk *c = r->chunks; c; c = c->next) { + /* 64 KiB steps cover every page size, as in scr_rgn_chunk_retire. */ + char *from = (char *)(((uintptr_t)c->bump + 0xffff) & ~(uintptr_t)0xffff); + if (c->touched > from) scr_mem_release_ahead(from, c->touched); + c->touched = c->bump; + } +#endif for (ScrRgnSeg *s = r->segs, *next; s; s = next) { next = s->next; free(s);