From 767c33ebdb1f242c8b6aa0d4cdc1d7846239bef1 Mon Sep 17 00:00:00 2001 From: Malte Ubl Date: Sun, 11 Oct 2026 12:58:50 -0700 Subject: [PATCH] Give glibc's large blocks transparent huge pages Blocks above the small-object classes (array storage, hash tables, link pages, large strings) come from glibc, whose brk heap and per-block mappings stay on 4 KiB pages where THP is in madvise mode; glibc advises its own memory only under the glibc.malloc.hugetlb tunable, which a program cannot set for itself. Once the brk heap passes 32 MiB, every large allocation now advises the heap grown since the previous one (and the aligned interior of big blocks mapped outside it), and glibc's top pad is raised to 32 MiB so heap growth no longer leaves each 2 MiB range straddling the heap end when it is first touched; the mmap and trim thresholds are pinned to their adaptive maxima, which setting the pad would otherwise freeze at their defaults. Linux and glibc only. --- packages/runtime/src/scr_alloc.c | 105 +++++++++++++++++++++++++- packages/runtime/src/scr_alloc.test.c | 37 +++++++++ 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index 32352d9dd..f8888e086 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -82,6 +82,101 @@ static char *scr_sa_extend(char *start, char *lim, char *end) { return end; } +/* Blocks above SCR_SA_MAX (array storage, hash tables, link pages, large + * strings) come from glibc, whose heap and per-block mappings use small + * pages where THP is in madvise mode (the common distribution default); + * in large compiler-like programs that is most of the page faults and a + * share of the TLB misses. glibc only advises huge pages for its own + * memory when the glibc.malloc.hugetlb tunable is set in the environment, + * which a program cannot do for itself. So once the brk heap has grown + * past SCR_SA_HEAP_HUGE_AFTER (small programs keep small pages and their + * RSS), every large allocation advises the heap grown since the previous + * one, and the aligned interior of a block mapped outside the heap. + * glibc grows the heap by its top pad beyond each request; the default + * (128 KiB) is smaller than a huge page, so every 2 MiB range would be + * first touched while it still straddles the heap end, where it can only + * get small pages. The policy therefore raises the top pad. Setting it + * fixes glibc's otherwise adaptive mmap and trim thresholds, so they are + * set to the adaptive maxima the heap of such a program reaches anyway. */ +#if defined(SCR_SA_THP) && defined(__GLIBC__) +#include +#include +#include +#include +#include +#define SCR_SA_HEAP_HUGE_AFTER ((uintptr_t)32 << 20) +#define SCR_SA_HEAP_TOP_PAD (32 << 20) +#define SCR_SA_HEAP_MMAP_THRESHOLD (32 << 20) /* glibc's adaptive maximum */ +#define SCR_SA_HEAP_TRIM_THRESHOLD (64 << 20) /* twice that, as glibc keeps it */ +/* 0 = undecided, 1 = following, 2 = off (THP not in madvise mode). */ +static _Atomic(int) scr_sa_heap_state; +static _Atomic(uintptr_t) scr_sa_heap_lo; /* heap start seen at the first large block */ +static _Atomic(uintptr_t) scr_sa_heap_hi; /* end of the advised heap prefix */ +static _Atomic(uintptr_t) scr_sa_heap_page; /* the page size, read with the mode */ + +static bool scr_sa_thp_madvise_mode(void) { + FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r"); + if (!f) return false; + char buf[96] = {0}; + size_t n = fread(buf, 1, sizeof buf - 1, f); + fclose(f); + buf[n] = 0; + return strstr(buf, "[madvise]") != NULL; +} + +static void scr_sa_large_note(void *p, size_t n) { + int state = atomic_load_explicit(&scr_sa_heap_state, memory_order_relaxed); + if (state == 2 || !p) return; + uintptr_t top = (uintptr_t)sbrk(0); + if (state == 0) { + uintptr_t lo = atomic_load_explicit(&scr_sa_heap_lo, memory_order_relaxed); + if (!lo) { + long page = sysconf(_SC_PAGESIZE); + if (page <= 0 || !scr_sa_thp_madvise_mode()) { + atomic_store_explicit(&scr_sa_heap_state, 2, memory_order_relaxed); + return; + } + atomic_store_explicit(&scr_sa_heap_page, (uintptr_t)page, memory_order_relaxed); + lo = (top + (uintptr_t)page - 1) & ~((uintptr_t)page - 1); + uintptr_t none = 0; + if (!atomic_compare_exchange_strong(&scr_sa_heap_lo, &none, lo)) lo = none; + } + if (top < lo + SCR_SA_HEAP_HUGE_AFTER) return; + uintptr_t unset = 0; /* the whole heap seen so far gets the advice */ + (void)atomic_compare_exchange_strong(&scr_sa_heap_hi, &unset, lo); + int undecided = 0; + if (atomic_compare_exchange_strong(&scr_sa_heap_state, &undecided, 1)) { + (void)mallopt(M_MMAP_THRESHOLD, SCR_SA_HEAP_MMAP_THRESHOLD); + (void)mallopt(M_TRIM_THRESHOLD, SCR_SA_HEAP_TRIM_THRESHOLD); + (void)mallopt(M_TOP_PAD, SCR_SA_HEAP_TOP_PAD); + } + } + /* The heap end moved since the last look: advise what it gained (pages + * already touched keep their small pages), or follow a trim down. */ + uintptr_t hi = atomic_load_explicit(&scr_sa_heap_hi, memory_order_relaxed); + if (!hi) return; /* another thread is switching the policy on */ + if (top > hi) { + if (atomic_compare_exchange_strong(&scr_sa_heap_hi, &hi, top)) { + /* madvise wants a page-aligned start */ + uintptr_t from = hi & ~(atomic_load_explicit(&scr_sa_heap_page, memory_order_relaxed) - 1); + (void)madvise((void *)from, top - from, MADV_HUGEPAGE); + } + } else if (top < hi) { + (void)atomic_compare_exchange_strong(&scr_sa_heap_hi, &hi, top); + } + /* A block glibc mapped by itself (or carved from a thread's heap). */ + uintptr_t a = (uintptr_t)p, lo = atomic_load_explicit(&scr_sa_heap_lo, memory_order_relaxed); + if ((a < lo || a >= top) && n >= 2 * SCR_SA_HUGE_PAGE) { + uintptr_t s = (a + SCR_SA_HUGE_PAGE - 1) & ~(uintptr_t)(SCR_SA_HUGE_PAGE - 1); + uintptr_t e = (a + n) & ~(uintptr_t)(SCR_SA_HUGE_PAGE - 1); + if (e > s) (void)madvise((void *)s, e - s, MADV_HUGEPAGE); + } +} +#define SCR_SA_LARGE_NOTE(p, n) scr_sa_large_note((p), (n)) +#else +#define SCR_SA_LARGE_NOTE(p, n) ((void)0) +#endif + #ifdef SCR_WORKERS #include #include @@ -547,7 +642,9 @@ __attribute__((cold, noinline)) void *scr_sa_slow(size_t n, bool zero) { scr_sa.bump[c] = p; } /* Large or zero-sized, an exhausted slice, or no reservation. */ - return zero ? calloc(1, n) : malloc(n); + void *q = zero ? calloc(1, n) : malloc(n); + if (n > SCR_SA_MAX) SCR_SA_LARGE_NOTE(q, n); + return q; } void *scr_mem_realloc(void *p, size_t n) { @@ -560,7 +657,11 @@ void *scr_mem_realloc(void *p, size_t n) { /* Another arena's block: move it into this thread's allocator. */ } #endif - else return realloc(p, n); + else { + void *q = realloc(p, n); + if (n > SCR_SA_MAX) SCR_SA_LARGE_NOTE(q, n); + return q; + } if (n && n <= have) return p; /* still fits its block */ void *q = scr_mem_alloc(n ? n : 1); if (!q) return NULL; diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index 9978c819e..ec4f38277 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -154,6 +154,40 @@ static void check_slice_growth(void) { assert(scr_mem_alloc(SZ) == first); scr_mem_free(first); } + +#if defined(__linux__) && defined(__GLIBC__) +#include +/* Blocks above SCR_SA_MAX come from glibc; once its brk heap has grown past + * 32 MiB, the heap is advised huge pages where THP is in madvise mode. */ +static void check_heap_huge_pages(void) { + enum { N = 1024, SZ = 64 << 10 }; + static unsigned char *block[N]; + uintptr_t start = (uintptr_t)sbrk(0); + for (size_t i = 0; i < N; i++) { + block[i] = scr_mem_alloc(SZ); + assert(block[i] && !owned(block[i])); + block[i][0] = (unsigned char)i; + } + uintptr_t top = (uintptr_t)sbrk(0); + char mode[96] = {0}; + FILE *thp = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r"); + if (thp) { + size_t n = fread(mode, 1, sizeof mode - 1, thp); + mode[n] = '\0'; + fclose(thp); + } + /* Only where glibc's heap served them (a sanitizer may replace malloc). */ + if (top - start >= ((size_t)48 << 20) && strstr(mode, "[madvise]")) { + char flags[256]; + vm_flags(block[N - 1], flags, sizeof flags); + assert(strstr(flags, " hg")); + } + for (size_t i = 0; i < N; i++) { + assert(block[i][0] == (unsigned char)i); + scr_mem_free(block[i]); + } +} +#endif #endif #if defined(SCR_WORKERS) && SCR_SMALL_ALLOC @@ -310,6 +344,9 @@ int main(void) { check_realloc(); check_system_pointers(); check_churn(); +#if SCR_SMALL_ALLOC && defined(__linux__) && defined(__GLIBC__) + check_heap_huge_pages(); +#endif #if defined(SCR_WORKERS) && SCR_SMALL_ALLOC check_threads(); #ifdef __linux__