From 5ab2defc8553ce1fc3d9dbdf7a7f07eadf00d5fb Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Mon, 12 Oct 2026 17:26:24 -0500 Subject: [PATCH] Keep the allocator's untouched reservation out of core dumps --- packages/runtime/src/scr_alloc.c | 75 +++++++++++++++++++++++++-- packages/runtime/src/scr_alloc.test.c | 16 +++++- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/packages/runtime/src/scr_alloc.c b/packages/runtime/src/scr_alloc.c index fa0ba6f47..a6b632095 100644 --- a/packages/runtime/src/scr_alloc.c +++ b/packages/runtime/src/scr_alloc.c @@ -61,6 +61,29 @@ static char scr_sa_thp_mode(void) { } #endif +/* Core dumps. The reservation is tens of GiB of address space, almost none + * of it ever touched. A core written to a file skips untouched pages, but a + * core piped to a handler (a kernel.core_pattern of "|...", such as apport + * or systemd-coredump) carries every page of a dumped mapping, untouched + * ones as zeros, and RLIMIT_CORE does not bound it: an aborting program + * would take seconds to minutes to die. The reservation is therefore mapped + * MADV_DONTDUMP, and whenever the slow path raises a slice's bump limit, + * the slice up to that limit becomes dumpable again (scr_sa_dumpable), so a + * core still holds every carved block. Without pre-faulting, limits advance + * in bounded steps for this (SCR_SA_DUMP_STEP on small pages, one huge page + * past the switch point). The dumpable part ends on a grid that coarsens as + * the slice grows (scr_sa_dump_edge): its untouched tail is at most + * SCR_SA_DUMP_GRAIN or a quarter of what the slice has carved, and a slice + * makes a few madvise calls per doubling, not one per step (each takes the + * mmap lock for writing, which would wait for the pre-fault helper). Grid + * points are huge-page aligned: a mapping boundary inside a huge page would + * split it. */ +#if defined(__linux__) && defined(MADV_DONTDUMP) +#define SCR_SA_DUMP 1 +#define SCR_SA_DUMP_STEP ((size_t)1 << 20) +#define SCR_SA_DUMP_GRAIN ((size_t)2 << 20) /* a multiple of the page size */ +#endif + /* Maps span bytes of address space for slices (huge-page aligned where the * policy above applies), or NULL. */ static void *scr_sa_map(size_t span) { @@ -81,10 +104,36 @@ static void *scr_sa_map(size_t span) { (void)munmap(a + span, (size_t)(p + pad - a)); /* a < p + pad */ p = a; (void)madvise(p, span, MADV_NOHUGEPAGE); +#endif +#ifdef SCR_SA_DUMP + (void)madvise(p, span, MADV_DONTDUMP); #endif return p; } +#ifdef SCR_SA_DUMP +/* The end of the dumpable part of the class slice [start, end) whose + * limit is p: p rounded up to a multiple of SCR_SA_DUMP_GRAIN or of a + * quarter of the largest power of two at or below p - start, whichever is + * larger (see above). Monotonic in p. */ +static char *scr_sa_dump_edge(char *start, char *end, char *p) { + size_t off = (size_t)(p - start), grain = SCR_SA_DUMP_GRAIN; + if (off) { + size_t quarter = ((size_t)1 << (63 - __builtin_clzll((unsigned long long)off))) >> 2; + if (quarter > grain) grain = quarter; + } + off = (off + grain - 1) & ~(grain - 1); + return off < (size_t)(end - start) ? start + off : end; +} + +/* The slice [start, end) raised its bump limit from lim to next: makes + * the memory up to next dumpable (see above). */ +static void scr_sa_dumpable(char *start, char *end, char *lim, char *next) { + char *lo = scr_sa_dump_edge(start, end, lim), *hi = scr_sa_dump_edge(start, end, next); + if (hi > lo) (void)madvise(lo, (size_t)(hi - lo), MADV_DODUMP); +} +#endif + /* Pre-faulting (Linux executables). First touch of a fresh slice page costs * a fault and a page clear on the allocating thread. With pre-faulting the * bump limits advance in steps, and each step asks a helper thread to fault @@ -292,8 +341,8 @@ static void scr_sa_prefault(char *lo, char *hi) { /* The next bump limit of a class slice [start, end) whose limit is lim * (< end): its small-page prefix ends at SCR_SA_HUGE_AFTER, where the rest - * of the slice switches to huge pages. With pre-faulting, limits advance in - * steps (see above). */ + * of the slice switches to huge pages. Limits advance in steps: the + * pre-fault steps (see above), or else the core-dump steps (SCR_SA_DUMP). */ static char *scr_sa_extend(char *start, char *lim, char *end) { #ifdef SCR_SA_THP char *sw = start + SCR_SA_HUGE_AFTER; @@ -320,11 +369,23 @@ static char *scr_sa_extend(char *start, char *lim, char *end) { return next; } #endif - if ((size_t)(lim - start) < SCR_SA_HUGE_AFTER && tiered) return sw; - if (lim > start) (void)madvise(lim, (size_t)(end - lim), MADV_HUGEPAGE); + if (tiered && lim < sw) { +#ifdef SCR_SA_DUMP + if ((size_t)(sw - lim) > SCR_SA_DUMP_STEP) return lim + SCR_SA_DUMP_STEP; +#endif + return sw; + } + if (tiered && lim == sw) (void)madvise(sw, (size_t)(end - sw), MADV_HUGEPAGE); +#ifdef SCR_SA_DUMP + if (tiered && (size_t)(end - lim) > SCR_SA_HUGE_PAGE) return lim + SCR_SA_HUGE_PAGE; +#endif #else (void)start; +#ifdef SCR_SA_DUMP + if ((size_t)(end - lim) > SCR_SA_DUMP_STEP) return lim + SCR_SA_DUMP_STEP; +#else (void)lim; +#endif #endif return end; } @@ -878,7 +939,13 @@ __attribute__((cold, noinline)) void *scr_sa_slow(size_t n, bool zero) { char *end = start + ((size_t)1 << scr_sa.shift); char *p = scr_sa.lim[c] ? scr_sa.bump[c] : start; char *lim = scr_sa.lim[c] ? scr_sa.lim[c] : start; +#ifdef SCR_SA_DUMP + char *from = lim; +#endif while ((size_t)(lim - p) < sz && lim < end) lim = scr_sa_extend(start, lim, end); +#ifdef SCR_SA_DUMP + if (lim > from) scr_sa_dumpable(start, end, from, lim); +#endif scr_sa.lim[c] = lim; if ((size_t)(lim - p) >= sz) { scr_sa.bump[c] = p + sz; diff --git a/packages/runtime/src/scr_alloc.test.c b/packages/runtime/src/scr_alloc.test.c index 58b9da0b8..2f858486e 100644 --- a/packages/runtime/src/scr_alloc.test.c +++ b/packages/runtime/src/scr_alloc.test.c @@ -123,8 +123,9 @@ static void vm_flags(const void *p, char *out, size_t cap) { /* A class slice carves contiguously from its start, across the points where * its limit is raised; on Linux the slices are huge-page aligned, the first - * 8 MiB of a slice stay on small pages and the rest may use huge pages, and - * a helper thread faults in memory ahead of the limit. Every fresh block + * 8 MiB of a slice stay on small pages and the rest may use huge pages, a + * helper thread faults in memory ahead of the limit, and only the slice up + * to its limit is included in core dumps. Every fresh block * still reads as zero, wherever the limit steps and whoever faulted it. */ static void check_slice_growth(void) { enum { SZ = 496, AFTER = 8 << 20, PAST = 3 * (2 << 20) + (1 << 20) }; @@ -152,6 +153,17 @@ static void check_slice_growth(void) { vm_flags((void *)(start + AFTER), flags, sizeof flags); assert(strstr(flags, " hg") && !strstr(flags, " nh")); } + /* Core dumps hold the carved blocks but not the untouched rest of the + * reservation (" dd" is VM_DONTDUMP). */ + char flags[256]; + vm_flags(first, flags, sizeof flags); + assert(!strstr(flags, " dd")); + vm_flags(prev + SZ - 1, flags, sizeof flags); + assert(!strstr(flags, " dd")); + vm_flags((void *)(start + ((uintptr_t)1 << scr_sa.shift) - 1), flags, sizeof flags); + assert(strstr(flags, " dd")); + vm_flags((void *)(scr_sa.base + scr_sa.span - 1), flags, sizeof flags); + assert(strstr(flags, " dd")); #endif /* The blocks recycle through the free list like any others. */ for (size_t i = blocks; i-- > 0;) scr_mem_free(first + i * SZ);