mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-11-09 20:53:14 +08:00
Chris Tate
4fe392cd25
On Linux, a worker thread that exits without having made any object it allocated reachable from another thread gives back all of its memory: the pages it carved, including ProgramLifetime instances and other objects it never freed, and the system blocks it still holds. The memory goes back once no other worker thread is running, so that returning it does not slow the workers that are still running. A worker carves past each slice's frontier while the arena's free lists and returned regions are set aside, takes from those free lists only once a slice has no bump room left, and records the system blocks it allocates. Messages and intern tables that share published graphs by pointer, and the process-wide key arrays of typed records, mark the thread as sharing; so does a remote free of one of its blocks. A worker that did not share parks with its arena claimed and its discard queued: returning pages makes the kernel interrupt every running thread of the process to flush its TLB, and glibc's trim locks the arenas other threads allocate from. The last worker to park starts one helper that returns each queued worker's pages, frees its system blocks, and restores its arena to its state at the claim, with the free blocks the worker took free again. A thread that finds every arena claimed runs the queued discards itself. A worker that shared keeps its memory as before. SCR_SA_DISCARD=0 keeps every worker's memory; SCR_SA_DISCARD=poison makes the discarded pages inaccessible instead, for tests.
1780 lines
69 KiB
C
1780 lines
69 KiB
C
/* scriptc runtime — small-object allocator state and slow paths.
|
|
*
|
|
* The fast paths (free-list pop, bump carve, free-list push) are inline in
|
|
* scr_runtime.h; see the contract there. This unit owns the process-wide
|
|
* state, the address-space reservation, the refill/fallback path, realloc,
|
|
* and the out-of-line entry points the LLVM backend calls for acyclic
|
|
* records and class instances.
|
|
*
|
|
* Freed blocks are recycled within their class; the reservation itself
|
|
* costs address space only, and pages are committed as the bump pointers
|
|
* first touch them. Only an exiting worker thread returns pages, those
|
|
* holding nothing but free blocks (scr_sa_trim, Linux). */
|
|
#include "scr_runtime.h"
|
|
|
|
/* Defined in every build: the emitted fast paths read it (see the layout
|
|
* contract in scr_runtime.h); without the allocator it stays all-zero.
|
|
* Explicitly initialized so it is an ordinary (localizable) definition: a
|
|
* tentative definition becomes a COMMON symbol, which library localization
|
|
* deliberately keeps global (abi.localize_runtime), leaking it from every
|
|
* localized library archive. */
|
|
#ifdef SCR_WORKERS
|
|
_Thread_local ScrThreadHot scr_thread_hot = {{0}};
|
|
#else
|
|
ScrSaState scr_sa = {0};
|
|
#endif
|
|
|
|
/* Counts a worker's large blocks (scr_sa_thread_large_note, below). */
|
|
#if defined(SCR_WORKERS) && defined(__GLIBC__) && !defined(SCR_LIB)
|
|
static void scr_sa_thread_large_note(void);
|
|
#define SCR_SA_THREAD_LARGE_NOTE() scr_sa_thread_large_note()
|
|
#else
|
|
#define SCR_SA_THREAD_LARGE_NOTE() ((void)0)
|
|
#endif
|
|
|
|
#if SCR_SMALL_ALLOC
|
|
#include <sys/mman.h>
|
|
|
|
/* Page policy. Slice pages are committed by first touch, which on 4 KiB
|
|
* pages is one fault (and page clear) per 4 KiB the bump pointers cross.
|
|
* Transparent huge pages fault 2 MiB at a time and shrink the TLB working
|
|
* set, but make the whole 2 MiB page around a bump pointer resident at once,
|
|
* so every class slice in use (per arena, in worker executables) would cost
|
|
* up to 2 MiB more than it holds. A slice therefore starts on small pages
|
|
* and moves the rest of its range to huge pages once it has carved
|
|
* SCR_SA_HUGE_AFTER bytes: only slices that already hold that much pay the
|
|
* partial last huge page. The reservation is huge-page aligned so the
|
|
* switch point and every slice start on a huge-page boundary. */
|
|
#if defined(__linux__) && defined(MADV_HUGEPAGE) && defined(MADV_NOHUGEPAGE)
|
|
#define SCR_SA_THP 1
|
|
#define SCR_SA_HUGE_PAGE ((size_t)2 << 20)
|
|
#define SCR_SA_HUGE_AFTER ((size_t)8 << 20)
|
|
#endif
|
|
|
|
#if defined(SCR_SA_THP) && defined(__GLIBC__)
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
/* The system's THP mode: 'a' (always), 'm' (madvise), 'n' (never), or 0
|
|
* when the kernel has no THP setting to read. */
|
|
static char scr_sa_thp_mode(void) {
|
|
FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r");
|
|
if (!f) return 0;
|
|
char buf[96] = {0};
|
|
size_t n = fread(buf, 1, sizeof buf - 1, f);
|
|
fclose(f);
|
|
buf[n] = 0;
|
|
if (strstr(buf, "[always]")) return 'a';
|
|
if (strstr(buf, "[madvise]")) return 'm';
|
|
return strstr(buf, "[never]") ? 'n' : 0;
|
|
}
|
|
#endif
|
|
|
|
/* Core dumps. The reservation is tens of GiB of address space, almost none
|
|
* of it ever touched. A core written to a file skips untouched pages, but a
|
|
* core piped to a handler (a kernel.core_pattern of "|...", such as apport
|
|
* or systemd-coredump) carries every page of a dumped mapping, untouched
|
|
* ones as zeros, and RLIMIT_CORE does not bound it: an aborting program
|
|
* would take seconds to minutes to die. The reservation is therefore mapped
|
|
* MADV_DONTDUMP, and whenever the slow path raises a slice's bump limit,
|
|
* the slice up to that limit becomes dumpable again (scr_sa_dumpable), so a
|
|
* core still holds every carved block. Without pre-faulting, limits advance
|
|
* in bounded steps for this (SCR_SA_DUMP_STEP on small pages, one huge page
|
|
* past the switch point). The dumpable part ends on a grid that coarsens as
|
|
* the slice grows (scr_sa_dump_edge): its untouched tail is at most
|
|
* SCR_SA_DUMP_GRAIN or a quarter of what the slice has carved, and a slice
|
|
* makes a few madvise calls per doubling, not one per step (each takes the
|
|
* mmap lock for writing, which would wait for the pre-fault helper). Grid
|
|
* points are huge-page aligned: a mapping boundary inside a huge page would
|
|
* split it. */
|
|
#if defined(__linux__) && defined(MADV_DONTDUMP)
|
|
#define SCR_SA_DUMP 1
|
|
#define SCR_SA_DUMP_STEP ((size_t)1 << 20)
|
|
#define SCR_SA_DUMP_GRAIN ((size_t)2 << 20) /* a multiple of the page size */
|
|
#endif
|
|
|
|
/* Maps span bytes of address space for slices (huge-page aligned where the
|
|
* policy above applies), or NULL. */
|
|
static void *scr_sa_map(size_t span) {
|
|
int flags = MAP_PRIVATE | MAP_ANON;
|
|
#ifdef MAP_NORESERVE
|
|
flags |= MAP_NORESERVE;
|
|
#endif
|
|
#ifdef SCR_SA_THP
|
|
size_t pad = SCR_SA_HUGE_PAGE;
|
|
#else
|
|
size_t pad = 0;
|
|
#endif
|
|
char *p = mmap(NULL, span + pad, PROT_READ | PROT_WRITE, flags, -1, 0);
|
|
if (p == MAP_FAILED) return NULL;
|
|
#ifdef SCR_SA_THP
|
|
char *a = (char *)(((uintptr_t)p + pad - 1) & ~(uintptr_t)(pad - 1));
|
|
if (a > p) (void)munmap(p, (size_t)(a - p));
|
|
(void)munmap(a + span, (size_t)(p + pad - a)); /* a < p + pad */
|
|
p = a;
|
|
(void)madvise(p, span, MADV_NOHUGEPAGE);
|
|
#endif
|
|
#ifdef SCR_SA_DUMP
|
|
(void)madvise(p, span, MADV_DONTDUMP);
|
|
#endif
|
|
return p;
|
|
}
|
|
|
|
#ifdef SCR_SA_DUMP
|
|
/* The end of the dumpable part of the class slice [start, end) whose
|
|
* limit is p: p rounded up to a multiple of SCR_SA_DUMP_GRAIN or of a
|
|
* quarter of the largest power of two at or below p - start, whichever is
|
|
* larger (see above). Monotonic in p. */
|
|
static char *scr_sa_dump_edge(char *start, char *end, char *p) {
|
|
size_t off = (size_t)(p - start), grain = SCR_SA_DUMP_GRAIN;
|
|
if (off) {
|
|
size_t quarter = ((size_t)1 << (63 - __builtin_clzll((unsigned long long)off))) >> 2;
|
|
if (quarter > grain) grain = quarter;
|
|
}
|
|
off = (off + grain - 1) & ~(grain - 1);
|
|
return off < (size_t)(end - start) ? start + off : end;
|
|
}
|
|
|
|
/* The slice [start, end) raised its bump limit from lim to next: makes
|
|
* the memory up to next dumpable (see above). */
|
|
static void scr_sa_dumpable(char *start, char *end, char *lim, char *next) {
|
|
char *lo = scr_sa_dump_edge(start, end, lim), *hi = scr_sa_dump_edge(start, end, next);
|
|
if (hi > lo) (void)madvise(lo, (size_t)(hi - lo), MADV_DODUMP);
|
|
}
|
|
#endif
|
|
|
|
/* Pre-faulting (Linux executables). First touch of a fresh slice page costs
|
|
* a fault and a page clear on the allocating thread. With pre-faulting the
|
|
* bump limits advance in steps, and each step asks a helper thread to fault
|
|
* in the memory just ahead of it (MADV_POPULATE_WRITE: the kernel commits
|
|
* zeroed pages as for a write, without storing anything, so it cannot race
|
|
* with the owner's carving). The owner still faults any page the helper has
|
|
* not reached; the fast paths are unchanged. Small-page prefixes advance in
|
|
* SCR_SA_PF_STEP chunks and fault one chunk ahead; huge-page ranges advance
|
|
* in half huge pages and fault the next huge page once the bump is in the
|
|
* second half of the current one, so a slice keeps at most about one step
|
|
* of untouched committed memory. Nothing is faulted ahead before a slice
|
|
* reaches SCR_SA_PF_START: a lightly used class commits nothing extra.
|
|
*
|
|
* Worker threads fault ahead only once they have shown sustained
|
|
* allocation, and within one budget that all workers share. A worker's
|
|
* limits must have advanced SCR_SA_PF_WORKER_MIN bytes, and the ranges
|
|
* faulted ahead of all workers together stay within SCR_SA_PF_WORKERS_MAX.
|
|
* A range counts against the budget until its class's limit reaches it or
|
|
* its thread parks its arena, and a class has one range in flight at a
|
|
* time. A step outside the budget is not faulted ahead: the owner faults
|
|
* those pages itself. Many worker threads (parse and check pools) would
|
|
* otherwise each hold faulted-in ranges, up to a huge page per hot class,
|
|
* that they never carve, and all of them would queue on the one helper,
|
|
* whose populates also hold the address-space lock their own mmap and brk
|
|
* calls need. A program whose allocation runs on one or two workers keeps
|
|
* most of the pre-faulting. The main thread keeps the policy above, and
|
|
* executables without worker threads compile only that policy.
|
|
* SCR_SA_PREFAULT=0 turns pre-faulting off. Library archives run on
|
|
* embedder threads and keep the plain policy. */
|
|
#if defined(SCR_SA_THP) && !defined(SCR_LIB)
|
|
#ifndef MADV_POPULATE_WRITE
|
|
#define MADV_POPULATE_WRITE 23 /* Linux 5.14 */
|
|
#endif
|
|
#define SCR_SA_PREFAULT 1
|
|
#define SCR_SA_PF_STEP ((size_t)256 << 10)
|
|
#define SCR_SA_PF_START ((size_t)1 << 20)
|
|
#define SCR_SA_PF_HUGE_STEP (SCR_SA_HUGE_PAGE / 2)
|
|
#define SCR_SA_PF_WORKER_MIN ((size_t)4 << 20)
|
|
#define SCR_SA_PF_WORKERS_MAX ((size_t)8 << 20)
|
|
#include <errno.h>
|
|
#include <pthread.h>
|
|
#ifdef SCR_WORKERS
|
|
#include <sys/syscall.h>
|
|
#include <unistd.h>
|
|
#endif
|
|
#include <signal.h>
|
|
#include <stdatomic.h>
|
|
|
|
typedef struct {
|
|
char *lo, *hi;
|
|
} ScrSaPfRange;
|
|
#define SCR_SA_PF_RING 64u
|
|
static pthread_mutex_t scr_sa_pf_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
static pthread_cond_t scr_sa_pf_cond = PTHREAD_COND_INITIALIZER;
|
|
static ScrSaPfRange scr_sa_pf_ring[SCR_SA_PF_RING];
|
|
static unsigned scr_sa_pf_head, scr_sa_pf_tail; /* under the lock */
|
|
static bool scr_sa_pf_started, scr_sa_pf_waiting;
|
|
/* 0 = not decided yet, 1 = on, -1 = off (environment, no thread, or a
|
|
* kernel without MADV_POPULATE_WRITE). */
|
|
static _Atomic(int) scr_sa_pf_state;
|
|
|
|
static void scr_sa_pf_populate(char *lo, char *hi) {
|
|
if (madvise(lo, (size_t)(hi - lo), MADV_POPULATE_WRITE) != 0 && errno == EINVAL)
|
|
atomic_store_explicit(&scr_sa_pf_state, -1, memory_order_relaxed);
|
|
}
|
|
|
|
/* The system allocator's heap (glibc): blocks above SCR_SA_MAX come from
|
|
* malloc, whose main arena grows by brk. Once the heap has grown by
|
|
* SCR_SA_PF_HEAP_MIN since the helper first looked, the helper gives malloc
|
|
* a larger top pad (SCR_SA_PF_TOP_PAD), so every extension runs that far
|
|
* ahead of use, and faults in each new extension as soon as it sees the
|
|
* break move. Smaller heaps keep glibc's defaults (a top pad also fixes
|
|
* glibc's otherwise dynamic mmap threshold), so a small program commits
|
|
* nothing extra. The helper checks after every request and then polls while
|
|
* the program keeps allocating (SCR_SA_PF_POLL_NS, up to SCR_SA_PF_POLLS
|
|
* times without growth before it sleeps until the next request), so an idle
|
|
* program does not wake it. Only the helper reads the break; a stale read
|
|
* only delays a fault-in, and a shrunk break (malloc trimming its top)
|
|
* restarts the watermarks there. Past the threshold the pad costs at most
|
|
* SCR_SA_PF_TOP_PAD of committed memory beyond what malloc would have;
|
|
* SCR_SA_PREFAULT_HEAP=0 keeps glibc's defaults and no polling.
|
|
*
|
|
* Where THP is in madvise mode none of this runs. The heap policy below
|
|
* moves the heap to huge pages at the same 32 MiB, so the owner takes one
|
|
* fault per 2 MiB and faulting ahead saves it little, while the helper's
|
|
* populates cost more: ranges it reaches before the owner advises them (and
|
|
* the partial huge pages at the ends of each extension) stay on small pages,
|
|
* 230-620 MB on large type checks; the owner's brk and madvise
|
|
* wait for the mmap lock a multi-megabyte populate holds; and the helper's
|
|
* top pad would replace the heap policy's larger one. */
|
|
#if defined(__GLIBC__)
|
|
#include <malloc.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
#define SCR_SA_PF_HEAP 1
|
|
#define SCR_SA_PF_HEAP_MIN ((size_t)32 << 20)
|
|
#define SCR_SA_PF_TOP_PAD ((size_t)16 << 20)
|
|
#define SCR_SA_PF_POLL_NS 1000000L
|
|
#define SCR_SA_PF_POLLS 64u
|
|
static int scr_sa_pf_heap_mode; /* 0 off, 1 on; decided before the helper starts */
|
|
static uintptr_t scr_sa_pf_page; /* likewise */
|
|
static char *scr_sa_pf_brk0, *scr_sa_pf_brk; /* helper only: first and last break seen */
|
|
static bool scr_sa_pf_heap_big; /* helper only: past SCR_SA_PF_HEAP_MIN */
|
|
|
|
/* Faults in the heap's growth since the last check; true if it grew. */
|
|
static bool scr_sa_pf_heap(void) {
|
|
char *cur = sbrk(0);
|
|
if (cur == (char *)-1) return false;
|
|
char *seen = scr_sa_pf_brk;
|
|
scr_sa_pf_brk = cur;
|
|
if (!seen || cur < scr_sa_pf_brk0) scr_sa_pf_brk0 = cur;
|
|
if (!seen || cur <= seen) return false;
|
|
if (!scr_sa_pf_heap_big) {
|
|
if ((size_t)(cur - scr_sa_pf_brk0) < SCR_SA_PF_HEAP_MIN) return true;
|
|
scr_sa_pf_heap_big = true;
|
|
(void)mallopt(M_TOP_PAD, (int)SCR_SA_PF_TOP_PAD);
|
|
}
|
|
scr_sa_pf_populate((char *)((uintptr_t)seen & ~(scr_sa_pf_page - 1)), cur);
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
static void *scr_sa_pf_main(void *arg) {
|
|
(void)arg;
|
|
#ifdef SCR_SA_PF_HEAP
|
|
unsigned idle = SCR_SA_PF_POLLS;
|
|
#endif
|
|
pthread_mutex_lock(&scr_sa_pf_lock);
|
|
for (;;) {
|
|
while (scr_sa_pf_head == scr_sa_pf_tail) {
|
|
scr_sa_pf_waiting = true;
|
|
#ifdef SCR_SA_PF_HEAP
|
|
if (scr_sa_pf_heap_mode && idle < SCR_SA_PF_POLLS) {
|
|
struct timespec t;
|
|
clock_gettime(CLOCK_MONOTONIC, &t);
|
|
t.tv_nsec += SCR_SA_PF_POLL_NS;
|
|
if (t.tv_nsec >= 1000000000L) {
|
|
t.tv_sec++;
|
|
t.tv_nsec -= 1000000000L;
|
|
}
|
|
int rc = pthread_cond_timedwait(&scr_sa_pf_cond, &scr_sa_pf_lock, &t);
|
|
scr_sa_pf_waiting = false;
|
|
if (rc != 0 && scr_sa_pf_head == scr_sa_pf_tail) {
|
|
pthread_mutex_unlock(&scr_sa_pf_lock);
|
|
idle = scr_sa_pf_heap() ? 0 : idle + 1;
|
|
pthread_mutex_lock(&scr_sa_pf_lock);
|
|
}
|
|
continue;
|
|
}
|
|
#endif
|
|
pthread_cond_wait(&scr_sa_pf_cond, &scr_sa_pf_lock);
|
|
scr_sa_pf_waiting = false;
|
|
}
|
|
ScrSaPfRange r = scr_sa_pf_ring[scr_sa_pf_tail++ % SCR_SA_PF_RING];
|
|
pthread_mutex_unlock(&scr_sa_pf_lock);
|
|
scr_sa_pf_populate(r.lo, r.hi);
|
|
#ifdef SCR_SA_PF_HEAP
|
|
if (scr_sa_pf_heap_mode) {
|
|
(void)scr_sa_pf_heap();
|
|
idle = 0;
|
|
}
|
|
#endif
|
|
pthread_mutex_lock(&scr_sa_pf_lock);
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static bool scr_sa_pf_on(void) {
|
|
int s = atomic_load_explicit(&scr_sa_pf_state, memory_order_relaxed);
|
|
if (s == 0) {
|
|
const char *v = getenv("SCR_SA_PREFAULT");
|
|
s = v && v[0] == '0' ? -1 : 1;
|
|
atomic_store_explicit(&scr_sa_pf_state, s, memory_order_relaxed);
|
|
}
|
|
return s > 0;
|
|
}
|
|
|
|
/* Asks the helper to fault in [lo, hi) (page aligned). Dropped when the
|
|
* ring is full: the owner then faults those pages itself. */
|
|
static void scr_sa_prefault(char *lo, char *hi) {
|
|
if (hi <= lo || !scr_sa_pf_on()) return;
|
|
pthread_mutex_lock(&scr_sa_pf_lock);
|
|
if (!scr_sa_pf_started) {
|
|
scr_sa_pf_started = true;
|
|
#ifdef SCR_SA_PF_HEAP
|
|
/* Nobody waits on the condition yet: give it the monotonic clock for
|
|
* the helper's polls (no polls without it). */
|
|
const char *v = getenv("SCR_SA_PREFAULT_HEAP");
|
|
long page = sysconf(_SC_PAGESIZE);
|
|
char thp = scr_sa_thp_mode();
|
|
pthread_condattr_t ca;
|
|
if (!(v && v[0] == '0') && thp != 'm' && page > 0 && pthread_condattr_init(&ca) == 0) {
|
|
if (pthread_condattr_setclock(&ca, CLOCK_MONOTONIC) == 0) {
|
|
pthread_cond_destroy(&scr_sa_pf_cond);
|
|
pthread_cond_init(&scr_sa_pf_cond, &ca);
|
|
scr_sa_pf_page = (uintptr_t)page;
|
|
scr_sa_pf_heap_mode = 1;
|
|
}
|
|
pthread_condattr_destroy(&ca);
|
|
}
|
|
#endif
|
|
/* The helper takes no signals: they stay with the program's threads. */
|
|
pthread_attr_t attr;
|
|
pthread_t helper;
|
|
sigset_t all, old;
|
|
sigfillset(&all);
|
|
bool ok = pthread_attr_init(&attr) == 0;
|
|
if (ok) {
|
|
bool masked = pthread_sigmask(SIG_SETMASK, &all, &old) == 0;
|
|
ok = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0 &&
|
|
pthread_attr_setstacksize(&attr, (size_t)64 << 10) == 0 &&
|
|
pthread_create(&helper, &attr, scr_sa_pf_main, NULL) == 0;
|
|
if (masked) (void)pthread_sigmask(SIG_SETMASK, &old, NULL);
|
|
pthread_attr_destroy(&attr);
|
|
}
|
|
if (!ok) atomic_store_explicit(&scr_sa_pf_state, -1, memory_order_relaxed);
|
|
}
|
|
if (atomic_load_explicit(&scr_sa_pf_state, memory_order_relaxed) > 0 &&
|
|
scr_sa_pf_head - scr_sa_pf_tail < SCR_SA_PF_RING) {
|
|
scr_sa_pf_ring[scr_sa_pf_head++ % SCR_SA_PF_RING] = (ScrSaPfRange){lo, hi};
|
|
if (scr_sa_pf_waiting) pthread_cond_signal(&scr_sa_pf_cond);
|
|
}
|
|
pthread_mutex_unlock(&scr_sa_pf_lock);
|
|
}
|
|
|
|
#ifdef SCR_WORKERS
|
|
/* The workers' budget (see above): bytes faulted ahead of all workers
|
|
* that their limits have not reached. Per thread: whether it is the main
|
|
* thread (0 not known yet, 1 main, 2 worker), bytes its slice limits have
|
|
* advanced, its share of the budget, and each class's range in flight. */
|
|
static _Atomic(size_t) scr_sa_pf_workers_ahead;
|
|
static _Thread_local int scr_sa_pf_role;
|
|
static _Thread_local size_t scr_sa_pf_advanced, scr_sa_pf_ahead;
|
|
static _Thread_local char *scr_sa_pf_lo[SCR_SA_NCLASS];
|
|
static _Thread_local size_t scr_sa_pf_len[SCR_SA_NCLASS];
|
|
|
|
/* Records class c's limit moving from lim to next: the advance counts
|
|
* toward SCR_SA_PF_WORKER_MIN, and a range in flight that the limit has
|
|
* reached goes back to the workers' budget. */
|
|
static void scr_sa_pf_advance(unsigned c, char *lim, char *next) {
|
|
scr_sa_pf_advanced += (size_t)(next - lim);
|
|
if (scr_sa_pf_len[c] && next > scr_sa_pf_lo[c]) {
|
|
scr_sa_pf_ahead -= scr_sa_pf_len[c];
|
|
atomic_fetch_sub_explicit(&scr_sa_pf_workers_ahead, scr_sa_pf_len[c], memory_order_relaxed);
|
|
scr_sa_pf_len[c] = 0;
|
|
}
|
|
}
|
|
|
|
/* Returns the calling thread's ranges in flight to the workers' budget
|
|
* when it parks its arena. */
|
|
static void scr_sa_pf_park(void) {
|
|
if (scr_sa_pf_ahead)
|
|
atomic_fetch_sub_explicit(&scr_sa_pf_workers_ahead, scr_sa_pf_ahead, memory_order_relaxed);
|
|
scr_sa_pf_ahead = 0;
|
|
scr_sa_pf_advanced = 0;
|
|
memset(scr_sa_pf_len, 0, sizeof scr_sa_pf_len);
|
|
}
|
|
|
|
/* scr_sa_prefault for class c: unconditionally on the main thread, within
|
|
* the budget on a worker. */
|
|
static void scr_sa_prefault_ahead(unsigned c, char *lo, char *hi) {
|
|
if (!scr_sa_pf_role) scr_sa_pf_role = (pid_t)syscall(SYS_gettid) == getpid() ? 1 : 2;
|
|
if (scr_sa_pf_role == 1) {
|
|
scr_sa_prefault(lo, hi);
|
|
return;
|
|
}
|
|
if (hi <= lo || scr_sa_pf_len[c] || scr_sa_pf_advanced < SCR_SA_PF_WORKER_MIN) return;
|
|
size_t n = (size_t)(hi - lo);
|
|
size_t ahead = atomic_load_explicit(&scr_sa_pf_workers_ahead, memory_order_relaxed);
|
|
do {
|
|
if (ahead + n > SCR_SA_PF_WORKERS_MAX) return;
|
|
} while (!atomic_compare_exchange_weak_explicit(&scr_sa_pf_workers_ahead, &ahead, ahead + n,
|
|
memory_order_relaxed, memory_order_relaxed));
|
|
scr_sa_pf_ahead += n;
|
|
scr_sa_pf_lo[c] = lo;
|
|
scr_sa_pf_len[c] = n;
|
|
scr_sa_prefault(lo, hi);
|
|
}
|
|
#else
|
|
/* Without worker threads every allocating thread is the main thread. */
|
|
#define scr_sa_pf_advance(c, lim, next) ((void)0)
|
|
#define scr_sa_prefault_ahead(c, lo, hi) scr_sa_prefault(lo, hi)
|
|
#endif
|
|
#endif
|
|
|
|
/* The next bump limit of class c's slice [start, end) whose limit is lim
|
|
* (< end): its small-page prefix ends at SCR_SA_HUGE_AFTER, where the rest
|
|
* of the slice switches to huge pages. Limits advance in steps: the
|
|
* pre-fault steps (see above), or else the core-dump steps (SCR_SA_DUMP). */
|
|
static char *scr_sa_extend(unsigned c, char *start, char *lim, char *end) {
|
|
(void)c;
|
|
#ifdef SCR_SA_THP
|
|
char *sw = start + SCR_SA_HUGE_AFTER;
|
|
bool tiered = (size_t)(end - start) > SCR_SA_HUGE_AFTER;
|
|
#ifdef SCR_SA_PREFAULT
|
|
if (scr_sa_pf_on()) {
|
|
if (!tiered) return end;
|
|
if (lim < sw) {
|
|
char *next = lim + SCR_SA_PF_STEP < sw ? lim + SCR_SA_PF_STEP : sw;
|
|
scr_sa_pf_advance(c, lim, next);
|
|
if (next == sw) {
|
|
/* The last small-page chunk: switch the rest now, so the first huge
|
|
* page can be faulted ahead like every later one. */
|
|
(void)madvise(sw, (size_t)(end - sw), MADV_HUGEPAGE);
|
|
scr_sa_prefault_ahead(c, sw, sw + SCR_SA_HUGE_PAGE < end ? sw + SCR_SA_HUGE_PAGE : end);
|
|
} else if ((size_t)(next - start) >= SCR_SA_PF_START) {
|
|
scr_sa_prefault_ahead(c, next, next + SCR_SA_PF_STEP < sw ? next + SCR_SA_PF_STEP : sw);
|
|
}
|
|
return next;
|
|
}
|
|
char *next = lim + SCR_SA_PF_HUGE_STEP < end ? lim + SCR_SA_PF_HUGE_STEP : end;
|
|
scr_sa_pf_advance(c, lim, next);
|
|
/* Entering the second half of a huge page: fault the next one. */
|
|
if (((uintptr_t)next & (SCR_SA_HUGE_PAGE - 1)) == 0 && next < end)
|
|
scr_sa_prefault_ahead(c, next, next + SCR_SA_HUGE_PAGE < end ? next + SCR_SA_HUGE_PAGE : end);
|
|
return next;
|
|
}
|
|
#endif
|
|
if (tiered && lim < sw) {
|
|
#ifdef SCR_SA_DUMP
|
|
if ((size_t)(sw - lim) > SCR_SA_DUMP_STEP) return lim + SCR_SA_DUMP_STEP;
|
|
#endif
|
|
return sw;
|
|
}
|
|
if (tiered && lim == sw) (void)madvise(sw, (size_t)(end - sw), MADV_HUGEPAGE);
|
|
#ifdef SCR_SA_DUMP
|
|
if (tiered && (size_t)(end - lim) > SCR_SA_HUGE_PAGE) return lim + SCR_SA_HUGE_PAGE;
|
|
#endif
|
|
#else
|
|
(void)start;
|
|
#ifdef SCR_SA_DUMP
|
|
if ((size_t)(end - lim) > SCR_SA_DUMP_STEP) return lim + SCR_SA_DUMP_STEP;
|
|
#else
|
|
(void)lim;
|
|
#endif
|
|
#endif
|
|
return end;
|
|
}
|
|
|
|
/* Blocks above SCR_SA_MAX (array storage, hash tables, link pages, large
|
|
* strings) come from glibc, whose heap and per-block mappings use small
|
|
* pages where THP is in madvise mode (the common distribution default);
|
|
* in large compiler-like programs that is most of the page faults and a
|
|
* share of the TLB misses. glibc only advises huge pages for its own
|
|
* memory when the glibc.malloc.hugetlb tunable is set in the environment,
|
|
* which a program cannot do for itself. So once the brk heap has grown
|
|
* past SCR_SA_HEAP_HUGE_AFTER (small programs keep small pages and their
|
|
* RSS), every large allocation advises the heap grown since the previous
|
|
* one, and the aligned interior of a block mapped outside the heap.
|
|
* glibc grows the heap by its top pad beyond each request; the default
|
|
* (128 KiB) is smaller than a huge page, so every 2 MiB range would be
|
|
* first touched while it still straddles the heap end, where it can only
|
|
* get small pages. The policy therefore raises the top pad. Setting it
|
|
* fixes glibc's otherwise adaptive mmap and trim thresholds, so they are
|
|
* set to the adaptive maxima the heap of such a program reaches anyway. */
|
|
#if defined(SCR_SA_THP) && defined(__GLIBC__)
|
|
#include <malloc.h>
|
|
#include <stdatomic.h>
|
|
#include <unistd.h>
|
|
#define SCR_SA_HEAP_HUGE_AFTER ((uintptr_t)32 << 20)
|
|
#define SCR_SA_HEAP_TOP_PAD (32 << 20)
|
|
#define SCR_SA_HEAP_MMAP_THRESHOLD (32 << 20) /* glibc's adaptive maximum */
|
|
#define SCR_SA_HEAP_TRIM_THRESHOLD (64 << 20) /* twice that, as glibc keeps it */
|
|
/* 0 = undecided, 1 = following, 2 = off (THP not in madvise mode). */
|
|
static _Atomic(int) scr_sa_heap_state;
|
|
static _Atomic(uintptr_t) scr_sa_heap_lo; /* heap start seen at the first large block */
|
|
static _Atomic(uintptr_t) scr_sa_heap_hi; /* end of the advised heap prefix */
|
|
static _Atomic(uintptr_t) scr_sa_heap_page; /* the page size, read with the mode */
|
|
/* The page size is stored before scr_sa_heap_lo is published (the CAS is a
|
|
* release), and every thread loads scr_sa_heap_lo, then scr_sa_heap_hi, with
|
|
* acquire before reading it; each scr_sa_heap_hi update is an RMW by a thread
|
|
* that already sees it, so any thread that wins the advice CAS sees a nonzero
|
|
* page size. A zero read still skips the advice rather than align to 0.
|
|
* These loads run only on blocks too large for the small allocator, next to
|
|
* a glibc malloc call: plain loads on x86-64, LDAR (or LDAPR) on arm64. */
|
|
|
|
static void scr_sa_large_note(void *p, size_t n) {
|
|
int state = atomic_load_explicit(&scr_sa_heap_state, memory_order_relaxed);
|
|
if (state == 2 || !p) return;
|
|
uintptr_t top = (uintptr_t)sbrk(0);
|
|
if (state == 0) {
|
|
uintptr_t lo = atomic_load_explicit(&scr_sa_heap_lo, memory_order_acquire);
|
|
if (!lo) {
|
|
long page = sysconf(_SC_PAGESIZE);
|
|
if (page <= 0 || scr_sa_thp_mode() != 'm') {
|
|
atomic_store_explicit(&scr_sa_heap_state, 2, memory_order_relaxed);
|
|
return;
|
|
}
|
|
atomic_store_explicit(&scr_sa_heap_page, (uintptr_t)page, memory_order_relaxed);
|
|
lo = (top + (uintptr_t)page - 1) & ~((uintptr_t)page - 1);
|
|
uintptr_t none = 0;
|
|
if (!atomic_compare_exchange_strong(&scr_sa_heap_lo, &none, lo)) lo = none;
|
|
}
|
|
if (top < lo + SCR_SA_HEAP_HUGE_AFTER) return;
|
|
uintptr_t unset = 0; /* the whole heap seen so far gets the advice */
|
|
(void)atomic_compare_exchange_strong(&scr_sa_heap_hi, &unset, lo);
|
|
int undecided = 0;
|
|
if (atomic_compare_exchange_strong(&scr_sa_heap_state, &undecided, 1)) {
|
|
(void)mallopt(M_MMAP_THRESHOLD, SCR_SA_HEAP_MMAP_THRESHOLD);
|
|
(void)mallopt(M_TRIM_THRESHOLD, SCR_SA_HEAP_TRIM_THRESHOLD);
|
|
(void)mallopt(M_TOP_PAD, SCR_SA_HEAP_TOP_PAD);
|
|
}
|
|
}
|
|
/* The heap end moved since the last look: advise what it gained (pages
|
|
* already touched keep their small pages), or follow a trim down. */
|
|
uintptr_t hi = atomic_load_explicit(&scr_sa_heap_hi, memory_order_acquire);
|
|
if (!hi) return; /* another thread is switching the policy on */
|
|
if (top > hi) {
|
|
if (atomic_compare_exchange_strong(&scr_sa_heap_hi, &hi, top)) {
|
|
/* madvise wants a page-aligned start */
|
|
uintptr_t page = atomic_load_explicit(&scr_sa_heap_page, memory_order_relaxed);
|
|
if (page) {
|
|
uintptr_t from = hi & ~(page - 1);
|
|
(void)madvise((void *)from, top - from, MADV_HUGEPAGE);
|
|
}
|
|
}
|
|
} else if (top < hi) {
|
|
(void)atomic_compare_exchange_strong(&scr_sa_heap_hi, &hi, top);
|
|
}
|
|
/* A block glibc mapped by itself (or carved from a thread's heap). */
|
|
uintptr_t a = (uintptr_t)p, lo = atomic_load_explicit(&scr_sa_heap_lo, memory_order_relaxed);
|
|
if ((a < lo || a >= top) && n >= 2 * SCR_SA_HUGE_PAGE) {
|
|
uintptr_t s = (a + SCR_SA_HUGE_PAGE - 1) & ~(uintptr_t)(SCR_SA_HUGE_PAGE - 1);
|
|
uintptr_t e = (a + n) & ~(uintptr_t)(SCR_SA_HUGE_PAGE - 1);
|
|
if (e > s) (void)madvise((void *)s, e - s, MADV_HUGEPAGE);
|
|
}
|
|
}
|
|
#define SCR_SA_LARGE_NOTE(p, n) scr_sa_large_note((p), (n))
|
|
#else
|
|
#define SCR_SA_LARGE_NOTE(p, n) ((void)0)
|
|
#endif
|
|
|
|
#ifdef SCR_WORKERS
|
|
#include <pthread.h>
|
|
#include <stdatomic.h>
|
|
#if defined(__linux__) && defined(MADV_DONTNEED)
|
|
#define SCR_SA_TRIM 1
|
|
#include <stdio.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
#if defined(__GLIBC__)
|
|
#include <malloc.h>
|
|
#endif
|
|
#endif
|
|
|
|
/* Per-thread arenas (contract in scr_runtime.h). The region is reserved
|
|
* once; arena i spans [region + i * arena_span, + arena_span) and is split
|
|
* into class slices exactly like the single-threaded reservation. */
|
|
#define SCR_SA_ARENAS 32u
|
|
static pthread_mutex_t scr_sa_region_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
static _Atomic(uintptr_t) scr_sa_region; /* 0 until reserved */
|
|
static uintptr_t scr_sa_arena_span;
|
|
static unsigned scr_sa_region_shift;
|
|
static bool scr_sa_region_refused;
|
|
static _Atomic(uint32_t) scr_sa_claimed; /* one bit per arena */
|
|
static ScrSaState scr_sa_parked[SCR_SA_ARENAS]; /* unowned arenas */
|
|
static _Atomic(ScrSaBlock *) scr_sa_remote[SCR_SA_ARENAS]; /* foreign frees */
|
|
static _Thread_local int scr_sa_arena = -1;
|
|
static _Thread_local bool scr_sa_refused; /* this thread got no arena */
|
|
_Static_assert(SCR_SA_ARENAS <= 32, "one claim bit per arena");
|
|
|
|
static bool scr_sa_reserve_region(void) {
|
|
pthread_mutex_lock(&scr_sa_region_lock);
|
|
if (!atomic_load_explicit(&scr_sa_region, memory_order_acquire) && !scr_sa_region_refused) {
|
|
/* Same per-class slice ladder as the single-threaded reservation, for
|
|
* every arena at once; hosts that limit address space get smaller
|
|
* slices (each slice that fills falls back to the system). */
|
|
for (unsigned shift = 30; shift >= 22; shift -= 2) {
|
|
size_t arena = (size_t)SCR_SA_NCLASS << shift;
|
|
size_t span = arena * SCR_SA_ARENAS;
|
|
void *p = scr_sa_map(span);
|
|
if (!p) continue;
|
|
scr_sa_arena_span = arena;
|
|
scr_sa_region_shift = shift;
|
|
atomic_store_explicit(&scr_sa_region, (uintptr_t)p, memory_order_release);
|
|
break;
|
|
}
|
|
if (!atomic_load_explicit(&scr_sa_region, memory_order_relaxed)) scr_sa_region_refused = true;
|
|
}
|
|
pthread_mutex_unlock(&scr_sa_region_lock);
|
|
return atomic_load_explicit(&scr_sa_region, memory_order_acquire) != 0;
|
|
}
|
|
|
|
#ifdef SCR_SA_TRIM
|
|
/* The arena's carved bytes when the current thread claimed it (see
|
|
* scr_sa_trim). */
|
|
static size_t scr_sa_claim_carved[SCR_SA_ARENAS];
|
|
static size_t scr_sa_carved(void);
|
|
/* A worker thread's memory, discarded when it parks (see "Discarding a
|
|
* worker's memory" below). */
|
|
typedef struct ScrSaKeep ScrSaKeep;
|
|
static _Thread_local bool scr_sa_worker; /* set by scr_sa_thread_starting */
|
|
static _Thread_local ScrSaKeep *scr_sa_keep; /* while the worker's memory is kept apart */
|
|
static void scr_sa_keep_begin(unsigned i);
|
|
static void scr_sa_keep_drained(ScrSaKeep *k, ScrSaBlock *b, unsigned c);
|
|
static bool scr_sa_drain(void);
|
|
#endif
|
|
|
|
/* Moves every block other threads freed into this arena's free lists. */
|
|
static void scr_sa_drain_remote(void) {
|
|
ScrSaBlock *b = atomic_exchange_explicit(&scr_sa_remote[scr_sa_arena], NULL, memory_order_acquire);
|
|
while (b) {
|
|
ScrSaBlock *next = b->next;
|
|
unsigned c = (unsigned)(((uintptr_t)b - scr_sa.base) >> scr_sa.shift);
|
|
#ifdef SCR_SA_TRIM
|
|
if (scr_sa_keep) {
|
|
scr_sa_keep_drained(scr_sa_keep, b, c);
|
|
b = next;
|
|
continue;
|
|
}
|
|
#endif
|
|
b->next = scr_sa.free[c];
|
|
scr_sa.free[c] = b;
|
|
b = next;
|
|
}
|
|
}
|
|
|
|
/* Claims an arena for the calling thread: a parked one continues with its
|
|
* free lists and bump pointers, a fresh one starts empty. */
|
|
static bool scr_sa_reserve(void) {
|
|
if (!atomic_load_explicit(&scr_sa_region, memory_order_acquire) && !scr_sa_reserve_region()) {
|
|
scr_sa_refused = true;
|
|
return false;
|
|
}
|
|
uintptr_t region = atomic_load_explicit(&scr_sa_region, memory_order_acquire);
|
|
uint32_t claimed = atomic_load_explicit(&scr_sa_claimed, memory_order_relaxed);
|
|
for (;;) {
|
|
if (claimed == UINT32_MAX) {
|
|
#ifdef SCR_SA_TRIM
|
|
/* Arenas held by discards that wait (see scr_sa_thread_park): this
|
|
* thread discards them and claims one. */
|
|
if (scr_sa_drain()) {
|
|
scr_sa_refused = false;
|
|
claimed = atomic_load_explicit(&scr_sa_claimed, memory_order_relaxed);
|
|
continue;
|
|
}
|
|
#endif
|
|
scr_sa_refused = true;
|
|
return false;
|
|
}
|
|
unsigned i = (unsigned)__builtin_ctz(~claimed);
|
|
if (atomic_compare_exchange_weak_explicit(&scr_sa_claimed, &claimed, claimed | (1u << i),
|
|
memory_order_acquire, memory_order_relaxed)) {
|
|
scr_sa_arena = (int)i;
|
|
scr_sa = scr_sa_parked[i];
|
|
if (!scr_sa.span) {
|
|
memset(&scr_sa, 0, sizeof scr_sa);
|
|
scr_sa.base = region + (uintptr_t)i * scr_sa_arena_span;
|
|
scr_sa.span = scr_sa_arena_span;
|
|
scr_sa.shift = scr_sa_region_shift;
|
|
}
|
|
scr_sa_drain_remote();
|
|
#ifdef SCR_SA_TRIM
|
|
scr_sa_claim_carved[i] = scr_sa_carved();
|
|
if (scr_sa_worker) scr_sa_keep_begin(i);
|
|
#endif
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
#ifdef SCR_SA_TRIM
|
|
/* Returning an exited thread's free memory (Linux). When a script thread
|
|
* exits, every page of its arena that holds only free blocks goes back to
|
|
* the system (MADV_DONTNEED, after which it reads as zero), so other
|
|
* threads can use that memory while the arena is parked. The free blocks on
|
|
* the kept pages are relinked in address order; the returned runs of pages
|
|
* become zeroed carve regions (`ScrSaHoles`), which the next thread to claim
|
|
* the arena carves before the slice's own frontier: the bump fast paths
|
|
* already rely on carved memory being zero. A thread that carved less than
|
|
* SCR_SA_TRIM_MIN, or whose arena holds less than that in free blocks,
|
|
* leaves the arena as it is. */
|
|
#define SCR_SA_TRIM_MIN ((size_t)16 << 20)
|
|
typedef struct {
|
|
char *lo, *hi;
|
|
} ScrSaRun;
|
|
typedef struct {
|
|
ScrSaRun *runs; /* zeroed regions left to carve, carved from the end */
|
|
size_t n, cap;
|
|
char *bump, *lim; /* the frontier while a region is carved (lim != NULL) */
|
|
} ScrSaHoles;
|
|
static ScrSaHoles scr_sa_holes[SCR_SA_ARENAS][SCR_SA_NCLASS];
|
|
|
|
/* Bytes carved from the calling thread's slices: up to each frontier. */
|
|
static size_t scr_sa_carved(void) {
|
|
size_t carved = 0;
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
if (!scr_sa.lim[c]) continue;
|
|
const ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
|
|
char *top = h->lim ? h->bump : scr_sa.bump[c];
|
|
carved += (size_t)(top - ((char *)scr_sa.base + ((size_t)c << scr_sa.shift)));
|
|
}
|
|
return carved;
|
|
}
|
|
|
|
/* For the slow path of class c, whose bump room is below sz: moves the
|
|
* bump pointer to the next zeroed region, or back to the frontier. */
|
|
static void scr_sa_next_hole(unsigned c, size_t sz) {
|
|
ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
|
|
if ((size_t)(scr_sa.lim[c] - scr_sa.bump[c]) >= sz || (!h->n && !h->lim)) return;
|
|
if (h->n) {
|
|
if (!h->lim) {
|
|
h->bump = scr_sa.bump[c];
|
|
h->lim = scr_sa.lim[c];
|
|
}
|
|
ScrSaRun r = h->runs[--h->n];
|
|
scr_sa.bump[c] = r.lo;
|
|
scr_sa.lim[c] = r.hi;
|
|
} else {
|
|
scr_sa.bump[c] = h->bump;
|
|
scr_sa.lim[c] = h->lim;
|
|
h->bump = h->lim = NULL;
|
|
}
|
|
}
|
|
|
|
static bool scr_sa_push_run(ScrSaHoles *h, char *lo, char *hi) {
|
|
if (h->n == h->cap) {
|
|
size_t cap = h->cap ? 2 * h->cap : 64;
|
|
ScrSaRun *runs = realloc(h->runs, cap * sizeof *runs);
|
|
if (!runs) return false;
|
|
h->runs = runs;
|
|
h->cap = cap;
|
|
}
|
|
h->runs[h->n++] = (ScrSaRun){lo, hi};
|
|
return true;
|
|
}
|
|
|
|
/* With SCR_SA_STATS=1: carved, free and returned bytes of the parking thread. */
|
|
static _Thread_local size_t scr_sa_trim_stats[3];
|
|
|
|
/* Returns the pages of class c that hold only free blocks (see above). */
|
|
static void scr_sa_trim_class(unsigned c, size_t page) {
|
|
ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
|
|
size_t sz = scr_sa_class_size(c);
|
|
char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift);
|
|
/* Close the region being carved: its rest is still zero. */
|
|
if (h->lim) {
|
|
char *lo = scr_sa.bump[c], *hi = scr_sa.lim[c];
|
|
scr_sa.bump[c] = h->bump;
|
|
scr_sa.lim[c] = h->lim;
|
|
h->bump = h->lim = NULL;
|
|
if ((size_t)(hi - lo) >= sz && !scr_sa_push_run(h, lo, hi)) return;
|
|
}
|
|
char *top = scr_sa.bump[c];
|
|
size_t nblocks = (size_t)(top - start) / sz;
|
|
if (!nblocks) return;
|
|
size_t words = (nblocks + 63) / 64;
|
|
uint64_t *bits = mmap(NULL, words * 8, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
|
|
if (bits == MAP_FAILED) return;
|
|
size_t nfree = 0;
|
|
for (ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) {
|
|
size_t k = (size_t)((char *)b - start) / sz;
|
|
bits[k / 64] |= (uint64_t)1 << (k % 64);
|
|
nfree++;
|
|
}
|
|
/* Uncarved zeroed regions count as free; they are rebuilt below. */
|
|
for (size_t r = 0; r < h->n; r++) {
|
|
for (char *q = h->runs[r].lo; q + sz <= h->runs[r].hi; q += sz) {
|
|
size_t k = (size_t)(q - start) / sz;
|
|
bits[k / 64] |= (uint64_t)1 << (k % 64);
|
|
}
|
|
}
|
|
scr_sa_trim_stats[0] += nblocks * sz;
|
|
scr_sa_trim_stats[1] += nfree * sz;
|
|
#define SCR_SA_BIT(k) ((bits[(k) / 64] >> ((k) % 64)) & 1)
|
|
/* Whole pages below the frontier whose blocks are all free. */
|
|
uintptr_t first = ((uintptr_t)start + page - 1) & ~(uintptr_t)(page - 1);
|
|
uintptr_t last = (uintptr_t)top & ~(uintptr_t)(page - 1);
|
|
h->n = 0;
|
|
char *run = NULL;
|
|
for (uintptr_t pg = first; pg <= last; pg += page) {
|
|
bool empty = false;
|
|
if (pg < last) {
|
|
size_t k0 = (size_t)((char *)pg - start) / sz;
|
|
size_t k1 = (size_t)((char *)pg + page - 1 - start) / sz;
|
|
if (k1 >= nblocks) k1 = nblocks - 1;
|
|
empty = true;
|
|
for (size_t k = k0; k <= k1 && empty; k++) empty = SCR_SA_BIT(k);
|
|
}
|
|
if (empty && !run) run = (char *)pg;
|
|
if (!empty && run) {
|
|
(void)madvise(run, (size_t)((char *)pg - run), MADV_DONTNEED);
|
|
scr_sa_trim_stats[2] += (size_t)((char *)pg - run);
|
|
/* Blocks wholly inside the returned pages carve from it later. */
|
|
char *lo = start + ((size_t)(run - start) + sz - 1) / sz * sz;
|
|
char *hi = start + ((size_t)((char *)pg - start) / sz) * sz;
|
|
if (hi > lo) (void)scr_sa_push_run(h, lo, hi);
|
|
for (char *q = lo; q < hi; q += sz) {
|
|
size_t k = (size_t)(q - start) / sz;
|
|
bits[k / 64] &= ~((uint64_t)1 << (k % 64));
|
|
}
|
|
/* A block starting in the returned pages but ending on a kept one is
|
|
* dropped: relinking it would fault its page back in. */
|
|
if (hi < (char *)pg && hi >= run) {
|
|
size_t k = (size_t)(hi - start) / sz;
|
|
bits[k / 64] &= ~((uint64_t)1 << (k % 64));
|
|
}
|
|
run = NULL;
|
|
}
|
|
}
|
|
#undef SCR_SA_BIT
|
|
/* The remaining free blocks, in address order. */
|
|
ScrSaBlock *head = NULL, **tail = &head;
|
|
for (size_t w = 0; w < words; w++) {
|
|
for (uint64_t m = bits[w]; m; m &= m - 1) {
|
|
size_t k = w * 64 + (size_t)__builtin_ctzll(m);
|
|
ScrSaBlock *b = (ScrSaBlock *)(start + k * sz);
|
|
*tail = b;
|
|
tail = &b->next;
|
|
}
|
|
}
|
|
*tail = NULL;
|
|
scr_sa.free[c] = head;
|
|
munmap(bits, words * 8);
|
|
/* The next owner carves the returned regions before the frontier. */
|
|
if (h->n) {
|
|
h->bump = scr_sa.bump[c];
|
|
h->lim = scr_sa.lim[c];
|
|
ScrSaRun r = h->runs[--h->n];
|
|
scr_sa.bump[c] = r.lo;
|
|
scr_sa.lim[c] = r.hi;
|
|
}
|
|
}
|
|
|
|
/* Whether the calling thread carved at least SCR_SA_TRIM_MIN since it
|
|
* claimed its arena: a short-lived thread reusing the arena's free blocks
|
|
* leaves them for the next one. */
|
|
static bool scr_sa_trim_worth(void) {
|
|
return scr_sa_carved() - scr_sa_claim_carved[scr_sa_arena] >= SCR_SA_TRIM_MIN;
|
|
}
|
|
|
|
/* Returns the free pages of the calling thread's arena (see above). */
|
|
static void scr_sa_trim(void) {
|
|
scr_sa_drain_remote();
|
|
size_t free_bytes = 0;
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
size_t n = 0, sz = scr_sa_class_size(c);
|
|
for (ScrSaBlock *b = scr_sa.free[c]; b && n * sz < SCR_SA_TRIM_MIN; b = b->next) n++;
|
|
free_bytes += n * sz;
|
|
if (free_bytes >= SCR_SA_TRIM_MIN) break;
|
|
}
|
|
const char *stats = getenv("SCR_SA_STATS");
|
|
const char *off = getenv("SCR_SA_TRIM");
|
|
bool trim = free_bytes >= SCR_SA_TRIM_MIN && !(off && off[0] == '0');
|
|
memset(scr_sa_trim_stats, 0, sizeof scr_sa_trim_stats);
|
|
if (!trim && stats && stats[0] == '1')
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
if (!scr_sa.lim[c]) continue;
|
|
size_t sz = scr_sa_class_size(c);
|
|
char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift);
|
|
scr_sa_trim_stats[0] += (size_t)(scr_sa.bump[c] - start);
|
|
for (ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) scr_sa_trim_stats[1] += sz;
|
|
}
|
|
if (trim) {
|
|
long page = sysconf(_SC_PAGESIZE);
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++)
|
|
if (scr_sa.lim[c]) scr_sa_trim_class(c, page > 0 ? (size_t)page : 4096);
|
|
}
|
|
#if defined(__GLIBC__)
|
|
/* Large blocks: the system allocator's free pages too. */
|
|
if (trim) (void)malloc_trim(0);
|
|
#endif
|
|
if (stats && stats[0] == '1')
|
|
fprintf(stderr, "scr_sa: park arena %d trim %d carved %zu MB free %zu MB returned %zu MB\n",
|
|
scr_sa_arena, (int)trim, scr_sa_trim_stats[0] >> 20, scr_sa_trim_stats[1] >> 20,
|
|
scr_sa_trim_stats[2] >> 20);
|
|
}
|
|
|
|
/* Discarding a worker's memory (Linux). Script objects stay on the thread
|
|
* that allocated them: other threads reach one only through a message or an
|
|
* intern table that shares a published (immortal) graph by pointer, or a
|
|
* process-wide immortal cache, and each of those calls
|
|
* scr_sa_thread_shares. Runtime state that outlives a thread (messages,
|
|
* mailboxes, shared bytes, intern table entries) comes from the system
|
|
* allocator directly. So when a worker thread parks without having shared,
|
|
* nothing it allocated through scr_mem_* can be reachable from another
|
|
* thread, and what it never freed (ProgramLifetime instances, published
|
|
* module constants) is garbage. Such a thread's memory goes back whole:
|
|
* - when a worker claims an arena, the arena's free lists and returned
|
|
* regions are set aside, and the worker carves only past each slice's
|
|
* frontier, so its blocks of class c are [start[c], bump[c]), plus the
|
|
* blocks it takes from the free lists set aside once a slice has no bump
|
|
* room left, which it records;
|
|
* - the system blocks it allocates (large blocks, an exhausted slice) are
|
|
* recorded until it frees them;
|
|
* - a remote free into its blocks means another thread held one: the same
|
|
* as sharing;
|
|
* - when it parks without having shared, its arena stays claimed until no
|
|
* other worker thread is running (scr_sa_thread_park): returning pages
|
|
* interrupts every running thread of the process to flush its TLB, and
|
|
* glibc's trim locks the arenas other threads allocate from, so discards
|
|
* wait for the last worker of a group, which hands them to one helper;
|
|
* - the helper then returns the pages
|
|
* it carved (MADV_DONTNEED; the start of the first page, which it may
|
|
* share with older blocks, is cleared), frees the recorded system blocks,
|
|
* and parks the arena as the worker found it, with the blocks it took
|
|
* free again. A worker that shared parks
|
|
* its arena with the blocks set aside added to its free lists, and the
|
|
* ordinary trim applies.
|
|
* SCR_SA_DISCARD=0 keeps every worker's memory. SCR_SA_DISCARD=poison makes
|
|
* the carved pages inaccessible instead, and keeps the system blocks, so
|
|
* that a reference to discarded memory faults (a check for tests). */
|
|
#define SCR_SA_SYS_GONE ((void *)1) /* a removed entry of ScrSaKeep.sys */
|
|
|
|
struct ScrSaKeep {
|
|
/* The arena as the worker claimed it: free lists (to which remote frees
|
|
* of older blocks are added), carve cursors and returned regions. */
|
|
ScrSaBlock *free[SCR_SA_NCLASS];
|
|
char *bump[SCR_SA_NCLASS], *lim[SCR_SA_NCLASS];
|
|
ScrSaHoles holes[SCR_SA_NCLASS];
|
|
/* Each slice's frontier then (bump and limit, NULL limits for a slice not
|
|
* opened yet), and the first address the worker carves in it. */
|
|
char *front_bump[SCR_SA_NCLASS], *front_lim[SCR_SA_NCLASS];
|
|
char *start[SCR_SA_NCLASS];
|
|
/* The worker's system blocks: an open-addressing set of `cap` slots (a
|
|
* power of two, or 0), NULL for an empty slot and SCR_SA_SYS_GONE for a
|
|
* removed entry; `used` counts both kinds of filled slots. */
|
|
void **sys;
|
|
size_t cap, used, live;
|
|
/* The blocks of the free lists set aside that the worker took once a slice
|
|
* had no bump room left, so that older free blocks are used before the
|
|
* slice grows; they go back to those lists with the worker's memory. */
|
|
ScrSaBlock **taken;
|
|
size_t ntaken, cap_taken;
|
|
bool shared;
|
|
bool poison;
|
|
};
|
|
|
|
static _Thread_local bool scr_sa_discarded; /* this thread's memory went back */
|
|
static ScrSaKeep *scr_sa_parked_keep[SCR_SA_ARENAS]; /* of discards that wait */
|
|
|
|
/* SCR_SA_DISCARD: 0 off, 1 on (the default), 2 poison. */
|
|
static int scr_sa_discard_mode(void) {
|
|
const char *mode = getenv("SCR_SA_DISCARD");
|
|
if (!mode) return 1;
|
|
if (strcmp(mode, "0") == 0) return 0;
|
|
return strcmp(mode, "poison") == 0 ? 2 : 1;
|
|
}
|
|
|
|
static void scr_sa_keep_begin(unsigned i) {
|
|
int mode = scr_sa_discard_mode();
|
|
if (!mode) return;
|
|
ScrSaKeep *k = calloc(1, sizeof *k);
|
|
if (!k) return;
|
|
k->poison = mode == 2;
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
ScrSaHoles *h = &scr_sa_holes[i][c];
|
|
k->free[c] = scr_sa.free[c];
|
|
k->bump[c] = scr_sa.bump[c];
|
|
k->lim[c] = scr_sa.lim[c];
|
|
k->holes[c] = *h;
|
|
memset(h, 0, sizeof *h);
|
|
scr_sa.free[c] = NULL;
|
|
/* While the slice carves a returned region, its frontier is kept apart. */
|
|
if (k->holes[c].lim) {
|
|
scr_sa.bump[c] = k->holes[c].bump;
|
|
scr_sa.lim[c] = k->holes[c].lim;
|
|
}
|
|
k->front_bump[c] = scr_sa.bump[c];
|
|
k->front_lim[c] = scr_sa.lim[c];
|
|
k->start[c] = scr_sa.lim[c] ? scr_sa.bump[c] : (char *)scr_sa.base + ((size_t)c << scr_sa.shift);
|
|
}
|
|
scr_sa_keep = k;
|
|
}
|
|
|
|
/* A remote free of class c, drained while the worker keeps its memory
|
|
* apart: one of its own blocks makes it share; an older block goes to the
|
|
* free lists set aside. Once the worker has taken blocks from those lists,
|
|
* any older block might be one of them, so it shares as well. */
|
|
static void scr_sa_keep_drained(ScrSaKeep *k, ScrSaBlock *b, unsigned c) {
|
|
if ((char *)b >= k->start[c] && (char *)b < scr_sa.bump[c]) {
|
|
k->shared = true;
|
|
b->next = scr_sa.free[c];
|
|
scr_sa.free[c] = b;
|
|
} else {
|
|
if (k->ntaken) k->shared = true;
|
|
b->next = k->free[c];
|
|
k->free[c] = b;
|
|
}
|
|
}
|
|
|
|
/* A block of class c from the free lists set aside, for a worker whose
|
|
* slice has no bump room left; NULL when there is none or it cannot be
|
|
* recorded. */
|
|
static ScrSaBlock *scr_sa_keep_take(ScrSaKeep *k, unsigned c) {
|
|
ScrSaBlock *b = k->free[c];
|
|
if (!b) return NULL;
|
|
if (k->ntaken == k->cap_taken) {
|
|
size_t cap = k->cap_taken ? 2 * k->cap_taken : 256;
|
|
ScrSaBlock **taken = realloc(k->taken, cap * sizeof *taken);
|
|
if (!taken) return NULL;
|
|
k->taken = taken;
|
|
k->cap_taken = cap;
|
|
}
|
|
k->free[c] = b->next;
|
|
k->taken[k->ntaken++] = b;
|
|
return b;
|
|
}
|
|
|
|
static size_t scr_sa_sys_slot(const void *p, size_t cap) {
|
|
return (size_t)((((uint64_t)(uintptr_t)p >> 4) * UINT64_C(0x9E3779B97F4A7C15)) >> 32) & (cap - 1);
|
|
}
|
|
|
|
/* Records system block p. A worker whose set cannot grow keeps its memory. */
|
|
static void scr_sa_keep_add(ScrSaKeep *k, void *p) {
|
|
if ((k->used + 1) * 2 > k->cap) {
|
|
size_t cap = 64;
|
|
while (cap < (k->live + 1) * 4) cap *= 2;
|
|
void **sys = calloc(cap, sizeof *sys);
|
|
if (!sys) {
|
|
k->shared = true;
|
|
return;
|
|
}
|
|
for (size_t s = 0; s < k->cap; s++) {
|
|
void *q = k->sys[s];
|
|
if (!q || q == SCR_SA_SYS_GONE) continue;
|
|
size_t j = scr_sa_sys_slot(q, cap);
|
|
while (sys[j]) j = (j + 1) & (cap - 1);
|
|
sys[j] = q;
|
|
}
|
|
free(k->sys);
|
|
k->sys = sys;
|
|
k->cap = cap;
|
|
k->used = k->live;
|
|
}
|
|
size_t j = scr_sa_sys_slot(p, k->cap);
|
|
while (k->sys[j] && k->sys[j] != SCR_SA_SYS_GONE) j = (j + 1) & (k->cap - 1);
|
|
if (!k->sys[j]) k->used++;
|
|
k->sys[j] = p;
|
|
k->live++;
|
|
}
|
|
|
|
/* Forgets system block p; whether it was recorded. */
|
|
static bool scr_sa_keep_remove(ScrSaKeep *k, const void *p) {
|
|
if (!k->live) return false;
|
|
for (size_t j = scr_sa_sys_slot(p, k->cap); k->sys[j]; j = (j + 1) & (k->cap - 1)) {
|
|
if (k->sys[j] != p) continue;
|
|
k->sys[j] = SCR_SA_SYS_GONE;
|
|
k->live--;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/* The calling thread's arena keeps the worker's memory (it shared): the
|
|
* free lists set aside go back, followed by the worker's own free blocks,
|
|
* and so do the returned regions, past which the worker carved. */
|
|
static void scr_sa_keep_merge(ScrSaKeep *k) {
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
ScrSaBlock *kept = k->free[c];
|
|
if (kept) {
|
|
ScrSaBlock *tail = kept;
|
|
while (tail->next) tail = tail->next;
|
|
tail->next = scr_sa.free[c];
|
|
scr_sa.free[c] = kept;
|
|
}
|
|
ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
|
|
*h = k->holes[c];
|
|
if (h->lim) {
|
|
h->bump = scr_sa.bump[c];
|
|
h->lim = scr_sa.lim[c];
|
|
scr_sa.bump[c] = k->bump[c];
|
|
scr_sa.lim[c] = k->lim[c];
|
|
}
|
|
}
|
|
free(k->sys);
|
|
free(k->taken);
|
|
free(k);
|
|
}
|
|
|
|
/* Returns the memory of the worker whose arena the calling thread (the
|
|
* trim helper) holds, and restores the arena to its state at the worker's
|
|
* claim; with poison, makes the pages inaccessible and moves the frontier
|
|
* past them instead. */
|
|
static void scr_sa_discard(ScrSaKeep *k) {
|
|
/* Remote frees since the worker's last drain. */
|
|
scr_sa_keep = k;
|
|
scr_sa_drain_remote();
|
|
scr_sa_keep = NULL;
|
|
if (k->shared) {
|
|
scr_sa_keep_merge(k);
|
|
scr_sa_trim();
|
|
return;
|
|
}
|
|
uintptr_t page = (uintptr_t)sysconf(_SC_PAGESIZE);
|
|
if (page == 0 || page == (uintptr_t)-1) page = 4096;
|
|
size_t returned = 0, sys_bytes = 0, sys_blocks = 0;
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
char *lo = k->start[c], *top = scr_sa.bump[c], *lim = scr_sa.lim[c];
|
|
char *bump = k->front_bump[c], *limit = k->front_lim[c];
|
|
if (lim && top > lo) {
|
|
char *first = (char *)(((uintptr_t)lo + page - 1) & ~(page - 1));
|
|
char *used = (char *)(((uintptr_t)top + page - 1) & ~(page - 1));
|
|
/* The part of a page that may also hold older blocks. */
|
|
if (first > lo) memset(lo, 0, (size_t)((first < top ? first : top) - lo));
|
|
if (k->poison) {
|
|
if (used > first) (void)mprotect(first, (size_t)(used - first), PROT_NONE);
|
|
bump = used > first ? used : top;
|
|
limit = lim > bump ? lim : bump;
|
|
} else {
|
|
char *end = (char *)(((uintptr_t)lim + page - 1) & ~(page - 1));
|
|
if (end > first) {
|
|
(void)madvise(first, (size_t)(end - first), MADV_DONTNEED);
|
|
returned += (size_t)(end - first);
|
|
}
|
|
}
|
|
} else {
|
|
bump = top;
|
|
limit = lim;
|
|
}
|
|
scr_sa.free[c] = k->free[c];
|
|
ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
|
|
*h = k->holes[c];
|
|
if (h->lim) {
|
|
h->bump = bump;
|
|
h->lim = limit;
|
|
scr_sa.bump[c] = k->bump[c];
|
|
scr_sa.lim[c] = k->lim[c];
|
|
} else {
|
|
scr_sa.bump[c] = bump;
|
|
scr_sa.lim[c] = limit;
|
|
}
|
|
}
|
|
/* The older free blocks the worker took are free again. */
|
|
if (!k->poison)
|
|
for (size_t t = 0; t < k->ntaken; t++) {
|
|
ScrSaBlock *b = k->taken[t];
|
|
unsigned c = (unsigned)(((uintptr_t)b - scr_sa.base) >> scr_sa.shift);
|
|
b->next = scr_sa.free[c];
|
|
scr_sa.free[c] = b;
|
|
}
|
|
if (!k->poison)
|
|
for (size_t s = 0; s < k->cap; s++) {
|
|
void *p = k->sys[s];
|
|
if (!p || p == SCR_SA_SYS_GONE) continue;
|
|
#if defined(__GLIBC__)
|
|
sys_bytes += malloc_usable_size(p);
|
|
#endif
|
|
sys_blocks++;
|
|
free(p);
|
|
}
|
|
#if defined(__GLIBC__)
|
|
if (sys_blocks) (void)malloc_trim(0);
|
|
#endif
|
|
const char *stats = getenv("SCR_SA_STATS");
|
|
if (stats && stats[0] == '1')
|
|
fprintf(stderr, "scr_sa: discard arena %d returned %zu MB, %zu system blocks (%zu MB)\n", scr_sa_arena,
|
|
returned >> 20, sys_blocks, sys_bytes >> 20);
|
|
free(k->sys);
|
|
free(k->taken);
|
|
free(k);
|
|
}
|
|
|
|
/* Hooks of the allocator's system paths and of sharing (scr_rc.h). */
|
|
static void scr_sa_keep_note_system(void *p) {
|
|
if (p && scr_sa_keep) scr_sa_keep_add(scr_sa_keep, p);
|
|
}
|
|
|
|
void scr_sa_thread_shares(void) {
|
|
if (scr_sa_keep) scr_sa_keep->shared = true;
|
|
}
|
|
|
|
/* Worker threads that started and have not parked. */
|
|
static _Atomic(unsigned) scr_sa_workers_live;
|
|
|
|
/* Library archives run on their embedder's threads and keep their memory. */
|
|
static void scr_sa_worker_starting(void) {
|
|
#ifndef SCR_LIB
|
|
scr_sa_worker = true;
|
|
atomic_fetch_add_explicit(&scr_sa_workers_live, 1, memory_order_relaxed);
|
|
#endif
|
|
}
|
|
#define SCR_SA_WORKER_STARTING() scr_sa_worker_starting()
|
|
#endif
|
|
|
|
#ifdef SCR_SA_TRIM
|
|
/* Trims run on a detached helper thread: the parent of a worker joins it
|
|
* when it handles the worker's exit, which must not wait for the trim (at
|
|
* the end of a program every worker exits at once, and the process ends
|
|
* right after). The helper takes over the exiting thread's claim on the
|
|
* arena, so no thread claims it during the trim. */
|
|
static _Atomic(unsigned) scr_sa_trimming; /* helpers running */
|
|
|
|
static void scr_sa_release(unsigned i) {
|
|
scr_sa_parked[i] = scr_sa;
|
|
memset(&scr_sa, 0, sizeof scr_sa);
|
|
scr_sa_arena = -1;
|
|
scr_sa_refused = true;
|
|
atomic_fetch_and_explicit(&scr_sa_claimed, ~(1u << i), memory_order_release);
|
|
}
|
|
|
|
static void *scr_sa_trim_main(void *arg) {
|
|
unsigned i = (unsigned)(uintptr_t)arg;
|
|
scr_sa_arena = (int)i;
|
|
scr_sa = scr_sa_parked[i];
|
|
scr_sa_trim();
|
|
scr_sa_release(i);
|
|
atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_release);
|
|
return NULL;
|
|
}
|
|
|
|
/* Hands the calling thread's arena to a trim helper; false if no helper
|
|
* could start. */
|
|
static bool scr_sa_start_helper(unsigned i) {
|
|
scr_sa_parked[i] = scr_sa;
|
|
atomic_fetch_add_explicit(&scr_sa_trimming, 1, memory_order_relaxed);
|
|
pthread_attr_t attr;
|
|
pthread_t helper;
|
|
bool started = pthread_attr_init(&attr) == 0;
|
|
if (started) {
|
|
started = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0 &&
|
|
pthread_create(&helper, &attr, scr_sa_trim_main, (void *)(uintptr_t)i) == 0;
|
|
pthread_attr_destroy(&attr);
|
|
}
|
|
if (!started) atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_relaxed);
|
|
return started;
|
|
}
|
|
|
|
/* Discards that wait for the last running worker: one bit per arena, whose
|
|
* state and ScrSaKeep are in scr_sa_parked and scr_sa_parked_keep. */
|
|
static pthread_mutex_t scr_sa_pending_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
static uint32_t scr_sa_pending;
|
|
|
|
/* Queues the discard of the calling worker's arena i; the arena stays
|
|
* claimed until it is done. */
|
|
static void scr_sa_defer_discard(unsigned i, ScrSaKeep *k) {
|
|
atomic_fetch_add_explicit(&scr_sa_trimming, 1, memory_order_relaxed);
|
|
pthread_mutex_lock(&scr_sa_pending_lock);
|
|
scr_sa_parked[i] = scr_sa;
|
|
scr_sa_parked_keep[i] = k;
|
|
scr_sa_pending |= 1u << i;
|
|
pthread_mutex_unlock(&scr_sa_pending_lock);
|
|
}
|
|
|
|
/* Runs the waiting discards on the calling thread, which ends with no arena;
|
|
* returns whether there were any. */
|
|
static bool scr_sa_drain(void) {
|
|
bool any = false;
|
|
for (;;) {
|
|
pthread_mutex_lock(&scr_sa_pending_lock);
|
|
uint32_t pending = scr_sa_pending;
|
|
unsigned i = pending ? (unsigned)__builtin_ctz(pending) : 0;
|
|
scr_sa_pending &= ~(1u << i);
|
|
pthread_mutex_unlock(&scr_sa_pending_lock);
|
|
if (!pending) return any;
|
|
any = true;
|
|
scr_sa_arena = (int)i;
|
|
scr_sa = scr_sa_parked[i];
|
|
ScrSaKeep *k = scr_sa_parked_keep[i];
|
|
scr_sa_parked_keep[i] = NULL;
|
|
scr_sa_discard(k);
|
|
scr_sa_release(i);
|
|
atomic_fetch_sub_explicit(&scr_sa_trimming, 1, memory_order_release);
|
|
}
|
|
}
|
|
|
|
static void *scr_sa_drain_main(void *arg) {
|
|
(void)arg;
|
|
scr_sa_drain();
|
|
return NULL;
|
|
}
|
|
|
|
/* A worker parked: the last one running starts the waiting discards. */
|
|
static void scr_sa_worker_parked(void) {
|
|
if (atomic_fetch_sub_explicit(&scr_sa_workers_live, 1, memory_order_acq_rel) != 1) return;
|
|
pthread_mutex_lock(&scr_sa_pending_lock);
|
|
bool pending = scr_sa_pending != 0;
|
|
pthread_mutex_unlock(&scr_sa_pending_lock);
|
|
if (!pending) return;
|
|
pthread_attr_t attr;
|
|
pthread_t helper;
|
|
bool started = pthread_attr_init(&attr) == 0;
|
|
if (started) {
|
|
started = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0 &&
|
|
pthread_create(&helper, &attr, scr_sa_drain_main, NULL) == 0;
|
|
pthread_attr_destroy(&attr);
|
|
}
|
|
if (!started) scr_sa_drain();
|
|
}
|
|
|
|
void scr_sa_wait_trims(void) {
|
|
while (atomic_load_explicit(&scr_sa_trimming, memory_order_acquire)) {
|
|
struct timespec pause = {0, 1000000};
|
|
nanosleep(&pause, NULL);
|
|
}
|
|
}
|
|
#else
|
|
void scr_sa_wait_trims(void) {}
|
|
void scr_sa_thread_shares(void) {}
|
|
#endif
|
|
|
|
static void scr_sa_park_arena(void);
|
|
|
|
void scr_sa_thread_park(void) {
|
|
#ifdef SCR_SA_TRIM
|
|
bool worker = scr_sa_worker;
|
|
scr_sa_worker = false;
|
|
scr_sa_park_arena();
|
|
if (worker) scr_sa_worker_parked();
|
|
#else
|
|
scr_sa_park_arena();
|
|
#endif
|
|
}
|
|
|
|
static void scr_sa_park_arena(void) {
|
|
if (scr_sa_arena < 0) return;
|
|
unsigned i = (unsigned)scr_sa_arena;
|
|
#if defined(SCR_SA_PREFAULT) && defined(SCR_WORKERS)
|
|
scr_sa_pf_park();
|
|
#endif
|
|
#ifdef SCR_SA_TRIM
|
|
ScrSaKeep *k = scr_sa_keep;
|
|
scr_sa_keep = NULL;
|
|
if (k && !k->shared) {
|
|
/* Nothing this thread allocated is reachable from another one: its
|
|
* memory goes back once no other worker runs (scr_sa_worker_parked),
|
|
* and later frees on this thread are dropped. */
|
|
scr_sa_discarded = true;
|
|
scr_sa_defer_discard(i, k);
|
|
memset(&scr_sa, 0, sizeof scr_sa);
|
|
scr_sa_arena = -1;
|
|
scr_sa_refused = true;
|
|
return;
|
|
}
|
|
if (k) scr_sa_keep_merge(k);
|
|
if (scr_sa_trim_worth()) {
|
|
if (scr_sa_start_helper(i)) {
|
|
/* Later frees on this thread take the foreign path into the arena's
|
|
* remote list; the helper releases the claim. */
|
|
memset(&scr_sa, 0, sizeof scr_sa);
|
|
scr_sa_arena = -1;
|
|
scr_sa_refused = true;
|
|
return;
|
|
}
|
|
scr_sa_trim();
|
|
}
|
|
#endif
|
|
scr_sa_parked[i] = scr_sa;
|
|
/* Later frees on this thread (thread-exit destructors) take the foreign
|
|
* path into the parked arena's remote list. */
|
|
memset(&scr_sa, 0, sizeof scr_sa);
|
|
scr_sa_arena = -1;
|
|
scr_sa_refused = true;
|
|
atomic_fetch_and_explicit(&scr_sa_claimed, ~(1u << i), memory_order_release);
|
|
}
|
|
|
|
/* The class size of a block in another thread's arena, or 0 outside the
|
|
* region (a system block). */
|
|
static size_t scr_sa_foreign_size(const void *p) {
|
|
uintptr_t region = atomic_load_explicit(&scr_sa_region, memory_order_acquire);
|
|
uintptr_t off = (uintptr_t)p - region;
|
|
if (!region || off >= scr_sa_arena_span * SCR_SA_ARENAS) return 0;
|
|
return scr_sa_class_size((unsigned)((off % scr_sa_arena_span) >> scr_sa_region_shift));
|
|
}
|
|
|
|
void scr_sa_free_foreign(void *p) {
|
|
#ifdef SCR_SA_TRIM
|
|
if (scr_sa_discarded) return; /* the block went back with the thread's memory */
|
|
#endif
|
|
uintptr_t region = atomic_load_explicit(&scr_sa_region, memory_order_acquire);
|
|
uintptr_t off = (uintptr_t)p - region;
|
|
if (region && off < scr_sa_arena_span * SCR_SA_ARENAS) {
|
|
unsigned i = (unsigned)(off / scr_sa_arena_span);
|
|
ScrSaBlock *b = p;
|
|
ScrSaBlock *head = atomic_load_explicit(&scr_sa_remote[i], memory_order_relaxed);
|
|
do b->next = head;
|
|
while (!atomic_compare_exchange_weak_explicit(&scr_sa_remote[i], &head, b, memory_order_release,
|
|
memory_order_relaxed));
|
|
return;
|
|
}
|
|
#ifdef SCR_SA_TRIM
|
|
if (scr_sa_keep) (void)scr_sa_keep_remove(scr_sa_keep, p);
|
|
#endif
|
|
free(p);
|
|
}
|
|
#else
|
|
static bool scr_sa_refused;
|
|
|
|
/* Reserve address space for every class slice at once; smaller slices are
|
|
* tried when the host limits address space or overcommit. */
|
|
static bool scr_sa_reserve(void) {
|
|
for (unsigned shift = 30; shift >= 24; shift -= 2) {
|
|
size_t span = (size_t)SCR_SA_NCLASS << shift;
|
|
void *p = scr_sa_map(span);
|
|
if (!p) continue;
|
|
scr_sa.base = (uintptr_t)p;
|
|
scr_sa.shift = shift;
|
|
scr_sa.span = span;
|
|
return true;
|
|
}
|
|
scr_sa_refused = true;
|
|
return false;
|
|
}
|
|
#endif /* SCR_WORKERS */
|
|
|
|
/* Runs once per class (to reserve and open its slice) and then only for
|
|
* blocks the fast path does not serve. */
|
|
__attribute__((cold, noinline)) void *scr_sa_slow(size_t n, bool zero) {
|
|
if (n - 1 < SCR_SA_MAX && (scr_sa.span || (!scr_sa_refused && scr_sa_reserve()))) {
|
|
unsigned c = scr_sa_class(n);
|
|
#ifdef SCR_WORKERS
|
|
/* A claimed arena may be a parked one with free blocks and bump room,
|
|
* and other threads may have returned blocks since the last miss. */
|
|
if (atomic_load_explicit(&scr_sa_remote[scr_sa_arena], memory_order_relaxed))
|
|
scr_sa_drain_remote();
|
|
ScrSaBlock *b = scr_sa.free[c];
|
|
if (b) {
|
|
scr_sa.free[c] = b->next;
|
|
if (zero) memset(b, 0, n);
|
|
return b;
|
|
}
|
|
#ifdef SCR_SA_TRIM
|
|
/* A worker that keeps its memory apart uses the arena's older free
|
|
* blocks before its slice grows. */
|
|
if (scr_sa_keep && (size_t)(scr_sa.lim[c] - scr_sa.bump[c]) < scr_sa_class_size(c) &&
|
|
(b = scr_sa_keep_take(scr_sa_keep, c)) != NULL) {
|
|
if (zero) memset(b, 0, n);
|
|
return b;
|
|
}
|
|
#endif
|
|
#endif
|
|
/* Carve from the slice, opening it or raising its limit as needed;
|
|
* fresh pages are zero. */
|
|
size_t sz = scr_sa_class_size(c);
|
|
#ifdef SCR_SA_TRIM
|
|
if (scr_sa.lim[c]) scr_sa_next_hole(c, sz);
|
|
#endif
|
|
char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift);
|
|
char *end = start + ((size_t)1 << scr_sa.shift);
|
|
char *p = scr_sa.lim[c] ? scr_sa.bump[c] : start;
|
|
char *lim = scr_sa.lim[c] ? scr_sa.lim[c] : start;
|
|
#ifdef SCR_SA_DUMP
|
|
char *from = lim;
|
|
#endif
|
|
while ((size_t)(lim - p) < sz && lim < end) lim = scr_sa_extend(c, start, lim, end);
|
|
#ifdef SCR_SA_DUMP
|
|
if (lim > from) scr_sa_dumpable(start, end, from, lim);
|
|
#endif
|
|
scr_sa.lim[c] = lim;
|
|
if ((size_t)(lim - p) >= sz) {
|
|
scr_sa.bump[c] = p + sz;
|
|
return p;
|
|
}
|
|
scr_sa.bump[c] = p;
|
|
}
|
|
/* Large or zero-sized, an exhausted slice, or no reservation. */
|
|
void *q = zero ? calloc(1, n) : malloc(n);
|
|
if (n > SCR_SA_MAX) {
|
|
SCR_SA_LARGE_NOTE(q, n);
|
|
SCR_SA_THREAD_LARGE_NOTE();
|
|
}
|
|
#ifdef SCR_SA_TRIM
|
|
scr_sa_keep_note_system(q);
|
|
#endif
|
|
return q;
|
|
}
|
|
|
|
void *scr_mem_realloc(void *p, size_t n) {
|
|
if (!p) return scr_mem_alloc(n);
|
|
uintptr_t off = (uintptr_t)p - scr_sa.base;
|
|
size_t have;
|
|
if (off < scr_sa.span) have = scr_sa_class_size((unsigned)(off >> scr_sa.shift));
|
|
#ifdef SCR_WORKERS
|
|
else if ((have = scr_sa_foreign_size(p)) != 0) {
|
|
/* Another arena's block: move it into this thread's allocator. */
|
|
}
|
|
#endif
|
|
else {
|
|
#ifdef SCR_SA_TRIM
|
|
bool kept = scr_sa_keep && scr_sa_keep_remove(scr_sa_keep, p);
|
|
#endif
|
|
void *q = realloc(p, n);
|
|
if (n > SCR_SA_MAX) SCR_SA_LARGE_NOTE(q, n);
|
|
#ifdef SCR_SA_TRIM
|
|
if (kept) scr_sa_keep_note_system(q ? q : p); /* a failed realloc leaves p */
|
|
#endif
|
|
return q;
|
|
}
|
|
/* Still fits its block, which is aligned for n (see the contract). */
|
|
if (n && n <= have && (n % 16 || have % 16 == 0)) return p;
|
|
void *q = scr_mem_alloc(n ? n : 1);
|
|
if (!q) return NULL;
|
|
memcpy(q, p, have < n ? have : n);
|
|
scr_mem_free(p);
|
|
return q;
|
|
}
|
|
#else
|
|
void *scr_mem_realloc(void *p, size_t n) { return realloc(p, n); }
|
|
#ifdef SCR_WORKERS
|
|
void scr_sa_thread_park(void) {}
|
|
void scr_sa_wait_trims(void) {}
|
|
void scr_sa_thread_shares(void) {}
|
|
void scr_sa_free_foreign(void *p) { free(p); }
|
|
#endif
|
|
#endif
|
|
|
|
#ifndef SCR_SA_WORKER_STARTING
|
|
#define SCR_SA_WORKER_STARTING() ((void)0)
|
|
#endif
|
|
|
|
#ifdef SCR_WORKERS
|
|
/* Blocks above SCR_SA_MAX come from glibc, which gives each thread that
|
|
* allocates its own arena: a heap reserved up front and made usable with
|
|
* mprotect as it grows. Each extension takes the process's mmap lock for
|
|
* writing, which stalls every other thread's page faults and mapping calls
|
|
* until it is done, and a heap grows in steps barely larger than the
|
|
* request: worker threads that parse or build large data extend their
|
|
* heaps around a hundred times each, while the other threads fault pages.
|
|
* A heap keeps that range accessible, though: trimming returns the pages
|
|
* of its free top but leaves them mapped read-write (unless the system
|
|
* forbids overcommit), and growing into them later needs no mprotect. So a
|
|
* worker that allocates many large blocks (SCR_SA_THREAD_RESERVE_AFTER)
|
|
* raises its heap's high-water mark once, by allocating
|
|
* SCR_SA_THREAD_HEAP_RESERVE in blocks below glibc's smallest mmap
|
|
* threshold and freeing them. The heap top stays as small as glibc leaves
|
|
* it, so large blocks are still mapped on their own, and only the blocks'
|
|
* headers are written. A worker with few large blocks never pays for the
|
|
* extensions. The reservation stays small because glibc's calloc clears
|
|
* all of a block that lies below the high-water mark, so a larger one
|
|
* makes zeroed blocks touch pages they would otherwise leave untouched.
|
|
* Library archives run on their embedder's threads and leave its heaps
|
|
* alone. */
|
|
#if defined(__GLIBC__) && !defined(SCR_LIB)
|
|
#define SCR_SA_THREAD_HEAP_RESERVE ((size_t)4 << 20)
|
|
#define SCR_SA_THREAD_HEAP_STEP ((size_t)120 << 10) /* below 128 KiB, glibc's lowest threshold */
|
|
#define SCR_SA_THREAD_RESERVE_AFTER 256u
|
|
/* Large blocks this worker allocated before its reservation; 0 on threads
|
|
* that are not workers (they never reserve). */
|
|
static _Thread_local unsigned scr_sa_thread_large;
|
|
|
|
void scr_sa_thread_starting(void) {
|
|
scr_sa_thread_large = 1;
|
|
SCR_SA_WORKER_STARTING();
|
|
}
|
|
|
|
static void scr_sa_thread_reserve(void) {
|
|
void *blocks[SCR_SA_THREAD_HEAP_RESERVE / SCR_SA_THREAD_HEAP_STEP + 1];
|
|
size_t n = 0;
|
|
for (size_t total = 0; total < SCR_SA_THREAD_HEAP_RESERVE; total += SCR_SA_THREAD_HEAP_STEP) {
|
|
void *p = malloc(SCR_SA_THREAD_HEAP_STEP);
|
|
if (!p) break;
|
|
blocks[n++] = p;
|
|
}
|
|
while (n) free(blocks[--n]);
|
|
}
|
|
|
|
static void scr_sa_thread_large_note(void) {
|
|
unsigned count = scr_sa_thread_large;
|
|
if (count == 0 || count > SCR_SA_THREAD_RESERVE_AFTER) return;
|
|
scr_sa_thread_large = count + 1;
|
|
if (count == SCR_SA_THREAD_RESERVE_AFTER) scr_sa_thread_reserve();
|
|
}
|
|
#else
|
|
void scr_sa_thread_starting(void) { SCR_SA_WORKER_STARTING(); }
|
|
#endif
|
|
#endif
|
|
|
|
void *scr_rt_calloc(size_t n) { return scr_mem_calloc(n); }
|
|
void scr_rt_free(void *p) { scr_mem_free(p); }
|
|
|
|
void scr_mem_prefault(void *lo, void *hi) {
|
|
#if SCR_SMALL_ALLOC && defined(SCR_SA_PREFAULT)
|
|
scr_sa_prefault(lo, hi);
|
|
#else
|
|
(void)lo;
|
|
(void)hi;
|
|
#endif
|
|
}
|
|
|
|
/* Free order. Free lists are LIFO, so blocks freed in descending address
|
|
* order come back in ascending order, the order the bump pointers carve
|
|
* fresh memory in, and the order the program then touches them in. The
|
|
* order within a page matters as much as across pages: blocks of a page
|
|
* handed out in no particular order defeat the hardware prefetcher.
|
|
*
|
|
* The pointers are marked in a bitmap of block indices per class, over the
|
|
* range of blocks they span, and read back from the highest block down.
|
|
* That takes one bit per block in that range, at most one per block in
|
|
* use, and reading the bitmaps back costs a step per word; a set spread so
|
|
* thinly that they would take more than SCR_SA_ORDER_SPARSE words per
|
|
* pointer is left as it is. Pointers outside the reservation, not at
|
|
* `offset` in a block, or repeated, come first, in the order they had. */
|
|
#define SCR_SA_ORDER_SPARSE 8
|
|
|
|
#if SCR_SMALL_ALLOC
|
|
/* The class and block index of a pointer `off` bytes past the reservation
|
|
* base plus the block offset, if it is at that offset in a block. Slices
|
|
* span at most 1 << 30 bytes, so an offset within one fits in 32 bits. */
|
|
static inline bool scr_sa_order_block(uintptr_t off, uintptr_t span, unsigned shift,
|
|
const uint32_t *size, unsigned *c, uint32_t *idx) {
|
|
if (off >= span) return false;
|
|
*c = (unsigned)(off >> shift);
|
|
uint32_t in = (uint32_t)(off & (((uintptr_t)1 << shift) - 1));
|
|
*idx = in / size[*c];
|
|
return in == *idx * size[*c];
|
|
}
|
|
#endif
|
|
|
|
void scr_mem_order_for_free(void **ptrs, size_t n, size_t offset) {
|
|
#if SCR_SMALL_ALLOC
|
|
if (n < 2 || !scr_sa.span) return;
|
|
uintptr_t base = scr_sa.base + offset, span = scr_sa.span;
|
|
unsigned shift = scr_sa.shift, c;
|
|
uint32_t idx;
|
|
uint32_t size[SCR_SA_NCLASS], lo[SCR_SA_NCLASS], hi[SCR_SA_NCLASS];
|
|
size_t at[SCR_SA_NCLASS], words = 0;
|
|
for (c = 0; c < SCR_SA_NCLASS; c++) {
|
|
size[c] = (uint32_t)scr_sa_class_size(c);
|
|
lo[c] = UINT32_MAX;
|
|
hi[c] = 0;
|
|
}
|
|
for (size_t i = 0; i < n; i++) {
|
|
if (!scr_sa_order_block((uintptr_t)ptrs[i] - base, span, shift, size, &c, &idx)) continue;
|
|
if (idx < lo[c]) lo[c] = idx;
|
|
if (idx > hi[c]) hi[c] = idx;
|
|
}
|
|
for (c = 0; c < SCR_SA_NCLASS; c++) {
|
|
at[c] = words;
|
|
if (lo[c] <= hi[c]) words += hi[c] / 64 - lo[c] / 64 + 1;
|
|
}
|
|
if (words == 0 || words / SCR_SA_ORDER_SPARSE > n) return;
|
|
uint64_t *bits = calloc(words, sizeof *bits);
|
|
if (!bits) return; /* the order only affects performance */
|
|
size_t k = 0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
void *p = ptrs[i];
|
|
if (scr_sa_order_block((uintptr_t)p - base, span, shift, size, &c, &idx)) {
|
|
uint64_t *word = &bits[at[c] + idx / 64 - lo[c] / 64], bit = (uint64_t)1 << (idx % 64);
|
|
if (!(*word & bit)) {
|
|
*word |= bit;
|
|
continue;
|
|
}
|
|
}
|
|
ptrs[k++] = p;
|
|
}
|
|
for (c = 0; c < SCR_SA_NCLASS; c++) {
|
|
if (lo[c] > hi[c]) continue;
|
|
char *first = (char *)base + ((uintptr_t)c << shift) + (size_t)(lo[c] / 64) * 64 * size[c];
|
|
for (size_t w = hi[c] / 64 - lo[c] / 64 + 1; w-- > 0;) {
|
|
uint64_t word = bits[at[c] + w];
|
|
while (word) {
|
|
unsigned b = 63 - (unsigned)__builtin_clzll(word);
|
|
word &= ~((uint64_t)1 << b);
|
|
ptrs[k++] = first + (w * 64 + b) * size[c];
|
|
}
|
|
}
|
|
}
|
|
free(bits);
|
|
#else
|
|
(void)ptrs;
|
|
(void)n;
|
|
(void)offset;
|
|
#endif
|
|
}
|
|
|
|
#if defined(__SANITIZE_ADDRESS__)
|
|
#define SCR_HEAP_SANITIZER 1
|
|
#elif defined(__has_feature)
|
|
#if __has_feature(address_sanitizer)
|
|
#define SCR_HEAP_SANITIZER 1
|
|
#endif
|
|
#endif
|
|
|
|
#if defined(SCR_HEAP_SANITIZER)
|
|
#include <sanitizer/allocator_interface.h>
|
|
#elif defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 33))
|
|
#include <malloc.h>
|
|
#define SCR_HEAP_MALLINFO2 1
|
|
#elif defined(__APPLE__)
|
|
#include <malloc/malloc.h>
|
|
#elif defined(_WIN32)
|
|
#include <malloc.h>
|
|
#include <windows.h>
|
|
#endif
|
|
|
|
/* process.memoryUsage's heap figures: the calling thread's small-object
|
|
* slices (the bytes carved from them; in use is that less the blocks on
|
|
* their free lists and the zeroed holes left to carve) plus the system
|
|
* allocator's own totals where the platform reports them (glibc's
|
|
* mallinfo2, the Darwin malloc zones, the Windows CRT heap, or the
|
|
* sanitizer's allocator in sanitized builds). Walks the free lists, so
|
|
* the cost grows with the number of free blocks. */
|
|
void scr_heap_stats(double *total, double *used) {
|
|
size_t held = 0, idle = 0;
|
|
#if SCR_SMALL_ALLOC
|
|
for (unsigned c = 0; c < SCR_SA_NCLASS; c++) {
|
|
if (!scr_sa.lim[c]) continue;
|
|
size_t sz = scr_sa_class_size(c);
|
|
char *start = (char *)scr_sa.base + ((size_t)c << scr_sa.shift);
|
|
char *top = scr_sa.bump[c];
|
|
#ifdef SCR_SA_TRIM
|
|
const ScrSaHoles *h = &scr_sa_holes[scr_sa_arena][c];
|
|
if (h->lim) {
|
|
top = h->bump;
|
|
idle += (size_t)(scr_sa.lim[c] - scr_sa.bump[c]);
|
|
}
|
|
for (size_t i = 0; i < h->n; i++) idle += (size_t)(h->runs[i].hi - h->runs[i].lo);
|
|
#endif
|
|
held += (size_t)(top - start);
|
|
for (const ScrSaBlock *b = scr_sa.free[c]; b; b = b->next) idle += sz;
|
|
}
|
|
#endif
|
|
size_t sys_held = 0, sys_used = 0;
|
|
#if defined(SCR_HEAP_SANITIZER)
|
|
sys_held = __sanitizer_get_heap_size();
|
|
sys_used = __sanitizer_get_current_allocated_bytes();
|
|
#elif defined(SCR_HEAP_MALLINFO2)
|
|
struct mallinfo2 info = mallinfo2();
|
|
sys_held = info.arena + info.hblkhd;
|
|
sys_used = info.uordblks + info.hblkhd;
|
|
#elif defined(__APPLE__)
|
|
malloc_statistics_t stats;
|
|
malloc_zone_statistics(NULL, &stats);
|
|
sys_held = stats.size_allocated;
|
|
sys_used = stats.size_in_use;
|
|
#elif defined(_WIN32)
|
|
typedef struct {
|
|
DWORD cb;
|
|
SIZE_T allocated;
|
|
SIZE_T committed;
|
|
SIZE_T reserved;
|
|
SIZE_T max_reserve;
|
|
} ScrHeapSummary;
|
|
typedef BOOL(WINAPI * SummaryFn)(HANDLE, DWORD, ScrHeapSummary *);
|
|
HMODULE kernel = GetModuleHandleA("kernel32.dll");
|
|
SummaryFn summary = kernel ? (SummaryFn)(void *)GetProcAddress(kernel, "HeapSummary") : NULL;
|
|
ScrHeapSummary s = {sizeof s, 0, 0, 0, 0};
|
|
if (summary && summary((HANDLE)_get_heap_handle(), 0, &s)) {
|
|
sys_held = s.committed;
|
|
sys_used = s.allocated;
|
|
}
|
|
#endif
|
|
*total = (double)(held + sys_held);
|
|
*used = (double)(held - (idle < held ? idle : held) + sys_used);
|
|
}
|