mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-11-08 08:44:37 +08:00
Chris Tate
3999311319
check_discard ran trim_allocate again for its returned-regions case, which writes the blocks into trim_blocks, the array check_trim had freed. Where glibc maps an 8 MB block by itself (its adaptive mmap threshold, which the allocator's heap policy leaves alone unless transparent huge pages are in madvise mode), the free unmaps it and the worker thread's first write faults. The case now allocates the array again and frees it. The check that a worker takes the free blocks set aside compared the highest block it carved with that of the worker before it, which took its blocks from free lists below the slice's frontier, and it passed when a worker never took them. Each run now counts the blocks it allocates past the highest block of the class so far: at most the bump room the slice had left, against all of them when a worker does not take the free blocks. The residency bound after a discard allows for pre-fault requests still in flight.
667 lines
23 KiB
C
667 lines
23 KiB
C
/* White-box checks for the small-object allocator (scr_alloc.c). Built by
|
|
* scr_alloc.test.ts without ASan (the size-class allocator is live), with
|
|
* ASan (every call must reach the system allocator), and as a worker
|
|
* executable's allocator (per-thread arenas, also under ThreadSanitizer). */
|
|
#include "scr_runtime.h"
|
|
#include <assert.h>
|
|
#include <stdio.h>
|
|
|
|
#if SCR_SMALL_ALLOC
|
|
static bool owned(const void *p) { return (uintptr_t)p - scr_sa.base < scr_sa.span; }
|
|
#endif
|
|
|
|
static void fill(unsigned char *p, size_t n, unsigned char seed) {
|
|
for (size_t i = 0; i < n; i++) p[i] = (unsigned char)(seed + i * 7);
|
|
}
|
|
|
|
static void verify(const unsigned char *p, size_t n, unsigned char seed) {
|
|
for (size_t i = 0; i < n; i++) assert(p[i] == (unsigned char)(seed + i * 7));
|
|
}
|
|
|
|
/* Blocks are 8-byte aligned, and 16-byte aligned for multiples of 16. */
|
|
static bool aligned(const void *p, size_t n) {
|
|
return ((uintptr_t)p & (n % 16 ? 7 : 15)) == 0;
|
|
}
|
|
|
|
/* Classes step by 8 bytes up to SCR_SA_FINE_MAX and by 16 above it; the
|
|
* class of 49 to 56-byte requests holds 64-byte blocks. */
|
|
static void check_classes(void) {
|
|
unsigned prev = 0;
|
|
for (size_t n = 1; n <= SCR_SA_MAX; n++) {
|
|
unsigned c = scr_sa_class(n);
|
|
size_t sz = scr_sa_class_size(c), step = n <= SCR_SA_FINE_MAX ? 8u : 16u;
|
|
assert(c < SCR_SA_NCLASS && (c == prev || c == prev + 1));
|
|
if (n > 48 && n <= 56) assert(sz == 64);
|
|
else assert(sz >= n && sz - n < step && sz % step == 0);
|
|
prev = c;
|
|
}
|
|
assert(scr_sa_class(SCR_SA_MAX) == SCR_SA_NCLASS - 1);
|
|
assert(scr_sa_class_size(SCR_SA_NCLASS - 1) == SCR_SA_MAX);
|
|
}
|
|
|
|
static void check_alignment_and_zeroing(void) {
|
|
for (size_t n = 1; n <= 600; n++) {
|
|
unsigned char *p = scr_mem_calloc(n);
|
|
assert(p && aligned(p, n));
|
|
for (size_t i = 0; i < n; i++) assert(p[i] == 0);
|
|
fill(p, n, (unsigned char)n);
|
|
scr_mem_free(p);
|
|
/* A recycled block (same class, LIFO) must come back zeroed. */
|
|
unsigned char *q = scr_mem_calloc(n);
|
|
assert(q && aligned(q, n));
|
|
for (size_t i = 0; i < n; i++) assert(q[i] == 0);
|
|
#if SCR_SMALL_ALLOC
|
|
if (n <= SCR_SA_MAX) assert(q == p && owned(q));
|
|
else assert(!owned(q));
|
|
#endif
|
|
scr_mem_free(q);
|
|
}
|
|
}
|
|
|
|
static void check_realloc(void) {
|
|
unsigned char *p = scr_mem_realloc(NULL, 10);
|
|
fill(p, 10, 3);
|
|
size_t sizes[] = {12, 16, 17, 100, 512, 513, 4000, 300, 40, 24, 16, 7};
|
|
size_t live = 10;
|
|
for (size_t i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) {
|
|
size_t n = sizes[i];
|
|
p = scr_mem_realloc(p, n);
|
|
assert(p && aligned(p, n));
|
|
verify(p, live < n ? live : n, 3);
|
|
fill(p, n, 3);
|
|
live = n;
|
|
}
|
|
scr_mem_free(p);
|
|
}
|
|
|
|
static void check_system_pointers(void) {
|
|
/* scr_mem_free / scr_mem_realloc accept blocks the system allocated. */
|
|
void *a = malloc(32), *b = calloc(1, 4096), *c = malloc(24);
|
|
assert(a && b && c);
|
|
scr_mem_free(a);
|
|
b = scr_mem_realloc(b, 8192);
|
|
assert(b);
|
|
scr_mem_free(b);
|
|
c = scr_mem_realloc(c, 64);
|
|
assert(c);
|
|
scr_mem_free(c);
|
|
scr_mem_free(NULL);
|
|
scr_mem_free(scr_mem_alloc(0)); /* malloc(0) contract: NULL or freeable */
|
|
}
|
|
|
|
/* Interleaved alloc/free across every class: blocks never overlap and a
|
|
* live block's contents survive unrelated traffic. */
|
|
static void check_churn(void) {
|
|
enum { SLOTS = 4096 };
|
|
static unsigned char *slot[SLOTS];
|
|
static size_t len[SLOTS];
|
|
uint32_t x = 0x9e3779b9u;
|
|
for (unsigned round = 0; round < 400000; round++) {
|
|
x ^= x << 13;
|
|
x ^= x >> 17;
|
|
x ^= x << 5;
|
|
unsigned i = x % SLOTS;
|
|
if (slot[i]) {
|
|
verify(slot[i], len[i], (unsigned char)i);
|
|
scr_rt_free(slot[i]);
|
|
slot[i] = NULL;
|
|
} else {
|
|
len[i] = 1 + (x >> 12) % 700;
|
|
slot[i] = (x & 1) ? scr_rt_calloc(len[i]) : scr_mem_alloc(len[i]);
|
|
assert(slot[i]);
|
|
fill(slot[i], len[i], (unsigned char)i);
|
|
}
|
|
}
|
|
for (unsigned i = 0; i < SLOTS; i++) {
|
|
if (!slot[i]) continue;
|
|
verify(slot[i], len[i], (unsigned char)i);
|
|
scr_mem_free(slot[i]);
|
|
}
|
|
}
|
|
|
|
#if SCR_SMALL_ALLOC
|
|
#include <string.h>
|
|
#ifdef __linux__
|
|
/* The VmFlags line of the mapping that contains p, or "". */
|
|
static void vm_flags(const void *p, char *out, size_t cap) {
|
|
out[0] = '\0';
|
|
FILE *f = fopen("/proc/self/smaps", "r");
|
|
assert(f);
|
|
char line[512];
|
|
bool inside = false;
|
|
while (fgets(line, sizeof line, f)) {
|
|
unsigned long lo, hi;
|
|
if (sscanf(line, "%lx-%lx ", &lo, &hi) == 2 && strchr(line, '-') < strchr(line, ' ')) {
|
|
inside = (uintptr_t)p >= lo && (uintptr_t)p < hi;
|
|
} else if (inside && !strncmp(line, "VmFlags:", 8)) {
|
|
snprintf(out, cap, "%s", line + 8);
|
|
break;
|
|
}
|
|
}
|
|
fclose(f);
|
|
}
|
|
#endif
|
|
|
|
/* A class slice carves contiguously from its start, across the points where
|
|
* its limit is raised; on Linux the slices are huge-page aligned, the first
|
|
* 8 MiB of a slice stay on small pages and the rest may use huge pages, a
|
|
* helper thread faults in memory ahead of the limit, and only the slice up
|
|
* to its limit is included in core dumps. Every fresh block
|
|
* still reads as zero, wherever the limit steps and whoever faulted it. */
|
|
static void check_slice_growth(void) {
|
|
enum { SZ = 496, AFTER = 8 << 20, PAST = 3 * (2 << 20) + (1 << 20) };
|
|
unsigned char *first = scr_mem_alloc(SZ), *prev = first;
|
|
assert(owned(first));
|
|
uintptr_t start = scr_sa.base + ((uintptr_t)scr_sa_class(SZ) << scr_sa.shift);
|
|
assert((uintptr_t)first == start); /* a class nothing else has touched */
|
|
size_t blocks = (AFTER + PAST) / SZ;
|
|
for (size_t i = 1; i < blocks; i++) {
|
|
unsigned char *p = scr_mem_alloc(SZ);
|
|
assert(p == prev + SZ);
|
|
for (size_t k = 0; k < SZ; k += 16) assert(p[k] == 0);
|
|
p[0] = (unsigned char)i; /* commit the pages */
|
|
p[SZ - 1] = (unsigned char)i;
|
|
prev = p;
|
|
}
|
|
#ifdef __linux__
|
|
assert((scr_sa.base & ((2u << 20) - 1)) == 0);
|
|
FILE *thp = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r");
|
|
if (thp) { /* a kernel without THP refuses both advices */
|
|
fclose(thp);
|
|
char flags[256];
|
|
vm_flags(first, flags, sizeof flags);
|
|
assert(strstr(flags, " nh") && !strstr(flags, " hg"));
|
|
vm_flags((void *)(start + AFTER), flags, sizeof flags);
|
|
assert(strstr(flags, " hg") && !strstr(flags, " nh"));
|
|
}
|
|
/* Core dumps hold the carved blocks but not the untouched rest of the
|
|
* reservation (" dd" is VM_DONTDUMP). */
|
|
char flags[256];
|
|
vm_flags(first, flags, sizeof flags);
|
|
assert(!strstr(flags, " dd"));
|
|
vm_flags(prev + SZ - 1, flags, sizeof flags);
|
|
assert(!strstr(flags, " dd"));
|
|
vm_flags((void *)(start + ((uintptr_t)1 << scr_sa.shift) - 1), flags, sizeof flags);
|
|
assert(strstr(flags, " dd"));
|
|
vm_flags((void *)(scr_sa.base + scr_sa.span - 1), flags, sizeof flags);
|
|
assert(strstr(flags, " dd"));
|
|
#endif
|
|
/* The blocks recycle through the free list like any others. */
|
|
for (size_t i = blocks; i-- > 0;) scr_mem_free(first + i * SZ);
|
|
assert(scr_mem_alloc(SZ) == first);
|
|
scr_mem_free(first);
|
|
}
|
|
|
|
#if defined(__linux__) && defined(__GLIBC__)
|
|
#include <unistd.h>
|
|
/* Blocks above SCR_SA_MAX come from glibc; once its brk heap has grown past
|
|
* 32 MiB, the heap is advised huge pages where THP is in madvise mode. */
|
|
static void check_heap_huge_pages(void) {
|
|
enum { N = 1024, SZ = 64 << 10 };
|
|
static unsigned char *block[N];
|
|
uintptr_t start = (uintptr_t)sbrk(0);
|
|
for (size_t i = 0; i < N; i++) {
|
|
block[i] = scr_mem_alloc(SZ);
|
|
assert(block[i] && !owned(block[i]));
|
|
block[i][0] = (unsigned char)i;
|
|
}
|
|
uintptr_t top = (uintptr_t)sbrk(0);
|
|
char mode[96] = {0};
|
|
FILE *thp = fopen("/sys/kernel/mm/transparent_hugepage/enabled", "r");
|
|
if (thp) {
|
|
size_t n = fread(mode, 1, sizeof mode - 1, thp);
|
|
mode[n] = '\0';
|
|
fclose(thp);
|
|
}
|
|
/* Only where glibc's heap served them (a sanitizer may replace malloc). */
|
|
if (top - start >= ((size_t)48 << 20) && strstr(mode, "[madvise]")) {
|
|
char flags[256];
|
|
vm_flags(block[N - 1], flags, sizeof flags);
|
|
assert(strstr(flags, " hg"));
|
|
}
|
|
for (size_t i = 0; i < N; i++) {
|
|
assert(block[i][0] == (unsigned char)i);
|
|
scr_mem_free(block[i]);
|
|
}
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
#if defined(SCR_WORKERS) && SCR_SMALL_ALLOC
|
|
#include <pthread.h>
|
|
|
|
/* Script threads own disjoint arenas; blocks freed by another thread return
|
|
* to their owner through its remote list, and an exiting thread's arena
|
|
* (free lists included) passes to the next thread that claims one. */
|
|
enum { HANDOFF = 256 };
|
|
static void *handoff[HANDOFF];
|
|
static void *handed[HANDOFF]; /* the addresses, kept after the frees */
|
|
static uintptr_t worker_base;
|
|
|
|
static void *alloc_and_hand_off(void *arg) {
|
|
(void)arg;
|
|
for (unsigned i = 0; i < HANDOFF; i++) {
|
|
handoff[i] = scr_mem_calloc(48);
|
|
assert(handoff[i] && owned(handoff[i]));
|
|
handed[i] = handoff[i];
|
|
fill(handoff[i], 48, (unsigned char)i);
|
|
}
|
|
worker_base = scr_sa.base;
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
static void *reclaim_arena(void *arg) {
|
|
(void)arg;
|
|
/* The parked arena is the only free one besides main's: claimed again,
|
|
* with the main thread's remote frees drained into its free lists. */
|
|
void *p = scr_mem_alloc(48);
|
|
assert(scr_sa.base == worker_base);
|
|
bool recycled = false;
|
|
for (unsigned i = 0; i < HANDOFF; i++) recycled |= p == handed[i];
|
|
assert(recycled);
|
|
scr_mem_free(p);
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
static void check_threads(void) {
|
|
void *mine = scr_mem_alloc(48);
|
|
assert(owned(mine));
|
|
pthread_t t;
|
|
assert(pthread_create(&t, NULL, alloc_and_hand_off, NULL) == 0);
|
|
assert(pthread_join(t, NULL) == 0);
|
|
assert(worker_base != scr_sa.base);
|
|
for (unsigned i = 0; i < HANDOFF; i++) {
|
|
assert(!owned(handoff[i]));
|
|
verify(handoff[i], 48, (unsigned char)i);
|
|
if (i == 0) {
|
|
handoff[0] = scr_mem_realloc(handoff[0], 600); /* moves to the system */
|
|
verify(handoff[0], 48, 0);
|
|
free(handoff[0]);
|
|
handoff[0] = NULL;
|
|
} else scr_mem_free(handoff[i]); /* remote free */
|
|
}
|
|
assert(pthread_create(&t, NULL, reclaim_arena, NULL) == 0);
|
|
assert(pthread_join(t, NULL) == 0);
|
|
scr_mem_free(mine);
|
|
}
|
|
#endif
|
|
|
|
#if defined(SCR_WORKERS) && SCR_SMALL_ALLOC && defined(__linux__)
|
|
#include <sys/mman.h>
|
|
#include <unistd.h>
|
|
|
|
/* An exiting thread returns the pages that hold only free blocks; the next
|
|
* thread to claim the arena carves the returned range again, zeroed, before
|
|
* growing the slice. */
|
|
enum { TRIM_BLOCKS = 1 << 20, TRIM_KEEP = 4096, TRIM_SIZE = 48 };
|
|
static unsigned char **trim_blocks;
|
|
static unsigned char *trim_kept[TRIM_BLOCKS / TRIM_KEEP];
|
|
static uintptr_t trim_lo, trim_hi;
|
|
static size_t trim_outside;
|
|
|
|
static size_t resident_pages(uintptr_t lo, uintptr_t hi) {
|
|
size_t page = (size_t)sysconf(_SC_PAGESIZE);
|
|
lo &= ~(uintptr_t)(page - 1);
|
|
size_t n = (hi - lo + page - 1) / page, count = 0;
|
|
unsigned char *vec = malloc(n);
|
|
assert(vec && mincore((void *)lo, n * page, vec) == 0);
|
|
for (size_t i = 0; i < n; i++) count += vec[i] & 1;
|
|
free(vec);
|
|
return count;
|
|
}
|
|
|
|
static void *trim_allocate(void *arg) {
|
|
(void)arg;
|
|
trim_lo = UINTPTR_MAX;
|
|
trim_hi = 0;
|
|
for (size_t i = 0; i < TRIM_BLOCKS; i++) {
|
|
trim_blocks[i] = scr_mem_alloc(TRIM_SIZE);
|
|
assert(trim_blocks[i] && owned(trim_blocks[i]));
|
|
fill(trim_blocks[i], TRIM_SIZE, (unsigned char)i);
|
|
if ((uintptr_t)trim_blocks[i] < trim_lo) trim_lo = (uintptr_t)trim_blocks[i];
|
|
if ((uintptr_t)trim_blocks[i] + TRIM_SIZE > trim_hi) trim_hi = (uintptr_t)trim_blocks[i] + TRIM_SIZE;
|
|
}
|
|
for (size_t i = 0; i < TRIM_BLOCKS; i++) {
|
|
if (i % TRIM_KEEP == 0) trim_kept[i / TRIM_KEEP] = trim_blocks[i];
|
|
else scr_mem_free(trim_blocks[i]);
|
|
}
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
static void *trim_reuse(void *arg) {
|
|
(void)arg;
|
|
trim_outside = 0;
|
|
for (size_t i = 0; i < TRIM_BLOCKS; i++) {
|
|
unsigned char *p = scr_mem_calloc(TRIM_SIZE);
|
|
assert(p && owned(p));
|
|
for (size_t k = 0; k < TRIM_SIZE; k++) assert(p[k] == 0);
|
|
fill(p, TRIM_SIZE, (unsigned char)i);
|
|
if ((uintptr_t)p < trim_lo || (uintptr_t)p >= trim_hi) trim_outside++;
|
|
trim_blocks[i] = p;
|
|
}
|
|
for (size_t i = 0; i < TRIM_BLOCKS; i++) scr_mem_free(trim_blocks[i]);
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
static void check_trim(void) {
|
|
trim_blocks = malloc(TRIM_BLOCKS * sizeof *trim_blocks);
|
|
assert(trim_blocks);
|
|
pthread_t t;
|
|
assert(pthread_create(&t, NULL, trim_allocate, NULL) == 0);
|
|
assert(pthread_join(t, NULL) == 0);
|
|
scr_sa_wait_trims(); /* a helper thread returns the pages */
|
|
/* One page per kept block stays, give or take the pages at run ends. */
|
|
size_t total = (trim_hi - trim_lo) / (size_t)sysconf(_SC_PAGESIZE);
|
|
size_t resident = resident_pages(trim_lo, trim_hi);
|
|
assert(resident < total / 8);
|
|
for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++)
|
|
verify(trim_kept[k], TRIM_SIZE, (unsigned char)(k * TRIM_KEEP));
|
|
assert(pthread_create(&t, NULL, trim_reuse, NULL) == 0);
|
|
assert(pthread_join(t, NULL) == 0);
|
|
scr_sa_wait_trims();
|
|
/* Nearly every block came from the returned range: the slice did not grow. */
|
|
assert(trim_outside < 4 * TRIM_BLOCKS / TRIM_KEEP + 256);
|
|
for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++) {
|
|
verify(trim_kept[k], TRIM_SIZE, (unsigned char)(k * TRIM_KEEP));
|
|
scr_mem_free(trim_kept[k]); /* remote free into the parked arena */
|
|
}
|
|
free(trim_blocks);
|
|
}
|
|
#endif
|
|
|
|
#if defined(SCR_WORKERS) && SCR_SMALL_ALLOC && defined(__linux__)
|
|
#include <malloc.h>
|
|
#include <stdatomic.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <time.h>
|
|
|
|
/* A worker thread (scr_sa_thread_starting) that exits without sharing
|
|
* gives its memory back: the pages it carved, and the system blocks it
|
|
* still holds. Older blocks of the arena, and the memory of a worker that
|
|
* shared (or got a remote free of its own blocks), stay. */
|
|
/* DISCARD_SIZE and POISON_SIZE are classes the earlier checks leave unused
|
|
* in the workers' arena, so their state there is known. */
|
|
enum { DISCARD_BLOCKS = 1 << 20, DISCARD_SIZE = 40, POISON_SIZE = 56, DISCARD_OLD = 64, DISCARD_LARGE = 64 };
|
|
enum { DISCARD_LARGE_SIZE = 64 << 10 };
|
|
static unsigned char *old_blocks[DISCARD_OLD];
|
|
static unsigned char **discard_blocks;
|
|
static unsigned char *discard_large[DISCARD_LARGE];
|
|
static uintptr_t discard_base, discard_lo, discard_hi;
|
|
static size_t block_size = DISCARD_SIZE; /* of the workers' and reusing threads' blocks */
|
|
/* The end of the highest block of class DISCARD_SIZE so far, and how many blocks of a run lay past the mark
|
|
* at its start: those the slice grew for. */
|
|
static uintptr_t class_hi;
|
|
static size_t discard_beyond, reuse_beyond;
|
|
static bool discard_shares;
|
|
static atomic_int discard_step;
|
|
|
|
static void wait_step(int step) {
|
|
while (atomic_load(&discard_step) < step) {
|
|
struct timespec pause = {0, 100000};
|
|
nanosleep(&pause, NULL);
|
|
}
|
|
}
|
|
|
|
/* A plain script thread whose blocks stay live in its parked arena. */
|
|
static void *discard_old_owner(void *arg) {
|
|
(void)arg;
|
|
for (unsigned i = 0; i < DISCARD_OLD; i++) {
|
|
old_blocks[i] = scr_mem_alloc(DISCARD_SIZE);
|
|
assert(owned(old_blocks[i]));
|
|
fill(old_blocks[i], DISCARD_SIZE, (unsigned char)i);
|
|
}
|
|
discard_base = scr_sa.base;
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
static void allocate_discardable(void) {
|
|
discard_lo = UINTPTR_MAX;
|
|
discard_hi = 0;
|
|
discard_beyond = 0;
|
|
for (size_t i = 0; i < DISCARD_BLOCKS; i++) {
|
|
unsigned char *p = scr_mem_alloc(block_size);
|
|
assert(p && owned(p));
|
|
fill(p, block_size, (unsigned char)i);
|
|
discard_blocks[i] = p;
|
|
if ((uintptr_t)p >= class_hi) discard_beyond++;
|
|
if ((uintptr_t)p < discard_lo) discard_lo = (uintptr_t)p;
|
|
if ((uintptr_t)p + block_size > discard_hi) discard_hi = (uintptr_t)p + block_size;
|
|
}
|
|
if (block_size == DISCARD_SIZE && discard_hi > class_hi) class_hi = discard_hi;
|
|
/* System blocks: some freed, some moved by realloc, the rest kept. */
|
|
for (unsigned i = 0; i < DISCARD_LARGE; i++) {
|
|
discard_large[i] = scr_mem_alloc(DISCARD_LARGE_SIZE);
|
|
assert(discard_large[i] && !owned(discard_large[i]));
|
|
fill(discard_large[i], 64, (unsigned char)i);
|
|
}
|
|
for (unsigned i = 0; i < DISCARD_LARGE; i += 4) {
|
|
scr_mem_free(discard_large[i]);
|
|
discard_large[i] = NULL;
|
|
}
|
|
for (unsigned i = 1; i < DISCARD_LARGE; i += 4) discard_large[i] = scr_mem_realloc(discard_large[i], 2 * DISCARD_LARGE_SIZE);
|
|
}
|
|
|
|
static void *discard_worker(void *arg) {
|
|
(void)arg;
|
|
scr_sa_thread_starting();
|
|
allocate_discardable();
|
|
assert(scr_sa.base == discard_base);
|
|
/* None of the arena's older blocks or free blocks is reused. */
|
|
for (unsigned i = 0; i < DISCARD_OLD; i++) assert(discard_lo > (uintptr_t)old_blocks[i]);
|
|
if (discard_shares) scr_sa_thread_shares();
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
/* A worker one of whose blocks the main thread frees while it runs. */
|
|
static void *remote_freed_worker(void *arg) {
|
|
(void)arg;
|
|
scr_sa_thread_starting();
|
|
allocate_discardable();
|
|
atomic_store(&discard_step, 1);
|
|
wait_step(2);
|
|
/* A class this thread has not used: its first block takes the slow path,
|
|
* which drains the remote free. */
|
|
void *p = scr_mem_alloc(200);
|
|
assert(p);
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
/* A plain thread that takes as many blocks again from the arena: carved
|
|
* ones must be zero (the fast path does not clear them). */
|
|
static size_t reuse_outside;
|
|
static uintptr_t reuse_hi;
|
|
static void *discard_reuse(void *arg) {
|
|
(void)arg;
|
|
reuse_outside = 0;
|
|
reuse_hi = 0;
|
|
reuse_beyond = 0;
|
|
for (size_t i = 0; i < DISCARD_BLOCKS; i++) {
|
|
unsigned char *p = scr_mem_calloc(block_size);
|
|
assert(p && owned(p));
|
|
assert(scr_sa.base == discard_base);
|
|
for (size_t k = 0; k < block_size; k++) assert(p[k] == 0);
|
|
for (unsigned j = 0; j < DISCARD_OLD; j++) assert(p != old_blocks[j]);
|
|
if ((uintptr_t)p < discard_lo || (uintptr_t)p >= discard_hi) reuse_outside++;
|
|
if ((uintptr_t)p + block_size > reuse_hi) reuse_hi = (uintptr_t)p + block_size;
|
|
if ((uintptr_t)p >= class_hi) reuse_beyond++;
|
|
discard_blocks[i] = p;
|
|
}
|
|
if (block_size == DISCARD_SIZE && reuse_hi > class_hi) class_hi = reuse_hi;
|
|
for (size_t i = 0; i < DISCARD_BLOCKS; i++) scr_mem_free(discard_blocks[i]);
|
|
scr_sa_thread_park();
|
|
return NULL;
|
|
}
|
|
|
|
static void run(void *(*fn)(void *)) {
|
|
pthread_t t;
|
|
assert(pthread_create(&t, NULL, fn, NULL) == 0);
|
|
assert(pthread_join(t, NULL) == 0);
|
|
scr_sa_wait_trims();
|
|
}
|
|
|
|
static void verify_old_blocks(void) {
|
|
for (unsigned i = 0; i < DISCARD_OLD; i++) verify(old_blocks[i], DISCARD_SIZE, (unsigned char)i);
|
|
}
|
|
|
|
/* The worker's blocks stayed as it left them, and the next thread to claim
|
|
* the arena carves past them; then they are freed (remote frees). */
|
|
static void verify_kept(void) {
|
|
for (size_t i = 0; i < DISCARD_BLOCKS; i++) verify(discard_blocks[i], block_size, (unsigned char)i);
|
|
for (unsigned i = 0; i < DISCARD_LARGE; i++) {
|
|
if (!discard_large[i]) continue;
|
|
verify(discard_large[i], 64, (unsigned char)i);
|
|
free(discard_large[i]);
|
|
}
|
|
for (size_t i = 0; i < DISCARD_BLOCKS; i++) scr_mem_free(discard_blocks[i]);
|
|
}
|
|
|
|
/* The protection of the mapping that holds address a, from /proc/self/maps. */
|
|
static bool readable(uintptr_t a) {
|
|
FILE *maps = fopen("/proc/self/maps", "r");
|
|
assert(maps);
|
|
char line[512];
|
|
bool found = false, result = false;
|
|
while (!found && fgets(line, sizeof line, maps)) {
|
|
unsigned long lo, hi;
|
|
char perms[8];
|
|
if (sscanf(line, "%lx-%lx %7s", &lo, &hi, perms) != 3) continue;
|
|
if (a >= lo && a < hi) {
|
|
found = true;
|
|
result = perms[0] == 'r';
|
|
}
|
|
}
|
|
fclose(maps);
|
|
assert(found);
|
|
return result;
|
|
}
|
|
|
|
static void check_discard(void) {
|
|
discard_blocks = malloc(DISCARD_BLOCKS * sizeof *discard_blocks);
|
|
assert(discard_blocks);
|
|
run(discard_old_owner);
|
|
size_t page = (size_t)sysconf(_SC_PAGESIZE);
|
|
|
|
/* Discarded: its pages are returned, and carved again (zeroed) by the next
|
|
* thread, which does not grow the slice. The arena's older blocks stay. */
|
|
discard_shares = false;
|
|
run(discard_worker);
|
|
verify_old_blocks();
|
|
uintptr_t first = (discard_lo + page - 1) & ~(uintptr_t)(page - 1);
|
|
/* Pre-fault requests still in flight may fault in a few megabytes again. */
|
|
size_t total = (discard_hi - first) / page;
|
|
assert(resident_pages(first, discard_hi) < total / 2);
|
|
run(discard_reuse);
|
|
assert(reuse_outside < 256);
|
|
verify_old_blocks();
|
|
|
|
/* SCR_SA_DISCARD=poison makes the pages inaccessible; the next thread
|
|
* carves past them. */
|
|
block_size = POISON_SIZE;
|
|
setenv("SCR_SA_DISCARD", "poison", 1);
|
|
run(discard_worker);
|
|
unsetenv("SCR_SA_DISCARD");
|
|
verify_old_blocks();
|
|
first = (discard_lo + page - 1) & ~(uintptr_t)(page - 1);
|
|
assert(!readable(first));
|
|
assert(!readable((discard_hi - 1) & ~(uintptr_t)(page - 1)));
|
|
run(discard_reuse);
|
|
assert(reuse_outside == DISCARD_BLOCKS);
|
|
verify_old_blocks();
|
|
block_size = DISCARD_SIZE;
|
|
|
|
/* A worker that shared keeps its memory. */
|
|
discard_shares = true;
|
|
run(discard_worker);
|
|
verify_old_blocks();
|
|
verify_kept();
|
|
|
|
/* So does one that got a remote free of one of its blocks. */
|
|
atomic_store(&discard_step, 0);
|
|
pthread_t t;
|
|
assert(pthread_create(&t, NULL, remote_freed_worker, NULL) == 0);
|
|
wait_step(1);
|
|
scr_mem_free(discard_blocks[0]);
|
|
discard_blocks[0] = scr_mem_alloc(DISCARD_SIZE); /* main's own block, for verify_kept */
|
|
fill(discard_blocks[0], DISCARD_SIZE, 0);
|
|
atomic_store(&discard_step, 2);
|
|
assert(pthread_join(t, NULL) == 0);
|
|
scr_sa_wait_trims();
|
|
verify_old_blocks();
|
|
verify_kept();
|
|
|
|
/* SCR_SA_DISCARD=0 keeps every worker's memory. */
|
|
discard_shares = false;
|
|
setenv("SCR_SA_DISCARD", "0", 1);
|
|
run(discard_worker);
|
|
unsetenv("SCR_SA_DISCARD");
|
|
verify_old_blocks();
|
|
verify_kept();
|
|
|
|
/* The kept workers' blocks are free now (remote frees): a worker takes
|
|
* them once its slice has no bump room left, and they are free again
|
|
* after its memory goes back, so the next thread does not grow the slice
|
|
* either. Past the highest block so far, each carves at most the bump room
|
|
* the slice had left (a few megabytes). */
|
|
size_t room = ((size_t)4 << 20) / DISCARD_SIZE;
|
|
run(discard_worker);
|
|
verify_old_blocks();
|
|
assert(discard_beyond < room);
|
|
run(discard_reuse);
|
|
assert(reuse_beyond < room);
|
|
verify_old_blocks();
|
|
|
|
/* Returned regions (holes) of an earlier trim are kept apart while a
|
|
* worker runs, and carved again afterwards. */
|
|
trim_blocks = malloc(TRIM_BLOCKS * sizeof *trim_blocks); /* check_trim freed its own */
|
|
assert(trim_blocks);
|
|
run(trim_allocate);
|
|
uintptr_t hole_lo = trim_lo, hole_hi = trim_hi;
|
|
discard_shares = false;
|
|
run(discard_worker);
|
|
verify_old_blocks();
|
|
for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++)
|
|
verify(trim_kept[k], TRIM_SIZE, (unsigned char)(k * TRIM_KEEP));
|
|
trim_lo = hole_lo;
|
|
trim_hi = hole_hi;
|
|
run(trim_reuse);
|
|
assert(trim_outside < 4 * TRIM_BLOCKS / TRIM_KEEP + 256);
|
|
for (size_t k = 0; k < TRIM_BLOCKS / TRIM_KEEP; k++) scr_mem_free(trim_kept[k]);
|
|
free(trim_blocks);
|
|
for (unsigned i = 0; i < DISCARD_OLD; i++) scr_mem_free(old_blocks[i]);
|
|
free(discard_blocks);
|
|
}
|
|
#endif
|
|
|
|
int main(void) {
|
|
#if SCR_SMALL_ALLOC
|
|
check_slice_growth(); /* first: its class must be untouched */
|
|
#endif
|
|
check_classes();
|
|
check_alignment_and_zeroing();
|
|
check_realloc();
|
|
check_system_pointers();
|
|
check_churn();
|
|
#if SCR_SMALL_ALLOC && defined(__linux__) && defined(__GLIBC__)
|
|
check_heap_huge_pages();
|
|
#endif
|
|
#if defined(SCR_WORKERS) && SCR_SMALL_ALLOC
|
|
check_threads();
|
|
#ifdef __linux__
|
|
check_trim();
|
|
check_discard();
|
|
#endif
|
|
printf("allocator checks passed: size-class workers\n");
|
|
#else
|
|
printf("allocator checks passed: %s\n", SCR_SMALL_ALLOC ? "size-class" : "system");
|
|
#endif
|
|
return 0;
|
|
}
|