diff --git a/packages/runtime/src/scr_array.c b/packages/runtime/src/scr_array.c index c64b85947..c00d1b4f6 100644 --- a/packages/runtime/src/scr_array.c +++ b/packages/runtime/src/scr_array.c @@ -131,13 +131,6 @@ uint32_t scr_arr_ops_register(ScrArrOps *ops) { const ScrArrOps *scr_arr_ops_overflow(const ScrArr *a) { return scr_arr_ext_of(a)->ops; } -/* Record `ops` as the element ops of `a` (NULL for none). */ -static void scr_arr_set_ops(ScrArr *a, const ScrArrOps *ops) { - uint32_t id = scr_arr_ops_id(ops); - a->spare = (uint16_t)((a->spare & ((1u << SCR_ARR_OPS_SHIFT) - 1)) | (id << SCR_ARR_OPS_SHIFT)); - if (id == SCR_ARR_OPS_OVERFLOW) scr_arr_ext(a)->ops = ops; -} - /* Canonical array indices grow indexed storage and may leave holes. Negative, * fractional, NaN, infinity, and 2^32-1 numeric keys use the ordinary * property vector; only a missing property or a low-level read of a hole @@ -402,9 +395,10 @@ static void scr_arr_sparse_drop_prefix(ScrArr *a, size_t count) { #define SCR_ARR_MIN_GROWTH ((size_t)4) /* Inline slots: an array built by scr_arr_new_small / scr_arr_new_ops_small - * (empty literals the compiler sees appended to) keeps its first dense - * slots and states in its own block, right after the ScrArr, so it costs - * one allocation and its first elements share the header's cache lines. + * (empty literals the compiler sees appended to), or constructed with a + * known element count of up to SCR_ARR_KNOWN_INLINE_MAX, keeps its first + * dense slots and states in its own block, right after the ScrArr, so it + * costs one allocation and its first elements share the header's cache lines. * `spare` records it. Growth past the inline capacity moves the slots to an * ordinary allocation; the inline room then stays unused until the array is * freed. */ @@ -526,6 +520,30 @@ static size_t scr_arr_small_cap(size_t prefix, bool bools) { #endif } +/* The most slots a construction with a known element count (array + * literals, copies, slices and the arrays the runtime builds) keeps inline. + * Larger counts, and blocks past the small-object allocator, keep separate + * storage: an array that then grows leaves its inline room unused, which + * this bound keeps small. */ +#define SCR_ARR_KNOWN_INLINE_MAX ((size_t)16) + +/* Inline capacity of a construction of `need` elements with `prefix` bytes + * of block before the slots, rounded up to fill the block; 0 when its slots + * stay separate. */ +static size_t scr_arr_known_cap(size_t prefix, size_t need, bool bools) { + if (need == 0 || need > SCR_ARR_KNOWN_INLINE_MAX) return 0; +#if SCR_SMALL_ALLOC + size_t unit = bools ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; + size_t bytes = prefix + need * unit; + if (bytes > SCR_SA_MAX) return 0; + return (scr_sa_class_size(scr_sa_class(bytes)) - prefix) / unit; +#else + (void)prefix; + (void)bools; + return need; +#endif +} + static void scr_arr_use_inline(ScrArr *a, size_t cap) { a->data = (uint64_t *)((char *)a + SCR_ARR_INLINE_OFFSET); a->cap = (uint32_t)cap; @@ -1180,7 +1198,8 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind el size_t initial_cap, bool small) { size_t unit = elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES; - size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL) : 0; + size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL) + : scr_arr_known_cap(SCR_ARR_INLINE_OFFSET, initial_cap, elem == SCR_ELEM_BOOL); ScrArr *a = scr_mem_alloc(inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr)); if (!a) scr_arr_oom(); a->rc = 1; @@ -1310,11 +1329,17 @@ static void scr_arr_gc_free(void *a0) { scr_cyc_free_inline(a); } -static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool small) { +/* Inlined into each constructor like scr_arr_make: with `small` and the + * count constant, scr_arr_new_ops_small's size class folds to a table + * index. */ +static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops(const ScrArrOps *ops, + size_t initial_cap, + bool small) { ScrArr *a; bool cyc = ops->trace != NULL; + size_t prefix = SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0); size_t inline_cap = - small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0), false) : 0; + small ? scr_arr_small_cap(prefix, false) : scr_arr_known_cap(prefix, initial_cap, false); size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * SCR_ARR_SLOT_BYTES : sizeof(ScrArr); if (cyc) { static ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY }; @@ -1328,12 +1353,19 @@ static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool s a->cap = 0; a->elem = SCR_ELEM_REF; a->cyc = cyc; - a->spare = 0; + /* The ops id is written with the other header fields: reading `spare` + * back after scr_arr_use_inline sets its inline flag stalls the + * store-to-load forward. */ + uint32_t id = scr_arr_ops_id(ops); + a->spare = (uint16_t)(id << SCR_ARR_OPS_SHIFT); a->filled = 0; a->data = NULL; - scr_arr_set_ops(a, ops); if (inline_cap) scr_arr_use_inline(a, inline_cap); else if (initial_cap > 0) scr_arr_grow_dense_to(a, initial_cap, SCR_ARR_GROW_EXACT); + /* After the storage: ops past the registry live in the ext, whose + * pointer precedes the dense allocation, so inline slots move out first + * (scr_arr_ext). */ + if (id == SCR_ARR_OPS_OVERFLOW) scr_arr_ext(a)->ops = ops; #ifdef SCR_RC_AUDIT scr_live_arrays++; #endif diff --git a/packages/runtime/test/test_array.c b/packages/runtime/test/test_array.c index 02da2fac2..d943acfb2 100644 --- a/packages/runtime/test/test_array.c +++ b/packages/runtime/test/test_array.c @@ -398,6 +398,8 @@ static MockRec *mock_cyc_new(int value) { static ScrArrOps mock_ops = { &mock_retain, &mock_release, NULL }; static ScrArrOps mock_cyc_ops = { &mock_cyc_retain, &mock_cyc_release, &mock_trace }; +/* First used after test_ops_overflow fills the registry: its arrays keep the ops in their ext. */ +static ScrArrOps mock_cyc_ops_late = { &mock_cyc_retain, &mock_cyc_release, &mock_trace }; static void test_ref_elements(void) { #ifdef SCR_RC_AUDIT @@ -514,6 +516,38 @@ static void test_ref_cycle(void) { #endif } +static void test_ref_known_count_cycle(ScrArrOps *ops) { +#ifdef SCR_RC_AUDIT + long arrays0 = scr_arr_live_count(); +#endif + /* A known-count construction of cycle-capable elements keeps its slots + * after the collector header: the collector traces them, and both the + * collected array and one released by its count free the one block. */ + ScrArr *arr = scr_arr_new_ops(ops, 2); + bool in_block = arr->data == (uint64_t *)((char *)arr + ((sizeof(ScrArr) + 7) & ~(size_t)7)); + /* Ops past the registry (once test_ops_overflow has filled it) live in + * the ext, which moves the slots to their own allocation. */ + bool overflow = (arr->spare >> SCR_ARR_OPS_SHIFT) == SCR_ARR_OPS_OVERFLOW; + check(arr->cap >= 2 && in_block == !overflow && scr_arr_ops(arr) == ops, + "known-count cycle-capable construction is inline unless its ops need the ext"); + MockRec *first = mock_cyc_new(1); + first->owner = (ScrArr *)scr_arr_retain(arr); + scr_arr_push_ref(arr, first); + scr_arr_push_ref(arr, mock_cyc_new(2)); + scr_arr_release(arr); + scr_collect_cycles(); + check(mock_live == 0, "cycle through an inline array collected"); + ScrArr *plain = scr_arr_new_ops(ops, 3); + for (int i = 0; i < 4; i++) scr_arr_push_ref(plain, mock_cyc_new(i)); + check(scr_arr_len(plain) == 4, "a cycle-capable inline array grows out of its block"); + scr_arr_release(plain); + scr_collect_cycles(); + check(mock_live == 0, "released inline cycle-capable array frees its elements"); +#ifdef SCR_RC_AUDIT + check(scr_arr_live_count() == arrays0, "known-count cycle: no arrays leaked"); +#endif +} + static void test_ref_truncate_cycle(void) { #ifdef SCR_RC_AUDIT long arrays0 = scr_arr_live_count(); @@ -1543,9 +1577,33 @@ static void test_inline_slots(void) { long strings0 = scr_str_live_count(); long arrays0 = scr_arr_live_count(); #endif - ScrArr *plain = scr_arr_new(SCR_ELEM_F64, 4); - check(!inline_slots(plain), "plain constructions keep separate storage"); + ScrArr *plain = scr_arr_new(SCR_ELEM_F64, 0); + check(!inline_slots(plain) && plain->data == NULL, "empty plain constructions allocate no slots"); scr_arr_release(plain); + /* A construction with a known count up to 16 slots keeps them in its own + * block; growth past them moves them out like any inline array. */ + ScrArr *known = scr_arr_new(SCR_ELEM_F64, 4); + check(inline_slots(known) && known->cap >= 4 && known->len == 0 && !scr_arr_has(known, 0) && + scr_arr_states_end(known) == (uint8_t *)(known->data + known->cap) + known->cap, + "a known-count construction places its slots in the array block"); + size_t known_cap = known->cap; + for (size_t i = 0; i <= known_cap; i++) scr_arr_push_f64(known, (double)i); + check(!inline_slots(known) && scr_arr_len(known) == known_cap + 1 && scr_arr_get_f64(known, 0) == 0 && + scr_arr_get_f64(known, (double)known_cap) == (double)known_cap, + "growth past a known-count construction moves the slots out"); + scr_arr_release(known); + ScrArr *sixteen = scr_arr_new(SCR_ELEM_STR, 16); + for (int i = 0; i < 16; i++) scr_arr_push_ref(sixteen, scr_str_new("s", 1)); + check(inline_slots(sixteen) && scr_arr_len(sixteen) == 16, "sixteen known slots stay inline"); + scr_arr_release(sixteen); + ScrArr *large = scr_arr_new(SCR_ELEM_F64, 17); + check(!inline_slots(large) && large->cap >= 17, "larger known counts keep separate storage"); + scr_arr_release(large); + ScrArr *known_flags = scr_arr_new(SCR_ELEM_BOOL, 5); + scr_arr_push_bool(known_flags, true); + check(inline_slots(known_flags) && known_flags->cap >= 5 && scr_arr_get_bool(known_flags, 0), + "known-count boolean constructions are inline too"); + scr_arr_release(known_flags); ScrArr *a = scr_arr_new_small(SCR_ELEM_F64); check(inline_slots(a) && a->cap >= 4 && a->len == 0 && scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap, @@ -2205,7 +2263,9 @@ int main(int argc, char **argv) { test_nested_rc(); test_index_of_includes(); test_ref_elements(); + test_ref_known_count_cycle(&mock_cyc_ops); test_ops_overflow(); + test_ref_known_count_cycle(&mock_cyc_ops_late); test_ext_through_storage_changes(); test_ref_cycle(); test_ref_truncate_cycle();