Keep the slots of short known-count arrays in the array's block

An array constructed with a known element count of up to 16 (array literals, copies, slices and the arrays the runtime builds) keeps its slots in its own block, as empty arrays built for appends already do, so it costs one allocation and one free. scr_arr_make_ops is inlined into its two constructors, so the size class of scr_arr_new_ops_small folds to a table index. It writes the element ops id with the other header fields and records ops past the registry in the array's ext after the storage is set up: the ext pointer precedes the dense allocation, so inline slots must move out before the ext is created.
2026-10-27 06:43:19 +08:00 · 2026-10-21 23:19:57 -05:00
parent f686669aa4
commit 56880e110c
2 changed files with 109 additions and 17 deletions
@@ -131,13 +131,6 @@ uint32_t scr_arr_ops_register(ScrArrOps *ops) {

 const ScrArrOps *scr_arr_ops_overflow(const ScrArr *a) { return scr_arr_ext_of(a)->ops; }

-/* Record `ops` as the element ops of `a` (NULL for none). */
-static void scr_arr_set_ops(ScrArr *a, const ScrArrOps *ops) {
-  uint32_t id = scr_arr_ops_id(ops);
-  a->spare = (uint16_t)((a->spare & ((1u << SCR_ARR_OPS_SHIFT) - 1)) | (id << SCR_ARR_OPS_SHIFT));
-  if (id == SCR_ARR_OPS_OVERFLOW) scr_arr_ext(a)->ops = ops;
-}
-
 /* Canonical array indices grow indexed storage and may leave holes. Negative,
 * fractional, NaN, infinity, and 2^32-1 numeric keys use the ordinary
 * property vector; only a missing property or a low-level read of a hole
@@ -402,9 +395,10 @@ static void scr_arr_sparse_drop_prefix(ScrArr *a, size_t count) {
 #define SCR_ARR_MIN_GROWTH ((size_t)4)

 /* Inline slots: an array built by scr_arr_new_small / scr_arr_new_ops_small
- * (empty literals the compiler sees appended to) keeps its first dense
- * slots and states in its own block, right after the ScrArr, so it costs
- * one allocation and its first elements share the header's cache lines.
+ * (empty literals the compiler sees appended to), or constructed with a
+ * known element count of up to SCR_ARR_KNOWN_INLINE_MAX, keeps its first
+ * dense slots and states in its own block, right after the ScrArr, so it
+ * costs one allocation and its first elements share the header's cache lines.
 * `spare` records it. Growth past the inline capacity moves the slots to an
 * ordinary allocation; the inline room then stays unused until the array is
 * freed. */
@@ -526,6 +520,30 @@ static size_t scr_arr_small_cap(size_t prefix, bool bools) {
 #endif
 }

+/* The most slots a construction with a known element count (array
+ * literals, copies, slices and the arrays the runtime builds) keeps inline.
+ * Larger counts, and blocks past the small-object allocator, keep separate
+ * storage: an array that then grows leaves its inline room unused, which
+ * this bound keeps small. */
+#define SCR_ARR_KNOWN_INLINE_MAX ((size_t)16)
+
+/* Inline capacity of a construction of `need` elements with `prefix` bytes
+ * of block before the slots, rounded up to fill the block; 0 when its slots
+ * stay separate. */
+static size_t scr_arr_known_cap(size_t prefix, size_t need, bool bools) {
+  if (need == 0 || need > SCR_ARR_KNOWN_INLINE_MAX) return 0;
+#if SCR_SMALL_ALLOC
+  size_t unit = bools ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES;
+  size_t bytes = prefix + need * unit;
+  if (bytes > SCR_SA_MAX) return 0;
+  return (scr_sa_class_size(scr_sa_class(bytes)) - prefix) / unit;
+#else
+  (void)prefix;
+  (void)bools;
+  return need;
+#endif
+}
+
 static void scr_arr_use_inline(ScrArr *a, size_t cap) {
  a->data = (uint64_t *)((char *)a + SCR_ARR_INLINE_OFFSET);
  a->cap = (uint32_t)cap;
@@ -1180,7 +1198,8 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind el
                                                                  size_t initial_cap,
                                                                  bool small) {
  size_t unit = elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES;
-  size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL) : 0;
+  size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL)
+                            : scr_arr_known_cap(SCR_ARR_INLINE_OFFSET, initial_cap, elem == SCR_ELEM_BOOL);
  ScrArr *a = scr_mem_alloc(inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr));
  if (!a) scr_arr_oom();
  a->rc = 1;
@@ -1310,11 +1329,17 @@ static void scr_arr_gc_free(void *a0) {
  scr_cyc_free_inline(a);
 }

-static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool small) {
+/* Inlined into each constructor like scr_arr_make: with `small` and the
+ * count constant, scr_arr_new_ops_small's size class folds to a table
+ * index. */
+static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops(const ScrArrOps *ops,
+                                                                      size_t initial_cap,
+                                                                      bool small) {
  ScrArr *a;
  bool cyc = ops->trace != NULL;
+  size_t prefix = SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0);
  size_t inline_cap =
-      small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0), false) : 0;
+      small ? scr_arr_small_cap(prefix, false) : scr_arr_known_cap(prefix, initial_cap, false);
  size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * SCR_ARR_SLOT_BYTES : sizeof(ScrArr);
  if (cyc) {
    static ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY };
@@ -1328,12 +1353,19 @@ static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool s
  a->cap = 0;
  a->elem = SCR_ELEM_REF;
  a->cyc = cyc;
-  a->spare = 0;
+  /* The ops id is written with the other header fields: reading `spare`
+   * back after scr_arr_use_inline sets its inline flag stalls the
+   * store-to-load forward. */
+  uint32_t id = scr_arr_ops_id(ops);
+  a->spare = (uint16_t)(id << SCR_ARR_OPS_SHIFT);
  a->filled = 0;
  a->data = NULL;
-  scr_arr_set_ops(a, ops);
  if (inline_cap) scr_arr_use_inline(a, inline_cap);
  else if (initial_cap > 0) scr_arr_grow_dense_to(a, initial_cap, SCR_ARR_GROW_EXACT);
+  /* After the storage: ops past the registry live in the ext, whose
+   * pointer precedes the dense allocation, so inline slots move out first
+   * (scr_arr_ext). */
+  if (id == SCR_ARR_OPS_OVERFLOW) scr_arr_ext(a)->ops = ops;
 #ifdef SCR_RC_AUDIT
  scr_live_arrays++;
 #endif
@@ -398,6 +398,8 @@ static MockRec *mock_cyc_new(int value) {

 static ScrArrOps mock_ops = { &mock_retain, &mock_release, NULL };
 static ScrArrOps mock_cyc_ops = { &mock_cyc_retain, &mock_cyc_release, &mock_trace };
+/* First used after test_ops_overflow fills the registry: its arrays keep the ops in their ext. */
+static ScrArrOps mock_cyc_ops_late = { &mock_cyc_retain, &mock_cyc_release, &mock_trace };

 static void test_ref_elements(void) {
 #ifdef SCR_RC_AUDIT
@@ -514,6 +516,38 @@ static void test_ref_cycle(void) {
 #endif
 }

+static void test_ref_known_count_cycle(ScrArrOps *ops) {
+#ifdef SCR_RC_AUDIT
+  long arrays0 = scr_arr_live_count();
+#endif
+  /* A known-count construction of cycle-capable elements keeps its slots
+   * after the collector header: the collector traces them, and both the
+   * collected array and one released by its count free the one block. */
+  ScrArr *arr = scr_arr_new_ops(ops, 2);
+  bool in_block = arr->data == (uint64_t *)((char *)arr + ((sizeof(ScrArr) + 7) & ~(size_t)7));
+  /* Ops past the registry (once test_ops_overflow has filled it) live in
+   * the ext, which moves the slots to their own allocation. */
+  bool overflow = (arr->spare >> SCR_ARR_OPS_SHIFT) == SCR_ARR_OPS_OVERFLOW;
+  check(arr->cap >= 2 && in_block == !overflow && scr_arr_ops(arr) == ops,
+        "known-count cycle-capable construction is inline unless its ops need the ext");
+  MockRec *first = mock_cyc_new(1);
+  first->owner = (ScrArr *)scr_arr_retain(arr);
+  scr_arr_push_ref(arr, first);
+  scr_arr_push_ref(arr, mock_cyc_new(2));
+  scr_arr_release(arr);
+  scr_collect_cycles();
+  check(mock_live == 0, "cycle through an inline array collected");
+  ScrArr *plain = scr_arr_new_ops(ops, 3);
+  for (int i = 0; i < 4; i++) scr_arr_push_ref(plain, mock_cyc_new(i));
+  check(scr_arr_len(plain) == 4, "a cycle-capable inline array grows out of its block");
+  scr_arr_release(plain);
+  scr_collect_cycles();
+  check(mock_live == 0, "released inline cycle-capable array frees its elements");
+#ifdef SCR_RC_AUDIT
+  check(scr_arr_live_count() == arrays0, "known-count cycle: no arrays leaked");
+#endif
+}
+
 static void test_ref_truncate_cycle(void) {
 #ifdef SCR_RC_AUDIT
  long arrays0 = scr_arr_live_count();
@@ -1543,9 +1577,33 @@ static void test_inline_slots(void) {
  long strings0 = scr_str_live_count();
  long arrays0 = scr_arr_live_count();
 #endif
-  ScrArr *plain = scr_arr_new(SCR_ELEM_F64, 4);
-  check(!inline_slots(plain), "plain constructions keep separate storage");
+  ScrArr *plain = scr_arr_new(SCR_ELEM_F64, 0);
+  check(!inline_slots(plain) && plain->data == NULL, "empty plain constructions allocate no slots");
  scr_arr_release(plain);
+  /* A construction with a known count up to 16 slots keeps them in its own
+   * block; growth past them moves them out like any inline array. */
+  ScrArr *known = scr_arr_new(SCR_ELEM_F64, 4);
+  check(inline_slots(known) && known->cap >= 4 && known->len == 0 && !scr_arr_has(known, 0) &&
+            scr_arr_states_end(known) == (uint8_t *)(known->data + known->cap) + known->cap,
+        "a known-count construction places its slots in the array block");
+  size_t known_cap = known->cap;
+  for (size_t i = 0; i <= known_cap; i++) scr_arr_push_f64(known, (double)i);
+  check(!inline_slots(known) && scr_arr_len(known) == known_cap + 1 && scr_arr_get_f64(known, 0) == 0 &&
+            scr_arr_get_f64(known, (double)known_cap) == (double)known_cap,
+        "growth past a known-count construction moves the slots out");
+  scr_arr_release(known);
+  ScrArr *sixteen = scr_arr_new(SCR_ELEM_STR, 16);
+  for (int i = 0; i < 16; i++) scr_arr_push_ref(sixteen, scr_str_new("s", 1));
+  check(inline_slots(sixteen) && scr_arr_len(sixteen) == 16, "sixteen known slots stay inline");
+  scr_arr_release(sixteen);
+  ScrArr *large = scr_arr_new(SCR_ELEM_F64, 17);
+  check(!inline_slots(large) && large->cap >= 17, "larger known counts keep separate storage");
+  scr_arr_release(large);
+  ScrArr *known_flags = scr_arr_new(SCR_ELEM_BOOL, 5);
+  scr_arr_push_bool(known_flags, true);
+  check(inline_slots(known_flags) && known_flags->cap >= 5 && scr_arr_get_bool(known_flags, 0),
+        "known-count boolean constructions are inline too");
+  scr_arr_release(known_flags);
  ScrArr *a = scr_arr_new_small(SCR_ELEM_F64);
  check(inline_slots(a) && a->cap >= 4 && a->len == 0 &&
            scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap,
@@ -2205,7 +2263,9 @@ int main(int argc, char **argv) {
  test_nested_rc();
  test_index_of_includes();
  test_ref_elements();
+  test_ref_known_count_cycle(&mock_cyc_ops);
  test_ops_overflow();
+  test_ref_known_count_cycle(&mock_cyc_ops_late);
  test_ext_through_storage_changes();
  test_ref_cycle();
  test_ref_truncate_cycle();