Keep the slots of short known-count arrays in the array's block (#1746)

2026-10-30 07:14:39 +08:00 · 2026-10-21 23:26:08 -05:00
parent 46bee64635 56880e110c
commit e790de2f68
2 changed files with 109 additions and 17 deletions
@@ -131,13 +131,6 @@ uint32_t scr_arr_ops_register(ScrArrOps *ops) {

 const ScrArrOps *scr_arr_ops_overflow(const ScrArr *a) { return scr_arr_ext_of(a)->ops; }

-/* Record `ops` as the element ops of `a` (NULL for none). */
-static void scr_arr_set_ops(ScrArr *a, const ScrArrOps *ops) {
-  uint32_t id = scr_arr_ops_id(ops);
-  a->spare = (uint16_t)((a->spare & ((1u << SCR_ARR_OPS_SHIFT) - 1)) | (id << SCR_ARR_OPS_SHIFT));
-  if (id == SCR_ARR_OPS_OVERFLOW) scr_arr_ext(a)->ops = ops;
-}
-
 /* Canonical array indices grow indexed storage and may leave holes. Negative,
 * fractional, NaN, infinity, and 2^32-1 numeric keys use the ordinary
 * property vector; only a missing property or a low-level read of a hole
@@ -402,9 +395,10 @@ static void scr_arr_sparse_drop_prefix(ScrArr *a, size_t count) {
 #define SCR_ARR_MIN_GROWTH ((size_t)4)

 /* Inline slots: an array built by scr_arr_new_small / scr_arr_new_ops_small
- * (empty literals the compiler sees appended to) keeps its first dense
- * slots and states in its own block, right after the ScrArr, so it costs
- * one allocation and its first elements share the header's cache lines.
+ * (empty literals the compiler sees appended to), or constructed with a
+ * known element count of up to SCR_ARR_KNOWN_INLINE_MAX, keeps its first
+ * dense slots and states in its own block, right after the ScrArr, so it
+ * costs one allocation and its first elements share the header's cache lines.
 * `spare` records it. Growth past the inline capacity moves the slots to an
 * ordinary allocation; the inline room then stays unused until the array is
 * freed. */
@@ -526,6 +520,30 @@ static size_t scr_arr_small_cap(size_t prefix, bool bools) {
 #endif
 }

+/* The most slots a construction with a known element count (array
+ * literals, copies, slices and the arrays the runtime builds) keeps inline.
+ * Larger counts, and blocks past the small-object allocator, keep separate
+ * storage: an array that then grows leaves its inline room unused, which
+ * this bound keeps small. */
+#define SCR_ARR_KNOWN_INLINE_MAX ((size_t)16)
+
+/* Inline capacity of a construction of `need` elements with `prefix` bytes
+ * of block before the slots, rounded up to fill the block; 0 when its slots
+ * stay separate. */
+static size_t scr_arr_known_cap(size_t prefix, size_t need, bool bools) {
+  if (need == 0 || need > SCR_ARR_KNOWN_INLINE_MAX) return 0;
+#if SCR_SMALL_ALLOC
+  size_t unit = bools ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES;
+  size_t bytes = prefix + need * unit;
+  if (bytes > SCR_SA_MAX) return 0;
+  return (scr_sa_class_size(scr_sa_class(bytes)) - prefix) / unit;
+#else
+  (void)prefix;
+  (void)bools;
+  return need;
+#endif
+}
+
 static void scr_arr_use_inline(ScrArr *a, size_t cap) {
  a->data = (uint64_t *)((char *)a + SCR_ARR_INLINE_OFFSET);
  a->cap = (uint32_t)cap;
@@ -1180,7 +1198,8 @@ static inline __attribute__((always_inline)) ScrArr *scr_arr_make(ScrElemKind el
                                                                  size_t initial_cap,
                                                                  bool small) {
  size_t unit = elem == SCR_ELEM_BOOL ? SCR_ARR_BOOL_SLOT_BYTES : SCR_ARR_SLOT_BYTES;
-  size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL) : 0;
+  size_t inline_cap = small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET, elem == SCR_ELEM_BOOL)
+                            : scr_arr_known_cap(SCR_ARR_INLINE_OFFSET, initial_cap, elem == SCR_ELEM_BOOL);
  ScrArr *a = scr_mem_alloc(inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * unit : sizeof(ScrArr));
  if (!a) scr_arr_oom();
  a->rc = 1;
@@ -1310,11 +1329,17 @@ static void scr_arr_gc_free(void *a0) {
  scr_cyc_free_inline(a);
 }

-static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool small) {
+/* Inlined into each constructor like scr_arr_make: with `small` and the
+ * count constant, scr_arr_new_ops_small's size class folds to a table
+ * index. */
+static inline __attribute__((always_inline)) ScrArr *scr_arr_make_ops(const ScrArrOps *ops,
+                                                                      size_t initial_cap,
+                                                                      bool small) {
  ScrArr *a;
  bool cyc = ops->trace != NULL;
+  size_t prefix = SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0);
  size_t inline_cap =
-      small ? scr_arr_small_cap(SCR_ARR_INLINE_OFFSET + (cyc ? sizeof(ScrCycHdr) : 0), false) : 0;
+      small ? scr_arr_small_cap(prefix, false) : scr_arr_known_cap(prefix, initial_cap, false);
  size_t size = inline_cap ? SCR_ARR_INLINE_OFFSET + inline_cap * SCR_ARR_SLOT_BYTES : sizeof(ScrArr);
  if (cyc) {
    static ScrCycType cyc_type = { &scr_arr_trace_v, &scr_arr_gc_free, SCR_CYC_KIND_ARRAY };
@@ -1328,12 +1353,19 @@ static ScrArr *scr_arr_make_ops(const ScrArrOps *ops, size_t initial_cap, bool s
  a->cap = 0;
  a->elem = SCR_ELEM_REF;
  a->cyc = cyc;
-  a->spare = 0;
+  /* The ops id is written with the other header fields: reading `spare`
+   * back after scr_arr_use_inline sets its inline flag stalls the
+   * store-to-load forward. */
+  uint32_t id = scr_arr_ops_id(ops);
+  a->spare = (uint16_t)(id << SCR_ARR_OPS_SHIFT);
  a->filled = 0;
  a->data = NULL;
-  scr_arr_set_ops(a, ops);
  if (inline_cap) scr_arr_use_inline(a, inline_cap);
  else if (initial_cap > 0) scr_arr_grow_dense_to(a, initial_cap, SCR_ARR_GROW_EXACT);
+  /* After the storage: ops past the registry live in the ext, whose
+   * pointer precedes the dense allocation, so inline slots move out first
+   * (scr_arr_ext). */
+  if (id == SCR_ARR_OPS_OVERFLOW) scr_arr_ext(a)->ops = ops;
 #ifdef SCR_RC_AUDIT
  scr_live_arrays++;
 #endif
@@ -398,6 +398,8 @@ static MockRec *mock_cyc_new(int value) {

 static ScrArrOps mock_ops = { &mock_retain, &mock_release, NULL };
 static ScrArrOps mock_cyc_ops = { &mock_cyc_retain, &mock_cyc_release, &mock_trace };
+/* First used after test_ops_overflow fills the registry: its arrays keep the ops in their ext. */
+static ScrArrOps mock_cyc_ops_late = { &mock_cyc_retain, &mock_cyc_release, &mock_trace };

 static void test_ref_elements(void) {
 #ifdef SCR_RC_AUDIT
@@ -514,6 +516,38 @@ static void test_ref_cycle(void) {
 #endif
 }

+static void test_ref_known_count_cycle(ScrArrOps *ops) {
+#ifdef SCR_RC_AUDIT
+  long arrays0 = scr_arr_live_count();
+#endif
+  /* A known-count construction of cycle-capable elements keeps its slots
+   * after the collector header: the collector traces them, and both the
+   * collected array and one released by its count free the one block. */
+  ScrArr *arr = scr_arr_new_ops(ops, 2);
+  bool in_block = arr->data == (uint64_t *)((char *)arr + ((sizeof(ScrArr) + 7) & ~(size_t)7));
+  /* Ops past the registry (once test_ops_overflow has filled it) live in
+   * the ext, which moves the slots to their own allocation. */
+  bool overflow = (arr->spare >> SCR_ARR_OPS_SHIFT) == SCR_ARR_OPS_OVERFLOW;
+  check(arr->cap >= 2 && in_block == !overflow && scr_arr_ops(arr) == ops,
+        "known-count cycle-capable construction is inline unless its ops need the ext");
+  MockRec *first = mock_cyc_new(1);
+  first->owner = (ScrArr *)scr_arr_retain(arr);
+  scr_arr_push_ref(arr, first);
+  scr_arr_push_ref(arr, mock_cyc_new(2));
+  scr_arr_release(arr);
+  scr_collect_cycles();
+  check(mock_live == 0, "cycle through an inline array collected");
+  ScrArr *plain = scr_arr_new_ops(ops, 3);
+  for (int i = 0; i < 4; i++) scr_arr_push_ref(plain, mock_cyc_new(i));
+  check(scr_arr_len(plain) == 4, "a cycle-capable inline array grows out of its block");
+  scr_arr_release(plain);
+  scr_collect_cycles();
+  check(mock_live == 0, "released inline cycle-capable array frees its elements");
+#ifdef SCR_RC_AUDIT
+  check(scr_arr_live_count() == arrays0, "known-count cycle: no arrays leaked");
+#endif
+}
+
 static void test_ref_truncate_cycle(void) {
 #ifdef SCR_RC_AUDIT
  long arrays0 = scr_arr_live_count();
@@ -1543,9 +1577,33 @@ static void test_inline_slots(void) {
  long strings0 = scr_str_live_count();
  long arrays0 = scr_arr_live_count();
 #endif
-  ScrArr *plain = scr_arr_new(SCR_ELEM_F64, 4);
-  check(!inline_slots(plain), "plain constructions keep separate storage");
+  ScrArr *plain = scr_arr_new(SCR_ELEM_F64, 0);
+  check(!inline_slots(plain) && plain->data == NULL, "empty plain constructions allocate no slots");
  scr_arr_release(plain);
+  /* A construction with a known count up to 16 slots keeps them in its own
+   * block; growth past them moves them out like any inline array. */
+  ScrArr *known = scr_arr_new(SCR_ELEM_F64, 4);
+  check(inline_slots(known) && known->cap >= 4 && known->len == 0 && !scr_arr_has(known, 0) &&
+            scr_arr_states_end(known) == (uint8_t *)(known->data + known->cap) + known->cap,
+        "a known-count construction places its slots in the array block");
+  size_t known_cap = known->cap;
+  for (size_t i = 0; i <= known_cap; i++) scr_arr_push_f64(known, (double)i);
+  check(!inline_slots(known) && scr_arr_len(known) == known_cap + 1 && scr_arr_get_f64(known, 0) == 0 &&
+            scr_arr_get_f64(known, (double)known_cap) == (double)known_cap,
+        "growth past a known-count construction moves the slots out");
+  scr_arr_release(known);
+  ScrArr *sixteen = scr_arr_new(SCR_ELEM_STR, 16);
+  for (int i = 0; i < 16; i++) scr_arr_push_ref(sixteen, scr_str_new("s", 1));
+  check(inline_slots(sixteen) && scr_arr_len(sixteen) == 16, "sixteen known slots stay inline");
+  scr_arr_release(sixteen);
+  ScrArr *large = scr_arr_new(SCR_ELEM_F64, 17);
+  check(!inline_slots(large) && large->cap >= 17, "larger known counts keep separate storage");
+  scr_arr_release(large);
+  ScrArr *known_flags = scr_arr_new(SCR_ELEM_BOOL, 5);
+  scr_arr_push_bool(known_flags, true);
+  check(inline_slots(known_flags) && known_flags->cap >= 5 && scr_arr_get_bool(known_flags, 0),
+        "known-count boolean constructions are inline too");
+  scr_arr_release(known_flags);
  ScrArr *a = scr_arr_new_small(SCR_ELEM_F64);
  check(inline_slots(a) && a->cap >= 4 && a->len == 0 &&
            scr_arr_states_end(a) == (uint8_t *)(a->data + a->cap) + a->cap,
@@ -2205,7 +2263,9 @@ int main(int argc, char **argv) {
  test_nested_rc();
  test_index_of_includes();
  test_ref_elements();
+  test_ref_known_count_cycle(&mock_cyc_ops);
  test_ops_overflow();
+  test_ref_known_count_cycle(&mock_cyc_ops_late);
  test_ext_through_storage_changes();
  test_ref_cycle();
  test_ref_truncate_cycle();