use thread local stats for abandoned statistics to reduce contention

include/mimalloc/internal.h

@@ -116,6 +116,7 @@ mi_subproc_t* _mi_subproc_main(void);
 mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
 mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
 size_t        _mi_thread_seq_id(void) mi_attr_noexcept;
+mi_tld_t*     _mi_thread_tld(void) mi_attr_noexcept;
 void          _mi_heap_guarded_init(mi_heap_t* heap);
 
 // os.c
@@ -171,7 +172,7 @@ void          _mi_arenas_unsafe_destroy_all(mi_tld_t* tld);
 
 mi_page_t*    _mi_arenas_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment);
 void          _mi_arenas_page_free(mi_page_t* page);
-void          _mi_arenas_page_abandon(mi_page_t* page);
+void          _mi_arenas_page_abandon(mi_page_t* page, mi_tld_t* tld);
 void          _mi_arenas_page_unabandon(mi_page_t* page);
 bool          _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
 

include/mimalloc/types.h

@@ -544,13 +544,20 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
 #define mi_subproc_stat_adjust_increase(subproc,stat,amnt,b)    __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amnt, b)
 #define mi_subproc_stat_adjust_decrease(subproc,stat,amnt,b)    __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amnt, b)
 
+#define mi_tld_stat_counter_increase(tld,stat,amount)           __mi_stat_counter_increase( &(tld)->stats.stat, amount)
+#define mi_tld_stat_increase(tld,stat,amount)                   __mi_stat_increase( &(tld)->stats.stat, amount)
+#define mi_tld_stat_decrease(tld,stat,amount)                   __mi_stat_decrease( &(tld)->stats.stat, amount)
+#define mi_tld_stat_adjust_increase(tld,stat,amnt,b)            __mi_stat_adjust_increase( &(tld)->stats.stat, amnt, b)
+#define mi_tld_stat_adjust_decrease(tld,stat,amnt,b)            __mi_stat_adjust_decrease( &(tld)->stats.stat, amnt, b)
+
+
 #define mi_os_stat_counter_increase(stat,amount)                mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)
 #define mi_os_stat_increase(stat,amount)                        mi_subproc_stat_increase(_mi_subproc(),stat,amount)
 #define mi_os_stat_decrease(stat,amount)                        mi_subproc_stat_decrease(_mi_subproc(),stat,amount)
 
-#define mi_heap_stat_counter_increase(heap,stat,amount)         __mi_stat_counter_increase( &(heap)->tld->stats.stat, amount)
+#define mi_heap_stat_counter_increase(heap,stat,amount)         mi_tld_stat_counter_increase(heap->tld, stat, amount)
-#define mi_heap_stat_increase(heap,stat,amount)                 __mi_stat_increase( &(heap)->tld->stats.stat, amount)
+#define mi_heap_stat_increase(heap,stat,amount)                 mi_tld_stat_increase( heap->tld, stat, amount)
-#define mi_heap_stat_decrease(heap,stat,amount)                 __mi_stat_decrease( &(heap)->tld->stats.stat, amount)
+#define mi_heap_stat_decrease(heap,stat,amount)                 mi_tld_stat_decrease( heap->tld, stat, amount)
 
 #define mi_debug_heap_stat_counter_increase(heap,stat,amount)   mi_debug_stat_counter_increase( (heap)->tld->stats.stat, amount)
 #define mi_debug_heap_stat_increase(heap,stat,amount)           mi_debug_stat_increase( (heap)->tld->stats.stat, amount)

src/arena.c

@@ -563,8 +563,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_subproc_t* subproc, size_
       mi_assert_internal(mi_page_is_abandoned(page));
       mi_assert_internal(mi_arena_has_page(arena,page));
       mi_atomic_decrement_relaxed(&subproc->abandoned_count[bin]);
-      mi_subproc_stat_decrease( arena->subproc, pages_abandoned, 1);
+      mi_tld_t* tld = _mi_thread_tld();
-      mi_subproc_stat_counter_increase(arena->subproc, pages_reclaim_on_alloc, 1);
+      mi_tld_stat_decrease( tld, pages_abandoned, 1);
+      mi_tld_stat_counter_increase( tld, pages_reclaim_on_alloc, 1);
 
       _mi_page_free_collect(page, false);  // update `used` count
       mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
@@ -855,7 +856,7 @@ void _mi_arenas_page_free(mi_page_t* page) {
   Arena abandon
 ----------------------------------------------------------- */
 
-void _mi_arenas_page_abandon(mi_page_t* page) {
+void _mi_arenas_page_abandon(mi_page_t* page, mi_tld_t* tld) {
   mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
   mi_assert_internal(_mi_ptr_page(page)==page);
   mi_assert_internal(mi_page_is_owned(page));
@@ -878,7 +879,7 @@ void _mi_arenas_page_abandon(mi_page_t* page) {
     const bool wasclear = mi_bitmap_set(arena->pages_abandoned[bin], slice_index);
     MI_UNUSED(wasclear); mi_assert_internal(wasclear);
     mi_atomic_increment_relaxed(&arena->subproc->abandoned_count[bin]);
-    mi_subproc_stat_increase(arena->subproc, pages_abandoned, 1);
+    mi_tld_stat_increase(tld, pages_abandoned, 1);
   }
   else {
     // page is full (or a singleton), or the page is OS/externally allocated
@@ -894,7 +895,7 @@ void _mi_arenas_page_abandon(mi_page_t* page) {
         subproc->os_abandoned_pages = page;
       }
     }
-    mi_subproc_stat_increase(_mi_subproc(), pages_abandoned, 1);
+    mi_tld_stat_increase(tld, pages_abandoned, 1);
   }
   _mi_page_unown(page);
 }
@@ -912,10 +913,10 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {
     return false;
   }
   else {
-    mi_subproc_t* subproc = _mi_subproc();
+    mi_tld_t* tld = _mi_thread_tld();
-    mi_subproc_stat_counter_increase( subproc, pages_reabandon_full, 1);
+    mi_tld_stat_counter_increase( tld, pages_reabandon_full, 1);
-    mi_subproc_stat_adjust_decrease( subproc, pages_abandoned, 1, true /* on alloc */);  // adjust as we are not abandoning fresh
+    mi_tld_stat_adjust_decrease( tld, pages_abandoned, 1, true /* on alloc */);  // adjust as we are not abandoning fresh
-    _mi_arenas_page_abandon(page);
+    _mi_arenas_page_abandon(page,tld);
     return true;
   }
 }
@@ -942,14 +943,14 @@ void _mi_arenas_page_unabandon(mi_page_t* page) {
     mi_bitmap_clear_once_set(arena->pages_abandoned[bin], slice_index);
     mi_page_clear_abandoned_mapped(page);
     mi_atomic_decrement_relaxed(&arena->subproc->abandoned_count[bin]);
-    mi_subproc_stat_decrease(arena->subproc, pages_abandoned, 1);
+    mi_tld_stat_decrease(_mi_thread_tld(), pages_abandoned, 1);
   }
   else {
     // page is full (or a singleton), page is OS allocated
-    mi_subproc_t* subproc = _mi_subproc();
+    mi_tld_stat_decrease(_mi_thread_tld(), pages_abandoned, 1);
-    mi_subproc_stat_decrease(_mi_subproc(), pages_abandoned, 1);
     // if not an arena page, remove from the subproc os pages list
     if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
+      mi_subproc_t* subproc = _mi_subproc();
       mi_lock(&subproc->os_abandoned_pages_lock) {
         if (page->prev != NULL) { page->prev->next = page->next; }
         if (page->next != NULL) { page->next->prev = page->prev; }

src/init.c

@@ -357,6 +357,18 @@ mi_subproc_t* _mi_subproc(void) {
 }
 
 
+mi_tld_t* _mi_thread_tld(void) mi_attr_noexcept {
+  // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`
+  mi_heap_t* heap = mi_prim_get_default_heap();
+  if (heap == NULL) {
+    return &tld_empty;
+  }
+  else {
+    return heap->tld;
+  }
+}
+
+
 /* -----------------------------------------------------------
   Sub process
 ----------------------------------------------------------- */

src/page.c

@@ -280,7 +280,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
     mi_page_queue_remove(pq, page);
     mi_tld_t* tld = page->heap->tld;
     mi_page_set_heap(page, NULL);
-    _mi_arenas_page_abandon(page);
+    _mi_arenas_page_abandon(page,tld);
     _mi_arenas_collect(false, false, tld); // allow purging
   }
 }

src/stats.c

@@ -152,6 +152,12 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
   mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1);
   mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1);
   mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count, 1);
+
+  mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1);
+  mi_stat_counter_add(&stats->pages_reclaim_on_alloc, &src->pages_reclaim_on_alloc, 1);
+  mi_stat_counter_add(&stats->pages_reclaim_on_free, &src->pages_reclaim_on_free, 1);
+  mi_stat_counter_add(&stats->pages_reabandon_full, &src->pages_reabandon_full, 1);
+  mi_stat_counter_add(&stats->pages_unabandon_busy_wait, &src->pages_unabandon_busy_wait, 1);
 #if MI_STAT>1
   for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
     if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) {