From 4d727ee6e62b726b22f0fe06866a1501ceb3a0bd Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 1 Mar 2025 18:30:24 -0800 Subject: [PATCH 1/7] avoid pthread allocation size in mstress bench --- test/test-stress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-stress.c b/test/test-stress.c index 41f2162b..c41954d3 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -116,7 +116,7 @@ static void* alloc_items(size_t items, random_t r) { else if (chance(10, r) && allow_large_objects) items *= 1000; // 0.1% huge else items *= 100; // 1% large objects; } - if (items == 40) items++; // pthreads uses that size for stack increases + if (items>=32 && items<=40) items*=2; // pthreads uses 320b allocations (this shows that more clearly in the stats) if (use_one_size > 0) items = (use_one_size / sizeof(uintptr_t)); if (items==0) items = 1; uintptr_t* p = (uintptr_t*)custom_calloc(items,sizeof(uintptr_t)); From bc5f636c5e010f4ad8da638af33a58722811ab25 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sat, 1 Mar 2025 19:46:01 -0800 Subject: [PATCH 2/7] nicer stats --- src/segment.c | 6 +++--- src/stats.c | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/segment.c b/src/segment.c index 27a3f295..e2730b7f 100644 --- a/src/segment.c +++ b/src/segment.c @@ -652,7 +652,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { MI_UNUSED(force); mi_assert(segment != NULL); - + // in `mi_segment_force_abandon` we set this to true to ensure the segment's memory stays valid if (segment->dont_free) return; @@ -664,7 +664,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t mi_assert_expensive(!mi_segment_queue_contains(&tld->medium_free, segment)); mi_assert(segment->next == NULL); mi_assert(segment->prev == NULL); - _mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size); + // _mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size); // return it to the OS mi_segment_os_free(segment, segment->segment_size, tld); @@ -1062,7 +1062,7 @@ static void mi_segment_force_abandon(mi_segment_t* segment, mi_segments_tld_t* t { mi_assert_internal(segment->abandoned < segment->used); mi_assert_internal(!segment->dont_free); - + // ensure the segment does not get free'd underneath us (so we can check if a page has been freed in `mi_page_force_abandon`) segment->dont_free = true; diff --git a/src/stats.c b/src/stats.c index 1edd89b1..c4ec92de 100644 --- a/src/stats.c +++ b/src/stats.c @@ -69,7 +69,7 @@ static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src) { if (src->current!=0) { mi_atomic_addi64_relaxed(&stat->current, src->current); } // peak scores do really not work across threads ... we use conservative max if (src->peak > stat->peak) { - mi_atomic_maxi64_relaxed(&stat->peak, src->peak); // or: mi_atomic_addi64_relaxed( &stat->peak, src->peak); + mi_atomic_maxi64_relaxed(&stat->peak, src->peak); // or: mi_atomic_addi64_relaxed( &stat->peak, src->peak); } } @@ -108,7 +108,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire); mi_stat_counter_add(&stats->searches, &src->searches); mi_stat_counter_add(&stats->normal_count, &src->normal_count); - mi_stat_counter_add(&stats->huge_count, &src->huge_count); + mi_stat_counter_add(&stats->huge_count, &src->huge_count); mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { @@ -305,21 +305,21 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) #endif #if MI_STAT mi_stat_print(&stats->normal, "normal", (stats->normal_count.total == 0 ? 1 : -1), out, arg); - mi_stat_print(&stats->huge, "huge", (stats->huge_count.total == 0 ? 1 : -1), out, arg); + mi_stat_print(&stats->huge, "huge", (stats->huge_count.total == 0 ? 1 : -1), out, arg); mi_stat_count_t total = { 0,0,0 }; mi_stat_add(&total, &stats->normal); mi_stat_add(&total, &stats->huge); - mi_stat_print(&total, "total", 1, out, arg); + mi_stat_print_ex(&total, "total", 1, out, arg, ""); #endif #if MI_STAT>1 - mi_stat_print(&stats->malloc, "malloc req", 1, out, arg); + mi_stat_print_ex(&stats->malloc, "malloc req", 1, out, arg, ""); _mi_fprintf(out, arg, "\n"); #endif mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); mi_stat_peak_print(&stats->reset, "reset", 1, out, arg ); mi_stat_peak_print(&stats->purged, "purged", 1, out, arg ); - mi_stat_print(&stats->page_committed, "touched", 1, out, arg); + mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, ""); mi_stat_print(&stats->segments, "segments", -1, out, arg); mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); @@ -445,7 +445,7 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s pinfo.page_faults = 0; _mi_prim_process_info(&pinfo); - + if (elapsed_msecs!=NULL) *elapsed_msecs = (pinfo.elapsed < 0 ? 0 : (pinfo.elapsed < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.elapsed : PTRDIFF_MAX)); if (user_msecs!=NULL) *user_msecs = (pinfo.utime < 0 ? 0 : (pinfo.utime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.utime : PTRDIFF_MAX)); if (system_msecs!=NULL) *system_msecs = (pinfo.stime < 0 ? 0 : (pinfo.stime < (mi_msecs_t)PTRDIFF_MAX ? (size_t)pinfo.stime : PTRDIFF_MAX)); From 1b749ea7d84fc9cc178addea9a47081f2f9f9a1e Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sat, 1 Mar 2025 20:51:44 -0800 Subject: [PATCH 3/7] clean up statistics --- include/mimalloc/internal.h | 2 +- include/mimalloc/types.h | 65 ++++++++++++++++++++++--------------- src/alloc-aligned.c | 2 +- src/alloc.c | 10 +++--- src/bitmap.c | 1 - src/free.c | 8 ++--- src/heap.c | 8 ++--- src/init.c | 28 +++++++--------- src/page.c | 8 ++--- src/stats.c | 63 ++++++++++++++++++++--------------- 10 files changed, 107 insertions(+), 88 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index d965c275..8fcaa22e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -311,7 +311,7 @@ bool _mi_page_is_valid(mi_page_t* page); #define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) #define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) #define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) - +#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x() #include // initialize a local variable to zero; use memset as compilers optimize constant sized memset's diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 64ebb160..c27489d0 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -573,37 +573,50 @@ typedef struct mi_stat_counter_s { } mi_stat_counter_t; typedef struct mi_stats_s { - mi_stat_count_t segments; - mi_stat_count_t pages; - mi_stat_count_t reserved; - mi_stat_count_t committed; - mi_stat_count_t reset; - mi_stat_count_t purged; - mi_stat_count_t page_committed; - mi_stat_count_t segments_abandoned; - mi_stat_count_t pages_abandoned; - mi_stat_count_t threads; - mi_stat_count_t normal; - mi_stat_count_t huge; - mi_stat_count_t giant; - mi_stat_count_t malloc; - mi_stat_count_t segments_cache; - mi_stat_counter_t pages_extended; + mi_stat_count_t pages; // count of mimalloc pages + mi_stat_count_t reserved; // reserved memory bytes + mi_stat_count_t committed; // committed bytes + mi_stat_count_t reset; // reset bytes + mi_stat_count_t purged; // purged bytes + mi_stat_count_t page_committed; // committed memory inside pages + mi_stat_count_t pages_abandoned; // abandonded pages count + mi_stat_count_t threads; // number of threads + mi_stat_count_t malloc_normal; // allocated bytes <= MI_LARGE_OBJ_SIZE_MAX + mi_stat_count_t malloc_huge; // allocated bytes in huge pages + mi_stat_count_t malloc_requested; // malloc requested bytes + mi_stat_counter_t mmap_calls; mi_stat_counter_t commit_calls; mi_stat_counter_t reset_calls; mi_stat_counter_t purge_calls; - mi_stat_counter_t page_no_retire; - mi_stat_counter_t searches; - mi_stat_counter_t normal_count; - mi_stat_counter_t huge_count; - mi_stat_counter_t arena_count; - mi_stat_counter_t arena_crossover_count; + mi_stat_counter_t arena_count; // number of memory arena's + mi_stat_counter_t malloc_normal_count; // number of blocks <= MI_LARGE_OBJ_SIZE_MAX + mi_stat_counter_t malloc_huge_count; // number of huge bloks + mi_stat_counter_t malloc_guarded_count; // number of allocations with guard pages + + // internal statistics mi_stat_counter_t arena_rollback_count; - mi_stat_counter_t guarded_alloc_count; -#if MI_STAT>1 - mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; -#endif + mi_stat_counter_t pages_extended; // number of page extensions + mi_stat_counter_t pages_retire; // number of pages that are retired + mi_stat_counter_t page_searches; // searches for a fresh page + // only on v1 and v2 + mi_stat_count_t segments; + mi_stat_count_t segments_abandoned; + mi_stat_count_t segments_cache; + mi_stat_count_t _segments_reserved; + // only on v3 + mi_stat_counter_t pages_reclaim_on_alloc; + mi_stat_counter_t pages_reclaim_on_free; + mi_stat_counter_t pages_reabandon_full; + mi_stat_counter_t pages_unabandon_busy_wait; + + // future extension + mi_stat_count_t _stat_reserved[4]; + mi_stat_counter_t _stat_counter_reserved[4]; + + // size segregated statistics + mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin + mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin } mi_stats_t; diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 7304eb1d..d0e691b3 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -192,7 +192,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t if mi_likely(is_aligned) { #if MI_STAT>1 - mi_heap_stat_increase(heap, malloc, size); + mi_heap_stat_increase(heap, malloc_requested, size); #endif void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen mi_assert_internal(p != NULL); diff --git a/src/alloc.c b/src/alloc.c index a093f108..badc2612 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -83,11 +83,11 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ #if (MI_STAT>0) const size_t bsize = mi_page_usable_block_size(page); if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_increase(heap, normal, bsize); - mi_heap_stat_counter_increase(heap, normal_count, 1); + mi_heap_stat_increase(heap, malloc_normal, bsize); + mi_heap_stat_counter_increase(heap, malloc_normal_count, 1); #if (MI_STAT>1) const size_t bin = _mi_bin(bsize); - mi_heap_stat_increase(heap, normal_bins[bin], 1); + mi_heap_stat_increase(heap, malloc_bins[bin], 1); #endif } #endif @@ -149,7 +149,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } - mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p)); } #endif #if MI_DEBUG>3 @@ -191,7 +191,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } - mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p)); } #endif #if MI_DEBUG>3 diff --git a/src/bitmap.c b/src/bitmap.c index 2def7bb0..9ef784d6 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -242,7 +242,6 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); // claimed! - mi_stat_counter_increase(_mi_stats_main.arena_crossover_count,1); *bitmap_idx = mi_bitmap_index_create(idx, initial_idx); return true; diff --git a/src/free.c b/src/free.c index f2e30b65..a1732e8c 100644 --- a/src/free.c +++ b/src/free.c @@ -521,17 +521,17 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { const size_t bsize = mi_page_usable_block_size(page); #if (MI_STAT>1) const size_t usize = mi_page_usable_size_of(page, block); - mi_heap_stat_decrease(heap, malloc, usize); + mi_heap_stat_decrease(heap, malloc_requested, usize); #endif if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, normal, bsize); + mi_heap_stat_decrease(heap, malloc_normal, bsize); #if (MI_STAT > 1) - mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1); + mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], 1); #endif } else { const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc - mi_heap_stat_decrease(heap, huge, bpsize); + mi_heap_stat_decrease(heap, malloc_huge, bpsize); } } #else diff --git a/src/heap.c b/src/heap.c index 58aa050c..28161266 100644 --- a/src/heap.c +++ b/src/heap.c @@ -324,18 +324,18 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ // stats const size_t bsize = mi_page_block_size(page); if (bsize > MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, huge, bsize); + mi_heap_stat_decrease(heap, malloc_huge, bsize); } #if (MI_STAT) _mi_page_free_collect(page, false); // update used count const size_t inuse = page->used; if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, normal, bsize * inuse); + mi_heap_stat_decrease(heap, malloc_normal, bsize * inuse); #if (MI_STAT>1) - mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse); + mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], inuse); #endif } - mi_heap_stat_decrease(heap, malloc, bsize * inuse); // todo: off for aligned blocks... + mi_heap_stat_decrease(heap, malloc_requested, bsize * inuse); // todo: off for aligned blocks... #endif /// pretend it is all free now diff --git a/src/init.c b/src/init.c index c25e7516..f91e279b 100644 --- a/src/init.c +++ b/src/init.c @@ -69,26 +69,22 @@ const mi_page_t _mi_page_empty = { #define MI_STAT_COUNT_NULL() {0,0,0} // Empty statistics -#if MI_STAT>1 -#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } -#else -#define MI_STAT_COUNT_END_NULL() -#endif - #define MI_STATS_NULL \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ { 0 }, { 0 }, { 0 }, { 0 }, \ { 0 }, { 0 }, { 0 }, { 0 }, \ + \ { 0 }, { 0 }, { 0 }, { 0 }, \ - { 0 } \ - MI_STAT_COUNT_END_NULL() + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + { 0 }, { 0 }, { 0 }, { 0 }, \ + \ + { MI_INIT4(MI_STAT_COUNT_NULL) }, \ + { { 0 }, { 0 }, { 0 }, { 0 } }, \ + \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, \ + { MI_INIT74(MI_STAT_COUNT_NULL) } // -------------------------------------------------------- // Statically allocate an empty heap as the initial diff --git a/src/page.c b/src/page.c index 6d0d6db7..df802c7d 100644 --- a/src/page.c +++ b/src/page.c @@ -473,7 +473,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { const size_t bsize = mi_page_block_size(page); if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? if (pq->last==page && pq->first==page) { // the only page in the queue? - mi_stat_counter_increase(_mi_stats_main.page_no_retire,1); + mi_stat_counter_increase(_mi_stats_main.pages_retire,1); page->retire_expire = (bsize <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(pq >= heap->pages); @@ -809,7 +809,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p page = next; } // for each page - mi_heap_stat_counter_increase(heap, searches, count); + mi_heap_stat_counter_increase(heap, page_searches, count); // set the page to the best candidate if (page_candidate != NULL) { @@ -922,8 +922,8 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_a mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue mi_page_set_heap(page, NULL); #endif - mi_heap_stat_increase(heap, huge, mi_page_block_size(page)); - mi_heap_stat_counter_increase(heap, huge_count, 1); + mi_heap_stat_increase(heap, malloc_huge, mi_page_block_size(page)); + mi_heap_stat_counter_increase(heap, malloc_huge_count, 1); } return page; } diff --git a/src/stats.c b/src/stats.c index c4ec92de..65d6d00b 100644 --- a/src/stats.c +++ b/src/stats.c @@ -81,39 +81,50 @@ static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t // must be thread safe as it is called from stats_merge static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { if (stats==src) return; - mi_stat_add(&stats->segments, &src->segments); mi_stat_add(&stats->pages, &src->pages); mi_stat_add(&stats->reserved, &src->reserved); mi_stat_add(&stats->committed, &src->committed); mi_stat_add(&stats->reset, &src->reset); mi_stat_add(&stats->purged, &src->purged); mi_stat_add(&stats->page_committed, &src->page_committed); - mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned); - mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned); mi_stat_add(&stats->threads, &src->threads); + mi_stat_add(&stats->malloc_normal, &src->malloc_normal); + mi_stat_add(&stats->malloc_huge, &src->malloc_huge); + mi_stat_add(&stats->malloc_requested, &src->malloc_requested); - mi_stat_add(&stats->malloc, &src->malloc); - mi_stat_add(&stats->segments_cache, &src->segments_cache); - mi_stat_add(&stats->normal, &src->normal); - mi_stat_add(&stats->huge, &src->huge); - mi_stat_add(&stats->giant, &src->giant); - - mi_stat_counter_add(&stats->pages_extended, &src->pages_extended); mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls); mi_stat_counter_add(&stats->commit_calls, &src->commit_calls); mi_stat_counter_add(&stats->reset_calls, &src->reset_calls); mi_stat_counter_add(&stats->purge_calls, &src->purge_calls); + mi_stat_counter_add(&stats->arena_count, &src->arena_count); + mi_stat_counter_add(&stats->malloc_normal_count, &src->malloc_normal_count); + mi_stat_counter_add(&stats->malloc_huge_count, &src->malloc_huge_count); + mi_stat_counter_add(&stats->malloc_guarded_count, &src->malloc_guarded_count); + + mi_stat_counter_add(&stats->arena_rollback_count, &src->arena_rollback_count); + mi_stat_counter_add(&stats->pages_extended, &src->pages_extended); + mi_stat_counter_add(&stats->pages_retire, &src->pages_retire); + mi_stat_counter_add(&stats->page_searches, &src->page_searches); + + mi_stat_add(&stats->segments, &src->segments); + mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned); + mi_stat_add(&stats->segments_cache, &src->segments_cache); + + mi_stat_counter_add(&stats->pages_reclaim_on_alloc, &src->pages_reclaim_on_alloc); + mi_stat_counter_add(&stats->pages_reclaim_on_free, &src->pages_reclaim_on_free); + mi_stat_counter_add(&stats->pages_reabandon_full, &src->pages_reabandon_full); + mi_stat_counter_add(&stats->pages_unabandon_busy_wait, &src->pages_unabandon_busy_wait); - mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire); - mi_stat_counter_add(&stats->searches, &src->searches); - mi_stat_counter_add(&stats->normal_count, &src->normal_count); - mi_stat_counter_add(&stats->huge_count, &src->huge_count); - mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { // if (src->normal_bins[i].total != 0 && src->normal_bins[i].current != 0) { - mi_stat_add(&stats->normal_bins[i], &src->normal_bins[i]); + mi_stat_add(&stats->malloc_bins[i], &src->malloc_bins[i]); + //} + } + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + // if (src->normal_bins[i].total != 0 && src->normal_bins[i].current != 0) { + mi_stat_add(&stats->page_bins[i], &src->page_bins[i]); //} } #endif @@ -301,18 +312,18 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) // and print using that mi_print_header(out,arg); #if MI_STAT>1 - mi_stats_print_bins(stats->normal_bins, MI_BIN_HUGE, "normal",out,arg); + mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, "normal",out,arg); #endif #if MI_STAT - mi_stat_print(&stats->normal, "normal", (stats->normal_count.total == 0 ? 1 : -1), out, arg); - mi_stat_print(&stats->huge, "huge", (stats->huge_count.total == 0 ? 1 : -1), out, arg); + mi_stat_print(&stats->malloc_normal, "normal", (stats->malloc_normal_count.total == 0 ? 1 : -1), out, arg); + mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? 1 : -1), out, arg); mi_stat_count_t total = { 0,0,0 }; - mi_stat_add(&total, &stats->normal); - mi_stat_add(&total, &stats->huge); + mi_stat_add(&total, &stats->malloc_normal); + mi_stat_add(&total, &stats->malloc_huge); mi_stat_print_ex(&total, "total", 1, out, arg, ""); #endif #if MI_STAT>1 - mi_stat_print_ex(&stats->malloc, "malloc req", 1, out, arg, ""); + mi_stat_print_ex(&stats->malloc_requested, "malloc req", 1, out, arg, ""); _mi_fprintf(out, arg, "\n"); #endif mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); @@ -326,17 +337,17 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_print(&stats->pages, "pages", -1, out, arg); mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg); mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg); - mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg); + mi_stat_counter_print(&stats->pages_retire, "-retire", out, arg); mi_stat_counter_print(&stats->arena_count, "arenas", out, arg); - mi_stat_counter_print(&stats->arena_crossover_count, "-crossover", out, arg); + // mi_stat_counter_print(&stats->arena_crossover_count, "-crossover", out, arg); mi_stat_counter_print(&stats->arena_rollback_count, "-rollback", out, arg); mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg); mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); - mi_stat_counter_print(&stats->guarded_alloc_count, "guarded", out, arg); + mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); - mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); + mi_stat_counter_print_avg(&stats->page_searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); size_t elapsed; From 09ad6d2819c1102c85e927eac859ff0e1349f972 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 2 Mar 2025 15:34:37 -0800 Subject: [PATCH 4/7] initial statistics api --- CMakeLists.txt | 1 + ide/vs2022/mimalloc-lib.vcxproj | 1 + ide/vs2022/mimalloc-lib.vcxproj.filters | 3 + include/mimalloc-stats.h | 102 +++++++++++ include/mimalloc/types.h | 200 ++++++++-------------- src/heap.c | 5 + src/init.c | 4 +- src/page-queue.c | 9 +- src/page.c | 5 +- src/stats.c | 217 +++++++++++++++++++----- test/test-stress.c | 12 +- 11 files changed, 380 insertions(+), 179 deletions(-) create mode 100644 include/mimalloc-stats.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dbd400c8..68153468 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -641,6 +641,7 @@ endif() install(FILES include/mimalloc.h DESTINATION ${mi_install_incdir}) install(FILES include/mimalloc-override.h DESTINATION ${mi_install_incdir}) install(FILES include/mimalloc-new-delete.h DESTINATION ${mi_install_incdir}) +install(FILES include/mimalloc-stats.h DESTINATION ${mi_install_incdir}) install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_cmakedir}) install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_cmakedir}) diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index c6c2cb5f..abdac1d1 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -486,6 +486,7 @@ + diff --git a/ide/vs2022/mimalloc-lib.vcxproj.filters b/ide/vs2022/mimalloc-lib.vcxproj.filters index 3acf4015..f88907ba 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -93,6 +93,9 @@ Headers + + Headers + diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h new file mode 100644 index 00000000..4dc4c381 --- /dev/null +++ b/include/mimalloc-stats.h @@ -0,0 +1,102 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_STATS_H +#define MIMALLOC_STATS_H + +#include +#include + +#define MI_STAT_VERSION 1 // increased on every backward incompatible change + +// count allocation over time +typedef struct mi_stat_count_s { + int64_t total; // total allocated + int64_t peak; // peak allocation + int64_t current; // current allocation +} mi_stat_count_t; + +// counters only increase +typedef struct mi_stat_counter_s { + int64_t total; // total count +} mi_stat_counter_t; + +#define MI_STAT_FIELDS() \ + MI_STAT_COUNT(pages) /* count of mimalloc pages */ \ + MI_STAT_COUNT(reserved) /* reserved memory bytes */ \ + MI_STAT_COUNT(committed) /* committed bytes */ \ + MI_STAT_COUNT(reset) /* reset bytes */ \ + MI_STAT_COUNT(purged) /* purged bytes */ \ + MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \ + MI_STAT_COUNT(pages_abandoned) /* abandonded pages count */ \ + MI_STAT_COUNT(threads) /* number of threads */ \ + MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ + MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ + MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \ + \ + MI_STAT_COUNTER(mmap_calls) \ + MI_STAT_COUNTER(commit_calls) \ + MI_STAT_COUNTER(reset_calls) \ + MI_STAT_COUNTER(purge_calls) \ + MI_STAT_COUNTER(arena_count) /* number of memory arena's */ \ + MI_STAT_COUNTER(malloc_normal_count) /* number of blocks <= MI_LARGE_OBJ_SIZE_MAX */ \ + MI_STAT_COUNTER(malloc_huge_count) /* number of huge bloks */ \ + MI_STAT_COUNTER(malloc_guarded_count) /* number of allocations with guard pages */ \ + \ + /* internal statistics */ \ + MI_STAT_COUNTER(arena_rollback_count) \ + MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \ + MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \ + MI_STAT_COUNTER(page_searches) /* searches for a fresh page */ \ + /* only on v1 and v2 */ \ + MI_STAT_COUNT(segments) \ + MI_STAT_COUNT(segments_abandoned) \ + MI_STAT_COUNT(segments_cache) \ + MI_STAT_COUNT(_segments_reserved) \ + /* only on v3 */ \ + MI_STAT_COUNTER(pages_reclaim_on_alloc) \ + MI_STAT_COUNTER(pages_reclaim_on_free) \ + MI_STAT_COUNTER(pages_reabandon_full) \ + MI_STAT_COUNTER(pages_unabandon_busy_wait) \ + + +// Define the statistics structure +#define MI_BIN_HUGE (73U) // see types.h +#define MI_STAT_COUNT(stat) mi_stat_count_t stat; +#define MI_STAT_COUNTER(stat) mi_stat_counter_t stat; + +typedef struct mi_stats_s +{ + int version; + + MI_STAT_FIELDS() + + // future extension + mi_stat_count_t _stat_reserved[4]; + mi_stat_counter_t _stat_counter_reserved[4]; + + // size segregated statistics + mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin + mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin +} mi_stats_t; + +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER + +// Exported definitions +#ifdef __cplusplus +extern "C" { +#endif + +mi_decl_export void mi_stats_get( size_t stats_size, mi_stats_t* stats ) mi_attr_noexcept; +mi_decl_export const char* mi_stats_get_json( size_t buf_size, char* buf ) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL + +#ifdef __cplusplus +} +#endif + +#endif // MIMALLOC_STATS_H \ No newline at end of file diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index c27489d0..4c912107 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -21,6 +21,7 @@ terms of the MIT license. A copy of the license can be found in the file // -------------------------------------------------------------------------- +#include #include // ptrdiff_t #include // uintptr_t, uint16_t, etc #include "atomic.h" // _Atomic @@ -515,135 +516,10 @@ struct mi_heap_s { }; - -// ------------------------------------------------------ -// Debug -// ------------------------------------------------------ - -#if !defined(MI_DEBUG_UNINIT) -#define MI_DEBUG_UNINIT (0xD0) -#endif -#if !defined(MI_DEBUG_FREED) -#define MI_DEBUG_FREED (0xDF) -#endif -#if !defined(MI_DEBUG_PADDING) -#define MI_DEBUG_PADDING (0xDE) -#endif - -#if (MI_DEBUG) -// use our own assertion to print without memory allocation -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); -#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) -#else -#define mi_assert(x) -#endif - -#if (MI_DEBUG>1) -#define mi_assert_internal mi_assert -#else -#define mi_assert_internal(x) -#endif - -#if (MI_DEBUG>2) -#define mi_assert_expensive mi_assert -#else -#define mi_assert_expensive(x) -#endif - -// ------------------------------------------------------ -// Statistics -// ------------------------------------------------------ - -#ifndef MI_STAT -#if (MI_DEBUG>0) -#define MI_STAT 2 -#else -#define MI_STAT 0 -#endif -#endif - -typedef struct mi_stat_count_s { - int64_t total; - int64_t peak; - int64_t current; -} mi_stat_count_t; - -typedef struct mi_stat_counter_s { - int64_t total; -} mi_stat_counter_t; - -typedef struct mi_stats_s { - mi_stat_count_t pages; // count of mimalloc pages - mi_stat_count_t reserved; // reserved memory bytes - mi_stat_count_t committed; // committed bytes - mi_stat_count_t reset; // reset bytes - mi_stat_count_t purged; // purged bytes - mi_stat_count_t page_committed; // committed memory inside pages - mi_stat_count_t pages_abandoned; // abandonded pages count - mi_stat_count_t threads; // number of threads - mi_stat_count_t malloc_normal; // allocated bytes <= MI_LARGE_OBJ_SIZE_MAX - mi_stat_count_t malloc_huge; // allocated bytes in huge pages - mi_stat_count_t malloc_requested; // malloc requested bytes - - mi_stat_counter_t mmap_calls; - mi_stat_counter_t commit_calls; - mi_stat_counter_t reset_calls; - mi_stat_counter_t purge_calls; - mi_stat_counter_t arena_count; // number of memory arena's - mi_stat_counter_t malloc_normal_count; // number of blocks <= MI_LARGE_OBJ_SIZE_MAX - mi_stat_counter_t malloc_huge_count; // number of huge bloks - mi_stat_counter_t malloc_guarded_count; // number of allocations with guard pages - - // internal statistics - mi_stat_counter_t arena_rollback_count; - mi_stat_counter_t pages_extended; // number of page extensions - mi_stat_counter_t pages_retire; // number of pages that are retired - mi_stat_counter_t page_searches; // searches for a fresh page - // only on v1 and v2 - mi_stat_count_t segments; - mi_stat_count_t segments_abandoned; - mi_stat_count_t segments_cache; - mi_stat_count_t _segments_reserved; - // only on v3 - mi_stat_counter_t pages_reclaim_on_alloc; - mi_stat_counter_t pages_reclaim_on_free; - mi_stat_counter_t pages_reabandon_full; - mi_stat_counter_t pages_unabandon_busy_wait; - - // future extension - mi_stat_count_t _stat_reserved[4]; - mi_stat_counter_t _stat_counter_reserved[4]; - - // size segregated statistics - mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin - mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin -} mi_stats_t; - - -// add to stat keeping track of the peak -void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); -void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); -// counters can just be increased -void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); - -#if (MI_STAT) -#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) -#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) -#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) -#else -#define mi_stat_increase(stat,amount) ((void)0) -#define mi_stat_decrease(stat,amount) ((void)0) -#define mi_stat_counter_increase(stat,amount) ((void)0) -#endif - -#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) -#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) -#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) - - // ------------------------------------------------------ // Sub processes do not reclaim or visit segments -// from other sub processes +// from other sub processes. These are essentially the +// static variables of a process. // ------------------------------------------------------ struct mi_subproc_s { @@ -656,6 +532,7 @@ struct mi_subproc_s { mi_memid_t memid; // provenance of this memory block }; + // ------------------------------------------------------ // Thread Local data // ------------------------------------------------------ @@ -693,4 +570,73 @@ struct mi_tld_s { mi_stats_t stats; // statistics }; + + +// ------------------------------------------------------ +// Debug +// ------------------------------------------------------ + +#if !defined(MI_DEBUG_UNINIT) +#define MI_DEBUG_UNINIT (0xD0) +#endif +#if !defined(MI_DEBUG_FREED) +#define MI_DEBUG_FREED (0xDF) +#endif +#if !defined(MI_DEBUG_PADDING) +#define MI_DEBUG_PADDING (0xDE) +#endif + +#if (MI_DEBUG) +// use our own assertion to print without memory allocation +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) +#else +#define mi_assert(x) +#endif + +#if (MI_DEBUG>1) +#define mi_assert_internal mi_assert +#else +#define mi_assert_internal(x) +#endif + +#if (MI_DEBUG>2) +#define mi_assert_expensive mi_assert +#else +#define mi_assert_expensive(x) +#endif + + +// ------------------------------------------------------ +// Statistics +// ------------------------------------------------------ +#ifndef MI_STAT +#if (MI_DEBUG>0) +#define MI_STAT 2 +#else +#define MI_STAT 0 +#endif +#endif + +// add to stat keeping track of the peak +void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); +void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); +// counters can just be increased +void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); + +#if (MI_STAT) +#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) +#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) +#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) +#else +#define mi_stat_increase(stat,amount) ((void)0) +#define mi_stat_decrease(stat,amount) ((void)0) +#define mi_stat_counter_increase(stat,amount) ((void)0) +#endif + +#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) +#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) +#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) + + #endif diff --git a/src/heap.c b/src/heap.c index 28161266..7c235a7b 100644 --- a/src/heap.c +++ b/src/heap.c @@ -167,6 +167,11 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // collect arenas (this is program wide so don't force purges on abandonment of threads) _mi_arenas_collect(collect == MI_FORCE /* force purge? */); + + // merge statistics + if (collect <= MI_FORCE) { + mi_stats_merge(); + } } void _mi_heap_collect_abandon(mi_heap_t* heap) { diff --git a/src/init.c b/src/init.c index f91e279b..f4fc6798 100644 --- a/src/init.c +++ b/src/init.c @@ -134,7 +134,7 @@ static mi_decl_cache_align mi_tld_t tld_main = { 0, 0, 0, 0, 0, &mi_subproc_default, &tld_main.stats }, // segments - { MI_STATS_NULL } // stats + { MI_STAT_VERSION, MI_STATS_NULL } // stats }; mi_decl_cache_align mi_heap_t _mi_heap_main = { @@ -159,7 +159,7 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = { bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. -mi_stats_t _mi_stats_main = { MI_STATS_NULL }; +mi_stats_t _mi_stats_main = { MI_STAT_VERSION, MI_STATS_NULL }; #if MI_GUARDED mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { diff --git a/src/page-queue.c b/src/page-queue.c index 9fc5f924..3507505d 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -136,10 +136,15 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* } #endif +static size_t mi_page_bin(const mi_page_t* page) { + const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); + mi_assert_internal(bin <= MI_BIN_FULL); + return bin; +} + static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { mi_assert_internal(heap!=NULL); - size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); - mi_assert_internal(bin <= MI_BIN_FULL); + const size_t bin = mi_page_bin(page); mi_page_queue_t* pq = &heap->pages[bin]; mi_assert_internal((mi_page_block_size(page) == pq->block_size) || (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) || diff --git a/src/page.c b/src/page.c index df802c7d..ea6535bd 100644 --- a/src/page.c +++ b/src/page.c @@ -290,6 +290,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size mi_assert_internal(full_block_size >= block_size); mi_page_init(heap, page, full_block_size, heap->tld); mi_heap_stat_increase(heap, pages, 1); + mi_heap_stat_increase(heap, page_bins[mi_page_bin(page)], 1); if (pq != NULL) { mi_page_queue_push(heap, pq, page); } mi_assert_expensive(_mi_page_is_valid(page)); return page; @@ -438,10 +439,12 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) - mi_segments_tld_t* segments_tld = &mi_page_heap(page)->tld->segments; + mi_heap_t* heap = mi_page_heap(page); + mi_segments_tld_t* segments_tld = &heap->tld->segments; mi_page_queue_remove(pq, page); // and free it + mi_heap_stat_decrease(heap, page_bins[mi_page_bin(page)], 1); mi_page_set_heap(page,NULL); _mi_segment_page_free(page, force, segments_tld); } diff --git a/src/stats.c b/src/stats.c index 65d6d00b..36c7f23c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -63,7 +63,7 @@ void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { // must be thread safe as it is called from stats_merge -static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src) { +static void mi_stat_count_add(mi_stat_count_t* stat, const mi_stat_count_t* src) { if (stat==src) return; if (src->total!=0) { mi_atomic_addi64_relaxed(&stat->total, src->total); } if (src->current!=0) { mi_atomic_addi64_relaxed(&stat->current, src->current); } @@ -78,58 +78,29 @@ static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t if (src->total!=0) { mi_atomic_addi64_relaxed(&stat->total, src->total); } } +#define MI_STAT_COUNT(stat) mi_stat_count_add(&stats->stat, &src->stat); +#define MI_STAT_COUNTER(stat) mi_stat_counter_add(&stats->stat, &src->stat); + // must be thread safe as it is called from stats_merge static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { if (stats==src) return; - mi_stat_add(&stats->pages, &src->pages); - mi_stat_add(&stats->reserved, &src->reserved); - mi_stat_add(&stats->committed, &src->committed); - mi_stat_add(&stats->reset, &src->reset); - mi_stat_add(&stats->purged, &src->purged); - mi_stat_add(&stats->page_committed, &src->page_committed); - mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned); - mi_stat_add(&stats->threads, &src->threads); - mi_stat_add(&stats->malloc_normal, &src->malloc_normal); - mi_stat_add(&stats->malloc_huge, &src->malloc_huge); - mi_stat_add(&stats->malloc_requested, &src->malloc_requested); - mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls); - mi_stat_counter_add(&stats->commit_calls, &src->commit_calls); - mi_stat_counter_add(&stats->reset_calls, &src->reset_calls); - mi_stat_counter_add(&stats->purge_calls, &src->purge_calls); - mi_stat_counter_add(&stats->arena_count, &src->arena_count); - mi_stat_counter_add(&stats->malloc_normal_count, &src->malloc_normal_count); - mi_stat_counter_add(&stats->malloc_huge_count, &src->malloc_huge_count); - mi_stat_counter_add(&stats->malloc_guarded_count, &src->malloc_guarded_count); + // copy all fields + MI_STAT_FIELDS() - mi_stat_counter_add(&stats->arena_rollback_count, &src->arena_rollback_count); - mi_stat_counter_add(&stats->pages_extended, &src->pages_extended); - mi_stat_counter_add(&stats->pages_retire, &src->pages_retire); - mi_stat_counter_add(&stats->page_searches, &src->page_searches); - - mi_stat_add(&stats->segments, &src->segments); - mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned); - mi_stat_add(&stats->segments_cache, &src->segments_cache); - - mi_stat_counter_add(&stats->pages_reclaim_on_alloc, &src->pages_reclaim_on_alloc); - mi_stat_counter_add(&stats->pages_reclaim_on_free, &src->pages_reclaim_on_free); - mi_stat_counter_add(&stats->pages_reabandon_full, &src->pages_reabandon_full); - mi_stat_counter_add(&stats->pages_unabandon_busy_wait, &src->pages_unabandon_busy_wait); - -#if MI_STAT>1 + #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { - // if (src->normal_bins[i].total != 0 && src->normal_bins[i].current != 0) { - mi_stat_add(&stats->malloc_bins[i], &src->malloc_bins[i]); - //} + mi_stat_count_add(&stats->malloc_bins[i], &src->malloc_bins[i]); } + #endif for (size_t i = 0; i <= MI_BIN_HUGE; i++) { - // if (src->normal_bins[i].total != 0 && src->normal_bins[i].current != 0) { - mi_stat_add(&stats->page_bins[i], &src->page_bins[i]); - //} - } -#endif + mi_stat_count_add(&stats->page_bins[i], &src->page_bins[i]); + } } +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER + /* ----------------------------------------------------------- Display statistics ----------------------------------------------------------- */ @@ -318,8 +289,8 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_print(&stats->malloc_normal, "normal", (stats->malloc_normal_count.total == 0 ? 1 : -1), out, arg); mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? 1 : -1), out, arg); mi_stat_count_t total = { 0,0,0 }; - mi_stat_add(&total, &stats->malloc_normal); - mi_stat_add(&total, &stats->malloc_huge); + mi_stat_count_add(&total, &stats->malloc_normal); + mi_stat_count_add(&total, &stats->malloc_huge); mi_stat_print_ex(&total, "total", 1, out, arg, ""); #endif #if MI_STAT>1 @@ -466,3 +437,159 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s if (peak_commit!=NULL) *peak_commit = pinfo.peak_commit; if (page_faults!=NULL) *page_faults = pinfo.page_faults; } + + +// -------------------------------------------------------- +// Return statistics +// -------------------------------------------------------- + +void mi_stats_get(size_t stats_size, mi_stats_t* stats) mi_attr_noexcept { + if (stats == NULL || stats_size == 0) return; + _mi_memzero(stats, stats_size); + const size_t size = (stats_size > sizeof(mi_stats_t) ? sizeof(mi_stats_t) : stats_size); + _mi_memcpy(stats, &_mi_stats_main, size); + stats->version = MI_STAT_VERSION; +} + + +// -------------------------------------------------------- +// Statics in json format +// -------------------------------------------------------- + +typedef struct mi_heap_buf_s { + char* buf; + size_t size; + size_t used; + bool can_realloc; +} mi_heap_buf_t; + +static bool mi_heap_buf_expand(mi_heap_buf_t* hbuf) { + if (hbuf==NULL) return false; + if (hbuf->buf != NULL && hbuf->size>0) { + hbuf->buf[hbuf->size-1] = 0; + } + if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false; + const size_t newsize = (hbuf->size == 0 ? 2*MI_KiB : 2*hbuf->size); + char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize); + if (newbuf == NULL) return false; + hbuf->buf = newbuf; + hbuf->size = newsize; + return true; +} + +static void mi_heap_buf_print(mi_heap_buf_t* hbuf, const char* msg) { + if (msg==NULL || hbuf==NULL) return; + if (hbuf->used + 1 >= hbuf->size && !hbuf->can_realloc) return; + for (const char* src = msg; *src != 0; src++) { + char c = *src; + if (hbuf->used + 1 >= hbuf->size) { + if (!mi_heap_buf_expand(hbuf)) return; + } + mi_assert_internal(hbuf->used < hbuf->size); + hbuf->buf[hbuf->used++] = c; + } + mi_assert_internal(hbuf->used < hbuf->size); + hbuf->buf[hbuf->used] = 0; +} + +static void mi_heap_buf_print_count_bin(mi_heap_buf_t* hbuf, const char* prefix, mi_stat_count_t* stat, size_t bin, bool add_comma) { + const size_t binsize = _mi_bin_size(bin); + const size_t pagesize = (binsize <= MI_SMALL_OBJ_SIZE_MAX ? MI_SMALL_PAGE_SIZE : + (binsize <= MI_MEDIUM_OBJ_SIZE_MAX ? MI_MEDIUM_PAGE_SIZE : + (binsize <= MI_LARGE_OBJ_SIZE_MAX ? MI_LARGE_PAGE_SIZE : 0))); + char buf[128]; + _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"block_size\": %zu, \"page_size\": %zu }%s\n", prefix, stat->total, stat->peak, stat->current, binsize, pagesize, (add_comma ? "," : "")); + buf[127] = 0; + mi_heap_buf_print(hbuf, buf); +} + +static void mi_heap_buf_print_count(mi_heap_buf_t* hbuf, const char* prefix, mi_stat_count_t* stat, bool add_comma) { + char buf[128]; + _mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld }%s\n", prefix, stat->total, stat->peak, stat->current, (add_comma ? "," : "")); + buf[127] = 0; + mi_heap_buf_print(hbuf, buf); +} + +static void mi_heap_buf_print_count_value(mi_heap_buf_t* hbuf, const char* name, mi_stat_count_t* stat) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": ", name); + buf[127] = 0; + mi_heap_buf_print(hbuf, buf); + mi_heap_buf_print_count(hbuf, "", stat, true); +} + +static void mi_heap_buf_print_value(mi_heap_buf_t* hbuf, const char* name, int64_t val) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": %lld,\n", name, val); + buf[127] = 0; + mi_heap_buf_print(hbuf, buf); +} + +static void mi_heap_buf_print_size(mi_heap_buf_t* hbuf, const char* name, size_t val, bool add_comma) { + char buf[128]; + _mi_snprintf(buf, 128, " \"%s\": %zu%s\n", name, val, (add_comma ? "," : "")); + buf[127] = 0; + mi_heap_buf_print(hbuf, buf); +} + +static void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* name, mi_stat_counter_t* stat) { + mi_heap_buf_print_value(hbuf, name, stat->total); +} + +#define MI_STAT_COUNT(stat) mi_heap_buf_print_count_value(&hbuf, #stat, &stats->stat); +#define MI_STAT_COUNTER(stat) mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat); + +const char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept { + mi_heap_buf_t hbuf = { NULL, 0, 0, true }; + if (output_size > 0 && output_buf != NULL) { + _mi_memzero(output_buf, output_size); + hbuf.buf = output_buf; + hbuf.size = output_size; + hbuf.can_realloc = false; + } + else { + if (!mi_heap_buf_expand(&hbuf)) return NULL; + } + mi_heap_buf_print(&hbuf, "{\n"); + mi_heap_buf_print_value(&hbuf, "version", MI_STAT_VERSION); + mi_heap_buf_print_value(&hbuf, "mimalloc_version", MI_MALLOC_VERSION); + + // process info + mi_heap_buf_print(&hbuf, " \"process\": {\n"); + size_t elapsed; + size_t user_time; + size_t sys_time; + size_t current_rss; + size_t peak_rss; + size_t current_commit; + size_t peak_commit; + size_t page_faults; + mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); + mi_heap_buf_print_size(&hbuf, "elapsed_msecs", elapsed, true); + mi_heap_buf_print_size(&hbuf, "user_msecs", user_time, true); + mi_heap_buf_print_size(&hbuf, "system_msecs", sys_time, true); + mi_heap_buf_print_size(&hbuf, "page_faults", page_faults, true); + mi_heap_buf_print_size(&hbuf, "rss_current", current_rss, true); + mi_heap_buf_print_size(&hbuf, "rss_peak", peak_rss, true); + mi_heap_buf_print_size(&hbuf, "commit_current", current_commit, true); + mi_heap_buf_print_size(&hbuf, "commit_peak", peak_commit, false); + mi_heap_buf_print(&hbuf, " },\n"); + + // statistics + mi_stats_t* stats = &_mi_stats_main; + MI_STAT_FIELDS() + + // size bins + mi_heap_buf_print(&hbuf, " \"malloc_bins\": [\n"); + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + mi_heap_buf_print_count_bin(&hbuf, " ", &stats->malloc_bins[i], i, i!=MI_BIN_HUGE); + } + mi_heap_buf_print(&hbuf, " ],\n"); + mi_heap_buf_print(&hbuf, " \"page_bins\": [\n"); + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + mi_heap_buf_print_count_bin(&hbuf, " ", &stats->page_bins[i], i, i!=MI_BIN_HUGE); + } + mi_heap_buf_print(&hbuf, " ]\n"); + mi_heap_buf_print(&hbuf, "}\n"); + return hbuf.buf; +} diff --git a/test/test-stress.c b/test/test-stress.c index c41954d3..bf95a0f5 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -36,7 +36,7 @@ static int ITER = 400; static int THREADS = 8; static int SCALE = 25; static int ITER = 20; -#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +#elif 1 || defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = 8; static int SCALE = 10; static int ITER = 10; @@ -61,6 +61,7 @@ static bool main_participates = false; // main thread participates as a #define custom_free(p) free(p) #else #include +#include #define custom_calloc(n,s) mi_calloc(n,s) #define custom_realloc(p,s) mi_realloc(p,s) #define custom_free(p) mi_free(p) @@ -330,8 +331,15 @@ int main(int argc, char** argv) { #ifndef NDEBUG mi_debug_show_arenas(); mi_collect(true); + + const char* json = mi_stats_get_json(0, NULL); + if (json != NULL) { + puts(json); + mi_free(json); + } + #endif - mi_stats_print(NULL); + mi_stats_print(NULL); #endif //bench_end_program(); return 0; From 5b685ef183093eb7e39d4e80c33ce31d2b0fec58 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 2 Mar 2025 15:35:12 -0800 Subject: [PATCH 5/7] fix test stress --- test/test-stress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index bf95a0f5..56aa3228 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -36,7 +36,7 @@ static int ITER = 400; static int THREADS = 8; static int SCALE = 25; static int ITER = 20; -#elif 1 || defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = 8; static int SCALE = 10; static int ITER = 10; @@ -331,13 +331,13 @@ int main(int argc, char** argv) { #ifndef NDEBUG mi_debug_show_arenas(); mi_collect(true); - + /* const char* json = mi_stats_get_json(0, NULL); if (json != NULL) { puts(json); mi_free(json); } - + */ #endif mi_stats_print(NULL); #endif From b319156c4f01ee2148841816991aae225dde932b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 2 Mar 2025 15:35:35 -0800 Subject: [PATCH 6/7] fix test stress --- test/test-stress.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index 56aa3228..43d06ad5 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -331,13 +331,11 @@ int main(int argc, char** argv) { #ifndef NDEBUG mi_debug_show_arenas(); mi_collect(true); - /* const char* json = mi_stats_get_json(0, NULL); if (json != NULL) { puts(json); mi_free(json); } - */ #endif mi_stats_print(NULL); #endif From c910750bbe5365b04ea60ca8d31c5a32d504f0de Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 2 Mar 2025 15:39:08 -0800 Subject: [PATCH 7/7] fix mi_stat_get_json signature --- include/mimalloc-stats.h | 4 ++-- src/stats.c | 2 +- test/test-stress.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index 4dc4c381..7c1ed770 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -92,8 +92,8 @@ typedef struct mi_stats_s extern "C" { #endif -mi_decl_export void mi_stats_get( size_t stats_size, mi_stats_t* stats ) mi_attr_noexcept; -mi_decl_export const char* mi_stats_get_json( size_t buf_size, char* buf ) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL +mi_decl_export void mi_stats_get( size_t stats_size, mi_stats_t* stats ) mi_attr_noexcept; +mi_decl_export char* mi_stats_get_json( size_t buf_size, char* buf ) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL #ifdef __cplusplus } diff --git a/src/stats.c b/src/stats.c index 36c7f23c..d9b26863 100644 --- a/src/stats.c +++ b/src/stats.c @@ -539,7 +539,7 @@ static void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* nam #define MI_STAT_COUNT(stat) mi_heap_buf_print_count_value(&hbuf, #stat, &stats->stat); #define MI_STAT_COUNTER(stat) mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat); -const char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept { +char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept { mi_heap_buf_t hbuf = { NULL, 0, 0, true }; if (output_size > 0 && output_buf != NULL) { _mi_memzero(output_buf, output_size); diff --git a/test/test-stress.c b/test/test-stress.c index 43d06ad5..9e041064 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -331,9 +331,9 @@ int main(int argc, char** argv) { #ifndef NDEBUG mi_debug_show_arenas(); mi_collect(true); - const char* json = mi_stats_get_json(0, NULL); + char* json = mi_stats_get_json(0, NULL); if (json != NULL) { - puts(json); + fputs(json,stderr); mi_free(json); } #endif