diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index d965c275..8fcaa22e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -311,7 +311,7 @@ bool _mi_page_is_valid(mi_page_t* page); #define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) #define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) #define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) - +#define MI_INIT74(x) MI_INIT64(x),MI_INIT8(x),x(),x() #include // initialize a local variable to zero; use memset as compilers optimize constant sized memset's diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 64ebb160..c27489d0 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -573,37 +573,50 @@ typedef struct mi_stat_counter_s { } mi_stat_counter_t; typedef struct mi_stats_s { - mi_stat_count_t segments; - mi_stat_count_t pages; - mi_stat_count_t reserved; - mi_stat_count_t committed; - mi_stat_count_t reset; - mi_stat_count_t purged; - mi_stat_count_t page_committed; - mi_stat_count_t segments_abandoned; - mi_stat_count_t pages_abandoned; - mi_stat_count_t threads; - mi_stat_count_t normal; - mi_stat_count_t huge; - mi_stat_count_t giant; - mi_stat_count_t malloc; - mi_stat_count_t segments_cache; - mi_stat_counter_t pages_extended; + mi_stat_count_t pages; // count of mimalloc pages + mi_stat_count_t reserved; // reserved memory bytes + mi_stat_count_t committed; // committed bytes + mi_stat_count_t reset; // reset bytes + mi_stat_count_t purged; // purged bytes + mi_stat_count_t page_committed; // committed memory inside pages + mi_stat_count_t pages_abandoned; // abandonded pages count + mi_stat_count_t threads; // number of threads + mi_stat_count_t malloc_normal; // allocated bytes <= MI_LARGE_OBJ_SIZE_MAX + mi_stat_count_t malloc_huge; // allocated bytes in huge pages + mi_stat_count_t malloc_requested; // malloc requested bytes + mi_stat_counter_t mmap_calls; mi_stat_counter_t commit_calls; mi_stat_counter_t reset_calls; mi_stat_counter_t purge_calls; - mi_stat_counter_t page_no_retire; - mi_stat_counter_t searches; - mi_stat_counter_t normal_count; - mi_stat_counter_t huge_count; - mi_stat_counter_t arena_count; - mi_stat_counter_t arena_crossover_count; + mi_stat_counter_t arena_count; // number of memory arena's + mi_stat_counter_t malloc_normal_count; // number of blocks <= MI_LARGE_OBJ_SIZE_MAX + mi_stat_counter_t malloc_huge_count; // number of huge bloks + mi_stat_counter_t malloc_guarded_count; // number of allocations with guard pages + + // internal statistics mi_stat_counter_t arena_rollback_count; - mi_stat_counter_t guarded_alloc_count; -#if MI_STAT>1 - mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; -#endif + mi_stat_counter_t pages_extended; // number of page extensions + mi_stat_counter_t pages_retire; // number of pages that are retired + mi_stat_counter_t page_searches; // searches for a fresh page + // only on v1 and v2 + mi_stat_count_t segments; + mi_stat_count_t segments_abandoned; + mi_stat_count_t segments_cache; + mi_stat_count_t _segments_reserved; + // only on v3 + mi_stat_counter_t pages_reclaim_on_alloc; + mi_stat_counter_t pages_reclaim_on_free; + mi_stat_counter_t pages_reabandon_full; + mi_stat_counter_t pages_unabandon_busy_wait; + + // future extension + mi_stat_count_t _stat_reserved[4]; + mi_stat_counter_t _stat_counter_reserved[4]; + + // size segregated statistics + mi_stat_count_t malloc_bins[MI_BIN_HUGE+1]; // allocation per size bin + mi_stat_count_t page_bins[MI_BIN_HUGE+1]; // pages allocated per size bin } mi_stats_t; diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 7304eb1d..d0e691b3 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -192,7 +192,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t if mi_likely(is_aligned) { #if MI_STAT>1 - mi_heap_stat_increase(heap, malloc, size); + mi_heap_stat_increase(heap, malloc_requested, size); #endif void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen mi_assert_internal(p != NULL); diff --git a/src/alloc.c b/src/alloc.c index a093f108..badc2612 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -83,11 +83,11 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ #if (MI_STAT>0) const size_t bsize = mi_page_usable_block_size(page); if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_increase(heap, normal, bsize); - mi_heap_stat_counter_increase(heap, normal_count, 1); + mi_heap_stat_increase(heap, malloc_normal, bsize); + mi_heap_stat_counter_increase(heap, malloc_normal_count, 1); #if (MI_STAT>1) const size_t bin = _mi_bin(bsize); - mi_heap_stat_increase(heap, normal_bins[bin], 1); + mi_heap_stat_increase(heap, malloc_bins[bin], 1); #endif } #endif @@ -149,7 +149,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } - mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p)); } #endif #if MI_DEBUG>3 @@ -191,7 +191,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } - mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p)); } #endif #if MI_DEBUG>3 diff --git a/src/bitmap.c b/src/bitmap.c index 2def7bb0..9ef784d6 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -242,7 +242,6 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit } while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap)); // claimed! - mi_stat_counter_increase(_mi_stats_main.arena_crossover_count,1); *bitmap_idx = mi_bitmap_index_create(idx, initial_idx); return true; diff --git a/src/free.c b/src/free.c index f2e30b65..a1732e8c 100644 --- a/src/free.c +++ b/src/free.c @@ -521,17 +521,17 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { const size_t bsize = mi_page_usable_block_size(page); #if (MI_STAT>1) const size_t usize = mi_page_usable_size_of(page, block); - mi_heap_stat_decrease(heap, malloc, usize); + mi_heap_stat_decrease(heap, malloc_requested, usize); #endif if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, normal, bsize); + mi_heap_stat_decrease(heap, malloc_normal, bsize); #if (MI_STAT > 1) - mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], 1); + mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], 1); #endif } else { const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc - mi_heap_stat_decrease(heap, huge, bpsize); + mi_heap_stat_decrease(heap, malloc_huge, bpsize); } } #else diff --git a/src/heap.c b/src/heap.c index 58aa050c..28161266 100644 --- a/src/heap.c +++ b/src/heap.c @@ -324,18 +324,18 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ // stats const size_t bsize = mi_page_block_size(page); if (bsize > MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, huge, bsize); + mi_heap_stat_decrease(heap, malloc_huge, bsize); } #if (MI_STAT) _mi_page_free_collect(page, false); // update used count const size_t inuse = page->used; if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - mi_heap_stat_decrease(heap, normal, bsize * inuse); + mi_heap_stat_decrease(heap, malloc_normal, bsize * inuse); #if (MI_STAT>1) - mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse); + mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], inuse); #endif } - mi_heap_stat_decrease(heap, malloc, bsize * inuse); // todo: off for aligned blocks... + mi_heap_stat_decrease(heap, malloc_requested, bsize * inuse); // todo: off for aligned blocks... #endif /// pretend it is all free now diff --git a/src/init.c b/src/init.c index c25e7516..f91e279b 100644 --- a/src/init.c +++ b/src/init.c @@ -69,26 +69,22 @@ const mi_page_t _mi_page_empty = { #define MI_STAT_COUNT_NULL() {0,0,0} // Empty statistics -#if MI_STAT>1 -#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } -#else -#define MI_STAT_COUNT_END_NULL() -#endif - #define MI_STATS_NULL \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ { 0 }, { 0 }, { 0 }, { 0 }, \ { 0 }, { 0 }, { 0 }, { 0 }, \ + \ { 0 }, { 0 }, { 0 }, { 0 }, \ - { 0 } \ - MI_STAT_COUNT_END_NULL() + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + { 0 }, { 0 }, { 0 }, { 0 }, \ + \ + { MI_INIT4(MI_STAT_COUNT_NULL) }, \ + { { 0 }, { 0 }, { 0 }, { 0 } }, \ + \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, \ + { MI_INIT74(MI_STAT_COUNT_NULL) } // -------------------------------------------------------- // Statically allocate an empty heap as the initial diff --git a/src/page.c b/src/page.c index 6d0d6db7..df802c7d 100644 --- a/src/page.c +++ b/src/page.c @@ -473,7 +473,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { const size_t bsize = mi_page_block_size(page); if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? if (pq->last==page && pq->first==page) { // the only page in the queue? - mi_stat_counter_increase(_mi_stats_main.page_no_retire,1); + mi_stat_counter_increase(_mi_stats_main.pages_retire,1); page->retire_expire = (bsize <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(pq >= heap->pages); @@ -809,7 +809,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p page = next; } // for each page - mi_heap_stat_counter_increase(heap, searches, count); + mi_heap_stat_counter_increase(heap, page_searches, count); // set the page to the best candidate if (page_candidate != NULL) { @@ -922,8 +922,8 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_a mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue mi_page_set_heap(page, NULL); #endif - mi_heap_stat_increase(heap, huge, mi_page_block_size(page)); - mi_heap_stat_counter_increase(heap, huge_count, 1); + mi_heap_stat_increase(heap, malloc_huge, mi_page_block_size(page)); + mi_heap_stat_counter_increase(heap, malloc_huge_count, 1); } return page; } diff --git a/src/stats.c b/src/stats.c index c4ec92de..65d6d00b 100644 --- a/src/stats.c +++ b/src/stats.c @@ -81,39 +81,50 @@ static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t // must be thread safe as it is called from stats_merge static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { if (stats==src) return; - mi_stat_add(&stats->segments, &src->segments); mi_stat_add(&stats->pages, &src->pages); mi_stat_add(&stats->reserved, &src->reserved); mi_stat_add(&stats->committed, &src->committed); mi_stat_add(&stats->reset, &src->reset); mi_stat_add(&stats->purged, &src->purged); mi_stat_add(&stats->page_committed, &src->page_committed); - mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned); - mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned); mi_stat_add(&stats->threads, &src->threads); + mi_stat_add(&stats->malloc_normal, &src->malloc_normal); + mi_stat_add(&stats->malloc_huge, &src->malloc_huge); + mi_stat_add(&stats->malloc_requested, &src->malloc_requested); - mi_stat_add(&stats->malloc, &src->malloc); - mi_stat_add(&stats->segments_cache, &src->segments_cache); - mi_stat_add(&stats->normal, &src->normal); - mi_stat_add(&stats->huge, &src->huge); - mi_stat_add(&stats->giant, &src->giant); - - mi_stat_counter_add(&stats->pages_extended, &src->pages_extended); mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls); mi_stat_counter_add(&stats->commit_calls, &src->commit_calls); mi_stat_counter_add(&stats->reset_calls, &src->reset_calls); mi_stat_counter_add(&stats->purge_calls, &src->purge_calls); + mi_stat_counter_add(&stats->arena_count, &src->arena_count); + mi_stat_counter_add(&stats->malloc_normal_count, &src->malloc_normal_count); + mi_stat_counter_add(&stats->malloc_huge_count, &src->malloc_huge_count); + mi_stat_counter_add(&stats->malloc_guarded_count, &src->malloc_guarded_count); + + mi_stat_counter_add(&stats->arena_rollback_count, &src->arena_rollback_count); + mi_stat_counter_add(&stats->pages_extended, &src->pages_extended); + mi_stat_counter_add(&stats->pages_retire, &src->pages_retire); + mi_stat_counter_add(&stats->page_searches, &src->page_searches); + + mi_stat_add(&stats->segments, &src->segments); + mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned); + mi_stat_add(&stats->segments_cache, &src->segments_cache); + + mi_stat_counter_add(&stats->pages_reclaim_on_alloc, &src->pages_reclaim_on_alloc); + mi_stat_counter_add(&stats->pages_reclaim_on_free, &src->pages_reclaim_on_free); + mi_stat_counter_add(&stats->pages_reabandon_full, &src->pages_reabandon_full); + mi_stat_counter_add(&stats->pages_unabandon_busy_wait, &src->pages_unabandon_busy_wait); - mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire); - mi_stat_counter_add(&stats->searches, &src->searches); - mi_stat_counter_add(&stats->normal_count, &src->normal_count); - mi_stat_counter_add(&stats->huge_count, &src->huge_count); - mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { // if (src->normal_bins[i].total != 0 && src->normal_bins[i].current != 0) { - mi_stat_add(&stats->normal_bins[i], &src->normal_bins[i]); + mi_stat_add(&stats->malloc_bins[i], &src->malloc_bins[i]); + //} + } + for (size_t i = 0; i <= MI_BIN_HUGE; i++) { + // if (src->normal_bins[i].total != 0 && src->normal_bins[i].current != 0) { + mi_stat_add(&stats->page_bins[i], &src->page_bins[i]); //} } #endif @@ -301,18 +312,18 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) // and print using that mi_print_header(out,arg); #if MI_STAT>1 - mi_stats_print_bins(stats->normal_bins, MI_BIN_HUGE, "normal",out,arg); + mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, "normal",out,arg); #endif #if MI_STAT - mi_stat_print(&stats->normal, "normal", (stats->normal_count.total == 0 ? 1 : -1), out, arg); - mi_stat_print(&stats->huge, "huge", (stats->huge_count.total == 0 ? 1 : -1), out, arg); + mi_stat_print(&stats->malloc_normal, "normal", (stats->malloc_normal_count.total == 0 ? 1 : -1), out, arg); + mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? 1 : -1), out, arg); mi_stat_count_t total = { 0,0,0 }; - mi_stat_add(&total, &stats->normal); - mi_stat_add(&total, &stats->huge); + mi_stat_add(&total, &stats->malloc_normal); + mi_stat_add(&total, &stats->malloc_huge); mi_stat_print_ex(&total, "total", 1, out, arg, ""); #endif #if MI_STAT>1 - mi_stat_print_ex(&stats->malloc, "malloc req", 1, out, arg, ""); + mi_stat_print_ex(&stats->malloc_requested, "malloc req", 1, out, arg, ""); _mi_fprintf(out, arg, "\n"); #endif mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); @@ -326,17 +337,17 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_print(&stats->pages, "pages", -1, out, arg); mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg); mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg); - mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg); + mi_stat_counter_print(&stats->pages_retire, "-retire", out, arg); mi_stat_counter_print(&stats->arena_count, "arenas", out, arg); - mi_stat_counter_print(&stats->arena_crossover_count, "-crossover", out, arg); + // mi_stat_counter_print(&stats->arena_crossover_count, "-crossover", out, arg); mi_stat_counter_print(&stats->arena_rollback_count, "-rollback", out, arg); mi_stat_counter_print(&stats->mmap_calls, "mmaps", out, arg); mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); - mi_stat_counter_print(&stats->guarded_alloc_count, "guarded", out, arg); + mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); - mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); + mi_stat_counter_print_avg(&stats->page_searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); size_t elapsed;