From 752dc834a9ae7c73afd83d23e9934c465d0fc2bd Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 20 May 2025 18:15:36 -0700 Subject: [PATCH] merge 3301ba0 to fix page_bin and pages statistics --- include/mimalloc/internal.h | 4 ++-- src/arena.c | 21 +++++++++++++++------ src/free.c | 4 ++-- src/heap.c | 9 ++++----- src/page.c | 10 +++------- src/stats.c | 3 ++- test/test-stress.c | 2 +- 7 files changed, 29 insertions(+), 24 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 50bd5d79..fce61435 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -184,7 +184,7 @@ void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld void _mi_arenas_unsafe_destroy_all(mi_tld_t* tld); mi_page_t* _mi_arenas_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment); -void _mi_arenas_page_free(mi_page_t* page); +void _mi_arenas_page_free(mi_page_t* page, mi_tld_t* tld); void _mi_arenas_page_abandon(mi_page_t* page, mi_tld_t* tld); void _mi_arenas_page_unabandon(mi_page_t* page); bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); @@ -843,7 +843,7 @@ static inline bool _mi_page_unown(mi_page_t* page) { _mi_page_free_collect(page, false); // update used if (mi_page_all_free(page)) { // it may become free just before unowning it _mi_arenas_page_unabandon(page); - _mi_arenas_page_free(page); + _mi_arenas_page_free(page,NULL); return true; } tf_old = mi_atomic_load_relaxed(&page->xthread_free); diff --git a/src/arena.c b/src/arena.c index 95cc8d63..b350d092 100644 --- a/src/arena.c +++ b/src/arena.c @@ -578,8 +578,8 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_subproc_t* subproc, size_ } // Allocate a fresh page -static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice_count, size_t block_size, size_t block_alignment, - mi_arena_t* req_arena, size_t tseq, int numa_node, bool commit) +static mi_page_t* mi_arenas_page_alloc_fresh(size_t slice_count, size_t block_size, size_t block_alignment, + mi_arena_t* req_arena, int numa_node, bool commit, mi_tld_t* tld) { const bool allow_large = (MI_SECURE < 2); // 2 = guard page at end of each arena page const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); @@ -593,7 +593,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice !os_align && // not large alignment slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large { - page = (mi_page_t*)mi_arenas_try_alloc(subproc, slice_count, page_alignment, commit, allow_large, req_arena, tseq, numa_node, &memid); + page = (mi_page_t*)mi_arenas_try_alloc(tld->subproc, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, &memid); if (page != NULL) { mi_assert_internal(mi_bitmap_is_clearN(memid.mem.arena.arena->pages, memid.mem.arena.slice_index, memid.mem.arena.slice_count)); mi_bitmap_set(memid.mem.arena.arena->pages, memid.mem.arena.slice_index); @@ -701,6 +701,11 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice // register in the page map _mi_page_map_register(page); + + // stats + mi_tld_stat_increase(tld, pages, 1); + mi_tld_stat_increase(tld, page_bins[_mi_page_bin(page)], 1); + mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_block_size(page) == block_size); @@ -724,7 +729,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_heap_t* heap, size_t slice_cou const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); - page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, 1, req_arena, tld->thread_seq, heap->numa_node, commit); + page = mi_arenas_page_alloc_fresh(slice_count, block_size, 1, req_arena, heap->numa_node, commit, tld); if (page != NULL) { mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); _mi_page_init(heap, page); @@ -746,7 +751,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_heap_t* heap, size_t block_s const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size()); #endif - mi_page_t* page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, block_alignment, req_arena, tld->thread_seq, heap->numa_node, true /* commit singletons always */); + mi_page_t* page = mi_arenas_page_alloc_fresh(slice_count, block_size, block_alignment, req_arena, heap->numa_node, true /* commit singletons always */, tld); if (page == NULL) return NULL; mi_assert(page->reserved == 1); @@ -785,7 +790,7 @@ mi_page_t* _mi_arenas_page_alloc(mi_heap_t* heap, size_t block_size, size_t bloc return page; } -void _mi_arenas_page_free(mi_page_t* page) { +void _mi_arenas_page_free(mi_page_t* page, mi_tld_t* tld /* can be NULL */) { mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(mi_page_is_owned(page)); @@ -793,6 +798,10 @@ void _mi_arenas_page_free(mi_page_t* page) { mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(page->next==NULL && page->prev==NULL); + if (tld==NULL) { tld = _mi_thread_tld(); } + mi_tld_stat_decrease(tld, page_bins[_mi_page_bin(page)], 1); + mi_tld_stat_decrease(tld, pages, 1); + #if MI_DEBUG>1 if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { size_t bin = _mi_bin(mi_page_block_size(page)); diff --git a/src/free.c b/src/free.c index 9ddd1f19..40e81380 100644 --- a/src/free.c +++ b/src/free.c @@ -217,7 +217,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish) _mi_arenas_page_unabandon(page); // we can free the page directly - _mi_arenas_page_free(page); + _mi_arenas_page_free(page,NULL); return; } @@ -300,7 +300,7 @@ static bool mi_page_unown_from_free(mi_page_t* page, mi_block_t* mt_free) { _mi_page_free_collect(page,false); // update used if (mi_page_all_free(page)) { // it may become free just before unowning it _mi_arenas_page_unabandon(page); - _mi_arenas_page_free(page); + _mi_arenas_page_free(page,NULL); return true; } tf_expect = mi_atomic_load_relaxed(&page->xthread_free); diff --git a/src/heap.c b/src/heap.c index 17dd1fdb..922433bf 100644 --- a/src/heap.c +++ b/src/heap.c @@ -117,11 +117,11 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // python/cpython#112532: we may be called from a thread that is not the owner of the heap // const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id()); + // if (_mi_is_main_thread()) { mi_debug_show_arenas(true, false, false); } + // collect retired pages _mi_heap_collect_retired(heap, force); - // if (_mi_is_main_thread()) { mi_debug_show_arenas(true, false, false); } - // collect all pages owned by this thread mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); @@ -130,7 +130,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, heap->tld); // merge statistics - if (collect <= MI_FORCE) { _mi_stats_merge_thread(heap->tld); } + _mi_stats_merge_from(&_mi_subproc()->stats, &heap->tld->stats); } void _mi_heap_collect_abandon(mi_heap_t* heap) { @@ -329,7 +329,6 @@ mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { MI_UNUSED(arg1); MI_UNUSED(arg2); - MI_UNUSED(heap); MI_UNUSED(pq); // ensure no more thread_delayed_free will be added @@ -361,7 +360,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ page->next = NULL; page->prev = NULL; mi_page_set_heap(page, NULL); - _mi_arenas_page_free(page); + _mi_arenas_page_free(page, heap->tld); return true; // keep going } diff --git a/src/page.c b/src/page.c index 103d11b4..e9b0e784 100644 --- a/src/page.c +++ b/src/page.c @@ -322,9 +322,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size else if (pq != NULL) { mi_page_queue_push(heap, pq, page); } - mi_heap_stat_increase(heap, pages, 1); mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size); - mi_heap_stat_increase(heap, page_bins[_mi_page_bin(page)], 1); mi_assert_expensive(_mi_page_is_valid(page)); return page; } @@ -394,12 +392,10 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { mi_page_queue_remove(pq, page); // and free it - mi_heap_t* heap = page->heap; - mi_heap_stat_decrease(heap, page_bins[_mi_page_bin(page)], 1); - mi_heap_stat_decrease(heap, pages, 1); + mi_tld_t* const tld = page->heap->tld; mi_page_set_heap(page,NULL); - _mi_arenas_page_free(page); - _mi_arenas_collect(false, false, heap->tld); // allow purging + _mi_arenas_page_free(page,tld); + _mi_arenas_collect(false, false, tld); // allow purging } #define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE diff --git a/src/stats.c b/src/stats.c index 93b68054..eac6a19c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -388,7 +388,7 @@ static mi_msecs_t mi_process_start; // = 0 // return thread local stats static mi_stats_t* mi_get_tld_stats(void) { - return &mi_heap_get_default()->tld->stats; + return &_mi_thread_tld()->stats; } void mi_stats_reset(void) mi_attr_noexcept { @@ -609,6 +609,7 @@ static void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* nam #define MI_STAT_COUNTER(stat) mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat); char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept { + mi_stats_merge(); mi_heap_buf_t hbuf = { NULL, 0, 0, true }; if (output_size > 0 && output_buf != NULL) { _mi_memzero(output_buf, output_size); diff --git a/test/test-stress.c b/test/test-stress.c index ba6270ba..a2e2b377 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -375,7 +375,7 @@ int main(int argc, char** argv) { } #endif mi_collect(true); - mi_stats_print(NULL); + mi_stats_print(NULL); #endif //bench_end_program(); return 0;