diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index aab7dffe..98908281 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -184,7 +184,7 @@ void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld void _mi_arenas_unsafe_destroy_all(mi_tld_t* tld); mi_page_t* _mi_arenas_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment); -void _mi_arenas_page_free(mi_page_t* page); +void _mi_arenas_page_free(mi_page_t* page, mi_tld_t* tld); void _mi_arenas_page_abandon(mi_page_t* page, mi_tld_t* tld); void _mi_arenas_page_unabandon(mi_page_t* page); bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); @@ -219,6 +219,7 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); void _mi_page_init(mi_heap_t* heap, mi_page_t* page); bool _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq); +size_t _mi_page_bin(const mi_page_t* page); // for stats size_t _mi_bin_size(size_t bin); // for stats size_t _mi_bin(size_t size); // for stats @@ -237,6 +238,7 @@ void _mi_heap_page_reclaim(mi_heap_t* heap, mi_page_t* page); // "stats.c" void _mi_stats_done(mi_stats_t* stats); +void _mi_stats_merge_thread(mi_tld_t* tld); void _mi_stats_merge_from(mi_stats_t* to, mi_stats_t* from); mi_msecs_t _mi_clock_now(void); mi_msecs_t _mi_clock_end(mi_msecs_t start); @@ -845,7 +847,7 @@ static inline bool _mi_page_unown(mi_page_t* page) { _mi_page_free_collect(page, false); // update used if (mi_page_all_free(page)) { // it may become free just before unowning it _mi_arenas_page_unabandon(page); - _mi_arenas_page_free(page); + _mi_arenas_page_free(page,NULL); return true; } tf_old = mi_atomic_load_relaxed(&page->xthread_free); diff --git a/src/arena.c b/src/arena.c index 95cc8d63..514a0b25 100644 --- a/src/arena.c +++ b/src/arena.c @@ -578,8 +578,8 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_subproc_t* subproc, size_ } // Allocate a fresh page -static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice_count, size_t block_size, size_t block_alignment, - mi_arena_t* req_arena, size_t tseq, int numa_node, bool commit) +static mi_page_t* mi_arenas_page_alloc_fresh(size_t slice_count, size_t block_size, size_t block_alignment, + mi_arena_t* req_arena, int numa_node, bool commit, mi_tld_t* tld) { const bool allow_large = (MI_SECURE < 2); // 2 = guard page at end of each arena page const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); @@ -593,7 +593,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice !os_align && // not large alignment slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large { - page = (mi_page_t*)mi_arenas_try_alloc(subproc, slice_count, page_alignment, commit, allow_large, req_arena, tseq, numa_node, &memid); + page = (mi_page_t*)mi_arenas_try_alloc(tld->subproc, slice_count, page_alignment, commit, allow_large, req_arena, tld->thread_seq, numa_node, &memid); if (page != NULL) { mi_assert_internal(mi_bitmap_is_clearN(memid.mem.arena.arena->pages, memid.mem.arena.slice_index, memid.mem.arena.slice_count)); mi_bitmap_set(memid.mem.arena.arena->pages, memid.mem.arena.slice_index); @@ -701,6 +701,11 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice // register in the page map _mi_page_map_register(page); + + // stats + mi_tld_stat_increase(tld, pages, 1); + mi_tld_stat_increase(tld, page_bins[_mi_page_bin(page)], 1); + mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_block_size(page) == block_size); @@ -724,7 +729,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_heap_t* heap, size_t slice_cou const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); - page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, 1, req_arena, tld->thread_seq, heap->numa_node, commit); + page = mi_arenas_page_alloc_fresh(slice_count, block_size, 1, req_arena, heap->numa_node, commit, tld); if (page != NULL) { mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); _mi_page_init(heap, page); @@ -746,7 +751,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_heap_t* heap, size_t block_s const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size()); #endif - mi_page_t* page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, block_alignment, req_arena, tld->thread_seq, heap->numa_node, true /* commit singletons always */); + mi_page_t* page = mi_arenas_page_alloc_fresh(slice_count, block_size, block_alignment, req_arena, heap->numa_node, true /* commit singletons always */, tld); if (page == NULL) return NULL; mi_assert(page->reserved == 1); @@ -785,7 +790,7 @@ mi_page_t* _mi_arenas_page_alloc(mi_heap_t* heap, size_t block_size, size_t bloc return page; } -void _mi_arenas_page_free(mi_page_t* page) { +void _mi_arenas_page_free(mi_page_t* page, mi_tld_t* stats_tld /* can be NULL */) { mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(mi_page_is_owned(page)); @@ -793,6 +798,15 @@ void _mi_arenas_page_free(mi_page_t* page) { mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(page->next==NULL && page->prev==NULL); + if (stats_tld != NULL) { + mi_tld_stat_decrease(stats_tld, page_bins[_mi_page_bin(page)], 1); + mi_tld_stat_decrease(stats_tld, pages, 1); + } + else { + mi_os_stat_decrease(page_bins[_mi_page_bin(page)], 1); + mi_os_stat_decrease(pages, 1); + } + #if MI_DEBUG>1 if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { size_t bin = _mi_bin(mi_page_block_size(page)); diff --git a/src/free.c b/src/free.c index 9ddd1f19..40e81380 100644 --- a/src/free.c +++ b/src/free.c @@ -217,7 +217,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish) _mi_arenas_page_unabandon(page); // we can free the page directly - _mi_arenas_page_free(page); + _mi_arenas_page_free(page,NULL); return; } @@ -300,7 +300,7 @@ static bool mi_page_unown_from_free(mi_page_t* page, mi_block_t* mt_free) { _mi_page_free_collect(page,false); // update used if (mi_page_all_free(page)) { // it may become free just before unowning it _mi_arenas_page_unabandon(page); - _mi_arenas_page_free(page); + _mi_arenas_page_free(page,NULL); return true; } tf_expect = mi_atomic_load_relaxed(&page->xthread_free); diff --git a/src/heap.c b/src/heap.c index 971aad68..fb0a6c9d 100644 --- a/src/heap.c +++ b/src/heap.c @@ -117,11 +117,11 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // python/cpython#112532: we may be called from a thread that is not the owner of the heap // const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id()); + // if (_mi_is_main_thread()) { mi_debug_show_arenas(true, false, false); } + // collect retired pages _mi_heap_collect_retired(heap, force); - // if (_mi_is_main_thread()) { mi_debug_show_arenas(true, false, false); } - // collect all pages owned by this thread mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); @@ -130,9 +130,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, heap->tld); // merge statistics - if (collect <= MI_FORCE) { - mi_stats_merge(); - } + if (collect <= MI_FORCE) { _mi_stats_merge_thread(heap->tld); } } void _mi_heap_collect_abandon(mi_heap_t* heap) { @@ -331,7 +329,6 @@ mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { MI_UNUSED(arg1); MI_UNUSED(arg2); - MI_UNUSED(heap); MI_UNUSED(pq); // ensure no more thread_delayed_free will be added @@ -363,7 +360,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ page->next = NULL; page->prev = NULL; mi_page_set_heap(page, NULL); - _mi_arenas_page_free(page); + _mi_arenas_page_free(page, heap->tld); return true; // keep going } diff --git a/src/page-queue.c b/src/page-queue.c index 3e2315cc..91bb0ef9 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -168,8 +168,7 @@ bool _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq) { return true; } - -static size_t mi_page_bin(const mi_page_t* page) { +size_t _mi_page_bin(const mi_page_t* page) { const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); mi_assert_internal(bin <= MI_BIN_FULL); return bin; @@ -177,7 +176,7 @@ static size_t mi_page_bin(const mi_page_t* page) { static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { mi_assert_internal(heap!=NULL); - const size_t bin = mi_page_bin(page); + const size_t bin = _mi_page_bin(page); mi_page_queue_t* pq = &heap->pages[bin]; mi_assert_internal((mi_page_block_size(page) == pq->block_size) || (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) || diff --git a/src/page.c b/src/page.c index 0d8e4e12..e9b0e784 100644 --- a/src/page.c +++ b/src/page.c @@ -322,9 +322,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size else if (pq != NULL) { mi_page_queue_push(heap, pq, page); } - mi_heap_stat_increase(heap, pages, 1); mi_assert_internal(pq!=NULL || mi_page_block_size(page) >= block_size); - mi_heap_stat_increase(heap, page_bins[mi_page_bin(page)], 1); mi_assert_expensive(_mi_page_is_valid(page)); return page; } @@ -394,12 +392,10 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { mi_page_queue_remove(pq, page); // and free it - mi_heap_t* heap = page->heap; - mi_heap_stat_decrease(heap, page_bins[mi_page_bin(page)], 1); - mi_heap_stat_decrease(heap, pages, 1); + mi_tld_t* const tld = page->heap->tld; mi_page_set_heap(page,NULL); - _mi_arenas_page_free(page); - _mi_arenas_collect(false, false, heap->tld); // allow purging + _mi_arenas_page_free(page,tld); + _mi_arenas_collect(false, false, tld); // allow purging } #define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE diff --git a/src/stats.c b/src/stats.c index 3b53d5a9..b1a7779d 100644 --- a/src/stats.c +++ b/src/stats.c @@ -388,7 +388,7 @@ static mi_msecs_t mi_process_start; // = 0 // return thread local stats static mi_stats_t* mi_get_tld_stats(void) { - return &mi_heap_get_default()->tld->stats; + return &_mi_thread_tld()->stats; } void mi_stats_reset(void) mi_attr_noexcept { @@ -400,6 +400,7 @@ void mi_stats_reset(void) mi_attr_noexcept { } void _mi_stats_merge_from(mi_stats_t* to, mi_stats_t* from) { + mi_assert_internal(to != NULL && from != NULL); if (to != from) { mi_stats_add(to, from); _mi_memzero(from, sizeof(mi_stats_t)); @@ -410,8 +411,13 @@ void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done` _mi_stats_merge_from(&_mi_subproc()->stats, stats); } +void _mi_stats_merge_thread(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && tld->subproc != NULL); + _mi_stats_merge_from( &tld->subproc->stats, &tld->stats ); +} + void mi_stats_merge(void) mi_attr_noexcept { - _mi_stats_done( mi_get_tld_stats() ); + _mi_stats_merge_thread( _mi_thread_tld() ); } void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { @@ -519,7 +525,7 @@ static bool mi_heap_buf_expand(mi_heap_buf_t* hbuf) { hbuf->buf[hbuf->size-1] = 0; } if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false; - const size_t newsize = (hbuf->size == 0 ? 2*MI_KiB : 2*hbuf->size); + const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size); char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize); if (newbuf == NULL) return false; hbuf->buf = newbuf; @@ -605,6 +611,7 @@ static void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* nam #define MI_STAT_COUNTER(stat) mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat); char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept { + mi_stats_merge(); mi_heap_buf_t hbuf = { NULL, 0, 0, true }; if (output_size > 0 && output_buf != NULL) { _mi_memzero(output_buf, output_size); diff --git a/test/test-api.c b/test/test-api.c index 20d85314..eda8ebf7 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -86,9 +86,12 @@ int main(void) { CHECK_BODY("malloc-nomem1") { result = (mi_malloc((size_t)PTRDIFF_MAX + (size_t)1) == NULL); }; - CHECK_BODY("malloc-null") { + CHECK_BODY("malloc-free-null") { mi_free(NULL); }; + CHECK_BODY("malloc-free-invalid-low") { + mi_free((void*)(MI_ZU(0x0000000003990080))); // issue #1087 + }; CHECK_BODY("calloc-overflow") { // use (size_t)&mi_calloc to get some number without triggering compiler warnings result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); diff --git a/test/test-stress.c b/test/test-stress.c index cd63b05e..a2e2b377 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -374,7 +374,8 @@ int main(int argc, char** argv) { mi_free(json); } #endif - mi_stats_print(NULL); + mi_collect(true); + mi_stats_print(NULL); #endif //bench_end_program(); return 0;