From 58d13f6a4fa7390e198a01dd4cc4a935724eb348 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 2 Mar 2025 17:06:25 -0800 Subject: [PATCH] collect every N generic calls --- include/mimalloc-stats.h | 2 +- include/mimalloc.h | 3 ++- include/mimalloc/types.h | 6 +++++- src/init.c | 2 ++ src/options.c | 1 + src/page.c | 20 ++++++++++++++++---- 6 files changed, 27 insertions(+), 7 deletions(-) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index c952af7e..44c4886f 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -100,4 +100,4 @@ mi_decl_export char* mi_stats_get_json( size_t buf_size, char* buf ) mi_attr_noe } #endif -#endif // MIMALLOC_STATS_H \ No newline at end of file +#endif // MIMALLOC_STATS_H diff --git a/include/mimalloc.h b/include/mimalloc.h index 136647e9..4e9c3156 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -310,7 +310,7 @@ mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t samp mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max); // Experimental: communicate that the thread is part of a threadpool -mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; +mi_decl_export void mi_thread_set_in_threadpool(void) mi_attr_noexcept; // Experimental: create a new heap with a specified heap tag. Set `allow_destroy` to false to allow the thread // to reclaim abandoned memory (with a compatible heap_tag and arena_id) but in that case `mi_heap_destroy` will @@ -385,6 +385,7 @@ typedef enum mi_option_e { mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_target_segments_per_thread, // experimental (=0) + mi_option_generic_collect, // collect heaps every N (=10000) generic allocation calls _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 4c912107..9f743149 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -202,7 +202,9 @@ typedef int32_t mi_ssize_t; #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) // Maximum number of size classes. (spaced exponentially in 12.5% increments) -#define MI_BIN_HUGE (73U) +#if MI_BIN_HUGE != 73U +#error "mimalloc internal: expecting 73 bins" +#endif #if (MI_LARGE_OBJ_WSIZE_MAX >= 655360) #error "mimalloc internal: define more bins" @@ -501,6 +503,8 @@ struct mi_heap_s { size_t page_count; // total number of pages in the `pages` queues. size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) size_t page_retired_max; // largest retired index into the `pages` array. + long generic_count; // how often is `_mi_malloc_generic` called? + long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting? mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom tag, can be used for separating heaps based on the object types diff --git a/src/init.c b/src/init.c index a759ac21..4ddc5bd1 100644 --- a/src/init.c +++ b/src/init.c @@ -105,6 +105,7 @@ mi_decl_hidden mi_decl_cache_align const mi_heap_t _mi_heap_empty = { { {0}, {0}, 0, true }, // random 0, // page count MI_BIN_FULL, 0, // page retired min/max + 0, 0, // generic count NULL, // next false, // can reclaim 0, // tag @@ -147,6 +148,7 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = { { {0x846ca68b}, {0}, 0, true }, // random 0, // page count MI_BIN_FULL, 0, // page retired min/max + 0, 0, // generic count NULL, // next heap false, // can reclaim 0, // tag diff --git a/src/options.c b/src/options.c index 33e2bbce..c8eb7d20 100644 --- a/src/options.c +++ b/src/options.c @@ -162,6 +162,7 @@ static mi_option_desc_t options[_mi_option_last] = UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000) { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable. + { 10000, UNINIT, MI_OPTION(generic_collect) }, // collect heaps every N (=10000) generic allocation calls }; static void mi_option_init(mi_option_desc_t* desc); diff --git a/src/page.c b/src/page.c index ea6535bd..6a693e89 100644 --- a/src/page.c +++ b/src/page.c @@ -970,11 +970,23 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al } mi_assert_internal(mi_heap_is_initialized(heap)); - // call potential deferred free routines - _mi_deferred_free(heap, false); + // do administrative tasks every N generic mallocs + if mi_unlikely(++heap->generic_count >= 100) { + heap->generic_collect_count += heap->generic_count; + heap->generic_count = 0; + // call potential deferred free routines + _mi_deferred_free(heap, false); - // free delayed frees from other threads (but skip contended ones) - _mi_heap_delayed_free_partial(heap); + // free delayed frees from other threads (but skip contended ones) + _mi_heap_delayed_free_partial(heap); + + // collect every once in a while (10000 by default) + const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); + if (heap->generic_collect_count >= generic_collect) { + heap->generic_collect_count = 0; + mi_heap_collect(heap, false /* force? */); + } + } // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(heap, size, huge_alignment);