From 86550d09bcf845034a81fb46acbab42dcaf26d23 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 7 Jan 2025 13:19:44 -0800 Subject: [PATCH 1/7] set more conservative options with increased medium and small object sizes --- include/mimalloc/types.h | 8 ++++---- src/options.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index e45da9a7..613bc69c 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -337,13 +337,13 @@ typedef struct mi_page_s { #endif // The max object size are checked to not waste more than 12.5% internally over the page sizes. -#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 8 KiB +#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_INFO_SIZE)/4) // < 16 KiB #if MI_ENABLE_LARGE_PAGES -#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 64 KiB +#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_INFO_SIZE)/4) // < 128 KiB #define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/8) // <= 256 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` #else -#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/8) // <= 64 KiB -#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // <= 64 KiB // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` +#define MI_MEDIUM_MAX_OBJ_SIZE (MI_MEDIUM_PAGE_SIZE/4) // <= 128 KiB +#define MI_LARGE_MAX_OBJ_SIZE MI_MEDIUM_MAX_OBJ_SIZE // note: this must be a nice power of 2 or we get rounding issues with `_mi_bin` #endif #define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE) diff --git a/src/options.c b/src/options.c index 9fcc6ef3..3d34d9b6 100644 --- a/src/options.c +++ b/src/options.c @@ -169,8 +169,8 @@ static mi_option_desc_t options[_mi_option_last] = UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000) { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable. - { 1, UNINIT, MI_OPTION_LEGACY(reclaim_on_free, abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free - { 2, UNINIT, MI_OPTION(page_full_retain) }, + { 0, UNINIT, MI_OPTION_LEGACY(reclaim_on_free, abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free + { 0, UNINIT, MI_OPTION(page_full_retain) }, { 4, UNINIT, MI_OPTION(page_max_candidates) }, { 0, UNINIT, MI_OPTION(max_vabits) }, { MI_DEFAULT_PAGEMAP_COMMIT, From 0caf80ec3c59de23dc5865de34d321df22e40fa4 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 7 Jan 2025 21:50:55 -0800 Subject: [PATCH 2/7] default purge delay to 100ms --- src/options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 3d34d9b6..a920fdcb 100644 --- a/src/options.c +++ b/src/options.c @@ -144,7 +144,7 @@ static mi_option_desc_t options[_mi_option_last] = #else { 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) #endif - { 0, UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds + { 100, UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose From df172843d13e357f58ea0e2bf9a9c5b5f54ad070 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 4 Feb 2025 20:15:38 -0800 Subject: [PATCH 3/7] call page_free_collect less often from a page search --- src/page.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/page.c b/src/page.c index dc3a6365..4b0c810c 100644 --- a/src/page.c +++ b/src/page.c @@ -175,7 +175,7 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); do { head = mi_tf_block(tfree); - if (head == NULL) return; // return if the list is empty + if mi_likely(head == NULL) return; // return if the list is empty tfreex = mi_tf_create(NULL,mi_tf_is_owned(tfree)); // set the thread free list to NULL } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); // release is enough? mi_assert_internal(head != NULL); @@ -717,14 +717,16 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, m count++; #endif candidate_limit--; - - // collect freed blocks by us and other threads - _mi_page_free_collect(page, false); - + // search up to N pages for a best candidate // is the local free list non-empty? - const bool immediate_available = mi_page_immediate_available(page); + bool immediate_available = mi_page_immediate_available(page); + if (!immediate_available) { + // collect freed blocks by us and other threads to we get a proper use count + _mi_page_free_collect(page, false); + immediate_available = mi_page_immediate_available(page); + } // if the page is completely full, move it to the `mi_pages_full` // queue so we don't visit long-lived pages too often. @@ -742,7 +744,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, m page_candidate = page; candidate_limit = _mi_option_get_fast(mi_option_page_max_candidates); } - else if (mi_page_all_free(page_candidate)) { + else if (mi_page_all_free(page_candidate)) { _mi_page_free(page_candidate, pq); page_candidate = page; } From 89d629317f986d2ef7605ced9fa5ec011adc1594 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 10 Feb 2025 12:45:38 -0800 Subject: [PATCH 4/7] limit page_reclaim to page queues of less than 4 pages; make page_commit_on_demand 0 by default. --- src/free.c | 10 +++++++++- src/options.c | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/free.c b/src/free.c index 1df10728..9ca71499 100644 --- a/src/free.c +++ b/src/free.c @@ -202,6 +202,14 @@ void mi_free(void* p) mi_attr_noexcept // Multi-threaded Free (`_mt`) // ------------------------------------------------------ static bool mi_page_unown_from_free(mi_page_t* page, mi_block_t* mt_free); +static bool inline mi_page_queue_len_is_atmost( mi_heap_t* heap, size_t block_size, size_t atmost) { + mi_page_queue_t* const pq = mi_page_queue(heap,block_size); + mi_assert_internal(pq!=NULL); + for(mi_page_t* p = pq->first; p!=NULL; p = p->next, atmost--) { + if (atmost == 0) { return false; } + } + return true; +} static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept { mi_assert_internal(mi_page_is_owned(page)); @@ -243,7 +251,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* } // can we reclaim? if (heap != NULL && heap->allow_page_reclaim) { - if (heap == page->heap || // only reclaim if we were the originating heap, + if ((heap == page->heap && mi_page_queue_len_is_atmost(heap, page->block_size, 4)) || // only reclaim if we were the originating heap, and we have at most N pages already (reclaim_on_free == 1 && // OR if the reclaim across heaps is allowed !mi_page_is_used_at_frac(page, 8) && // and the page is not too full !heap->tld->is_in_threadpool && // and not part of a threadpool diff --git a/src/options.c b/src/options.c index a61c2dc2..d1bdd716 100644 --- a/src/options.c +++ b/src/options.c @@ -172,9 +172,9 @@ static mi_option_desc_t options[_mi_option_last] = { 2, UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues { 4, UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate { 0, UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits - { MI_DEFAULT_PAGEMAP_COMMIT, + { MI_DEFAULT_PAGEMAP_COMMIT, UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront? - { 2, UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this on overcommit systems (like Linux)) + { 0, UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this on overcommit systems (like Linux)) }; static void mi_option_init(mi_option_desc_t* desc); From 44a4c83fbfda403ae25dd436fed4adf3197a62b3 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Feb 2025 13:56:58 -0800 Subject: [PATCH 5/7] maintain count in pagequeue for constant time test in free.c --- include/mimalloc/internal.h | 1 + include/mimalloc/types.h | 1 + src/free.c | 5 ++++- src/heap.c | 4 ++++ src/init.c | 2 +- src/page-queue.c | 37 +++++++++++++++++++++++++++++++++++-- 6 files changed, 46 insertions(+), 4 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c9f69a26..b45f7565 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -213,6 +213,7 @@ void _mi_deferred_free(mi_heap_t* heap, bool force); void _mi_page_free_collect(mi_page_t* page, bool force); void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); void _mi_page_init(mi_heap_t* heap, mi_page_t* page); +bool _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq); size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 5059ecd1..a743546e 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -389,6 +389,7 @@ typedef struct mi_tld_s mi_tld_t; typedef struct mi_page_queue_s { mi_page_t* first; mi_page_t* last; + size_t count; size_t block_size; } mi_page_queue_t; diff --git a/src/free.c b/src/free.c index 9ca71499..418acd02 100644 --- a/src/free.c +++ b/src/free.c @@ -202,13 +202,16 @@ void mi_free(void* p) mi_attr_noexcept // Multi-threaded Free (`_mt`) // ------------------------------------------------------ static bool mi_page_unown_from_free(mi_page_t* page, mi_block_t* mt_free); -static bool inline mi_page_queue_len_is_atmost( mi_heap_t* heap, size_t block_size, size_t atmost) { +static inline bool mi_page_queue_len_is_atmost( mi_heap_t* heap, size_t block_size, size_t atmost) { mi_page_queue_t* const pq = mi_page_queue(heap,block_size); mi_assert_internal(pq!=NULL); + return (pq->count <= atmost); + /* for(mi_page_t* p = pq->first; p!=NULL; p = p->next, atmost--) { if (atmost == 0) { return false; } } return true; + */ } static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept { diff --git a/src/heap.c b/src/heap.c index daad8afc..116d0589 100644 --- a/src/heap.c +++ b/src/heap.c @@ -63,6 +63,9 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ static bool mi_heap_is_valid(mi_heap_t* heap) { mi_assert_internal(heap!=NULL); mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL); + for (int bin = 0; bin < MI_BIN_COUNT; bin++) { + mi_assert_internal(_mi_page_queue_is_valid(heap, &heap->pages[bin])); + } return true; } #endif @@ -106,6 +109,7 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) { if (heap==NULL || !mi_heap_is_initialized(heap)) return; + mi_assert_expensive(mi_heap_is_valid(heap)); const bool force = (collect >= MI_FORCE); _mi_deferred_free(heap, force); diff --git a/src/init.c b/src/init.c index 5bedab85..4cac1c18 100644 --- a/src/init.c +++ b/src/init.c @@ -50,7 +50,7 @@ const mi_page_t _mi_page_empty = { // Empty page queues for every bin -#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } +#define QNULL(sz) { NULL, NULL, 0, (sz)*sizeof(uintptr_t) } #define MI_PAGE_QUEUES_EMPTY \ { QNULL(1), \ QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ diff --git a/src/page-queue.c b/src/page-queue.c index 9e3aaacc..5365c0b7 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -49,6 +49,10 @@ static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { return (pq->block_size > MI_LARGE_MAX_OBJ_SIZE); } +static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { + return pq->count; +} + /* ----------------------------------------------------------- Bins ----------------------------------------------------------- */ @@ -142,6 +146,25 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* } #endif +bool _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq) { + if (pq==NULL) return false; + size_t count = 0; + mi_page_t* prev = NULL; + for (mi_page_t* page = pq->first; page != NULL; page = page->next) { + mi_assert_internal(page->prev == prev); + mi_assert_internal(mi_page_block_size(page) == pq->block_size); + mi_assert_internal(page->heap == heap); + if (page->next == NULL) { + mi_assert_internal(pq->last == page); + } + count++; + prev = page; + } + mi_assert_internal(pq->count == count); + return true; +} + + static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { mi_assert_internal(heap!=NULL); uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); @@ -211,6 +234,7 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_expensive(mi_page_queue_contains(queue, page)); + mi_assert_internal(queue->count >= 1); mi_assert_internal(mi_page_block_size(page) == queue->block_size || (mi_page_is_huge(page) && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); @@ -225,6 +249,7 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { mi_heap_queue_first_update(heap,queue); } heap->page_count--; + queue->count--; page->next = NULL; page->prev = NULL; mi_page_set_in_full(page,false); @@ -253,6 +278,7 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_ else { queue->first = queue->last = page; } + queue->count++; // update direct mi_heap_queue_first_update(heap, queue); @@ -279,6 +305,7 @@ static void mi_page_queue_push_at_end(mi_heap_t* heap, mi_page_queue_t* queue, m else { queue->first = queue->last = page; } + queue->count++; // update direct if (queue->first == page) { @@ -298,6 +325,7 @@ static void mi_page_queue_move_to_front(mi_heap_t* heap, mi_page_queue_t* queue, static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { mi_assert_internal(page != NULL); + mi_assert_internal(from->count >= 1); mi_assert_expensive(mi_page_queue_contains(from, page)); mi_assert_expensive(!mi_page_queue_contains(to, page)); const size_t bsize = mi_page_block_size(page); @@ -320,8 +348,10 @@ static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* mi_assert_internal(mi_heap_contains_queue(heap, from)); mi_heap_queue_first_update(heap, from); } + from->count--; // insert into `to` + to->count++; if (enqueue_at_end) { // enqueue at the end page->prev = to->last; @@ -378,15 +408,16 @@ static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) { mi_assert_internal(mi_heap_contains_queue(heap,pq)); mi_assert_internal(pq->block_size == append->block_size); - + if (append->first==NULL) return 0; - + // set append pages to new heap and count size_t count = 0; for (mi_page_t* page = append->first; page != NULL; page = page->next) { mi_page_set_heap(page, heap); count++; } + mi_assert_internal(count == append->count); if (pq->last==NULL) { // take over afresh @@ -403,5 +434,7 @@ size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue append->first->prev = pq->last; pq->last = append->last; } + pq->count += append->count; + return count; } From 0cbdcfac94780061af20b3c39e9f21ab41ddd400 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 11 Feb 2025 16:07:35 -0800 Subject: [PATCH 6/7] fix signed warning --- src/heap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heap.c b/src/heap.c index 57bb2f52..ac67698a 100644 --- a/src/heap.c +++ b/src/heap.c @@ -63,7 +63,7 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ static bool mi_heap_is_valid(mi_heap_t* heap) { mi_assert_internal(heap!=NULL); mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL); - for (int bin = 0; bin < MI_BIN_COUNT; bin++) { + for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { mi_assert_internal(_mi_page_queue_is_valid(heap, &heap->pages[bin])); } return true; From cd2763aa3dbea905231798cec23c1ba0eaa1f7f7 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 11 Feb 2025 16:27:25 -0800 Subject: [PATCH 7/7] fix compile warnings and assertion --- src/page-queue.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/page-queue.c b/src/page-queue.c index 1ffbbf2a..6e8b0853 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -141,12 +141,21 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* #endif bool _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq) { + MI_UNUSED_RELEASE(heap); if (pq==NULL) return false; - size_t count = 0; - mi_page_t* prev = NULL; + size_t count = 0; MI_UNUSED_RELEASE(count); + mi_page_t* prev = NULL; MI_UNUSED_RELEASE(prev); for (mi_page_t* page = pq->first; page != NULL; page = page->next) { mi_assert_internal(page->prev == prev); - mi_assert_internal(mi_page_block_size(page) == pq->block_size); + if (mi_page_is_in_full(page)) { + mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); + } + else if (mi_page_is_huge(page)) { + mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); + } + else { + mi_assert_internal(mi_page_block_size(page) == pq->block_size); + } mi_assert_internal(page->heap == heap); if (page->next == NULL) { mi_assert_internal(pq->last == page);