From c84d996e884412b1fa58fa48ee6fc6e2fa841446 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 5 May 2025 10:23:52 -0700 Subject: [PATCH 1/9] fix TLS initialization for MI_WIN_USE_FIXED_TLS with redirection --- include/mimalloc/prim.h | 2 +- src/prim/windows/prim.c | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 60af4d59..d3157949 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -119,7 +119,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); - +#define MI_WIN_USE_FIXED_TLS 1 //------------------------------------------------------------------- // Access to TLS (thread local storage) slots. diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index d0fee4c2..535d34a6 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -628,18 +628,16 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { //---------------------------------------------------------------- #if MI_WIN_USE_FIXED_TLS==1 -mi_decl_cache_align size_t _mi_win_tls_offset = sizeof(void*); // use 2nd slot by default +mi_decl_cache_align size_t _mi_win_tls_offset = 0; #endif -static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { - MI_UNUSED(reserved); - MI_UNUSED(module); +static void mi_win_tls_init(DWORD reason) { #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation #if MI_WIN_USE_FIXED_TLS==1 - if (reason==DLL_PROCESS_ATTACH) { - const DWORD tls_slot = TlsAlloc(); - if (tls_slot == TLS_OUT_OF_INDEXES) { - _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); + if (reason==DLL_PROCESS_ATTACH && _mi_win_tls_offset == 0) { + const DWORD tls_slot = TlsAlloc(); // usually returns slot 1 + if (tls_slot == TLS_OUT_OF_INDEXES) { + _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); } _mi_win_tls_offset = (size_t)tls_slot * sizeof(void*); } @@ -653,7 +651,15 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { mi_assert_internal(p == (void*)&_mi_heap_empty); #endif } + #else + MI_UNUSED(reason); #endif +} + +static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { + MI_UNUSED(reserved); + MI_UNUSED(module); + mi_win_tls_init(reason); if (reason==DLL_PROCESS_ATTACH) { _mi_process_load(); } @@ -815,11 +821,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #endif mi_decl_export void _mi_redirect_entry(DWORD reason) { // called on redirection; careful as this may be called before DllMain - #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation - if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) { - _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); - } - #endif + mi_win_tls_init(reason); if (reason == DLL_PROCESS_ATTACH) { mi_redirected = true; } From 36e1cbfdbce01198eb1e0dbe9d7731dc18f5361d Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 12 May 2025 18:02:42 -0700 Subject: [PATCH 2/9] change default page_reclaim_max and change reclamation test to potentially address perf regresion in mimalloc v3 with respect to v2 -- see also https://github.com/leanprover/lean4/pull/7786 --- include/mimalloc.h | 2 +- src/free.c | 25 ++++++++++++++----------- src/options.c | 11 ++++++++--- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index c4a4eb19..ac89f89d 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -411,7 +411,7 @@ typedef enum mi_option_e { mi_option_max_vabits, // max user space virtual address bits to consider (=48) mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0) mi_option_page_commit_on_demand, // commit page memory on-demand - mi_option_page_reclaim_max, // don't reclaim pages if we already own N pages (in that size class) (=16) + mi_option_page_max_reclaim, // don't reclaim pages if we already own N pages (in that size class) (=16) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/src/free.c b/src/free.c index b892803c..5989d43d 100644 --- a/src/free.c +++ b/src/free.c @@ -148,7 +148,7 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) } mi_page_t* page = _mi_safe_ptr_page(p); if (p != NULL && page == NULL) { - _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); + _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } return page; #else @@ -163,7 +163,7 @@ void mi_free(void* p) mi_attr_noexcept mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); if mi_unlikely(page==NULL) return; // page will be NULL if p==NULL mi_assert_internal(p!=NULL && page!=NULL); - + const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page)); if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page @@ -219,7 +219,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // 1. free if the page is free now (this is updated by `_mi_page_free_collect_partly`) if (mi_page_all_free(page)) { - // first remove it from the abandoned pages in the arena (if mapped, this waits for any readers to finish) + // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish) _mi_arenas_page_unabandon(page); // we can free the page directly _mi_arenas_page_free(page); @@ -243,16 +243,19 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* } } // can we reclaim into this heap? - if (heap != NULL && heap->allow_page_reclaim) { - const long reclaim_max = _mi_option_get_fast(mi_option_page_reclaim_max); - if ((heap == page->heap && mi_page_queue_len_is_atmost(heap, page->block_size, reclaim_max)) || // only reclaim if we were the originating heap, and we have at most N pages already - (reclaim_on_free == 1 && // OR if the reclaim across heaps is allowed - !mi_page_is_used_at_frac(page, 8) && // and the page is not too full - !heap->tld->is_in_threadpool && // and not part of a threadpool - _mi_arena_memid_is_suitable(page->memid, heap->exclusive_arena)) // and the memory is suitable + const long max_reclaim = _mi_option_get_fast(mi_option_page_max_reclaim); + if (heap != NULL && heap->allow_page_reclaim && + (max_reclaim < 0 || mi_page_queue_len_is_atmost(heap, page->block_size, max_reclaim))) // we have less than N pages already + { + if ((heap == page->heap) // always reclaim if we were the originating heap, + || // OR: + (reclaim_on_free == 1 && // reclaim across heaps is allowed + !mi_page_is_used_at_frac(page,8) && // and the page is not too full + !heap->tld->is_in_threadpool && // and not part of a threadpool + _mi_arena_memid_is_suitable(page->memid, heap->exclusive_arena)) // and the memory is suitable ) { - // first remove it from the abandoned pages in the arena -- this waits for any readers to finish + // first remove it from the abandoned pages in the arena -- this might wait for any readers to finish _mi_arenas_page_unabandon(page); _mi_heap_page_reclaim(heap, page); mi_heap_stat_counter_increase(heap, pages_reclaim_on_free, 1); diff --git a/src/options.c b/src/options.c index f1d16d6b..b618d195 100644 --- a/src/options.c +++ b/src/options.c @@ -98,7 +98,11 @@ int mi_version(void) mi_attr_noexcept { #endif #endif -// Static options +#ifndef MI_DEFAULT_PAGE_MAX_RECLAIM +#define MI_DEFAULT_PAGE_MAX_RECLAIM 4096 +#endif + +// Static options static mi_option_desc_t mi_options[_mi_option_last] = { // stable options @@ -157,14 +161,15 @@ static mi_option_desc_t mi_options[_mi_option_last] = MI_OPTION_UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000) { 0, MI_OPTION_UNINIT, MI_OPTION(guarded_sample_seed)}, { 10000, MI_OPTION_UNINIT, MI_OPTION(generic_collect) }, // collect heaps every N (=10000) generic allocation calls - { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned pages on a free: -1 = disable completely, 0 = only reclaim into the originating heap, 1 = reclaim on free across heaps + { 0, MI_OPTION_UNINIT, MI_OPTION_LEGACY(page_reclaim_on_free, abandoned_reclaim_on_free) },// reclaim abandoned (small) pages on a free: -1 = disable completely, 0 = only reclaim into the originating heap, 1 = reclaim on free across heaps { 2, MI_OPTION_UNINIT, MI_OPTION(page_full_retain) }, // number of (small) pages to retain in the free page queues { 4, MI_OPTION_UNINIT, MI_OPTION(page_max_candidates) }, // max search to find a best page candidate { 0, MI_OPTION_UNINIT, MI_OPTION(max_vabits) }, // max virtual address space bits { MI_DEFAULT_PAGEMAP_COMMIT, MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront? { 0, MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux)) - { 16, MI_OPTION_UNINIT, MI_OPTION(page_reclaim_max) }, // don't reclaim pages if we already own N pages (in that size class) + { MI_DEFAULT_PAGE_MAX_RECLAIM, + MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages if we already own N pages in that size class }; static void mi_option_init(mi_option_desc_t* desc); From 3ef6784455479a665f1c8bfe11038237b757bc98 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 12 May 2025 22:05:15 -0700 Subject: [PATCH 3/9] destroy the page map as well if MIMALLOC_DESTROY_ON_EXIT is set; see issue #1041 --- include/mimalloc/bits.h | 1 + include/mimalloc/internal.h | 1 + src/free.c | 2 +- src/init.c | 1 + src/page-map.c | 42 +++++++++++++++++++++++++++++++++---- src/page.c | 1 + 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 0c62cd3b..371cb7ce 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -120,6 +120,7 @@ typedef int32_t mi_ssize_t; #define MI_MAX_VABITS (32) #endif + // use a flat page-map (or a 2-level one) #ifndef MI_PAGE_MAP_FLAT #if MI_MAX_VABITS <= 40 && !defined(__APPLE__) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index a5ab8162..9c5eb362 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -200,6 +200,7 @@ void _mi_page_map_register(mi_page_t* page); void _mi_page_map_unregister(mi_page_t* page); void _mi_page_map_unregister_range(void* start, size_t size); mi_page_t* _mi_safe_ptr_page(const void* p); +void _mi_page_map_unsafe_destroy(void); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; diff --git a/src/free.c b/src/free.c index 5989d43d..45ec5683 100644 --- a/src/free.c +++ b/src/free.c @@ -247,7 +247,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* if (heap != NULL && heap->allow_page_reclaim && (max_reclaim < 0 || mi_page_queue_len_is_atmost(heap, page->block_size, max_reclaim))) // we have less than N pages already { - if ((heap == page->heap) // always reclaim if we were the originating heap, + if ((heap == page->heap) // always reclaim if we were the originating heap (todo: maybe not if in a threadpool?) || // OR: (reclaim_on_free == 1 && // reclaim across heaps is allowed !mi_page_is_used_at_frac(page,8) && // and the page is not too full diff --git a/src/init.c b/src/init.c index 892f4988..c9c61dbb 100644 --- a/src/init.c +++ b/src/init.c @@ -780,6 +780,7 @@ void mi_cdecl _mi_process_done(void) { mi_heap_collect(heap, true /* force */); _mi_heap_unsafe_destroy_all(heap); // forcefully release all memory held by all heaps (of this thread only!) _mi_arenas_unsafe_destroy_all(heap->tld); + _mi_page_map_unsafe_destroy(); } if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { diff --git a/src/page-map.c b/src/page-map.c index c286d87e..0a6a2e36 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -71,6 +71,17 @@ bool _mi_page_map_init(void) { return true; } +void _mi_page_map_unsafe_destroy(void) { + mi_assert_internal(_mi_page_map != NULL); + if (_mi_page_map == NULL) return; + _mi_os_free(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, mi_page_map_memid); + _mi_page_map = NULL; + mi_page_map_commit = NULL; + mi_page_map_max_address = NULL; + mi_page_map_memid = _mi_memid_none(); +} + + static void mi_page_map_ensure_committed(size_t idx, size_t slice_count) { // is the page map area that contains the page address committed? // we always set the commit bits so we can track what ranges are in-use. @@ -163,11 +174,12 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att #define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) mi_decl_cache_align mi_page_t*** _mi_page_map; +static size_t mi_page_map_count; static void* mi_page_map_max_address; static mi_memid_t mi_page_map_memid; - static _Atomic(mi_bfield_t) mi_page_map_commit; +static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx); static mi_page_t** mi_page_map_ensure_committed(size_t idx); static mi_page_t** mi_page_map_ensure_at(size_t idx); static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count); @@ -184,10 +196,10 @@ bool _mi_page_map_init(void) { // Allocate the page map and commit bits mi_assert(MI_MAX_VABITS >= vbits); mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits)); - const size_t page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); - mi_assert(page_map_count <= MI_PAGE_MAP_COUNT); + mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT); const size_t os_page_size = _mi_os_page_size(); - const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size); + const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size); const size_t reserve_size = page_map_size + os_page_size; const bool commit = page_map_size <= 64*MI_KiB || mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); @@ -214,6 +226,28 @@ bool _mi_page_map_init(void) { return true; } +void _mi_page_map_unsafe_destroy(void) { + mi_assert_internal(_mi_page_map != NULL); + if (_mi_page_map == NULL) return; + for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 + // free all sub-maps + if (mi_page_map_is_committed(idx, NULL)) { + mi_page_t** sub = _mi_page_map[idx]; + if (sub != NULL) { + mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*), true, false, false); + _mi_os_free(memid.mem.os.base, memid.mem.os.size, memid); + _mi_page_map[idx] = NULL; + } + } + } + _mi_os_free(_mi_page_map, mi_page_map_memid.mem.os.size, mi_page_map_memid); + _mi_page_map = NULL; + mi_page_map_count = 0; + mi_page_map_memid = _mi_memid_none(); + mi_page_map_max_address = NULL; + mi_atomic_store_release(&mi_page_map_commit, 0); +} + #define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS) diff --git a/src/page.c b/src/page.c index 4b66841b..0d8e4e12 100644 --- a/src/page.c +++ b/src/page.c @@ -396,6 +396,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { // and free it mi_heap_t* heap = page->heap; mi_heap_stat_decrease(heap, page_bins[mi_page_bin(page)], 1); + mi_heap_stat_decrease(heap, pages, 1); mi_page_set_heap(page,NULL); _mi_arenas_page_free(page); _mi_arenas_collect(false, false, heap->tld); // allow purging From a92f86dc73d2a7ee7c51b7235bf65bef16bc8466 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 14:12:22 -0700 Subject: [PATCH 4/9] add page_cross_thread_max_reclaim option --- include/mimalloc.h | 3 ++- src/free.c | 15 ++++++++------- src/options.c | 10 ++++++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index ac89f89d..a2268850 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -411,7 +411,8 @@ typedef enum mi_option_e { mi_option_max_vabits, // max user space virtual address bits to consider (=48) mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0) mi_option_page_commit_on_demand, // commit page memory on-demand - mi_option_page_max_reclaim, // don't reclaim pages if we already own N pages (in that size class) (=16) + mi_option_page_max_reclaim, // don't reclaim pages of the same originating heap if we already own N pages (in that size class) (=-1 (unlimited)) + mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/src/free.c b/src/free.c index 45ec5683..f4f8666b 100644 --- a/src/free.c +++ b/src/free.c @@ -191,10 +191,11 @@ void mi_free(void* p) mi_attr_noexcept // Multi-threaded Free (`_mt`) // ------------------------------------------------------ static bool mi_page_unown_from_free(mi_page_t* page, mi_block_t* mt_free); -static inline bool mi_page_queue_len_is_atmost( mi_heap_t* heap, size_t block_size, size_t atmost) { +static inline bool mi_page_queue_len_is_atmost( mi_heap_t* heap, size_t block_size, long atmost) { + if (atmost < 0) return true; // unlimited mi_page_queue_t* const pq = mi_page_queue(heap,block_size); mi_assert_internal(pq!=NULL); - return (pq->count <= atmost); + return (pq->count <= (size_t)atmost); /* for(mi_page_t* p = pq->first; p!=NULL; p = p->next, atmost--) { if (atmost == 0) { return false; } @@ -242,14 +243,14 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* heap = _mi_heap_by_tag(heap, page->heap_tag); } } - // can we reclaim into this heap? - const long max_reclaim = _mi_option_get_fast(mi_option_page_max_reclaim); - if (heap != NULL && heap->allow_page_reclaim && - (max_reclaim < 0 || mi_page_queue_len_is_atmost(heap, page->block_size, max_reclaim))) // we have less than N pages already + // can we reclaim into this heap? + if (heap != NULL && heap->allow_page_reclaim) { - if ((heap == page->heap) // always reclaim if we were the originating heap (todo: maybe not if in a threadpool?) + if ((heap == page->heap && // always reclaim if we were the originating heap (todo: maybe not if in a threadpool?) + mi_page_queue_len_is_atmost(heap, page->block_size, _mi_option_get_fast(mi_option_page_max_reclaim))) || // OR: (reclaim_on_free == 1 && // reclaim across heaps is allowed + mi_page_queue_len_is_atmost(heap, page->block_size, _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim)) && !mi_page_is_used_at_frac(page,8) && // and the page is not too full !heap->tld->is_in_threadpool && // and not part of a threadpool _mi_arena_memid_is_suitable(page->memid, heap->exclusive_arena)) // and the memory is suitable diff --git a/src/options.c b/src/options.c index b618d195..5760ac5c 100644 --- a/src/options.c +++ b/src/options.c @@ -99,7 +99,11 @@ int mi_version(void) mi_attr_noexcept { #endif #ifndef MI_DEFAULT_PAGE_MAX_RECLAIM -#define MI_DEFAULT_PAGE_MAX_RECLAIM 4096 +#define MI_DEFAULT_PAGE_MAX_RECLAIM (-1) // unlimited +#endif + +#ifndef MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM +#define MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM 16 #endif // Static options @@ -169,7 +173,9 @@ static mi_option_desc_t mi_options[_mi_option_last] = MI_OPTION_UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront? { 0, MI_OPTION_UNINIT, MI_OPTION(page_commit_on_demand) }, // commit pages on-demand (2 disables this only on overcommit systems (like Linux)) { MI_DEFAULT_PAGE_MAX_RECLAIM, - MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages if we already own N pages in that size class + MI_OPTION_UNINIT, MI_OPTION(page_max_reclaim) }, // don't reclaim (small) pages of the same originating heap if we already own N pages in that size class + { MI_DEFAULT_PAGE_CROSS_THREAD_MAX_RECLAIM, + MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class }; static void mi_option_init(mi_option_desc_t* desc); From ad0764272a28c229f3fba6c4c16e0ad9643c3a23 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 14:41:40 -0700 Subject: [PATCH 5/9] potential fix for low address allocations, issue #1087 --- src/page-map.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/page-map.c b/src/page-map.c index 0a6a2e36..c8686924 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -218,10 +218,14 @@ bool _mi_page_map_init(void) { if (!mi_page_map_memid.initially_committed) { _mi_os_commit(&_mi_page_map[0], os_page_size, NULL); // commit first part of the map } - _mi_page_map[0] = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved 2 sub maps at the end already + _mi_page_map[0] = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already if (!mi_page_map_memid.initially_committed) { _mi_os_commit(_mi_page_map[0], os_page_size, NULL); // only first OS page } + if (!mi_page_map_memid.initially_zero) { // initialize first addresses with NULL + _mi_memzero_aligned(_mi_page_map[0], os_page_size); + } + mi_assert_internal(_mi_ptr_page(NULL)==NULL); return true; } @@ -271,19 +275,24 @@ static mi_page_t** mi_page_map_ensure_committed(size_t idx) { static mi_page_t** mi_page_map_ensure_at(size_t idx) { mi_page_t** sub = mi_page_map_ensure_committed(idx); - if mi_unlikely(sub == NULL) { + if mi_unlikely(sub == NULL || idx == 0 /* low addresses */) { // sub map not yet allocated, alloc now mi_memid_t memid; - sub = (mi_page_t**)_mi_os_alloc(MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*), &memid); - mi_page_t** expect = NULL; - if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, ((_Atomic(mi_page_t**)*)&_mi_page_map[idx]), &expect, sub)) { - // another thread already allocated it.. free and continue - _mi_os_free(sub, MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*), memid); - sub = expect; - mi_assert_internal(sub!=NULL); - } + mi_page_t** expect = sub; + const size_t submap_size = MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*); + sub = (mi_page_t**)_mi_os_alloc(submap_size, &memid); if (sub == NULL) { _mi_error_message(EFAULT, "internal error: unable to extend the page map\n"); + return NULL; + } + if (!memid.initially_zero) { + _mi_memzero_aligned(sub, submap_size); + } + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, ((_Atomic(mi_page_t**)*)&_mi_page_map[idx]), &expect, sub)) { + // another thread already allocated it.. free and continue + _mi_os_free(sub, submap_size, memid); + sub = expect; + mi_assert_internal(sub!=NULL); } } return sub; From 0184a86eaf4cf0018d544e5992b86f5ede688601 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:32:11 -0700 Subject: [PATCH 6/9] add alpine x86 docker file --- contrib/docker/alpine-arm32v7/Dockerfile | 2 +- contrib/docker/alpine-x86/Dockerfile | 28 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 contrib/docker/alpine-x86/Dockerfile diff --git a/contrib/docker/alpine-arm32v7/Dockerfile b/contrib/docker/alpine-arm32v7/Dockerfile index f74934fb..daa60f50 100644 --- a/contrib/docker/alpine-arm32v7/Dockerfile +++ b/contrib/docker/alpine-arm32v7/Dockerfile @@ -1,6 +1,6 @@ # install from an image # download first an appropriate tar.gz image into the current directory -# from: +# from FROM scratch # Substitute the image name that was downloaded diff --git a/contrib/docker/alpine-x86/Dockerfile b/contrib/docker/alpine-x86/Dockerfile new file mode 100644 index 00000000..a0f76c17 --- /dev/null +++ b/contrib/docker/alpine-x86/Dockerfile @@ -0,0 +1,28 @@ +# install from an image +# download first an appropriate tar.gz image into the current directory +# from +FROM scratch + +# Substitute the image name that was downloaded +ADD alpine-minirootfs-20250108-x86.tar.gz / + +# Install tools +RUN apk add build-base make cmake +RUN apk add git +RUN apk add vim + +RUN mkdir -p /home/dev +WORKDIR /home/dev + +# Get mimalloc +RUN git clone https://github.com/microsoft/mimalloc -b dev2 +RUN mkdir -p mimalloc/out/release +RUN mkdir -p mimalloc/out/debug + +# Build mimalloc debug +WORKDIR /home/dev/mimalloc/out/debug +RUN cmake ../.. -DMI_DEBUG_FULL=ON +# RUN make -j +# RUN make test + +CMD ["/bin/sh"] From 341149391fee496790a7fa916b1fd3fdd0cce1a1 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:33:29 -0700 Subject: [PATCH 7/9] fix include of prctl.h on alpine linux x86 --- src/prim/unix/prim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index f3ccb013..a90fa659 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -32,7 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__linux__) #include #include // THP disable, PR_SET_VMA - #if !defined(PR_SET_VMA) + #if defined(__GLIBC__) && !defined(PR_SET_VMA) #include #endif #if defined(__GLIBC__) From a6ecb5c299e65eb7dd6602b97235126acc01a868 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:35:29 -0700 Subject: [PATCH 8/9] fix format specifier (for alpine linux x86, issue #1086) --- src/arena.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/arena.c b/src/arena.c index bdae8da1..aa01ffcb 100644 --- a/src/arena.c +++ b/src/arena.c @@ -44,7 +44,7 @@ typedef struct mi_arena_s { mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited _Atomic(size_t) search_idx; // optimization to start the search for free blocks _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be purged from `blocks_purge`. - + mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) @@ -365,7 +365,7 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t *arena_id) { if (_mi_preloading()) return false; // use OS only while pre loading - + const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count); if (arena_count > (MI_MAX_ARENAS - 4)) return false; @@ -407,7 +407,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data) if (!mi_option_is_enabled(mi_option_disallow_arena_alloc)) { // is arena allocation allowed? - if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) + if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) { void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; @@ -487,7 +487,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) // we need to ensure we do not try to reset (as that may be invalid for uncommitted memory). mi_assert_internal(already_committed < blocks); mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits)); - needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, mi_arena_block_size(already_committed)); + needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, mi_arena_block_size(already_committed)); } // clear the purged blocks @@ -556,7 +556,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) { // check pre-conditions if (arena->memid.is_pinned) return false; - + // expired yet? mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); if (!force && (expire == 0 || expire > now)) return false; @@ -611,7 +611,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) return any_purged; } -static void mi_arenas_try_purge( bool force, bool visit_all ) +static void mi_arenas_try_purge( bool force, bool visit_all ) { if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled @@ -628,7 +628,7 @@ static void mi_arenas_try_purge( bool force, bool visit_all ) mi_atomic_guard(&purge_guard) { // increase global expire: at most one purge per delay cycle - mi_atomic_storei64_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay()); + mi_atomic_storei64_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay()); size_t max_purge_count = (visit_all ? max_arena : 2); bool all_visited = true; for (size_t i = 0; i < max_arena; i++) { @@ -947,7 +947,7 @@ void mi_debug_show_arenas(void) mi_attr_noexcept { for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); if (arena == NULL) break; - _mi_message("arena %zu: %zu blocks of size %zuMiB (in %zu fields) %s\n", i, arena->block_count, MI_ARENA_BLOCK_SIZE / MI_MiB, arena->field_count, (arena->memid.is_pinned ? ", pinned" : "")); + _mi_message("arena %zu: %zu blocks of size %zuMiB (in %zu fields) %s\n", i, arena->block_count, (size_t)(MI_ARENA_BLOCK_SIZE / MI_MiB), arena->field_count, (arena->memid.is_pinned ? ", pinned" : "")); if (show_inuse) { inuse_total += mi_debug_show_bitmap(" ", "inuse blocks", arena->block_count, arena->blocks_inuse, arena->field_count); } From 72f05e2f076b3e1b160b8aaca7bc220a2532ced0 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:58:45 -0700 Subject: [PATCH 9/9] fix guarded sample rate of 1 (issue #1085) --- include/mimalloc/types.h | 1 - src/init.c | 17 ++++++++--------- test/main-override-static.c | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ab697f23..e2b5d318 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -512,7 +512,6 @@ struct mi_heap_s { size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) - size_t guarded_sample_seed; // starting sample count size_t guarded_sample_count; // current sample count (counting down to 0) #endif mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. diff --git a/src/init.c b/src/init.c index 8a48ae5e..fe0acd8a 100644 --- a/src/init.c +++ b/src/init.c @@ -110,7 +110,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { false, // can reclaim 0, // tag #if MI_GUARDED - 0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) + 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY @@ -153,7 +153,7 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = { false, // can reclaim 0, // tag #if MI_GUARDED - 0, 0, 0, 0, 0, + 0, 0, 0, 0, #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY @@ -165,15 +165,14 @@ mi_stats_t _mi_stats_main = { MI_STAT_VERSION, MI_STATS_NULL }; #if MI_GUARDED mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { - heap->guarded_sample_seed = seed; - if (heap->guarded_sample_seed == 0) { - heap->guarded_sample_seed = _mi_heap_random_next(heap); - } heap->guarded_sample_rate = sample_rate; - if (heap->guarded_sample_rate >= 1) { - heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; + heap->guarded_sample_count = sample_rate; // count down samples + if (heap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_heap_random_next(heap); + } + heap->guarded_sample_count = (seed % heap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` } - heap->guarded_sample_count = heap->guarded_sample_seed; // count down samples } mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) { diff --git a/test/main-override-static.c b/test/main-override-static.c index 06d7baa5..c94b98f4 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -43,7 +43,7 @@ int main() { // corrupt_free(); // block_overflow1(); // block_overflow2(); - // test_canary_leak(); + test_canary_leak(); // test_aslr(); // invalid_free(); // test_reserved();