From c84d996e884412b1fa58fa48ee6fc6e2fa841446 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 5 May 2025 10:23:52 -0700 Subject: [PATCH 01/10] fix TLS initialization for MI_WIN_USE_FIXED_TLS with redirection --- include/mimalloc/prim.h | 2 +- src/prim/windows/prim.c | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 60af4d59..d3157949 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -119,7 +119,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); - +#define MI_WIN_USE_FIXED_TLS 1 //------------------------------------------------------------------- // Access to TLS (thread local storage) slots. diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index d0fee4c2..535d34a6 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -628,18 +628,16 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { //---------------------------------------------------------------- #if MI_WIN_USE_FIXED_TLS==1 -mi_decl_cache_align size_t _mi_win_tls_offset = sizeof(void*); // use 2nd slot by default +mi_decl_cache_align size_t _mi_win_tls_offset = 0; #endif -static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { - MI_UNUSED(reserved); - MI_UNUSED(module); +static void mi_win_tls_init(DWORD reason) { #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation #if MI_WIN_USE_FIXED_TLS==1 - if (reason==DLL_PROCESS_ATTACH) { - const DWORD tls_slot = TlsAlloc(); - if (tls_slot == TLS_OUT_OF_INDEXES) { - _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); + if (reason==DLL_PROCESS_ATTACH && _mi_win_tls_offset == 0) { + const DWORD tls_slot = TlsAlloc(); // usually returns slot 1 + if (tls_slot == TLS_OUT_OF_INDEXES) { + _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); } _mi_win_tls_offset = (size_t)tls_slot * sizeof(void*); } @@ -653,7 +651,15 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { mi_assert_internal(p == (void*)&_mi_heap_empty); #endif } + #else + MI_UNUSED(reason); #endif +} + +static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { + MI_UNUSED(reserved); + MI_UNUSED(module); + mi_win_tls_init(reason); if (reason==DLL_PROCESS_ATTACH) { _mi_process_load(); } @@ -815,11 +821,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #endif mi_decl_export void _mi_redirect_entry(DWORD reason) { // called on redirection; careful as this may be called before DllMain - #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation - if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) { - _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); - } - #endif + mi_win_tls_init(reason); if (reason == DLL_PROCESS_ATTACH) { mi_redirected = true; } From 0184a86eaf4cf0018d544e5992b86f5ede688601 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:32:11 -0700 Subject: [PATCH 02/10] add alpine x86 docker file --- contrib/docker/alpine-arm32v7/Dockerfile | 2 +- contrib/docker/alpine-x86/Dockerfile | 28 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 contrib/docker/alpine-x86/Dockerfile diff --git a/contrib/docker/alpine-arm32v7/Dockerfile b/contrib/docker/alpine-arm32v7/Dockerfile index f74934fb..daa60f50 100644 --- a/contrib/docker/alpine-arm32v7/Dockerfile +++ b/contrib/docker/alpine-arm32v7/Dockerfile @@ -1,6 +1,6 @@ # install from an image # download first an appropriate tar.gz image into the current directory -# from: +# from FROM scratch # Substitute the image name that was downloaded diff --git a/contrib/docker/alpine-x86/Dockerfile b/contrib/docker/alpine-x86/Dockerfile new file mode 100644 index 00000000..a0f76c17 --- /dev/null +++ b/contrib/docker/alpine-x86/Dockerfile @@ -0,0 +1,28 @@ +# install from an image +# download first an appropriate tar.gz image into the current directory +# from +FROM scratch + +# Substitute the image name that was downloaded +ADD alpine-minirootfs-20250108-x86.tar.gz / + +# Install tools +RUN apk add build-base make cmake +RUN apk add git +RUN apk add vim + +RUN mkdir -p /home/dev +WORKDIR /home/dev + +# Get mimalloc +RUN git clone https://github.com/microsoft/mimalloc -b dev2 +RUN mkdir -p mimalloc/out/release +RUN mkdir -p mimalloc/out/debug + +# Build mimalloc debug +WORKDIR /home/dev/mimalloc/out/debug +RUN cmake ../.. -DMI_DEBUG_FULL=ON +# RUN make -j +# RUN make test + +CMD ["/bin/sh"] From 341149391fee496790a7fa916b1fd3fdd0cce1a1 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:33:29 -0700 Subject: [PATCH 03/10] fix include of prctl.h on alpine linux x86 --- src/prim/unix/prim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index f3ccb013..a90fa659 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -32,7 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__linux__) #include #include // THP disable, PR_SET_VMA - #if !defined(PR_SET_VMA) + #if defined(__GLIBC__) && !defined(PR_SET_VMA) #include #endif #if defined(__GLIBC__) From a6ecb5c299e65eb7dd6602b97235126acc01a868 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:35:29 -0700 Subject: [PATCH 04/10] fix format specifier (for alpine linux x86, issue #1086) --- src/arena.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/arena.c b/src/arena.c index bdae8da1..aa01ffcb 100644 --- a/src/arena.c +++ b/src/arena.c @@ -44,7 +44,7 @@ typedef struct mi_arena_s { mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited _Atomic(size_t) search_idx; // optimization to start the search for free blocks _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be purged from `blocks_purge`. - + mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) @@ -365,7 +365,7 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t *arena_id) { if (_mi_preloading()) return false; // use OS only while pre loading - + const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count); if (arena_count > (MI_MAX_ARENAS - 4)) return false; @@ -407,7 +407,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data) if (!mi_option_is_enabled(mi_option_disallow_arena_alloc)) { // is arena allocation allowed? - if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) + if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) { void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid); if (p != NULL) return p; @@ -487,7 +487,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) // we need to ensure we do not try to reset (as that may be invalid for uncommitted memory). mi_assert_internal(already_committed < blocks); mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits)); - needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, mi_arena_block_size(already_committed)); + needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, mi_arena_block_size(already_committed)); } // clear the purged blocks @@ -556,7 +556,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) { // check pre-conditions if (arena->memid.is_pinned) return false; - + // expired yet? mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); if (!force && (expire == 0 || expire > now)) return false; @@ -611,7 +611,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) return any_purged; } -static void mi_arenas_try_purge( bool force, bool visit_all ) +static void mi_arenas_try_purge( bool force, bool visit_all ) { if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled @@ -628,7 +628,7 @@ static void mi_arenas_try_purge( bool force, bool visit_all ) mi_atomic_guard(&purge_guard) { // increase global expire: at most one purge per delay cycle - mi_atomic_storei64_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay()); + mi_atomic_storei64_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay()); size_t max_purge_count = (visit_all ? max_arena : 2); bool all_visited = true; for (size_t i = 0; i < max_arena; i++) { @@ -947,7 +947,7 @@ void mi_debug_show_arenas(void) mi_attr_noexcept { for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); if (arena == NULL) break; - _mi_message("arena %zu: %zu blocks of size %zuMiB (in %zu fields) %s\n", i, arena->block_count, MI_ARENA_BLOCK_SIZE / MI_MiB, arena->field_count, (arena->memid.is_pinned ? ", pinned" : "")); + _mi_message("arena %zu: %zu blocks of size %zuMiB (in %zu fields) %s\n", i, arena->block_count, (size_t)(MI_ARENA_BLOCK_SIZE / MI_MiB), arena->field_count, (arena->memid.is_pinned ? ", pinned" : "")); if (show_inuse) { inuse_total += mi_debug_show_bitmap(" ", "inuse blocks", arena->block_count, arena->blocks_inuse, arena->field_count); } From 72f05e2f076b3e1b160b8aaca7bc220a2532ced0 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 15:58:45 -0700 Subject: [PATCH 05/10] fix guarded sample rate of 1 (issue #1085) --- include/mimalloc/types.h | 1 - src/init.c | 17 ++++++++--------- test/main-override-static.c | 2 +- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ab697f23..e2b5d318 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -512,7 +512,6 @@ struct mi_heap_s { size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) - size_t guarded_sample_seed; // starting sample count size_t guarded_sample_count; // current sample count (counting down to 0) #endif mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. diff --git a/src/init.c b/src/init.c index 8a48ae5e..fe0acd8a 100644 --- a/src/init.c +++ b/src/init.c @@ -110,7 +110,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { false, // can reclaim 0, // tag #if MI_GUARDED - 0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) + 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY @@ -153,7 +153,7 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = { false, // can reclaim 0, // tag #if MI_GUARDED - 0, 0, 0, 0, 0, + 0, 0, 0, 0, #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY @@ -165,15 +165,14 @@ mi_stats_t _mi_stats_main = { MI_STAT_VERSION, MI_STATS_NULL }; #if MI_GUARDED mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { - heap->guarded_sample_seed = seed; - if (heap->guarded_sample_seed == 0) { - heap->guarded_sample_seed = _mi_heap_random_next(heap); - } heap->guarded_sample_rate = sample_rate; - if (heap->guarded_sample_rate >= 1) { - heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; + heap->guarded_sample_count = sample_rate; // count down samples + if (heap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_heap_random_next(heap); + } + heap->guarded_sample_count = (seed % heap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` } - heap->guarded_sample_count = heap->guarded_sample_seed; // count down samples } mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) { diff --git a/test/main-override-static.c b/test/main-override-static.c index 06d7baa5..c94b98f4 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -43,7 +43,7 @@ int main() { // corrupt_free(); // block_overflow1(); // block_overflow2(); - // test_canary_leak(); + test_canary_leak(); // test_aslr(); // invalid_free(); // test_reserved(); From 0ae310327f83abd3b354bf03b819f3595be0daf2 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 13 May 2025 16:22:08 -0700 Subject: [PATCH 06/10] fix debug assertion for windows TLS --- src/prim/windows/prim.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 535d34a6..6ab715e6 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -644,12 +644,12 @@ static void mi_win_tls_init(DWORD reason) { #endif if (reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) { if (mi_prim_get_default_heap() == NULL) { - _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); + _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); + #if MI_DEBUG && MI_WIN_USE_FIXED_TLS==1 + void* const p = TlsGetValue((DWORD)(_mi_win_tls_offset / sizeof(void*))); + mi_assert_internal(p == (void*)&_mi_heap_empty); + #endif } - #if MI_DEBUG && MI_WIN_USE_FIXED_TLS==1 - void* const p = TlsGetValue((DWORD)(_mi_win_tls_offset / sizeof(void*))); - mi_assert_internal(p == (void*)&_mi_heap_empty); - #endif } #else MI_UNUSED(reason); From df3e1916209b3783bb3d001013ce8fbba4815da6 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 13 May 2025 16:38:53 -0700 Subject: [PATCH 07/10] make windows fixed TLS opt-in --- include/mimalloc/prim.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index d3157949..fbf0cc74 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -118,9 +118,6 @@ void _mi_prim_thread_done_auto_done(void); void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); - -#define MI_WIN_USE_FIXED_TLS 1 - //------------------------------------------------------------------- // Access to TLS (thread local storage) slots. // We need fast access to both a unique thread id (in `free.c:mi_free`) and From 41cc1bfe5199fbfc4dc5e7c7ecb1453ad4e8ad7b Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 13 May 2025 17:38:16 -0700 Subject: [PATCH 08/10] add guarded TLS test for Windows fixed TLS --- ide/vs2022/mimalloc-test-stress.vcxproj | 4 +-- include/mimalloc/prim.h | 2 +- src/init.c | 1 - src/prim/windows/prim.c | 33 ++++++++++++++----------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ide/vs2022/mimalloc-test-stress.vcxproj b/ide/vs2022/mimalloc-test-stress.vcxproj index d6af71ce..128a4ff6 100644 --- a/ide/vs2022/mimalloc-test-stress.vcxproj +++ b/ide/vs2022/mimalloc-test-stress.vcxproj @@ -282,8 +282,8 @@ - - {abb5eae7-b3e6-432e-b636-333449892ea6} + + {abb5eae7-b3e6-432e-b636-333449892ea7} diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index fbf0cc74..3d8f1806 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -208,7 +208,7 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce // thread-local initialization checks in the fast path. // We allocate a user TLS slot at process initialization (see `windows/prim.c`) // and store the offset `_mi_win_tls_offset`. -#define MI_HAS_TLS_SLOT 2 // 2 = we can reliably initialize the slot (saving a test on each malloc) +#define MI_HAS_TLS_SLOT 1 // 2 = we can reliably initialize the slot (saving a test on each malloc) extern mi_decl_hidden size_t _mi_win_tls_offset; diff --git a/src/init.c b/src/init.c index fe0acd8a..ff6c5d29 100644 --- a/src/init.c +++ b/src/init.c @@ -225,7 +225,6 @@ mi_heap_t* _mi_heap_main_get(void) { return &_mi_heap_main; } - /* ----------------------------------------------------------- Sub process ----------------------------------------------------------- */ diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 6ab715e6..9ffacaa3 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -631,18 +631,23 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { mi_decl_cache_align size_t _mi_win_tls_offset = 0; #endif +//static void mi_debug_out(const char* s) { +// HANDLE h = GetStdHandle(STD_ERROR_HANDLE); +// WriteConsole(h, s, (DWORD)_mi_strlen(s), NULL, NULL); +//} + static void mi_win_tls_init(DWORD reason) { - #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation - #if MI_WIN_USE_FIXED_TLS==1 - if (reason==DLL_PROCESS_ATTACH && _mi_win_tls_offset == 0) { - const DWORD tls_slot = TlsAlloc(); // usually returns slot 1 - if (tls_slot == TLS_OUT_OF_INDEXES) { - _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); - } - _mi_win_tls_offset = (size_t)tls_slot * sizeof(void*); - } - #endif if (reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) { + #if MI_WIN_USE_FIXED_TLS==1 // we must allocate a TLS slot dynamically + if (_mi_win_tls_offset == 0 && reason=DLL_PROCESS_ATTACH) { + const DWORD tls_slot = TlsAlloc(); // usually returns slot 1 + if (tls_slot == TLS_OUT_OF_INDEXES) { + _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); + } + _mi_win_tls_offset = (size_t)tls_slot * sizeof(void*); + } + #endif + #if MI_HAS_TLS_SLOT >= 2 // we must initialize the TLS slot before any allocation if (mi_prim_get_default_heap() == NULL) { _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); #if MI_DEBUG && MI_WIN_USE_FIXED_TLS==1 @@ -650,10 +655,8 @@ static void mi_win_tls_init(DWORD reason) { mi_assert_internal(p == (void*)&_mi_heap_empty); #endif } - } - #else - MI_UNUSED(reason); - #endif + #endif + } } static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { @@ -676,7 +679,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #define MI_PRIM_HAS_PROCESS_ATTACH 1 // Windows DLL: easy to hook into process_init and thread_done - BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { + BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { mi_win_main((PVOID)inst,reason,reserved); return TRUE; } From 15c917ef1522957a260686cf027a9f294ba1c5cd Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 13 May 2025 17:45:10 -0700 Subject: [PATCH 09/10] fix syntax error --- src/prim/windows/prim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 9ffacaa3..b82918c1 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -639,7 +639,7 @@ mi_decl_cache_align size_t _mi_win_tls_offset = 0; static void mi_win_tls_init(DWORD reason) { if (reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) { #if MI_WIN_USE_FIXED_TLS==1 // we must allocate a TLS slot dynamically - if (_mi_win_tls_offset == 0 && reason=DLL_PROCESS_ATTACH) { + if (_mi_win_tls_offset == 0 && reason == DLL_PROCESS_ATTACH) { const DWORD tls_slot = TlsAlloc(); // usually returns slot 1 if (tls_slot == TLS_OUT_OF_INDEXES) { _mi_error_message(EFAULT, "unable to allocate the a TLS slot (rebuild without MI_WIN_USE_FIXED_TLS?)\n"); From 08c33768a5344e43a1ba95b88c3adcbb6a5c3498 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 21 May 2025 11:09:34 -0700 Subject: [PATCH 10/10] fix stats for pages and page_bins --- include/mimalloc/internal.h | 2 ++ src/heap.c | 4 +--- src/page-queue.c | 4 ++-- src/page.c | 5 ++--- src/segment.c | 1 + src/stats.c | 6 +++++- test/test-stress.c | 1 + 7 files changed, 14 insertions(+), 9 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index b11bd357..e277f0ff 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -217,6 +217,7 @@ void _mi_deferred_free(mi_heap_t* heap, bool force); void _mi_page_free_collect(mi_page_t* page,bool force); void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments +size_t _mi_page_bin(const mi_page_t* page); // for stats size_t _mi_bin_size(size_t bin); // for stats size_t _mi_bin(size_t size); // for stats @@ -233,6 +234,7 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* pa // "stats.c" void _mi_stats_done(mi_stats_t* stats); +void _mi_stats_merge_thread(mi_tld_t* tld); mi_msecs_t _mi_clock_now(void); mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); diff --git a/src/heap.c b/src/heap.c index 0ea9a2ff..118438b9 100644 --- a/src/heap.c +++ b/src/heap.c @@ -169,9 +169,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) _mi_arenas_collect(collect == MI_FORCE /* force purge? */); // merge statistics - if (collect <= MI_FORCE) { - mi_stats_merge(); - } + if (collect <= MI_FORCE) { _mi_stats_merge_thread(heap->tld); } } void _mi_heap_collect_abandon(mi_heap_t* heap) { diff --git a/src/page-queue.c b/src/page-queue.c index 3507505d..38b9aff4 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -136,7 +136,7 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* } #endif -static size_t mi_page_bin(const mi_page_t* page) { +size_t _mi_page_bin(const mi_page_t* page) { const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); mi_assert_internal(bin <= MI_BIN_FULL); return bin; @@ -144,7 +144,7 @@ static size_t mi_page_bin(const mi_page_t* page) { static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { mi_assert_internal(heap!=NULL); - const size_t bin = mi_page_bin(page); + const size_t bin = _mi_page_bin(page); mi_page_queue_t* pq = &heap->pages[bin]; mi_assert_internal((mi_page_block_size(page) == pq->block_size) || (mi_page_is_huge(page) && mi_page_queue_is_huge(pq)) || diff --git a/src/page.c b/src/page.c index 55150f33..89acb409 100644 --- a/src/page.c +++ b/src/page.c @@ -290,7 +290,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size mi_assert_internal(full_block_size >= block_size); mi_page_init(heap, page, full_block_size, heap->tld); mi_heap_stat_increase(heap, pages, 1); - mi_heap_stat_increase(heap, page_bins[mi_page_bin(page)], 1); + mi_heap_stat_increase(heap, page_bins[_mi_page_bin(page)], 1); if (pq != NULL) { mi_page_queue_push(heap, pq, page); } mi_assert_expensive(_mi_page_is_valid(page)); return page; @@ -443,8 +443,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_segments_tld_t* segments_tld = &heap->tld->segments; mi_page_queue_remove(pq, page); - // and free it - mi_heap_stat_decrease(heap, page_bins[mi_page_bin(page)], 1); + // and free it mi_page_set_heap(page,NULL); _mi_segment_page_free(page, force, segments_tld); } diff --git a/src/segment.c b/src/segment.c index 75f8dacb..708ddd00 100644 --- a/src/segment.c +++ b/src/segment.c @@ -718,6 +718,7 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_seg size_t inuse = page->capacity * mi_page_block_size(page); _mi_stat_decrease(&tld->stats->page_committed, inuse); _mi_stat_decrease(&tld->stats->pages, 1); + _mi_stat_decrease(&tld->stats->page_bins[_mi_page_bin(page)], 1); page->is_zero_init = false; page->segment_in_use = false; diff --git a/src/stats.c b/src/stats.c index 92bc049c..ec8b65a3 100644 --- a/src/stats.c +++ b/src/stats.c @@ -395,6 +395,10 @@ void mi_stats_merge(void) mi_attr_noexcept { mi_stats_merge_from( mi_stats_get_default() ); } +void _mi_stats_merge_thread(mi_tld_t* tld) { + mi_stats_merge_from( &tld->stats ); +} + void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done` mi_stats_merge_from(stats); } @@ -498,7 +502,7 @@ static bool mi_heap_buf_expand(mi_heap_buf_t* hbuf) { hbuf->buf[hbuf->size-1] = 0; } if (hbuf->size > SIZE_MAX/2 || !hbuf->can_realloc) return false; - const size_t newsize = (hbuf->size == 0 ? 2*MI_KiB : 2*hbuf->size); + const size_t newsize = (hbuf->size == 0 ? mi_good_size(12*MI_KiB) : 2*hbuf->size); char* const newbuf = (char*)mi_rezalloc(hbuf->buf, newsize); if (newbuf == NULL) return false; hbuf->buf = newbuf; diff --git a/test/test-stress.c b/test/test-stress.c index 9e041064..1abe56d2 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -337,6 +337,7 @@ int main(int argc, char** argv) { mi_free(json); } #endif + mi_collect(true); mi_stats_print(NULL); #endif //bench_end_program();