From 189b0ac416b94848bc0a7702f7e97fe6ebd91829 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 13 Jun 2025 09:29:18 -0700 Subject: [PATCH 1/5] count commit stats only if the commit succeeded --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 279b60fb..a2d58482 100644 --- a/src/os.c +++ b/src/os.c @@ -463,7 +463,6 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) { if (is_zero != NULL) { *is_zero = false; } - mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit mi_os_stat_counter_increase(commit_calls, 1); // page align range @@ -487,6 +486,7 @@ bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) if (os_is_zero) { mi_track_mem_defined(start,csize); } else { mi_track_mem_undefined(start,csize); } #endif + mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit return true; } From 3243a2b105c6859ed4de9c0e95b14012cd809a2b Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 10 Jun 2025 16:11:56 -0700 Subject: [PATCH 2/5] add mi_bitmap_popcountN --- src/bitmap.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/bitmap.h | 3 +++ 2 files changed, 54 insertions(+) diff --git a/src/bitmap.c b/src/bitmap.c index c07792d0..66ebc157 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -256,6 +256,11 @@ static inline bool mi_bfield_atomic_is_xset_mask(mi_xset_t set, const _Atomic(mi else return mi_bfield_atomic_is_clear_mask(b, mask); } +// Count bits in a mask +static inline size_t mi_bfield_atomic_popcount_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask) { + const mi_bfield_t x = mi_atomic_load_relaxed(b); + return mi_bfield_popcount(x & mask); +} /* -------------------------------------------------------------------------------- @@ -366,6 +371,38 @@ static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, b return mi_bchunk_xsetN_(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear); } +// Check if a sequence of `n` bits within a chunk are all set/cleared. +// This can cross bfield's +mi_decl_noinline static size_t mi_bchunk_popcountN_(mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) { + mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS); + size_t count = 0; + while (n > 0) { + size_t m = MI_BFIELD_BITS - idx; // m is the bits to xset in this field + if (m > n) { m = n; } + mi_assert_internal(idx + m <= MI_BFIELD_BITS); + mi_assert_internal(field_idx < MI_BCHUNK_FIELDS); + const size_t mask = mi_bfield_mask(m, idx); + count += mi_bfield_atomic_popcount_mask(&chunk->bfields[field_idx], mask); + // next field + field_idx++; + idx = 0; + n -= m; + } + return count; +} + +// Count set bits a sequence of `n` bits. +static inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) { + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); + mi_assert_internal(n>0); + if (n==0) return 0; + const size_t i = cidx / MI_BFIELD_BITS; + const size_t idx = cidx % MI_BFIELD_BITS; + if (n==1) { return (mi_bfield_atomic_is_set(&chunk->bfields[i], idx) ? 1 : 0); } + if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_popcount_mask(&chunk->bfields[i], mi_bfield_mask(n, idx)); } + return mi_bchunk_popcountN_(chunk, i, idx, n); +} + // ------- mi_bchunk_is_xset --------------------------------------- @@ -1110,6 +1147,20 @@ bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { return were_allset; } +// Count bits set in a range of `n` bits. +// `n` cannot cross chunk boundaries (and `n <= MI_BCHUNK_BITS`)! +size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) { + mi_assert_internal(n>0); + mi_assert_internal(n<=MI_BCHUNK_BITS); + + const size_t chunk_idx = idx / MI_BCHUNK_BITS; + const size_t cidx = idx % MI_BCHUNK_BITS; + mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); // don't cross chunks (for now) + mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); + if (cidx + n > MI_BCHUNK_BITS) { n = MI_BCHUNK_BITS - cidx; } // paranoia + return mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, n); +} + // Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) { diff --git a/src/bitmap.h b/src/bitmap.h index 45ae8fe5..b9c9dfb4 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -219,6 +219,9 @@ bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_a bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); +// Count all set bits in given range in the bitmap. (cannot cross chunks) +size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n); + /* ---------------------------------------------------------------------------- Binned concurrent bitmap Assigns a size class to each chunk such that small blocks don't cause too From b10c0f87ee185b8e2b4836d7e53b715ba48ebe24 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 10 Jun 2025 16:13:39 -0700 Subject: [PATCH 3/5] fix state of commit bits on commit failure --- src/arena.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/arena.c b/src/arena.c index 61903c5d..b26f4442 100644 --- a/src/arena.c +++ b/src/arena.c @@ -188,25 +188,26 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( } // set commit state - if (commit) { - memid->initially_committed = true; - + if (commit) { // commit requested, but the range may not be committed as a whole: ensure it is committed now - if (!mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)) { - // not fully committed: commit the full range and set the commit bits - // we set the bits first since we own these slices (they are no longer free) - size_t already_committed_count = 0; - mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count); - // now actually commit + const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count); + if (already_committed < slice_count) { + // not all committed, try to commit now bool commit_zero = false; - if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed_count))) { - // if the commit fails, we roll back - _mi_arenas_free( p, mi_size_of_slices(slice_count), *memid); // this will uncommit as well + if (!_mi_os_commit_ex(p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) { + // if the commit fails, release ownership, and return NULL; + // note: this does not roll back dirty bits but that is ok. + mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); return NULL; } + if (commit_zero) { + memid->initially_zero = true; + } + + // set the commit bits + mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL); // committed - if (commit_zero) { memid->initially_zero = true; } #if MI_DEBUG > 1 if (memid->initially_zero) { if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) { @@ -225,6 +226,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices)); } } + + mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + memid->initially_committed = true; + // tool support if (memid->initially_zero) { mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE); From dfa50c37d951128b1e77167dd9291081aa88eea4 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 13 Jun 2025 21:45:06 -0700 Subject: [PATCH 4/5] nicefy mi_popcount --- include/mimalloc/bits.h | 2 +- src/libc.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index e3038d11..1d4063bb 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -209,7 +209,7 @@ static inline size_t mi_popcount(size_t x) { return mi_builtinz(popcount)(x); #else #define MI_HAS_FAST_POPCOUNT 0 - return (x<=1 ? x : _mi_popcount_generic(x)); + return _mi_popcount_generic(x); #endif } diff --git a/src/libc.c b/src/libc.c index a54eec5b..b40d1204 100644 --- a/src/libc.c +++ b/src/libc.c @@ -383,6 +383,8 @@ static size_t mi_popcount_generic32(uint32_t x) { } mi_decl_noinline size_t _mi_popcount_generic(size_t x) { + if (x<=1) return x; + if (~x==0) return MI_SIZE_BITS; return mi_popcount_generic32(x); } @@ -407,6 +409,8 @@ static size_t mi_popcount_generic64(uint64_t x) { } mi_decl_noinline size_t _mi_popcount_generic(size_t x) { + if (x<=1) return x; + if (~x==0) return MI_SIZE_BITS; return mi_popcount_generic64(x); } #endif From dc84ea7866b14414b12f077df26c77e8e2d9a701 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 13 Jun 2025 22:11:38 -0700 Subject: [PATCH 5/5] bump version to v3.1.6 for further development --- cmake/mimalloc-config-version.cmake | 2 +- contrib/vcpkg/portfile.cmake | 8 ++++---- contrib/vcpkg/vcpkg.json | 2 +- include/mimalloc.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 97756d79..d88fe047 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 3) set(mi_version_minor 1) -set(mi_version_patch 5) +set(mi_version_patch 6) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/contrib/vcpkg/portfile.cmake b/contrib/vcpkg/portfile.cmake index a3be8101..e6df0f58 100644 --- a/contrib/vcpkg/portfile.cmake +++ b/contrib/vcpkg/portfile.cmake @@ -3,13 +3,13 @@ vcpkg_from_github( REPO microsoft/mimalloc HEAD_REF master - # The "REF" can be a commit hash, branch name (dev2), or a version (v2.2.1). + # The `REF` can be a commit hash, branch name (`dev3`), or a version (`v3.1.4`). + # Use `"v${VERSION}"` to use the version specified in `vcpkg.json`. REF "v${VERSION}" - # REF 6a89f8554eaab8d8d00e17b5b09f79e1d8dbf61b - + # The sha512 is the hash of the tar.gz bundle. # (To get the sha512, run `vcpkg install "mimalloc[override]" --overlay-ports=./contrib/vcpkg` and copy the sha from the error message.) - SHA512 1e1268292283eb3c07a1f4de1bb919ebfeb18cbd4af0ca01d197d4dc46972ea127527e95e18f73381bda03593bcca5d3e4744c3ab49876c68567abebff030690 + SHA512 616351e549707318c1f8b164251141684a73d5bf8205b905736f48ab21fbb19bfaa4d52c4e63642fcb144345b6a5331944b6c8e0827925000553e46f2c2c31e9 ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS diff --git a/contrib/vcpkg/vcpkg.json b/contrib/vcpkg/vcpkg.json index 65d57d2e..3b779d00 100644 --- a/contrib/vcpkg/vcpkg.json +++ b/contrib/vcpkg/vcpkg.json @@ -1,6 +1,6 @@ { "name": "mimalloc", - "version": "3.1.4", + "version": "3.1.5", "port-version": 2, "description": "Compact general purpose allocator with excellent performance", "homepage": "https://github.com/microsoft/mimalloc", diff --git a/include/mimalloc.h b/include/mimalloc.h index aa4222f6..f13862b1 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 315 // major + 2 digits minor +#define MI_MALLOC_VERSION 316 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes