From 2696627aafef1afc52ead72fe27c3a2a1347b27a Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 6 Jun 2025 20:07:37 -0700 Subject: [PATCH 1/4] add MI_UNUSED for unix _mi_prim_reuse --- src/prim/unix/prim.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 9ac855a5..8452b8c2 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -434,6 +434,7 @@ int _mi_prim_commit(void* start, size_t size, bool* is_zero) { } int _mi_prim_reuse(void* start, size_t size) { + MI_UNUSED(start); MI_UNUSED(size); #if defined(__APPLE__) && defined(MADV_FREE_REUSE) return unix_madvise(start, size, MADV_FREE_REUSE); #endif From d7431402c5ef192a5d9c277abdc2fb4640abc4c1 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 6 Jun 2025 20:15:16 -0700 Subject: [PATCH 2/4] fall back to MADV_DONTNEED if MADV_FREE_REUSABLE fails on macOS; disable use of MADV_FREE_REUSE on a reset (issue #1097) --- src/prim/unix/prim.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 8452b8c2..780d254f 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -443,18 +443,17 @@ int _mi_prim_reuse(void* start, size_t size) { int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int err = 0; - #if !MI_DEBUG && MI_SECURE<=2 - *needs_recommit = false; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) err = unix_madvise(start, size, MADV_FREE_REUSABLE); - #else + if (err) { err = unix_madvise(start, size, MADV_DONTNEED); } + #else // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) err = unix_madvise(start, size, MADV_DONTNEED); - #endif + #endif + #if !MI_DEBUG && MI_SECURE<=2 + *needs_recommit = false; #else - // note: don't use MADV_FREE_REUSABLE as the range may contain protected areas - err = unix_madvise(start, size, MADV_DONTNEED); *needs_recommit = true; mprotect(start, size, PROT_NONE); #endif @@ -470,10 +469,11 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int _mi_prim_reset(void* start, size_t size) { int err = 0; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // on macOS we try to use MADV_FREE_REUSABLE as it seems the fastest + + // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower) + #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) err = unix_madvise(start, size, MADV_FREE_REUSABLE); - if (err == 0) return 0; + if (err==0) return 0; // fall through #endif From 2f0540c4f9e57c55cca4e0d621dd8b3c74843ef0 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 6 Jun 2025 20:50:50 -0700 Subject: [PATCH 3/4] add _mi_os_zalloc --- include/mimalloc/internal.h | 1 + include/mimalloc/prim.h | 2 +- src/arena.c | 7 +----- src/init.c | 28 +++++++++-------------- src/os.c | 45 ++++++++++++++++++++++++++++++------- src/segment-map.c | 2 +- 6 files changed, 51 insertions(+), 34 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 3e57e252..7250d31a 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -154,6 +154,7 @@ void _mi_heap_guarded_init(mi_heap_t* heap); // os.c void _mi_os_init(void); // called from process init void* _mi_os_alloc(size_t size, mi_memid_t* memid); +void* _mi_os_zalloc(size_t size, mi_memid_t* memid); void _mi_os_free(void* p, size_t size, mi_memid_t memid); void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid); diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index c71678cc..1087d9b8 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -59,7 +59,7 @@ int _mi_prim_commit(void* addr, size_t size, bool* is_zero); // pre: needs_recommit != NULL int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); -// Reset memory. The range keeps being accessible but the content might be reset. +// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment. // Returns error code or 0 on success. int _mi_prim_reset(void* addr, size_t size); diff --git a/src/arena.c b/src/arena.c index 25cef886..ba36c415 100644 --- a/src/arena.c +++ b/src/arena.c @@ -188,14 +188,9 @@ void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) { if (p != NULL) return p; // or fall back to the OS - p = _mi_os_alloc(size, memid); + p = _mi_os_zalloc(size, memid); if (p == NULL) return NULL; - // zero the OS memory if needed - if (!memid->initially_zero) { - _mi_memzero_aligned(p, size); - memid->initially_zero = true; - } return p; } diff --git a/src/init.c b/src/init.c index ff6c5d29..05ff688e 100644 --- a/src/init.c +++ b/src/init.c @@ -298,7 +298,6 @@ static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; static mi_thread_data_t* mi_thread_data_zalloc(void) { // try to find thread metadata in the cache - bool is_zero = false; mi_thread_data_t* td = NULL; for (int i = 0; i < TD_CACHE_SIZE; i++) { td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); @@ -306,32 +305,25 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) { // found cached allocation, try use it td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); if (td != NULL) { - break; + _mi_memzero(td, offsetof(mi_thread_data_t,memid)); + return td; } } } // if that fails, allocate as meta data + mi_memid_t memid; + td = (mi_thread_data_t*)_mi_os_zalloc(sizeof(mi_thread_data_t), &memid); if (td == NULL) { - mi_memid_t memid; - td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid); + // if this fails, try once more. (issue #257) + td = (mi_thread_data_t*)_mi_os_zalloc(sizeof(mi_thread_data_t), &memid); if (td == NULL) { - // if this fails, try once more. (issue #257) - td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid); - if (td == NULL) { - // really out of memory - _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); - } - } - if (td != NULL) { - td->memid = memid; - is_zero = memid.initially_zero; + // really out of memory + _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); + return NULL; } } - - if (td != NULL && !is_zero) { - _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid)); - } + td->memid = memid; return td; } diff --git a/src/os.c b/src/os.c index 4c99d625..3c25ff59 100644 --- a/src/os.c +++ b/src/os.c @@ -182,6 +182,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me if (mi_memkind_is_os(memid.memkind)) { size_t csize = memid.mem.os.size; if (csize==0) { csize = _mi_os_good_alloc_size(size); } + mi_assert_internal(csize >= size); size_t commit_size = (still_committed ? csize : 0); void* base = addr; // different base? (due to alignment) @@ -341,9 +342,11 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid) { bool os_is_large = false; bool os_is_zero = false; void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero); - if (p != NULL) { - *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); - } + if (p == NULL) return NULL; + + *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); + mi_assert_internal(memid->mem.os.size >= size); + mi_assert_internal(memid->initially_committed); return p; } @@ -359,14 +362,40 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo bool os_is_zero = false; void* os_base = NULL; void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base ); - if (p != NULL) { - *memid = _mi_memid_create_os(p, size, commit, os_is_zero, os_is_large); - memid->mem.os.base = os_base; - memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned? - } + if (p == NULL) return NULL; + + *memid = _mi_memid_create_os(p, size, commit, os_is_zero, os_is_large); + memid->mem.os.base = os_base; + memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned? + + mi_assert_internal(memid->mem.os.size >= size); + mi_assert_internal(_mi_is_aligned(p,alignment)); + mi_assert_internal(!commit || memid->initially_committed); + mi_assert_internal(!memid->initially_zero || memid->initially_committed); return p; } + +mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) { + if (p==NULL || size==0 || memid->initially_zero) return p; + if (!memid->initially_committed) { + bool is_zero = false; + if (!_mi_os_commit(p, size, &is_zero)) { + _mi_os_free(p, size, *memid); + return NULL; + } + memid->initially_committed = true; + } + _mi_memzero_aligned(p,size); + memid->initially_zero = true; + return p; +} + +void* _mi_os_zalloc(size_t size, mi_memid_t* memid) { + void* p = _mi_os_alloc(size,memid); + return mi_os_ensure_zero(p, size, memid); +} + /* ----------------------------------------------------------- OS aligned allocation with an offset. This is used for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc diff --git a/src/segment-map.c b/src/segment-map.c index 2f68f8c4..bbcea28a 100644 --- a/src/segment-map.c +++ b/src/segment-map.c @@ -61,7 +61,7 @@ static mi_segmap_part_t* mi_segment_map_index_of(const mi_segment_t* segment, bo if mi_unlikely(part == NULL) { if (!create_on_demand) return NULL; mi_memid_t memid; - part = (mi_segmap_part_t*)_mi_os_alloc(sizeof(mi_segmap_part_t), &memid); + part = (mi_segmap_part_t*)_mi_os_zalloc(sizeof(mi_segmap_part_t), &memid); if (part == NULL) return NULL; part->memid = memid; mi_segmap_part_t* expected = NULL; From 57830a4b254673de60900ab83031b5b8454d947a Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 6 Jun 2025 21:09:32 -0700 Subject: [PATCH 4/4] fix assertion in mi_os_ensure_zero --- src/os.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index 3c25ff59..028a5ee5 100644 --- a/src/os.c +++ b/src/os.c @@ -370,14 +370,15 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo mi_assert_internal(memid->mem.os.size >= size); mi_assert_internal(_mi_is_aligned(p,alignment)); - mi_assert_internal(!commit || memid->initially_committed); - mi_assert_internal(!memid->initially_zero || memid->initially_committed); + if (commit) { mi_assert_internal(memid->initially_committed); } + if (memid->initially_zero) { mi_assert_internal(memid->initially_committed); } return p; } mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) { - if (p==NULL || size==0 || memid->initially_zero) return p; + if (p==NULL || size==0) return p; + // ensure committed if (!memid->initially_committed) { bool is_zero = false; if (!_mi_os_commit(p, size, &is_zero)) { @@ -386,6 +387,8 @@ mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_ } memid->initially_committed = true; } + // ensure zero'd + if (memid->initially_zero) return p; _mi_memzero_aligned(p,size); memid->initially_zero = true; return p;