diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 0b515281..8354b000 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -154,6 +154,7 @@ void _mi_heap_guarded_init(mi_heap_t* heap); // os.c void _mi_os_init(void); // called from process init void* _mi_os_alloc(size_t size, mi_memid_t* memid); +void* _mi_os_zalloc(size_t size, mi_memid_t* memid); void _mi_os_free(void* p, size_t size, mi_memid_t memid); void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid); diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index c71678cc..1087d9b8 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -59,7 +59,7 @@ int _mi_prim_commit(void* addr, size_t size, bool* is_zero); // pre: needs_recommit != NULL int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); -// Reset memory. The range keeps being accessible but the content might be reset. +// Reset memory. The range keeps being accessible but the content might be reset to zero at any moment. // Returns error code or 0 on success. int _mi_prim_reset(void* addr, size_t size); diff --git a/src/arena.c b/src/arena.c index 98b6b408..e97ca885 100644 --- a/src/arena.c +++ b/src/arena.c @@ -192,14 +192,9 @@ void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) { if (p != NULL) return p; // or fall back to the OS - p = _mi_os_alloc(size, memid); + p = _mi_os_zalloc(size, memid); if (p == NULL) return NULL; - // zero the OS memory if needed - if (!memid->initially_zero) { - _mi_memzero_aligned(p, size); - memid->initially_zero = true; - } return p; } diff --git a/src/init.c b/src/init.c index 792b79b5..0924ae1a 100644 --- a/src/init.c +++ b/src/init.c @@ -317,7 +317,6 @@ static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; static mi_thread_data_t* mi_thread_data_zalloc(void) { // try to find thread metadata in the cache - bool is_zero = false; mi_thread_data_t* td = NULL; for (int i = 0; i < TD_CACHE_SIZE; i++) { td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); @@ -325,32 +324,25 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) { // found cached allocation, try use it td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); if (td != NULL) { - break; + _mi_memzero(td, offsetof(mi_thread_data_t,memid)); + return td; } } } // if that fails, allocate as meta data + mi_memid_t memid; + td = (mi_thread_data_t*)_mi_os_zalloc(sizeof(mi_thread_data_t), &memid); if (td == NULL) { - mi_memid_t memid; - td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid); + // if this fails, try once more. (issue #257) + td = (mi_thread_data_t*)_mi_os_zalloc(sizeof(mi_thread_data_t), &memid); if (td == NULL) { - // if this fails, try once more. (issue #257) - td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid); - if (td == NULL) { - // really out of memory - _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); - } - } - if (td != NULL) { - td->memid = memid; - is_zero = memid.initially_zero; + // really out of memory + _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); + return NULL; } } - - if (td != NULL && !is_zero) { - _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid)); - } + td->memid = memid; return td; } diff --git a/src/os.c b/src/os.c index faca0d23..a67c6444 100644 --- a/src/os.c +++ b/src/os.c @@ -167,6 +167,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me if (mi_memkind_is_os(memid.memkind)) { size_t csize = memid.mem.os.size; if (csize==0) { csize = _mi_os_good_alloc_size(size); } + mi_assert_internal(csize >= size); size_t commit_size = (still_committed ? csize : 0); void* base = addr; // different base? (due to alignment) @@ -326,9 +327,11 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid) { bool os_is_large = false; bool os_is_zero = false; void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero); - if (p != NULL) { - *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); - } + if (p == NULL) return NULL; + + *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); + mi_assert_internal(memid->mem.os.size >= size); + mi_assert_internal(memid->initially_committed); return p; } @@ -344,14 +347,43 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo bool os_is_zero = false; void* os_base = NULL; void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base ); - if (p != NULL) { - *memid = _mi_memid_create_os(p, size, commit, os_is_zero, os_is_large); - memid->mem.os.base = os_base; - memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned? - } + if (p == NULL) return NULL; + + *memid = _mi_memid_create_os(p, size, commit, os_is_zero, os_is_large); + memid->mem.os.base = os_base; + memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned? + + mi_assert_internal(memid->mem.os.size >= size); + mi_assert_internal(_mi_is_aligned(p,alignment)); + if (commit) { mi_assert_internal(memid->initially_committed); } + if (memid->initially_zero) { mi_assert_internal(memid->initially_committed); } return p; } + +mi_decl_nodiscard static void* mi_os_ensure_zero(void* p, size_t size, mi_memid_t* memid) { + if (p==NULL || size==0) return p; + // ensure committed + if (!memid->initially_committed) { + bool is_zero = false; + if (!_mi_os_commit(p, size, &is_zero)) { + _mi_os_free(p, size, *memid); + return NULL; + } + memid->initially_committed = true; + } + // ensure zero'd + if (memid->initially_zero) return p; + _mi_memzero_aligned(p,size); + memid->initially_zero = true; + return p; +} + +void* _mi_os_zalloc(size_t size, mi_memid_t* memid) { + void* p = _mi_os_alloc(size,memid); + return mi_os_ensure_zero(p, size, memid); +} + /* ----------------------------------------------------------- OS aligned allocation with an offset. This is used for large alignments > MI_BLOCK_ALIGNMENT_MAX. We use a large mimalloc diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index c1fa9f7c..650aa657 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -430,6 +430,7 @@ int _mi_prim_commit(void* start, size_t size, bool* is_zero) { } int _mi_prim_reuse(void* start, size_t size) { + MI_UNUSED(start); MI_UNUSED(size); #if defined(__APPLE__) && defined(MADV_FREE_REUSE) return unix_madvise(start, size, MADV_FREE_REUSE); #endif @@ -438,18 +439,17 @@ int _mi_prim_reuse(void* start, size_t size) { int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int err = 0; - #if !MI_DEBUG && MI_SECURE<=2 - *needs_recommit = false; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) err = unix_madvise(start, size, MADV_FREE_REUSABLE); - #else + if (err) { err = unix_madvise(start, size, MADV_DONTNEED); } + #else // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) err = unix_madvise(start, size, MADV_DONTNEED); - #endif + #endif + #if !MI_DEBUG && MI_SECURE<=2 + *needs_recommit = false; #else - // note: don't use MADV_FREE_REUSABLE as the range may contain protected areas - err = unix_madvise(start, size, MADV_DONTNEED); *needs_recommit = true; mprotect(start, size, PROT_NONE); #endif @@ -465,10 +465,11 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int _mi_prim_reset(void* start, size_t size) { int err = 0; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // on macOS we try to use MADV_FREE_REUSABLE as it seems the fastest + + // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower) + #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) err = unix_madvise(start, size, MADV_FREE_REUSABLE); - if (err == 0) return 0; + if (err==0) return 0; // fall through #endif diff --git a/src/segment-map.c b/src/segment-map.c index 2f68f8c4..bbcea28a 100644 --- a/src/segment-map.c +++ b/src/segment-map.c @@ -61,7 +61,7 @@ static mi_segmap_part_t* mi_segment_map_index_of(const mi_segment_t* segment, bo if mi_unlikely(part == NULL) { if (!create_on_demand) return NULL; mi_memid_t memid; - part = (mi_segmap_part_t*)_mi_os_alloc(sizeof(mi_segmap_part_t), &memid); + part = (mi_segmap_part_t*)_mi_os_zalloc(sizeof(mi_segmap_part_t), &memid); if (part == NULL) return NULL; part->memid = memid; mi_segmap_part_t* expected = NULL;