From 34910664f1038c22df88ec37a822a47e872806f7 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 14:46:23 -0800 Subject: [PATCH 1/8] add mi_heap_new_ and mi_heap_new_n --- include/mimalloc.h | 3 +++ src/alloc.c | 47 ++++++++++++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 1fc10b2a..7dc66f5f 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -394,6 +394,9 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, s mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_new_(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); + #ifdef __cplusplus } #endif diff --git a/src/alloc.c b/src/alloc.c index 64708d7b..5e26f689 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -904,20 +904,46 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow ) { +static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool nothrow ) { void* p = NULL; while(p == NULL && mi_try_new_handler(nothrow)) { - p = mi_malloc(size); + p = mi_heap_malloc(heap,size); } return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { - void* p = mi_malloc(size); - if mi_unlikely(p == NULL) return mi_try_new(size,false); +static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { + return mi_heap_try_new(mi_get_default_heap(), size, nothrow); +} + + +mi_decl_nodiscard mi_decl_restrict inline void* mi_heap_new_(mi_heap_t* heap, size_t size) { + void* p = mi_heap_malloc(heap,size); + if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); return p; } +mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { + return mi_heap_new_(mi_get_default_heap(), size); +} + + +mi_decl_nodiscard mi_decl_restrict inline void* mi_heap_new_n(mi_heap_t* heap, size_t count, size_t size) { + size_t total; + if mi_unlikely(mi_count_size_overflow(count, size, &total)) { + mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc + return NULL; + } + else { + return mi_heap_new_(heap,total); + } +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { + return mi_heap_new_n(mi_get_default_heap(), size, count); +} + + mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { void* p = mi_malloc(size); if mi_unlikely(p == NULL) return mi_try_new(size, true); @@ -942,17 +968,6 @@ mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, siz return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { - size_t total; - if mi_unlikely(mi_count_size_overflow(count, size, &total)) { - mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc - return NULL; - } - else { - return mi_new(total); - } -} - mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { void* q; do { From 6e2b077b3565d8f6bb67d87300245878be11ceba Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 14:48:26 -0800 Subject: [PATCH 2/8] rename to heap_alloc_new and heap_alloc_new_n --- include/mimalloc.h | 4 ++-- src/alloc.c | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 7dc66f5f..57f31336 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -394,8 +394,8 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, s mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); -mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_new_(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); #ifdef __cplusplus } diff --git a/src/alloc.c b/src/alloc.c index 5e26f689..84a9fb43 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -917,30 +917,30 @@ static mi_decl_noinline void* mi_try_new(size_t size, bool nothrow) { } -mi_decl_nodiscard mi_decl_restrict inline void* mi_heap_new_(mi_heap_t* heap, size_t size) { +mi_decl_nodiscard mi_decl_restrict inline void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) { void* p = mi_heap_malloc(heap,size); if mi_unlikely(p == NULL) return mi_heap_try_new(heap, size, false); return p; } mi_decl_nodiscard mi_decl_restrict void* mi_new(size_t size) { - return mi_heap_new_(mi_get_default_heap(), size); + return mi_heap_alloc_new(mi_get_default_heap(), size); } -mi_decl_nodiscard mi_decl_restrict inline void* mi_heap_new_n(mi_heap_t* heap, size_t count, size_t size) { +mi_decl_nodiscard mi_decl_restrict inline void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc return NULL; } else { - return mi_heap_new_(heap,total); + return mi_heap_alloc_new(heap,total); } } mi_decl_nodiscard mi_decl_restrict void* mi_new_n(size_t count, size_t size) { - return mi_heap_new_n(mi_get_default_heap(), size, count); + return mi_heap_alloc_new_n(mi_get_default_heap(), size, count); } From 9617f16df95095de18564c5a8d3c421c3557d09b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 16:58:32 -0800 Subject: [PATCH 3/8] add STL allocators that use a specific heap and can destroy at the end; see original PR #625 by @vmarkovtsev --- include/mimalloc.h | 118 +++++++++++++++++++++++++++++++++++++++++ src/alloc.c | 5 +- test/main-override.cpp | 41 ++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 57f31336..d43a4419 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -456,6 +456,124 @@ template struct mi_stl_allocator { template bool operator==(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return true; } template bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; } + +#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 +#include // std::shared_ptr + +// STL allocator allocation in a specific heap +template struct mi_heap_stl_allocator { + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type& reference; + typedef value_type const& const_reference; + typedef value_type* pointer; + typedef value_type const* const_pointer; + template struct rebind { typedef mi_heap_stl_allocator other; }; + + mi_heap_stl_allocator() { + mi_heap_t* hp = mi_heap_new(); + this->heap.reset(hp, heap_delete); + } + mi_heap_stl_allocator(mi_heap_t* hp) : heap(hp) { } /* will not delete or destroy the passed in heap */ + mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : heap(x.heap) { } + template mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : heap(x.heap) { } + mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { mi_free(p); } + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + using is_always_equal = std::false_type; + template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); } + template void destroy(U* p) mi_attr_noexcept { p->~U(); } + #else + void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } + void destroy(pointer p) { p->~value_type(); } + #endif + + size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX / sizeof(value_type)); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } + + void collect(bool force) { mi_heap_collect(this->heap.get(), force); } + +protected: + std::shared_ptr heap; + +private: + static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } +}; + +template bool operator==(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (x.heap == y.heap); } +template bool operator!=(const mi_heap_stl_allocator& x, const mi_heap_stl_allocator& y) mi_attr_noexcept { return (x.heap != y.heap); } + + +// STL allocator allocation in a specific heap, where `free` does nothing and +// the heap is destroyed in one go on destruction -- use with care! +template struct mi_heap_destroy_stl_allocator { + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type& reference; + typedef value_type const& const_reference; + typedef value_type* pointer; + typedef value_type const* const_pointer; + template struct rebind { typedef mi_heap_destroy_stl_allocator other; }; + + mi_heap_destroy_stl_allocator() { + mi_heap_t* hp = mi_heap_new(); + this->heap.reset(hp, heap_destroy); + } + mi_heap_destroy_stl_allocator(mi_heap_t* hp) : heap(hp) { } /* will not delete or destroy the passed-in heap; nor free any allocated objects it allocates in the heap! */ + mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : heap(x.heap) { } + template mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : heap(x.heap) { } + mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } + void deallocate(T* p, size_type) { /* do nothing as we destroy the heap on destruct. */ } + + #if (__cplusplus >= 201703L) // C++17 + mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } + mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } + #else + mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } + #endif + + #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + using is_always_equal = std::false_type; + template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); } + template void destroy(U* p) mi_attr_noexcept { p->~U(); } + #else + void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } + void destroy(pointer p) { p->~value_type(); } + #endif + + size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX / sizeof(value_type)); } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } + +// protected: + std::shared_ptr heap; + +private: + static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } } +}; + +template bool operator==(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (x.heap == y.heap); } +template bool operator!=(const mi_heap_destroy_stl_allocator& x, const mi_heap_destroy_stl_allocator& y) mi_attr_noexcept { return (x.heap != y.heap); } + +#endif // C++11 + #endif // __cplusplus #endif diff --git a/src/alloc.c b/src/alloc.c index 84a9fb43..21ad3bb3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -91,7 +91,10 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { mi_assert(heap != NULL); - mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local + #if MI_DEBUG + const uintptr_t tid = _mi_thread_id(); + mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local + #endif mi_assert(size <= MI_SMALL_SIZE_MAX); #if (MI_PADDING) if (size == 0) { diff --git a/test/main-override.cpp b/test/main-override.cpp index f748c75a..f5cb3668 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -36,6 +36,8 @@ static void fail_aslr(); // issue #372 static void tsan_numa_test(); // issue #414 static void strdup_test(); // issue #445 +static void test_stl_allocators(); + int main() { mi_stats_reset(); // ignore earlier allocations heap_thread_free_large(); @@ -46,6 +48,8 @@ int main() { tsan_numa_test(); strdup_test(); + test_stl_allocators(); + test_mt_shutdown(); //fail_aslr(); mi_stats_print(NULL); @@ -122,6 +126,43 @@ static bool test_stl_allocator2() { return vec.size() == 0; } +static bool test_stl_allocator3() { + std::vector > vec; + vec.push_back(1); + vec.pop_back(); + return vec.size() == 0; +} + +static bool test_stl_allocator4() { + std::vector > vec; + vec.push_back(some_struct()); + vec.pop_back(); + return vec.size() == 0; +} + +static bool test_stl_allocator5() { + std::vector > vec; + vec.push_back(1); + vec.pop_back(); + return vec.size() == 0; +} + +static bool test_stl_allocator6() { + std::vector > vec; + vec.push_back(some_struct()); + vec.pop_back(); + return vec.size() == 0; +} + +static void test_stl_allocators() { + test_stl_allocator1(); + test_stl_allocator2(); + test_stl_allocator3(); + test_stl_allocator4(); + test_stl_allocator5(); + test_stl_allocator6(); +} + // issue 445 static void strdup_test() { #ifdef _MSC_VER From 061bbe25b0837f6a1cd073756c55d888571786df Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 17:17:14 -0800 Subject: [PATCH 4/8] update readme --- readme.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index fe2ead69..64cd0c4b 100644 --- a/readme.md +++ b/readme.md @@ -27,6 +27,8 @@ It also has an easy way to override the default allocator in [Windows](#override to integrate and adapt in other projects. For runtime systems it provides hooks for a monotonic _heartbeat_ and deferred freeing (for bounded worst-case times with reference counting). + Partly due to its simplicity, mimalloc has been ported to many systems (Windows, macOS, + Linux, WASM, various BSD's, Haiku, MUSL, etc) and has excellent support for dynamic overriding. - __free list sharding__: instead of one big free list (per size class) we have many smaller lists per "mimalloc page" which reduces fragmentation and increases locality -- @@ -42,7 +44,7 @@ It also has an easy way to override the default allocator in [Windows](#override similar to randomized algorithms like skip lists where adding a random oracle removes the need for a more complex algorithm. - __eager page reset__: when a "page" becomes empty (with increased chance - due to free list sharding) the memory is marked to the OS as unused ("reset" or "purged") + due to free list sharding) the memory is marked to the OS as unused (reset or decommitted) reducing (real) memory pressure and fragmentation, especially in long running programs. - __secure__: _mimalloc_ can be built in secure mode, adding guard pages, @@ -52,13 +54,12 @@ It also has an easy way to override the default allocator in [Windows](#override - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions. A heap can be destroyed at once instead of deallocating each object separately. - __bounded__: it does not suffer from _blowup_ \[1\], has bounded worst-case allocation - times (_wcat_), bounded space overhead (~0.2% meta-data, with low internal fragmentation), - and has no internal points of contention using only atomic operations. + times (_wcat_) (upto OS primitives), bounded space overhead (~0.2% meta-data, with low + internal fragmentation), and has no internal points of contention using only atomic operations. - __fast__: In our benchmarks (see [below](#performance)), _mimalloc_ outperforms other leading allocators (_jemalloc_, _tcmalloc_, _Hoard_, etc), - and often uses less memory. A nice property - is that it does consistently well over a wide range of benchmarks. There is also good huge OS page - support for larger server programs. + and often uses less memory. A nice property is that it does consistently well over a wide range + of benchmarks. There is also good huge OS page support for larger server programs. The [documentation](https://microsoft.github.io/mimalloc) gives a full overview of the API. You can read more on the design of _mimalloc_ in the [technical report](https://www.microsoft.com/en-us/research/publication/mimalloc-free-list-sharding-in-action) which also has detailed benchmark results. From 7dce31f74385f713749b4a902cc16d31f211c01b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 18:13:27 -0800 Subject: [PATCH 5/8] reenable decommitting of a huge aligned prefix --- src/segment.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/segment.c b/src/segment.c index b922a50c..b67fce87 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1261,7 +1261,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_segments_track_size(-(long)segment->segment_size, tld); mi_page_t* page = mi_segment_find_free(segment, tld); mi_assert_internal(page != NULL); -#if MI_DEBUG > 3 + if (page_alignment > 0) { size_t psize; size_t pre_size; @@ -1276,7 +1276,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, _mi_mem_decommit(decommit_start, decommit_size, os_tld); } } -#endif + // for huge pages we initialize the xblock_size as we may // overallocate to accommodate large alignments. size_t psize; From 78690fbec20d2bea57a82259a4ec20b6467a4777 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 18:41:51 -0800 Subject: [PATCH 6/8] fix proteced status in stl allocator (pr #625)# --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index d43a4419..86856900 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -506,7 +506,7 @@ template struct mi_heap_stl_allocator { void collect(bool force) { mi_heap_collect(this->heap.get(), force); } -protected: +// protected: std::shared_ptr heap; private: From 00a42bf379a166ccc64ee21fd0b751420ae271ef Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 18:42:05 -0800 Subject: [PATCH 7/8] refactor mi_segment_init --- src/segment.c | 158 +++++++++++++++++++------------------------------- 1 file changed, 59 insertions(+), 99 deletions(-) diff --git a/src/segment.c b/src/segment.c index b67fce87..e22417c9 100644 --- a/src/segment.c +++ b/src/segment.c @@ -496,11 +496,50 @@ void _mi_segment_thread_collect(mi_segments_tld_t* tld) { Segment allocation ----------------------------------------------------------- */ -// Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . -static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_page_kind_t page_kind, size_t page_shift, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignment, size_t pre_size, size_t info_size, + size_t* segment_size, bool* is_zero, bool* commit, mi_segments_tld_t* tld, mi_os_tld_t* tld_os) { - // the segment parameter is non-null if it came from our cache - mi_assert_internal(segment==NULL || (required==0 && page_kind <= MI_PAGE_LARGE)); + size_t memid; + bool mem_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy + bool is_pinned = false; + size_t align_offset = 0; + size_t alignment = MI_SEGMENT_SIZE; + if (page_alignment > 0) { + alignment = page_alignment; + align_offset = _mi_align_up(pre_size, MI_SEGMENT_SIZE); + *segment_size = *segment_size + (align_offset - pre_size); + } + + mi_segment_t* segment = (mi_segment_t*)_mi_mem_alloc_aligned(*segment_size, alignment, align_offset, commit, &mem_large, &is_pinned, is_zero, &memid, tld_os); + if (segment == NULL) return NULL; // failed to allocate + if (!commit) { + // ensure the initial info is committed + mi_assert_internal(!mem_large && !is_pinned); + bool commit_zero = false; + bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld_os); + if (commit_zero) *is_zero = true; + if (!ok) { + // commit failed; we cannot touch the memory: free the segment directly and return `NULL` + _mi_mem_free(segment, *segment_size, alignment, align_offset, memid, false, false, tld_os); + return NULL; + } + } + + mi_track_mem_undefined(segment, info_size); + segment->memid = memid; + segment->mem_is_pinned = (mem_large || is_pinned); + segment->mem_is_committed = commit; + segment->mem_alignment = alignment; + segment->mem_align_offset = align_offset; + mi_segments_track_size((long)(*segment_size), tld); + return segment; +} + +// Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` . +static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_shift, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) +{ + // required is only > 0 for huge page allocations + mi_assert_internal((required > 0 && page_kind > MI_PAGE_LARGE)|| (required==0 && page_kind <= MI_PAGE_LARGE)); // calculate needed sizes first size_t capacity; @@ -527,105 +566,29 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); - bool pages_still_good = false; bool is_zero = false; - - // Try to get it from our thread local cache first - if (segment != NULL) { - // came from cache - mi_track_mem_defined(segment,info_size); - mi_assert_internal(segment->segment_size == segment_size); - if (page_kind <= MI_PAGE_MEDIUM && segment->page_kind == page_kind && segment->segment_size == segment_size) { - pages_still_good = true; - } - else - { - if (MI_SECURE!=0) { - mi_assert_internal(!segment->mem_is_pinned); - mi_segment_protect(segment, false, tld->os); // reset protection if the page kind differs - } - // different page kinds; unreset any reset pages, and unprotect - // TODO: optimize cache pop to return fitting pages if possible? - for (size_t i = 0; i < segment->capacity; i++) { - mi_page_t* page = &segment->pages[i]; - if (page->is_reset) { - if (!commit && mi_option_is_enabled(mi_option_reset_decommits)) { - page->is_reset = false; - } - else { - mi_page_unreset(segment, page, 0, tld); // todo: only unreset the part that was reset? (instead of the full page) - } - } - } - // ensure the initial info is committed - if (segment->capacity < capacity) { - bool commit_zero = false; - bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld->os); - if (commit_zero) is_zero = true; - if (!ok) { - return NULL; - } - } - } - } - else { - // Allocate the segment from the OS - size_t memid; - bool mem_large = (!eager_delayed && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy - bool is_pinned = false; - size_t align_offset = 0; - size_t alignment = MI_SEGMENT_SIZE; - if (page_alignment > 0) { - alignment = page_alignment; - align_offset = _mi_align_up( pre_size, MI_SEGMENT_SIZE ); - segment_size += (align_offset - pre_size); - } - segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, alignment, align_offset, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld); - if (segment == NULL) return NULL; // failed to allocate - if (!commit) { - // ensure the initial info is committed - mi_assert_internal(!mem_large && !is_pinned); - bool commit_zero = false; - bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld->os); - if (commit_zero) is_zero = true; - if (!ok) { - // commit failed; we cannot touch the memory: free the segment directly and return `NULL` - _mi_mem_free(segment, segment_size, alignment, align_offset, memid, false, false, os_tld); - return NULL; - } - } - mi_track_mem_undefined(segment,info_size); - segment->memid = memid; - segment->mem_is_pinned = (mem_large || is_pinned); - segment->mem_is_committed = commit; - segment->mem_alignment = alignment; - segment->mem_align_offset = align_offset; - mi_segments_track_size((long)segment_size, tld); - } + + // Allocate the segment from the OS (segment_size can change due to alignment) + mi_segment_t* segment = mi_segment_os_alloc(eager_delayed, page_alignment, pre_size, info_size, &segment_size, &is_zero, &commit, tld, os_tld); + if (segment == NULL) return NULL; mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); mi_assert_internal(segment->mem_is_pinned ? segment->mem_is_committed : true); mi_atomic_store_ptr_release(mi_segment_t, &segment->abandoned_next, NULL); // tsan - if (!pages_still_good) { - // zero the segment info (but not the `mem` fields) - ptrdiff_t ofs = offsetof(mi_segment_t, next); - memset((uint8_t*)segment + ofs, 0, info_size - ofs); + + // zero the segment info (but not the `mem` fields) + ptrdiff_t ofs = offsetof(mi_segment_t, next); + memset((uint8_t*)segment + ofs, 0, info_size - ofs); - // initialize pages info - for (size_t i = 0; i < capacity; i++) { - mi_assert_internal(i <= 255); - segment->pages[i].segment_idx = (uint8_t)i; - segment->pages[i].is_reset = false; - segment->pages[i].is_committed = commit; - segment->pages[i].is_zero_init = is_zero; - } + // initialize pages info + for (size_t i = 0; i < capacity; i++) { + mi_assert_internal(i <= 255); + segment->pages[i].segment_idx = (uint8_t)i; + segment->pages[i].is_reset = false; + segment->pages[i].is_committed = commit; + segment->pages[i].is_zero_init = is_zero; } - else { - // zero the segment info but not the pages info (and mem fields) - ptrdiff_t ofs = offsetof(mi_segment_t, next); - memset((uint8_t*)segment + ofs, 0, offsetof(mi_segment_t,pages) - ofs); - } - + // initialize segment->page_kind = page_kind; segment->capacity = capacity; @@ -648,9 +611,6 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ return segment; } -static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_shift, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { - return mi_segment_init(NULL, required, page_kind, page_shift, page_alignment, tld, os_tld); -} static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { MI_UNUSED(force); From fed883c81f1a462e0d470f30a832824662ccf562 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 22 Nov 2022 18:44:27 -0800 Subject: [PATCH 8/8] refactor mi_segment_init fix --- src/segment.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/segment.c b/src/segment.c index e22417c9..fe53b8ce 100644 --- a/src/segment.c +++ b/src/segment.c @@ -512,7 +512,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme mi_segment_t* segment = (mi_segment_t*)_mi_mem_alloc_aligned(*segment_size, alignment, align_offset, commit, &mem_large, &is_pinned, is_zero, &memid, tld_os); if (segment == NULL) return NULL; // failed to allocate - if (!commit) { + if (!(*commit)) { // ensure the initial info is committed mi_assert_internal(!mem_large && !is_pinned); bool commit_zero = false; @@ -525,7 +525,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme } } - mi_track_mem_undefined(segment, info_size); + mi_track_mem_undefined(segment, info_size); MI_UNUSED(info_size); segment->memid = memid; segment->mem_is_pinned = (mem_large || is_pinned); segment->mem_is_committed = commit;