From b0003cb10107642456f17284b543ba53241ccb71 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Wed, 14 Aug 2019 15:07:13 +0100 Subject: [PATCH 1/8] linux build correction MAP_HUGE_2MB is linux specific and the related header is not automatically included. --- src/os.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/os.c b/src/os.c index d7fb38f9..9349019d 100644 --- a/src/os.c +++ b/src/os.c @@ -21,6 +21,9 @@ terms of the MIT license. A copy of the license can be found in the file #else #include // mmap #include // sysconf +#if defined(__linux__) +#include // linux mmap flags +#endif #if defined(__APPLE__) #include #endif From 770e752578ca1457767c8e51f8868a3758d6829d Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 19:15:04 -0700 Subject: [PATCH 2/8] fix parameter order on mul_overflow to fix static analysis warnings (pr #125) --- include/mimalloc-internal.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 151cd001..e4bb1387 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -150,15 +150,15 @@ bool _mi_page_is_valid(mi_page_t* page); // Overflow detecting multiply #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) -static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) { +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 #if (MI_INTPTR_SIZE == 4) - return __builtin_umul_overflow(size, count, total); + return __builtin_umul_overflow(count, size, total); #else - return __builtin_umull_overflow(size, count, total); + return __builtin_umull_overflow(count, size, total); #endif #else /* __builtin_umul_overflow is unavailable */ - *total = size * count; + *total = count * size; return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); #endif From b46bacc18000d7dd147ce696a01c8c5508126685 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 19:19:09 -0700 Subject: [PATCH 3/8] fix absorb heap (issue #132) --- src/heap.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/heap.c b/src/heap.c index 35c1d460..78860125 100644 --- a/src/heap.c +++ b/src/heap.c @@ -303,14 +303,14 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; - // unfull all full pages - mi_page_t* page = heap->pages[MI_BIN_FULL].first; + // unfull all full pages in the `from` heap + mi_page_t* page = from->pages[MI_BIN_FULL].first; while (page != NULL) { mi_page_t* next = page->next; _mi_page_unfull(page); page = next; } - mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL); + mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL); // free outstanding thread delayed free blocks _mi_heap_delayed_free(from); From 74299646bb3ff67f87b063a1590e9ad69b60b535 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 19:22:13 -0700 Subject: [PATCH 4/8] fix heap cookie and random order on initialization (issue #131) --- src/init.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/init.c b/src/init.c index 4fc5d602..13aba9b3 100644 --- a/src/init.c +++ b/src/init.c @@ -106,14 +106,14 @@ mi_heap_t _mi_heap_main = { MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, NULL, - 0, - 0, + 0, // thread id #if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!) 0xCDCDCDCDCDCDCDCDUL, #else 0xCDCDCDCDUL, #endif - 0, + 0, // random + 0, // page count false // can reclaim }; From 2066e2797d81210d239f6effd4615db5d0f1f115 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 19:29:27 -0700 Subject: [PATCH 5/8] don't allow allocations larger than PTRDIFF_MAX, issue #121 --- src/page.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page.c b/src/page.c index 627cb728..41e41b89 100644 --- a/src/page.c +++ b/src/page.c @@ -741,7 +741,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept // huge allocation? mi_page_t* page; if (mi_unlikely(size > MI_LARGE_OBJ_SIZE_MAX)) { - if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) { + if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see ) page = NULL; } else { From 48a7d64bdd23d60d51b97dec1c92f8b1b0a9990c Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 21:20:24 -0700 Subject: [PATCH 6/8] if region overcommitting due to contention save it to a further slot --- src/memory.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/memory.c b/src/memory.c index bf99ef42..d6d94a44 100644 --- a/src/memory.c +++ b/src/memory.c @@ -128,6 +128,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit size_t mask = mi_region_block_mask(blocks,bitidx); mi_assert_internal(mask != 0); mi_assert_internal((mask & mi_atomic_read(®ion->map)) == mask); + mi_assert_internal(®ions[idx] == region); // ensure the region is reserved void* start = mi_atomic_read_ptr(®ion->start); @@ -142,6 +143,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit } while (!mi_atomic_compare_exchange(®ion->map, map & ~mask, map)); return false; } + Sleep(10); // set the newly allocated region if (mi_atomic_compare_exchange_ptr(®ion->start, start, NULL)) { @@ -149,9 +151,23 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit mi_atomic_increment(®ions_count); } else { - // failed, another thread allocated just before us, free our allocated memory - // TODO: should we keep the allocated memory and assign it to some other region? - _mi_os_free(start, MI_REGION_SIZE, tld->stats); + // failed, another thread allocated just before us! + // we assign it to a later slot instead (up to 4 tries). + // note: we don't need to increment the region count, this will happen on another allocation + for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) { + void* s = mi_atomic_read_ptr(®ions[idx+i].start); + if (s == NULL) { // quick test + if (mi_atomic_compare_exchange_ptr(®ions[idx+i].start, start, s)) { + start = NULL; + break; + } + } + } + if (start != NULL) { + // free it if we didn't succeed to save it to some other region + _mi_os_free(start, MI_REGION_SIZE, tld->stats); + } + // and continue with the memory at our index start = mi_atomic_read_ptr(®ion->start); } } From 13ef8a049b2fc9bdb0046cf0ede9112c6c466cf2 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 18:20:51 -0700 Subject: [PATCH 7/8] allow larger large objects to better use segment space --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 94188cc0..c0778f87 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -93,7 +93,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit -#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1Mb on 64-bit +#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2Mb on 64-bit #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT) #define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) From d2324f1c2a6a8471fcf00623911234fa6e60d143 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 19 Aug 2019 11:17:00 -0700 Subject: [PATCH 8/8] always check large_os_pages enabled option so it can be changed during the running of the program --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 6c2a18dd..506e1e6d 100644 --- a/src/os.c +++ b/src/os.c @@ -70,7 +70,7 @@ size_t _mi_os_large_page_size() { static bool use_large_os_page(size_t size, size_t alignment) { // if we have access, check the size and alignment requirements - if (large_os_page_size == 0) return false; + if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false; return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0); }