From dcae918b849b3e0c05d3c2bd90bda0621fa4aed4 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 1 Feb 2021 09:49:12 -0800 Subject: [PATCH 001/134] always do ASLR in secure mode even in debug mode (issue #289) --- src/os.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index e2c93d72..3b4b4b4d 100644 --- a/src/os.c +++ b/src/os.c @@ -472,7 +472,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area - #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode + #if (MI_SECURE>0) // security: randomize start of aligned allocations uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif @@ -1013,7 +1013,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { if (start == 0) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address -#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode +#if (MI_SECURE>0) // security: randomize start of huge pages uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB #endif From 0091a641a75542d8b940196b2a01a32e3acb8e9c Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 1 Feb 2021 09:55:18 -0800 Subject: [PATCH 002/134] undo previous commit dcae918 due to wrong logic (issue #289) --- src/os.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index 3b4b4b4d..e2c93d72 100644 --- a/src/os.c +++ b/src/os.c @@ -472,7 +472,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area - #if (MI_SECURE>0) // security: randomize start of aligned allocations + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif @@ -1013,7 +1013,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { if (start == 0) { // Initialize the start address after the 32TiB area start = ((uintptr_t)32 << 40); // 32TiB virtual start address -#if (MI_SECURE>0) // security: randomize start of huge pages +#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB #endif From c426ab4ea2c0be6257409be2c35afda5c87c42f9 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 1 Feb 2021 15:41:41 -0800 Subject: [PATCH 003/134] add condition to avoid compilation error on vs2015 (#issue 353) --- src/heap.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/heap.c b/src/heap.c index a9799dde..275af755 100644 --- a/src/heap.c +++ b/src/heap.c @@ -359,7 +359,9 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a // the regular `_mi_free_delayed_block` which is safe. _mi_heap_delayed_free(from); + #if !defined(_MSC_VER) || (_MSC_VER > 1900) // somehow the following line gives an error in VS2015, issue #353 mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL); + #endif // and reset the `from` heap mi_heap_reset_pages(from); From a7c33a3b0eae509463bde8a492470472fad1ca39 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 1 Feb 2021 15:47:22 -0800 Subject: [PATCH 004/134] fix getting the unique thread id on the Apple M1, see issue #354. --- include/mimalloc-internal.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index e87732ac..cf4d80ea 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -707,7 +707,7 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { res = tcb[slot]; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); -#if defined(__APPLE__) // issue #343 +#if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); @@ -734,7 +734,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { tcb[slot] = value; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); -#if defined(__APPLE__) // issue #343 +#if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); @@ -744,8 +744,13 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { - // in all our targets, slot 0 is the pointer to the thread control block +#if defined(__aarch64__) && defined(__APPLE__) // M1 + // on macOS on the M1, slot 0 does not seem to work, so we fall back to portable C for now. See issue #354 + return (uintptr_t)&_mi_heap_default; +#else + // in all our other targets, slot 0 is the pointer to the thread control block return (uintptr_t)mi_tls_slot(0); +#endif } #else // otherwise use standard C From 331491e1e80da8f99321c9261b17463f3b32ade5 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 2 Feb 2021 10:46:30 -0800 Subject: [PATCH 005/134] build fix for Apple M1 (issue #354 and pr #356) --- include/mimalloc-internal.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf4d80ea..7160bc47 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -324,10 +324,14 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { #elif defined(MI_TLS_PTHREAD) #include extern pthread_key_t _mi_heap_default_key; -#else -extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from #endif +// Default heap to allocate from (if not using TLS- or pthread slots). +// Do not use this directly but use through `mi_heap_get_default()` (or the unchecked `mi_get_default_heap`). +// This thread local variable is only used when neither MI_TLS_SLOT, MI_TLS_PTHREAD, or MI_TLS_PTHREAD_SLOT_OFS are defined. +// However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356). +extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from + static inline mi_heap_t* mi_get_default_heap(void) { #if defined(MI_TLS_SLOT) mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT); From 7962420697ce2435e75f67bd3d2357f38183d201 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 22 Feb 2021 12:37:08 -0800 Subject: [PATCH 006/134] fix bug in bitmap is_claimed_across; issue #368 --- src/bitmap.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bitmap.c b/src/bitmap.c index 68ae3b2e..75da19df 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -372,13 +372,13 @@ static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_field if ((prev & pre_mask) != 0) any_ones = true; while (mid_count-- > 0) { prev = mi_atomic_load_relaxed(field++); - if ((prev & pre_mask) != pre_mask) all_ones = false; - if ((prev & pre_mask) != 0) any_ones = true; + if ((prev & mid_mask) != mid_mask) all_ones = false; + if ((prev & mid_mask) != 0) any_ones = true; } if (post_mask!=0) { prev = mi_atomic_load_relaxed(field); - if ((prev & pre_mask) != pre_mask) all_ones = false; - if ((prev & pre_mask) != 0) any_ones = true; + if ((prev & post_mask) != post_mask) all_ones = false; + if ((prev & post_mask) != 0) any_ones = true; } if (pany_ones != NULL) *pany_ones = any_ones; return all_ones; From 71ac98ab089fc5380335486eb449578fd6cf20c0 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 22 Feb 2021 13:04:11 -0800 Subject: [PATCH 007/134] rename include to for mingw compatibility (see pr #367) --- include/mimalloc-atomic.h | 2 +- src/alloc.c | 2 +- src/init.c | 2 +- src/options.c | 2 +- src/os.c | 2 +- src/random.c | 2 +- src/stats.c | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index 2d725a25..db885319 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -118,7 +118,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { // MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. #define WIN32_LEAN_AND_MEAN -#include +#include #include #ifdef _WIN64 typedef LONG64 msc_intptr_t; diff --git a/src/alloc.c b/src/alloc.c index 3faf6801..7fcd6a49 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -726,7 +726,7 @@ mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { #ifndef PATH_MAX #define PATH_MAX MAX_PATH #endif -#include +#include mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names char buf[PATH_MAX]; diff --git a/src/init.c b/src/init.c index f635cf98..df5efd03 100644 --- a/src/init.c +++ b/src/init.c @@ -284,7 +284,7 @@ static void _mi_thread_done(mi_heap_t* default_heap); // nothing to do as it is done in DllMain #elif defined(_WIN32) && !defined(MI_SHARED_LIB) // use thread local storage keys to detect thread ending - #include + #include #include #if (_WIN32_WINNT < 0x600) // before Windows Vista WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); diff --git a/src/options.c b/src/options.c index e6f81b28..6e229f9f 100644 --- a/src/options.c +++ b/src/options.c @@ -422,7 +422,7 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) { // reliably even when this is invoked before the C runtime is initialized. // i.e. when `_mi_preloading() == true`. // Note: on windows, environment names are not case sensitive. -#include +#include static bool mi_getenv(const char* name, char* result, size_t result_size) { result[0] = 0; size_t len = GetEnvironmentVariableA(name, result, (DWORD)result_size); diff --git a/src/os.c b/src/os.c index e2c93d72..4071f65c 100644 --- a/src/os.c +++ b/src/os.c @@ -28,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_WIN32) -#include +#include #elif defined(__wasi__) // stdlib.h is all we need, and has already been included in mimalloc.h #else diff --git a/src/random.c b/src/random.c index b6067ecf..113ba0fd 100644 --- a/src/random.c +++ b/src/random.c @@ -244,7 +244,7 @@ static bool os_random_buf(void* buf, size_t buf_len) { #endif #if defined(_WIN32) -#include +#include #elif defined(__APPLE__) #include #else diff --git a/src/stats.c b/src/stats.c index 091ad173..8cd74e41 100644 --- a/src/stats.c +++ b/src/stats.c @@ -393,7 +393,7 @@ void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { // Basic timer for convenience; use milli-seconds to avoid doubles // ---------------------------------------------------------------- #ifdef _WIN32 -#include +#include static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { static LARGE_INTEGER mfreq; // = 0 if (mfreq.QuadPart == 0LL) { @@ -448,7 +448,7 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) { // -------------------------------------------------------- #if defined(_WIN32) -#include +#include #include #pragma comment(lib,"psapi.lib") From 9f3c29c6424fb2a67c007434a7bbd0e371d89ca0 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 22 Feb 2021 13:09:41 -0800 Subject: [PATCH 008/134] remove -march=native flag; see pr #362 for discussion --- CMakeLists.txt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92dbbad2..f708e75a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,14 +190,6 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) list(APPEND mi_cflags /Zc:__cplusplus) endif() -# Architecture flags -if(${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "arm" AND NOT APPLE) - check_cxx_compiler_flag(-march=native CXX_SUPPORTS_MARCH_NATIVE) - if (CXX_SUPPORTS_MARCH_NATIVE) - list(APPEND mi_cflags -march=native) - endif() -endif() - # extra needed libraries if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) From 3228bb685fccba397c8d50d9411fe23be297ce44 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 22 Feb 2021 14:17:25 -0800 Subject: [PATCH 009/134] set errno ENOMEM for limited arena allocation (issue #295) --- src/arena.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index 6e1526ac..f697622f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -197,7 +197,10 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* } // finally, fall back to the OS - if (mi_option_is_enabled(mi_option_limit_os_alloc)) return NULL; + if (mi_option_is_enabled(mi_option_limit_os_alloc)) { + errno = ENOMEM; + return NULL; + } *is_zero = true; *memid = MI_MEMID_OS; void* p = _mi_os_alloc_aligned(size, alignment, *commit, large, tld->stats); From 9317256a4f3c76d4ce70741a357e5aadcda58529 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 24 Feb 2021 15:14:17 -0800 Subject: [PATCH 010/134] improved ASLR (issue #372) --- src/os.c | 11 +++++++++-- test/main-override.cpp | 10 ++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index 4071f65c..ebe5a6d1 100644 --- a/src/os.c +++ b/src/os.c @@ -465,10 +465,17 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; -// Return a 4MiB aligned address that is probably available -static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { +// Return a 4MiB aligned address that is probably available. +// If this returns NULL, the OS will determine the address but on some OS's that may not be +// properly aligned which can be more costly as it needs to be adjusted afterwards. +// For a size > 4GiB this always returns NULL in order to guarantee good ASLR randomization; +// otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses +// in the middle of the 4TiB - 8TiB address range (see issue #372). +static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) +{ if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if ((size%MI_SEGMENT_SIZE) != 0) return NULL; + if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(4<<40 / 1<<30) = 1/4096. uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area diff --git a/test/main-override.cpp b/test/main-override.cpp index fe5403d1..e8d2bfa5 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -32,6 +32,7 @@ void heap_late_free(); // issue #204 void padding_shrink(); // issue #209 void various_tests(); void test_mt_shutdown(); +void fail_aslr(); // issue #372 int main() { mi_stats_reset(); // ignore earlier allocations @@ -41,6 +42,7 @@ int main() { padding_shrink(); various_tests(); //test_mt_shutdown(); + //test_aslr(); mi_stats_print(NULL); return 0; } @@ -207,3 +209,11 @@ void test_mt_shutdown() std::cout << "done" << std::endl; } + +// issue #372 +void fail_aslr() { + size_t sz = (4ULL << 40); // 4TiB + void* p = malloc(sz); + printf("pointer p: %p: area up to %p\n", p, (uint8_t*)p + sz); + *(int*)0x7FFFFFFF000 = 0; // should segfault +} \ No newline at end of file From e64474e06b02dc28c0c8dd6c5c01ea4deafc47ae Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 24 Feb 2021 15:30:39 -0800 Subject: [PATCH 011/134] add virtiual gaps between hinted allocations in secure mode --- src/os.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/os.c b/src/os.c index ebe5a6d1..55738d9a 100644 --- a/src/os.c +++ b/src/os.c @@ -476,6 +476,10 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if ((size%MI_SEGMENT_SIZE) != 0) return NULL; if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(4<<40 / 1<<30) = 1/4096. + #if (MI_SECURE>0) + size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. + #endif + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area From 5f596056c97928b443d7fe229e445dd6ac17847b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 24 Feb 2021 15:49:43 -0800 Subject: [PATCH 012/134] use 2-6TiB area for hints to accommodate pre-windows8 better --- src/os.c | 23 ++++++++++++++--------- test/main-override.cpp | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/os.c b/src/os.c index 55738d9a..18105596 100644 --- a/src/os.c +++ b/src/os.c @@ -468,28 +468,33 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // Return a 4MiB aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be // properly aligned which can be more costly as it needs to be adjusted afterwards. -// For a size > 4GiB this always returns NULL in order to guarantee good ASLR randomization; -// otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses -// in the middle of the 4TiB - 8TiB address range (see issue #372). +// For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; +// (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses +// in the middle of the 2TiB - 6TiB address range (see issue #372)) + +#define KK_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start +#define KK_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) +#define KK_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) + static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if ((size%MI_SEGMENT_SIZE) != 0) return NULL; - if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(4<<40 / 1<<30) = 1/4096. + if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. #endif uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); - if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) - uintptr_t init = ((uintptr_t)4 << 40); // start at 4TiB area - #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode + if (hint == 0 || hint > KK_HINT_MAX) { // wrap or initialize + uintptr_t init = KK_HINT_BASE; + #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); - init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % KK_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > KK_HINT_MAX but that is ok, it is a hint after all } if (hint%try_alignment != 0) return NULL; return (void*)hint; diff --git a/test/main-override.cpp b/test/main-override.cpp index e8d2bfa5..890b79a6 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -42,7 +42,7 @@ int main() { padding_shrink(); various_tests(); //test_mt_shutdown(); - //test_aslr(); + //fail_aslr(); mi_stats_print(NULL); return 0; } @@ -215,5 +215,5 @@ void fail_aslr() { size_t sz = (4ULL << 40); // 4TiB void* p = malloc(sz); printf("pointer p: %p: area up to %p\n", p, (uint8_t*)p + sz); - *(int*)0x7FFFFFFF000 = 0; // should segfault + *(int*)0x5FFFFFFF000 = 0; // should segfault } \ No newline at end of file From 985f313b35bc709aaa15d0f01867f857923c2ca6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 6 Apr 2021 10:56:26 -0700 Subject: [PATCH 013/134] bump version to 1.7.1 --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index a2835d02..a7849f86 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 170 // major + 2 digits minor +#define MI_MALLOC_VERSION 171 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes From b19da8e362acae8944c60a40cf5c40c8f5ebeb44 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 6 Apr 2021 11:05:43 -0700 Subject: [PATCH 014/134] update readme for 1.7.1 and 2.0.1 --- readme.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 00c6fbd1..e3662368 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the run-time systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.0.0` (beta, 2021-01-31). -Latest stable tag: `v1.7.0` (2021-01-31). +Latest release tag: `v2.0.1` (beta, 2021-04-06). +Latest stable tag: `v1.7.1` (2021-04-06). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -71,8 +71,10 @@ Enjoy! * `dev`: development branch for mimalloc v1. * `dev-slice`: development branch for mimalloc v2 with a new algorithm for managing internal mimalloc pages. -### Release +### Releases +* 2021-04-06, `v1.7.1`, `v2.0.1` (beta): fix bug in arena allocation for huge pages, improved aslr on large allocations, improved M1 support (still experimental). + * 2021-01-31, `v2.0.0`: beta release 2.0: new algorithm for managing internal mimalloc pages that tends to use reduce memory usage and fragmentation compared to mimalloc v1 (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. From ad2fa2bf6f737ba5b8d86716dc96375e8aa56db7 Mon Sep 17 00:00:00 2001 From: Danny Lin Date: Wed, 7 Apr 2021 01:32:19 -0700 Subject: [PATCH 015/134] Fix thread ID getter on Android ARM/AArch64 Android's Bionic libc stores the thread ID in TLS slot 1 instead of 0 on 32-bit ARM and AArch64. Slot 0 contains a pointer to the ELF DTV (Dynamic Thread Vector) instead, which is constant for each loaded DSO. Because mimalloc uses the thread ID to determine whether operations are thread-local or cross-thread (atomic), all threads having the same ID causes internal data structures to get corrupted quickly when multiple threads are using the allocator: mimalloc: assertion failed: at "external/mimalloc/src/page.c":563, mi_page_extend_free assertion: "page->local_free == NULL" mimalloc: assertion failed: at "external/mimalloc/src/page.c":74, mi_page_is_valid_init assertion: "page->used <= page->capacity" mimalloc: assertion failed: at "external/mimalloc/src/page.c":100, mi_page_is_valid_init assertion: "page->used + free_count == page->capacity" mimalloc: assertion failed: at "external/mimalloc/src/page.c":74, mi_page_is_valid_init assertion: "page->used <= page->capacity" Add support for Android's alternate TLS layout to fix the crashes in multi-threaded use cases. Fixes #376. --- include/mimalloc-internal.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 7160bc47..d1b8f107 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -751,6 +751,9 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { #if defined(__aarch64__) && defined(__APPLE__) // M1 // on macOS on the M1, slot 0 does not seem to work, so we fall back to portable C for now. See issue #354 return (uintptr_t)&_mi_heap_default; +#elif defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) + // on Android, slot 1 is the thread ID (pointer to pthread internal struct) + return (uintptr_t)mi_tls_slot(1); #else // in all our other targets, slot 0 is the pointer to the thread control block return (uintptr_t)mi_tls_slot(0); From ad44f76598c2f86c266a7bcafbd4a8381d9c4de5 Mon Sep 17 00:00:00 2001 From: elbaro Date: Sun, 11 Apr 2021 03:09:23 +0900 Subject: [PATCH 016/134] commit --- src/alloc-override.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index 5906bd20..ed0dbb39 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -187,11 +187,14 @@ void* memalign(size_t alignment, size_t size) { return mi_memali int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -// on some glibc `aligned_alloc` is declared `static inline` so we cannot override it (e.g. Conda). This happens -// when _GLIBCXX_HAVE_ALIGNED_ALLOC is not defined. However, in those cases it will use `memalign`, `posix_memalign`, -// or `_aligned_malloc` and we can avoid overriding it ourselves. -// We should always override if using C compilation. (issue #276) -#if _GLIBCXX_HAVE_ALIGNED_ALLOC || !defined(__cplusplus) +// aligned_alloc is available when __USE_ISOC11 is defined. +// if aligned_alloc is not available, we will use `memalign`, `posix_memalign`, or `_aligned_malloc` +// and avoid overriding it ourselves. +// +// For example, Conda has a custom glibc where `aligned_alloc` is declared `static inline`. +// Both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. +// When compiling C codes, _ISOC11_SOURCE is undefined but __USE_ISOC11 is 1. +#if __USE_ISOC11 void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #endif From 8311cef0d11a343068a713900a879ff0cdc0d036 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 17 Apr 2021 15:08:25 -0300 Subject: [PATCH 017/134] Fix typo in comment it -> if in mimalloc-types.h --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 99024679..ddb0511d 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -270,7 +270,7 @@ typedef struct mi_segment_s { struct mi_segment_s* prev; size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) - size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) + size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim if it is too long) size_t used; // count of pages in use (`used <= capacity`) size_t capacity; // count of available pages (`#free + used`) From 3402c6cc3f439e8be04ff99afea2c810f27af035 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 19 Apr 2021 01:02:13 +0800 Subject: [PATCH 018/134] Revise the use of macOS predefined macro Quoted from "Porting UNIX/Linux Applications to OS X,"[1] * macro __MACH__ is defined if Mach system calls are supported; * macro __APPLE__ is defined in any Apple computer. __MACH__ is not specific to macOS since GNU/Hurd runs on a Mach-based microkernel (gnumach) [2]. __MACH__ is defined by the compiler, leading to potential confusions. The solution is just changing the checked identifier (i.e. __APPLE__), so it is really used only on macOS. [1] https://developer.apple.com/library/archive/documentation/Porting/Conceptual/PortingUnix/compiling/compiling.html [2] https://www.gnu.org/software/hurd/microkernel/mach/gnumach.html --- include/mimalloc-internal.h | 6 +++--- src/alloc-override.c | 8 ++++---- src/options.c | 4 ++-- src/stats.c | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 7160bc47..5655cd48 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -293,7 +293,7 @@ extern bool _mi_process_is_initialized; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap #if defined(MI_MALLOC_OVERRIDE) -#if defined(__MACH__) // OSX +#if defined(__APPLE__) // macOS #define MI_TLS_SLOT 89 // seems unused? // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) // see @@ -699,7 +699,7 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) +#elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI @@ -726,7 +726,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) +#elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) __asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI diff --git a/src/alloc-override.c b/src/alloc-override.c index 5906bd20..b4a661b8 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -13,13 +13,13 @@ terms of the MIT license. A copy of the license can be found in the file #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" #endif -#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__MACH__) && !defined(MI_INTERPOSE))) +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_INTERPOSE))) // ------------------------------------------------------ // Override system malloc // ------------------------------------------------------ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))) @@ -81,7 +81,7 @@ terms of the MIT license. A copy of the license can be found in the file void free(void* p) MI_FORWARD0(mi_free, p); #endif -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) #pragma GCC visibility push(default) #endif @@ -214,7 +214,7 @@ void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(a } #endif -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) #pragma GCC visibility pop #endif diff --git a/src/options.c b/src/options.c index 6e229f9f..14df8fba 100644 --- a/src/options.c +++ b/src/options.c @@ -259,7 +259,7 @@ static _Atomic(uintptr_t) warning_count; // = 0; // when >= max_warning_count s static mi_decl_thread bool recurse = false; static bool mi_recurse_enter(void) { - #if defined(__MACH__) || defined(MI_TLS_RECURSE_GUARD) + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) if (_mi_preloading()) return true; #endif if (recurse) return false; @@ -268,7 +268,7 @@ static bool mi_recurse_enter(void) { } static void mi_recurse_exit(void) { - #if defined(__MACH__) || defined(MI_TLS_RECURSE_GUARD) + #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) if (_mi_preloading()) return; #endif recurse = false; diff --git a/src/stats.c b/src/stats.c index 8cd74e41..a82eeb8a 100644 --- a/src/stats.c +++ b/src/stats.c @@ -479,12 +479,12 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec *page_faults = (size_t)info.PageFaultCount; } -#elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__HAIKU__) +#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) #include #include #include -#if defined(__APPLE__) && defined(__MACH__) +#if defined(__APPLE__) #include #endif @@ -520,7 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec while (get_next_area_info(tid.team, &c, &mem) == B_OK) { *peak_rss += mem.ram_size; } -#elif defined(__APPLE__) && defined(__MACH__) +#elif defined(__APPLE__) *peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; From 52943917ad0b154fe21d40496e32db15a23d6efb Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 21 Apr 2021 13:14:53 +0000 Subject: [PATCH 019/134] Rewrite align_down with bitwise operations mi_align_down_ptr was implemented with multiplication and division, which can be converted to equivalent and deterministic bit operations. --- src/os.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 18105596..fc971a44 100644 --- a/src/os.c +++ b/src/os.c @@ -65,7 +65,11 @@ static void* mi_align_up_ptr(void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } -static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { +static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + mi_assert_internal(alignment != 0); + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) // power of two? + return sz & ~mask; return (sz / alignment) * alignment; } From 5940d3bcce18e0247774f430502788cce265cea5 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 24 Apr 2021 16:35:11 +0000 Subject: [PATCH 020/134] Bump copyright date Each source file has been changed according to relevant Git activities. --- LICENSE | 2 +- doc/mimalloc-doc.h | 2 +- include/mimalloc-atomic.h | 2 +- include/mimalloc-internal.h | 2 +- include/mimalloc-new-delete.h | 2 +- include/mimalloc-override.h | 2 +- include/mimalloc-types.h | 2 +- include/mimalloc.h | 2 +- src/alloc-aligned.c | 2 +- src/alloc-override-osx.c | 4 ++-- src/alloc-override.c | 2 +- src/alloc-posix.c | 2 +- src/alloc.c | 2 +- src/arena.c | 2 +- src/bitmap.c | 2 +- src/bitmap.h | 2 +- src/heap.c | 2 +- src/init.c | 2 +- src/options.c | 2 +- src/os.c | 2 +- src/page-queue.c | 2 +- src/page.c | 2 +- src/random.c | 2 +- src/region.c | 2 +- src/segment.c | 2 +- src/static.c | 2 +- src/stats.c | 2 +- test/test-api.c | 2 +- test/test-stress.c | 2 +- 29 files changed, 30 insertions(+), 30 deletions(-) diff --git a/LICENSE b/LICENSE index 4151dbe4..670b668a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Microsoft Corporation, Daan Leijen +Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 59113402..b448f14a 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index db885319..d82bcfce 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2020 Microsoft Research, Daan Leijen +Copyright (c) 2018-2021 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index d8e293b4..449893b7 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc-new-delete.h b/include/mimalloc-new-delete.h index fded0c04..ba208f05 100644 --- a/include/mimalloc-new-delete.h +++ b/include/mimalloc-new-delete.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019 Microsoft Research, Daan Leijen +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index 2362bfbc..7d9f3e7d 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019 Microsoft Research, Daan Leijen +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index ddb0511d..18f1623c 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/include/mimalloc.h b/include/mimalloc.h index a7849f86..fe5aa8f3 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 4be651d4..724c0a1b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 4b77f631..3a46ecd9 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -278,4 +278,4 @@ static void __attribute__((constructor(0))) _mi_macos_override_malloc() { } -#endif // MI_MALLOC_OVERRIDE \ No newline at end of file +#endif // MI_MALLOC_OVERRIDE diff --git a/src/alloc-override.c b/src/alloc-override.c index b4a661b8..5ceab651 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/alloc-posix.c b/src/alloc-posix.c index eef70ab5..43931e56 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/alloc.c b/src/alloc.c index 7fcd6a49..25cc04ef 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/arena.c b/src/arena.c index f697622f..0e6615a4 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen +Copyright (c) 2019-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/bitmap.c b/src/bitmap.c index 75da19df..3b5c8199 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019,2020 Microsoft Research, Daan Leijen +Copyright (c) 2019-2021 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/bitmap.h b/src/bitmap.h index f7819803..21fd4e13 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019,2020 Microsoft Research, Daan Leijen +Copyright (c) 2019-2020 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/heap.c b/src/heap.c index 275af755..bda10699 100644 --- a/src/heap.c +++ b/src/heap.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/init.c b/src/init.c index df5efd03..c0f09b5e 100644 --- a/src/init.c +++ b/src/init.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/options.c b/src/options.c index 14df8fba..30025db2 100644 --- a/src/options.c +++ b/src/options.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/os.c b/src/os.c index 18105596..814622fc 100644 --- a/src/os.c +++ b/src/os.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/page-queue.c b/src/page-queue.c index 57e3d6a5..365257e7 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/page.c b/src/page.c index 4b7e9ffb..c08be9c0 100644 --- a/src/page.c +++ b/src/page.c @@ -1,5 +1,5 @@ /*---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/random.c b/src/random.c index 113ba0fd..255bede4 100644 --- a/src/random.c +++ b/src/random.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen +Copyright (c) 2019-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/region.c b/src/region.c index 663859c8..79540730 100644 --- a/src/region.c +++ b/src/region.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen +Copyright (c) 2019-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/segment.c b/src/segment.c index fb8e0fe1..1d59be9d 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/static.c b/src/static.c index 346aced1..4b3abc28 100644 --- a/src/static.c +++ b/src/static.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/stats.c b/src/stats.c index a82eeb8a..c94fbde9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2021, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/test/test-api.c b/test/test-api.c index e5827a93..d3344928 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen +Copyright (c) 2018-2020, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/test/test-stress.c b/test/test-stress.c index c4247abe..9b64d243 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018,2019 Microsoft Research, Daan Leijen +Copyright (c) 2018-2020 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. -----------------------------------------------------------------------------*/ From aca46242abbe61be9d1bb7df4adb9b3f044684da Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 28 Apr 2021 12:47:14 -0700 Subject: [PATCH 021/134] update comment for aligned_alloc --- src/alloc-override.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index a5fea3ca..798af77b 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -187,14 +187,12 @@ void* memalign(size_t alignment, size_t size) { return mi_memali int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -// aligned_alloc is available when __USE_ISOC11 is defined. -// if aligned_alloc is not available, we will use `memalign`, `posix_memalign`, or `_aligned_malloc` -// and avoid overriding it ourselves. -// -// For example, Conda has a custom glibc where `aligned_alloc` is declared `static inline`. -// Both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. -// When compiling C codes, _ISOC11_SOURCE is undefined but __USE_ISOC11 is 1. -#if __USE_ISOC11 +// `aligned_alloc` is only available when __USE_ISOC11 is defined. +// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot +// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. +// Fortunately, in the case where `aligned_alloc` is declared as `static inline` it +// uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. +#if __USE_ISOC11 void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #endif From 29ea7a89ab9c40ec7a2672cfb7d555b5e5057f06 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 28 Apr 2021 13:08:59 -0700 Subject: [PATCH 022/134] add braces --- src/os.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index fc971a44..e01d9d97 100644 --- a/src/os.c +++ b/src/os.c @@ -68,9 +68,12 @@ static void* mi_align_up_ptr(void* p, size_t alignment) { static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) // power of two? - return sz & ~mask; - return (sz / alignment) * alignment; + if ((alignment & mask) == 0) { // power of two? + return (sz & ~mask); + } + else { + return ((sz / alignment) * alignment); + } } static void* mi_align_down_ptr(void* p, size_t alignment) { From 73c339235c455e83b50a459370a26c1a76788aa2 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 28 Apr 2021 16:12:32 -0700 Subject: [PATCH 023/134] collect in debug mode in stress test --- test/test-stress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test-stress.c b/test/test-stress.c index 9b64d243..cf01dc40 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -250,7 +250,9 @@ int main(int argc, char** argv) { test_leak(); #endif - // mi_collect(true); +#ifndef NDEBUG + mi_collect(true); +#endif #ifndef USE_STD_MALLOC mi_stats_print(NULL); #endif From acba250e60a0d0c48d73da62423d3e4179cf89a8 Mon Sep 17 00:00:00 2001 From: Marek Kurdej Date: Tue, 4 May 2021 11:26:07 +0200 Subject: [PATCH 024/134] [Windows] Correctly choose 32-bit version of mimalloc-redirect{,32}.dll. --- CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f708e75a..f805823d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,10 +265,16 @@ if(MI_BUILD_SHARED) ) if(WIN32) # On windows copy the mimalloc redirection dll too. - target_link_libraries(mimalloc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect.lib) + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + set(MIMALLOC_REDIRECT_SUFFIX "32") + else() + set(MIMALLOC_REDIRECT_SUFFIX "") + endif() + + target_link_libraries(mimalloc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.lib) add_custom_command(TARGET mimalloc POST_BUILD - COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect.dll" $ - COMMENT "Copy mimalloc-redirect.dll to output directory") + COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" $ + COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory") endif() if (MI_INSTALL_TOPLEVEL MATCHES "ON") From 712e7d3de087dad03dd4a1129d8cadbc44f72c9e Mon Sep 17 00:00:00 2001 From: Yupeng Zhang Date: Mon, 10 May 2021 12:01:03 -0400 Subject: [PATCH 025/134] [CMake] Respect CMAKE_INSTALL_PREFIX at install time The standard way of cmake install to a destination folder is the following pattern: ```shell cd cmake cmake --build cmake --install --prefix ``` Right now, the `` folder passed in cmake --install command is ignored, and always installed into `C:/Program Files(x86)/...`, which is the default `CMAKE_INSTALL_PREFIX` value passed at the `cmake ` call. Thus, it is not possible to install the binaries into different folders without rerun the cmake/build process. The important thing here is, the cmake variable `CMAKE_INSTALL_PREFIX` is supposed to be passed at `cmake --install` time with the `--prefix` argument. In cmake file, `install` with relative path will use that prefix automaticlly. And it is the best practice to not include CMAKE_INSTALL_PREFIX in the `install(... DESTINATION )` argument: ``` In particular, there is no need to make paths absolute by prepending CMAKE_INSTALL_PREFIX; this prefix is used by default if the DESTINATION is a relative path. ``` referenced from: https://cmake.org/cmake/help/latest/command/install.html --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f708e75a..36cb7b2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,9 +208,9 @@ endif() # ----------------------------------------------------------------------------- if (MI_INSTALL_TOPLEVEL) - set(mi_install_dir "${CMAKE_INSTALL_PREFIX}") + set(mi_install_dir "") else() - set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") + set(mi_install_dir "lib/mimalloc-${mi_version}") endif() if(MI_SECURE) From 34172910e5016ed3b9dd82ba04bc7a98035fa39e Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 21 May 2021 13:01:11 -0700 Subject: [PATCH 026/134] fix symlink and --prefix option with delayed CMAKE_INSTALL_PREFIX; fix issue #398 --- CMakeLists.txt | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index abc7af17..3589d83f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,9 +208,11 @@ endif() # ----------------------------------------------------------------------------- if (MI_INSTALL_TOPLEVEL) - set(mi_install_dir "") + set(mi_install_dir "") + set(mi_install_libdir "lib") else() - set(mi_install_dir "lib/mimalloc-${mi_version}") + set(mi_install_dir "lib/mimalloc-${mi_version}") + set(mi_install_libdir "${mi_install_dir}") endif() if(MI_SECURE) @@ -277,14 +279,7 @@ if(MI_BUILD_SHARED) COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory") endif() - if (MI_INSTALL_TOPLEVEL MATCHES "ON") - install(TARGETS mimalloc EXPORT mimalloc LIBRARY - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) - else() - install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_dir} LIBRARY) - endif() + install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) install(EXPORT mimalloc DESTINATION ${mi_install_dir}/cmake) endif() @@ -308,7 +303,7 @@ if (MI_BUILD_STATIC) set_target_properties(mimalloc-static PROPERTIES OUTPUT_NAME ${mi_basename}) endif() - install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_dir}) + install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) endif() # install include files @@ -322,8 +317,8 @@ if(NOT WIN32 AND MI_BUILD_SHARED AND NOT MI_INSTALL_TOPLEVEL) # install a symlink in the /usr/local/lib to the versioned library set(mi_symlink "${CMAKE_SHARED_MODULE_PREFIX}${mi_basename}${CMAKE_SHARED_LIBRARY_SUFFIX}") set(mi_soname "mimalloc-${mi_version}/${mi_symlink}.${mi_version}") - install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink} WORKING_DIRECTORY ${mi_install_dir}/..)") - install(CODE "MESSAGE(\"-- Symbolic link: ${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink} -> ${mi_soname}\")") + install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") + install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink} -> ${mi_soname}\")") endif() # single object file for more predictable static overriding @@ -343,7 +338,7 @@ if (MI_BUILD_OBJECT) # the FILES expression can also be: $ # but that fails cmake versions less than 3.10 so we leave it as is for now install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION} - DESTINATION ${mi_install_dir} + DESTINATION ${mi_install_libdir} RENAME ${mi_basename}${CMAKE_C_OUTPUT_EXTENSION} ) endif() From e2c095fad2a1d506712be0b22616f97cc927c05b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 21 May 2021 15:15:50 -0700 Subject: [PATCH 027/134] fix installation directories on unix to use /lib, /include, /share; fix issues #399, #223, and #89 --- CMakeLists.txt | 35 ++++++++++++++++++++--------------- cmake/mimalloc-config.cmake | 11 ++++++++++- test/CMakeLists.txt | 6 +++--- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3589d83f..ca2ce3e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,11 +208,13 @@ endif() # ----------------------------------------------------------------------------- if (MI_INSTALL_TOPLEVEL) - set(mi_install_dir "") - set(mi_install_libdir "lib") + set(mi_install_libdir "lib") + set(mi_install_incdir "include") + set(mi_install_cmakedir "cmake") else() - set(mi_install_dir "lib/mimalloc-${mi_version}") - set(mi_install_libdir "${mi_install_dir}") + set(mi_install_libdir "lib/mimalloc-${mi_version}") + set(mi_install_incdir "include/mimalloc-${mi_version}") + set(mi_install_cmakedir "share/mimalloc-${mi_version}/cmake") endif() if(MI_SECURE) @@ -246,7 +248,7 @@ if(MI_USE_CXX) else() message(STATUS "Compiler : ${CMAKE_C_COMPILER}") endif() -message(STATUS "Install directory: ${mi_install_dir}") +message(STATUS "Version : ${mi_version}") message(STATUS "Build targets : ${mi_build_targets}") message(STATUS "") @@ -263,7 +265,7 @@ if(MI_BUILD_SHARED) target_link_libraries(mimalloc PUBLIC ${mi_libraries}) target_include_directories(mimalloc PUBLIC $ - $ + $ ) if(WIN32) # On windows copy the mimalloc redirection dll too. @@ -280,7 +282,7 @@ if(MI_BUILD_SHARED) endif() install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) - install(EXPORT mimalloc DESTINATION ${mi_install_dir}/cmake) + install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir}) endif() # static library @@ -292,7 +294,7 @@ if (MI_BUILD_STATIC) target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) target_include_directories(mimalloc-static PUBLIC $ - $ + $ ) if(WIN32) # When building both static and shared libraries on Windows, a static library should use a @@ -307,18 +309,21 @@ if (MI_BUILD_STATIC) endif() # install include files -install(FILES include/mimalloc.h DESTINATION ${mi_install_dir}/include) -install(FILES include/mimalloc-override.h DESTINATION ${mi_install_dir}/include) -install(FILES include/mimalloc-new-delete.h DESTINATION ${mi_install_dir}/include) -install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_dir}/cmake) -install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_dir}/cmake) +install(FILES include/mimalloc.h DESTINATION ${mi_install_incdir}) +install(FILES include/mimalloc-override.h DESTINATION ${mi_install_incdir}) +install(FILES include/mimalloc-new-delete.h DESTINATION ${mi_install_incdir}) +install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_cmakedir}) +install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_cmakedir}) if(NOT WIN32 AND MI_BUILD_SHARED AND NOT MI_INSTALL_TOPLEVEL) # install a symlink in the /usr/local/lib to the versioned library + # note: use delayed prefix expansion as \${CMAKE_INSTALL_PREFIX} set(mi_symlink "${CMAKE_SHARED_MODULE_PREFIX}${mi_basename}${CMAKE_SHARED_LIBRARY_SUFFIX}") set(mi_soname "mimalloc-${mi_version}/${mi_symlink}.${mi_version}") install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink} -> ${mi_soname}\")") + install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink}.${mi_version} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") + install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink}.${mi_version} -> ${mi_soname}\")") endif() # single object file for more predictable static overriding @@ -329,11 +334,11 @@ if (MI_BUILD_OBJECT) target_compile_options(mimalloc-obj PRIVATE ${mi_cflags}) target_include_directories(mimalloc-obj PUBLIC $ - $ + $ ) # the following seems to lead to cmake warnings/errors on some systems, disable for now :-( - # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_dir}) + # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_libdir}) # the FILES expression can also be: $ # but that fails cmake versions less than 3.10 so we leave it as is for now diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 12da076e..024c97d9 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -1,2 +1,11 @@ include(${CMAKE_CURRENT_LIST_DIR}/mimalloc.cmake) -get_filename_component(MIMALLOC_TARGET_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) +get_filename_component(MIMALLOC_SHARE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/share/mimalloc-2.0 +if (MIMALLOC_SHARE_DIR MATCHES "/share/") + string(REPLACE "/share/" "/lib/" MIMALLOC_LIBRARY_DIR ${MIMALLOC_SHARE_DIR}) + string(REPLACE "/share/" "/include/" MIMALLOC_INCLUDE_DIR ${MIMALLOC_SHARE_DIR}) +else() + # if MI_INSTALL_TOPLEVEL==ON + set(MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}/lib") + set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}/include") +endif() +set(MIMALLOC_TARGET_DIR "${MIMALLOC_LIBRARY_DIR}") # legacy diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7392d20e..7986d2da 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,7 +14,7 @@ endif() # Import mimalloc (if installed) find_package(mimalloc 1.7 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) -message(STATUS "Found mimalloc installed at: ${MIMALLOC_TARGET_DIR}") +message(STATUS "Found mimalloc installed at: ${MIMALLOC_LIBRARY_DIR}") # overriding with a dynamic library add_executable(dynamic-override main-override.c) @@ -26,8 +26,8 @@ target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) # overriding with a static object file works reliable as the symbols in the # object file have priority over those in library files -add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) -target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) +add_executable(static-override-obj main-override.c ${MIMALLOC_LIBRARY_DIR}/mimalloc.o) +target_include_directories(static-override-obj PUBLIC ${MIMALLOC_INCLUDE_DIR}) target_link_libraries(static-override-obj PUBLIC pthread) From 0f57425f808ea900ff26875f04dc5b25f6dbd204 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 30 May 2021 13:09:02 +0800 Subject: [PATCH 028/134] Distinguish SI and Binary Prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SI prefixes [the decimal prefixes] refer strictly to powers of 10. They should not be used to indicate powers of 2. e.g., one kilobit represents 1000 bits instead of 1024 bits. IEC 60027‐2 symbols are formed adding a "i" to the SI symbol (e.g. G + i = Gi). --- include/mimalloc-types.h | 22 +++++++++++----------- src/arena.c | 6 +++--- src/os.c | 6 +++--- src/segment.c | 8 ++++---- src/stats.c | 10 +++++----- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 18f1623c..502e41f8 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -105,10 +105,10 @@ terms of the MIT license. A copy of the license can be found in the file // Main tuning parameters for segment and page sizes // Sizes for 64-bit, divide by two for 32-bit -#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb -#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb -#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb -#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb +#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB +#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB +#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB // Derived constants #define MI_SEGMENT_SIZE (1UL<512kb) are put into a single page in a segment of the exact size (but still 2mb aligned) + MI_PAGE_HUGE // huge blocks (>512KiB) are put into a single page in a segment of the exact size (but still 2MiB aligned) } mi_page_kind_t; -// Segments are large allocated memory blocks (2mb on 64 bit) from +// Segments are large allocated memory blocks (2MiB on 64 bit) from // the OS. Inside segments we allocated fixed size _pages_ that // contain blocks. typedef struct mi_segment_s { diff --git a/src/arena.c b/src/arena.c index 0e6615a4..fb25a89d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -330,7 +330,7 @@ int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noe _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); return ENOMEM; } - _mi_verbose_message("reserved %zu kb memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); return 0; } @@ -347,10 +347,10 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec size_t pages_reserved = 0; void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize); if (p==NULL || pages_reserved==0) { - _mi_warning_message("failed to reserve %zu gb huge pages\n", pages); + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); return ENOMEM; } - _mi_verbose_message("numa node %i: reserved %zu gb huge pages (of the %zu gb requested)\n", numa_node, pages_reserved, pages); + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) { _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); diff --git a/src/os.c b/src/os.c index 2e2ec88d..4b87054e 100644 --- a/src/os.c +++ b/src/os.c @@ -210,7 +210,7 @@ void _mi_os_init(void) { } #elif defined(__wasi__) void _mi_os_init() { - os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KB + os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } #else @@ -964,7 +964,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) else { // fall back to regular large pages mi_huge_pages_available = false; // don't try further huge pages - _mi_warning_message("unable to allocate using huge (1gb) pages, trying large (2mb) pages instead (status 0x%lx)\n", err); + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); } } // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation @@ -1007,7 +1007,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) // see: long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { - _mi_warning_message("failed to bind huge (1gb) pages to numa node %d: %s\n", numa_node, strerror(errno)); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); } } return p; diff --git a/src/segment.c b/src/segment.c index 1d59be9d..bdf97019 100644 --- a/src/segment.c +++ b/src/segment.c @@ -17,14 +17,14 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_ /* -------------------------------------------------------------------------------- Segment allocation - We allocate pages inside bigger "segments" (4mb on 64-bit). This is to avoid + We allocate pages inside bigger "segments" (4MiB on 64-bit). This is to avoid splitting VMA's on Linux and reduce fragmentation on other OS's. Each thread owns its own segments. Currently we have: - - small pages (64kb), 64 in one segment - - medium pages (512kb), 8 in one segment - - large pages (4mb), 1 in one segment + - small pages (64KiB), 64 in one segment + - medium pages (512KiB), 8 in one segment + - large pages (4MiB), 1 in one segment - huge blocks > MI_LARGE_OBJ_SIZE_MAX become large segment with 1 page In any case the memory for a segment is virtual and usually committed on demand. diff --git a/src/stats.c b/src/stats.c index c94fbde9..07fce5e2 100644 --- a/src/stats.c +++ b/src/stats.c @@ -135,7 +135,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { char buf[32]; int len = 32; - const char* suffix = (unit <= 0 ? " " : "b"); + const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); if (unit>0) n *= unit; @@ -145,13 +145,13 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* } else { int64_t divider = base; - const char* magnitude = "k"; - if (pos >= divider*base) { divider *= base; magnitude = "m"; } - if (pos >= divider*base) { divider *= base; magnitude = "g"; } + const char* magnitude = "K"; + if (pos >= divider*base) { divider *= base; magnitude = "M"; } + if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, suffix); + snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } From d48c93af2c986f8c49ab8bb6ec35734e480816d8 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 31 May 2021 11:49:08 +0800 Subject: [PATCH 029/134] Add branch hint for _mi_os_good_alloc_size In _mi_os_good_alloc_size, overflow caused by alignment size is rare, and this patch added the appropriate branch hint during range checks. --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 2e2ec88d..5fed6b93 100644 --- a/src/os.c +++ b/src/os.c @@ -113,7 +113,7 @@ size_t _mi_os_good_alloc_size(size_t size) { else if (size < 8*MiB) align_size = 256*KiB; else if (size < 32*MiB) align_size = 1*MiB; else align_size = 4*MiB; - if (size >= (SIZE_MAX - align_size)) return size; // possible overflow? + if (mi_unlikely(size >= (SIZE_MAX - align_size))) return size; // possible overflow? return _mi_align_up(size, align_size); } From 1feb6123d90f5557a0fc1bc2afc72401e58f8cb0 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 6 Jun 2021 20:31:36 -0700 Subject: [PATCH 030/134] make all includes relative --- CMakeLists.txt | 15 +++------------ ide/vs2019/mimalloc-override-test.vcxproj | 3 ++- ide/vs2019/mimalloc-override.vcxproj | 12 ++++++++---- ide/vs2019/mimalloc.vcxproj | 12 ++++++++---- include/mimalloc-types.h | 2 +- src/alloc-aligned.c | 4 ++-- src/alloc-override-osx.c | 4 ++-- src/alloc-posix.c | 4 ++-- src/alloc.c | 6 +++--- src/arena.c | 6 +++--- src/bitmap.c | 4 ++-- src/heap.c | 6 +++--- src/init.c | 4 ++-- src/options.c | 6 +++--- src/os.c | 6 +++--- src/page.c | 6 +++--- src/random.c | 4 ++-- src/region.c | 6 +++--- src/segment.c | 6 +++--- src/static.c | 4 ++-- src/stats.c | 6 +++--- 21 files changed, 63 insertions(+), 63 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca2ce3e3..453a79c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,10 +263,7 @@ if(MI_BUILD_SHARED) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) target_link_libraries(mimalloc PUBLIC ${mi_libraries}) - target_include_directories(mimalloc PUBLIC - $ - $ - ) + target_include_directories(mimalloc PUBLIC $) if(WIN32) # On windows copy the mimalloc redirection dll too. if(CMAKE_SIZEOF_VOID_P EQUAL 4) @@ -292,10 +289,7 @@ if (MI_BUILD_STATIC) target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) - target_include_directories(mimalloc-static PUBLIC - $ - $ - ) + target_include_directories(mimalloc-static PUBLIC $) if(WIN32) # When building both static and shared libraries on Windows, a static library should use a # different output name to avoid the conflict with the import library of a shared one. @@ -332,10 +326,7 @@ if (MI_BUILD_OBJECT) set_property(TARGET mimalloc-obj PROPERTY POSITION_INDEPENDENT_CODE ON) target_compile_definitions(mimalloc-obj PRIVATE ${mi_defines}) target_compile_options(mimalloc-obj PRIVATE ${mi_cflags}) - target_include_directories(mimalloc-obj PUBLIC - $ - $ - ) + target_include_directories(mimalloc-obj PUBLIC $) # the following seems to lead to cmake warnings/errors on some systems, disable for now :-( # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_libdir}) diff --git a/ide/vs2019/mimalloc-override-test.vcxproj b/ide/vs2019/mimalloc-override-test.vcxproj index 7a9202f1..28731d40 100644 --- a/ide/vs2019/mimalloc-override-test.vcxproj +++ b/ide/vs2019/mimalloc-override-test.vcxproj @@ -177,7 +177,8 @@ - + + diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj index 182ddab1..9b703bf6 100644 --- a/ide/vs2019/mimalloc-override.vcxproj +++ b/ide/vs2019/mimalloc-override.vcxproj @@ -94,7 +94,8 @@ Disabled true true - ../../include + + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false @@ -122,7 +123,8 @@ Disabled true true - ../../include + + MI_DEBUG=3;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false @@ -151,7 +153,8 @@ true true true - ../../include + + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) @@ -183,7 +186,8 @@ true true true - ../../include + + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 6c7e276c..66cd8d2c 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -96,7 +96,8 @@ Disabled true true - ../../include + + MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false @@ -115,7 +116,8 @@ Disabled true true - ../../include + + MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false @@ -142,7 +144,8 @@ MaxSpeed true true - ../../include + + %(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) @@ -170,7 +173,8 @@ MaxSpeed true true - ../../include + + %(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 18f1623c..caf161d6 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include // _Atomic +#include "mimalloc-atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 724c0a1b..c8a52871 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" #include // memset diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 3a46ecd9..89c86745 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" #if defined(MI_MALLOC_OVERRIDE) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 43931e56..6567c61b 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -9,8 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file // mi prefixed publi definitions of various Posix, Unix, and C++ functions // for convenience and used when overriding these functions. // ------------------------------------------------------------------------ -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" // ------------------------------------------------------ // Posix & Unix functions definitions diff --git a/src/alloc.c b/src/alloc.c index 25cc04ef..1d3cd398 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // memset, strlen #include // malloc, exit diff --git a/src/arena.c b/src/arena.c index 0e6615a4..cb16e199 100644 --- a/src/arena.c +++ b/src/arena.c @@ -25,9 +25,9 @@ blocks of 2GiB (64*32MiB) and no object can cross the boundary. This can lead to fragmentation but fortunately most objects will be regions of 256MiB in practice. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // memset #include // ENOMEM diff --git a/src/bitmap.c b/src/bitmap.c index 3b5c8199..8c835bd0 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -17,8 +17,8 @@ The `_across` postfixed functions do allow sequences that can cross over between the fields. (This is used in arena allocation) ---------------------------------------------------------------------------- */ -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" #include "bitmap.h" /* ----------------------------------------------------------- diff --git a/src/heap.c b/src/heap.c index bda10699..0202fd8d 100644 --- a/src/heap.c +++ b/src/heap.c @@ -5,9 +5,9 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // memset, memcpy diff --git a/src/init.c b/src/init.c index c0f09b5e..ee5b99ec 100644 --- a/src/init.c +++ b/src/init.c @@ -4,8 +4,8 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" #include // memcpy, memset #include // atexit diff --git a/src/options.c b/src/options.c index 30025db2..41e885e6 100644 --- a/src/options.c +++ b/src/options.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include #include // strtol diff --git a/src/os.c b/src/os.c index 2e2ec88d..1d8ddcc7 100644 --- a/src/os.c +++ b/src/os.c @@ -16,9 +16,9 @@ terms of the MIT license. A copy of the license can be found in the file #undef _XOPEN_SOURCE #undef _POSIX_C_SOURCE #endif -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // strerror diff --git a/src/page.c b/src/page.c index c08be9c0..0f5b6d3a 100644 --- a/src/page.c +++ b/src/page.c @@ -11,9 +11,9 @@ terms of the MIT license. A copy of the license can be found in the file exported is `mi_malloc_generic`. ----------------------------------------------------------- */ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" /* ----------------------------------------------------------- Definition of page queues for each block size diff --git a/src/random.c b/src/random.c index 255bede4..88d2c893 100644 --- a/src/random.c +++ b/src/random.c @@ -4,8 +4,8 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" #include // memset diff --git a/src/region.c b/src/region.c index 79540730..53bf5df9 100644 --- a/src/region.c +++ b/src/region.c @@ -31,9 +31,9 @@ Possible issues: linearly. At what point will direct OS calls be faster? Is there a way to do this better without adding too much complexity? -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // memset diff --git a/src/segment.c b/src/segment.c index 1d59be9d..77f6aa44 100644 --- a/src/segment.c +++ b/src/segment.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // memset #include diff --git a/src/static.c b/src/static.c index 4b3abc28..5fe103b6 100644 --- a/src/static.c +++ b/src/static.c @@ -13,8 +13,8 @@ terms of the MIT license. A copy of the license can be found in the file #undef _POSIX_C_SOURCE #endif -#include "mimalloc.h" -#include "mimalloc-internal.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" // For a static override we create a single object file // containing the whole library. If it is linked first diff --git a/src/stats.c b/src/stats.c index c94fbde9..a7364762 100644 --- a/src/stats.c +++ b/src/stats.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include "../include/mimalloc.h" +#include "../include/mimalloc-internal.h" +#include "../include/mimalloc-atomic.h" #include // fputs, stderr #include // memset From 4ba32c31605cd872da9858744123cb35547ffa90 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 7 Jun 2021 16:47:57 -0700 Subject: [PATCH 031/134] Revert "make all includes relative" This reverts commit 1feb6123d90f5557a0fc1bc2afc72401e58f8cb0. --- CMakeLists.txt | 15 ++++++++++++--- ide/vs2019/mimalloc-override-test.vcxproj | 3 +-- ide/vs2019/mimalloc-override.vcxproj | 12 ++++-------- ide/vs2019/mimalloc.vcxproj | 12 ++++-------- include/mimalloc-types.h | 2 +- src/alloc-aligned.c | 4 ++-- src/alloc-override-osx.c | 4 ++-- src/alloc-posix.c | 4 ++-- src/alloc.c | 6 +++--- src/arena.c | 6 +++--- src/bitmap.c | 4 ++-- src/heap.c | 6 +++--- src/init.c | 4 ++-- src/options.c | 6 +++--- src/os.c | 6 +++--- src/page.c | 6 +++--- src/random.c | 4 ++-- src/region.c | 6 +++--- src/segment.c | 6 +++--- src/static.c | 4 ++-- src/stats.c | 6 +++--- 21 files changed, 63 insertions(+), 63 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 453a79c7..ca2ce3e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,7 +263,10 @@ if(MI_BUILD_SHARED) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) target_link_libraries(mimalloc PUBLIC ${mi_libraries}) - target_include_directories(mimalloc PUBLIC $) + target_include_directories(mimalloc PUBLIC + $ + $ + ) if(WIN32) # On windows copy the mimalloc redirection dll too. if(CMAKE_SIZEOF_VOID_P EQUAL 4) @@ -289,7 +292,10 @@ if (MI_BUILD_STATIC) target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) - target_include_directories(mimalloc-static PUBLIC $) + target_include_directories(mimalloc-static PUBLIC + $ + $ + ) if(WIN32) # When building both static and shared libraries on Windows, a static library should use a # different output name to avoid the conflict with the import library of a shared one. @@ -326,7 +332,10 @@ if (MI_BUILD_OBJECT) set_property(TARGET mimalloc-obj PROPERTY POSITION_INDEPENDENT_CODE ON) target_compile_definitions(mimalloc-obj PRIVATE ${mi_defines}) target_compile_options(mimalloc-obj PRIVATE ${mi_cflags}) - target_include_directories(mimalloc-obj PUBLIC $) + target_include_directories(mimalloc-obj PUBLIC + $ + $ + ) # the following seems to lead to cmake warnings/errors on some systems, disable for now :-( # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_libdir}) diff --git a/ide/vs2019/mimalloc-override-test.vcxproj b/ide/vs2019/mimalloc-override-test.vcxproj index 28731d40..7a9202f1 100644 --- a/ide/vs2019/mimalloc-override-test.vcxproj +++ b/ide/vs2019/mimalloc-override-test.vcxproj @@ -177,8 +177,7 @@ - - + diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj index 9b703bf6..182ddab1 100644 --- a/ide/vs2019/mimalloc-override.vcxproj +++ b/ide/vs2019/mimalloc-override.vcxproj @@ -94,8 +94,7 @@ Disabled true true - - + ../../include MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false @@ -123,8 +122,7 @@ Disabled true true - - + ../../include MI_DEBUG=3;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false @@ -153,8 +151,7 @@ true true true - - + ../../include MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) @@ -186,8 +183,7 @@ true true true - - + ../../include MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 66cd8d2c..6c7e276c 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -96,8 +96,7 @@ Disabled true true - - + ../../include MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false @@ -116,8 +115,7 @@ Disabled true true - - + ../../include MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false @@ -144,8 +142,7 @@ MaxSpeed true true - - + ../../include %(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) @@ -173,8 +170,7 @@ MaxSpeed true true - - + ../../include %(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index caf161d6..18f1623c 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include "mimalloc-atomic.h" // _Atomic +#include // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index c8a52871..724c0a1b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" #include // memset diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 89c86745..3a46ecd9 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -5,8 +5,8 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" #if defined(MI_MALLOC_OVERRIDE) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 6567c61b..43931e56 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -9,8 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file // mi prefixed publi definitions of various Posix, Unix, and C++ functions // for convenience and used when overriding these functions. // ------------------------------------------------------------------------ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" // ------------------------------------------------------ // Posix & Unix functions definitions diff --git a/src/alloc.c b/src/alloc.c index 1d3cd398..25cc04ef 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // memset, strlen #include // malloc, exit diff --git a/src/arena.c b/src/arena.c index cb16e199..0e6615a4 100644 --- a/src/arena.c +++ b/src/arena.c @@ -25,9 +25,9 @@ blocks of 2GiB (64*32MiB) and no object can cross the boundary. This can lead to fragmentation but fortunately most objects will be regions of 256MiB in practice. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // memset #include // ENOMEM diff --git a/src/bitmap.c b/src/bitmap.c index 8c835bd0..3b5c8199 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -17,8 +17,8 @@ The `_across` postfixed functions do allow sequences that can cross over between the fields. (This is used in arena allocation) ---------------------------------------------------------------------------- */ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" #include "bitmap.h" /* ----------------------------------------------------------- diff --git a/src/heap.c b/src/heap.c index 0202fd8d..bda10699 100644 --- a/src/heap.c +++ b/src/heap.c @@ -5,9 +5,9 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // memset, memcpy diff --git a/src/init.c b/src/init.c index ee5b99ec..c0f09b5e 100644 --- a/src/init.c +++ b/src/init.c @@ -4,8 +4,8 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" #include // memcpy, memset #include // atexit diff --git a/src/options.c b/src/options.c index 41e885e6..30025db2 100644 --- a/src/options.c +++ b/src/options.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include #include // strtol diff --git a/src/os.c b/src/os.c index 1d8ddcc7..2e2ec88d 100644 --- a/src/os.c +++ b/src/os.c @@ -16,9 +16,9 @@ terms of the MIT license. A copy of the license can be found in the file #undef _XOPEN_SOURCE #undef _POSIX_C_SOURCE #endif -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // strerror diff --git a/src/page.c b/src/page.c index 0f5b6d3a..c08be9c0 100644 --- a/src/page.c +++ b/src/page.c @@ -11,9 +11,9 @@ terms of the MIT license. A copy of the license can be found in the file exported is `mi_malloc_generic`. ----------------------------------------------------------- */ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" /* ----------------------------------------------------------- Definition of page queues for each block size diff --git a/src/random.c b/src/random.c index 88d2c893..255bede4 100644 --- a/src/random.c +++ b/src/random.c @@ -4,8 +4,8 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" #include // memset diff --git a/src/region.c b/src/region.c index 53bf5df9..79540730 100644 --- a/src/region.c +++ b/src/region.c @@ -31,9 +31,9 @@ Possible issues: linearly. At what point will direct OS calls be faster? Is there a way to do this better without adding too much complexity? -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // memset diff --git a/src/segment.c b/src/segment.c index 77f6aa44..1d59be9d 100644 --- a/src/segment.c +++ b/src/segment.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // memset #include diff --git a/src/static.c b/src/static.c index 5fe103b6..4b3abc28 100644 --- a/src/static.c +++ b/src/static.c @@ -13,8 +13,8 @@ terms of the MIT license. A copy of the license can be found in the file #undef _POSIX_C_SOURCE #endif -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" // For a static override we create a single object file // containing the whole library. If it is linked first diff --git a/src/stats.c b/src/stats.c index a7364762..c94fbde9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -4,9 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ -#include "../include/mimalloc.h" -#include "../include/mimalloc-internal.h" -#include "../include/mimalloc-atomic.h" +#include "mimalloc.h" +#include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // fputs, stderr #include // memset From aeb62c2711d0ffdea1e097d00b5972ccc2fc569b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 7 Jun 2021 16:50:31 -0700 Subject: [PATCH 032/134] fix double quote includes --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 18f1623c..caf161d6 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // ptrdiff_t #include // uintptr_t, uint16_t, etc -#include // _Atomic +#include "mimalloc-atomic.h" // _Atomic #ifdef _MSC_VER #pragma warning(disable:4214) // bitfield is not int From c8b5b745006a833189381eaab119df06bd1c8f44 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 7 Jun 2021 17:51:27 -0700 Subject: [PATCH 033/134] improve warnings --- CMakeLists.txt | 3 +++ src/alloc-override.c | 2 +- src/alloc.c | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca2ce3e3..403251cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,6 +172,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() + if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") + list(APPEND mi_cflags -Wpedantic -Wno-static-in-inline) + endif() endif() if(CMAKE_C_COMPILER_ID MATCHES "Intel") diff --git a/src/alloc-override.c b/src/alloc-override.c index 48fc33bb..084f8ae4 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -137,7 +137,7 @@ terms of the MIT license. A copy of the license can be found in the file void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } - typedef struct mi_nothrow_s { } mi_nothrow_t; + typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit diff --git a/src/alloc.c b/src/alloc.c index 25cc04ef..8acff783 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -557,6 +557,7 @@ void* _mi_externs[] = { (void*)&_mi_page_malloc, (void*)&mi_malloc, (void*)&mi_malloc_small, + (void*)&mi_zalloc_small, (void*)&mi_heap_malloc, (void*)&mi_heap_zalloc, (void*)&mi_heap_malloc_small From a83bca72b3f98b55c564713176c40e44698e2c43 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 17 Jun 2021 19:15:09 -0700 Subject: [PATCH 034/134] fixes for M1; disable interpose use zones; fix pedantic warnings --- CMakeLists.txt | 11 +++--- include/mimalloc-atomic.h | 4 +-- include/mimalloc-internal.h | 7 ++-- src/alloc-override-osx.c | 2 +- src/alloc-override.c | 68 ++++++++++++++++++------------------- test/test-stress.c | 7 ++-- 6 files changed, 48 insertions(+), 51 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 403251cf..b56953c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,8 +12,8 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" ON) -option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" OFF) # enables interpose as well +option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) +option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) @@ -76,15 +76,12 @@ if(MI_OVERRIDE) # use zone's on macOS message(STATUS " Use malloc zone to override malloc (MI_OSX_ZONE=ON)") list(APPEND mi_sources src/alloc-override-osx.c) - list(APPEND mi_defines MI_OSX_ZONE=1) - if(NOT MI_INTERPOSE) - message(STATUS " (enabling INTERPOSE as well since zone's require this)") - set(MI_INTERPOSE "ON") - endif() + list(APPEND mi_defines MI_OSX_ZONE=1) endif() if(MI_INTERPOSE) # use interpose on macOS message(STATUS " Use interpose to override malloc (MI_INTERPOSE=ON)") + message(STATUS " WARNING: interpose does not seem to work reliably on the M1; use -DMI_OSX_ZONE=ON instead") list(APPEND mi_defines MI_INTERPOSE) endif() endif() diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index d82bcfce..dc48f0a2 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -295,7 +295,7 @@ static inline void mi_atomic_yield(void) { } #elif defined(__aarch64__) static inline void mi_atomic_yield(void) { - asm volatile("wfe"); + __asm__ volatile("wfe"); } #elif (defined(__arm__) && __ARM_ARCH__ >= 7) static inline void mi_atomic_yield(void) { @@ -307,7 +307,7 @@ static inline void mi_atomic_yield(void) { } #elif defined(__armel__) || defined(__ARMEL__) static inline void mi_atomic_yield(void) { - asm volatile ("nop" ::: "memory"); + __asm__ volatile ("nop" ::: "memory"); } #endif #elif defined(__sun) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 449893b7..4a803ff5 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -713,6 +713,7 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { void** tcb; UNUSED(ofs); #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); + tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif @@ -740,6 +741,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { void** tcb; UNUSED(ofs); #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); + tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif @@ -748,10 +750,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__aarch64__) && defined(__APPLE__) // M1 - // on macOS on the M1, slot 0 does not seem to work, so we fall back to portable C for now. See issue #354 - return (uintptr_t)&_mi_heap_default; -#elif defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) +#if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) // on Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); #else diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 3a46ecd9..f506d30a 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -64,7 +64,7 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { static void zone_free(malloc_zone_t* zone, void* p) { UNUSED(zone); - return mi_free(p); + mi_free(p); } static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { diff --git a/src/alloc-override.c b/src/alloc-override.c index 084f8ae4..6a87e7bd 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -22,9 +22,9 @@ terms of the MIT license. A copy of the license can be found in the file #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))) + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); #else - #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))) + #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); #endif #define MI_FORWARD1(fun,x) MI_FORWARD(fun) #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) @@ -75,10 +75,10 @@ terms of the MIT license. A copy of the license can be found in the file // we just override new/delete which does work in a static library. #else // On all other systems forward to our API - void* malloc(size_t size) MI_FORWARD1(mi_malloc, size); - void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n); - void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize); - void free(void* p) MI_FORWARD0(mi_free, p); + void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) + void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) + void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) + void free(void* p) MI_FORWARD0(mi_free, p) #endif #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) @@ -96,18 +96,18 @@ terms of the MIT license. A copy of the license can be found in the file // see // ------------------------------------------------------ #include - void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); - void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } #if (__cplusplus >= 201402L || _MSC_VER >= 1916) - void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); - void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) #endif #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) @@ -128,10 +128,10 @@ terms of the MIT license. A copy of the license can be found in the file // used by GCC and CLang). // See // ------------------------------------------------------ - void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete - void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[] - void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); - void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } @@ -139,19 +139,19 @@ terms of the MIT license. A copy of the license can be found in the file typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) - void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit - void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit - void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); - void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) - void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit - void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit - void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); - void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } @@ -170,13 +170,13 @@ extern "C" { // Posix & Unix functions definitions // ------------------------------------------------------ -void cfree(void* p) MI_FORWARD0(mi_free, p); -void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); -size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p); +void cfree(void* p) MI_FORWARD0(mi_free, p) +void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) +size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) #if !defined(__ANDROID__) -size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); +size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else -size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p); +size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) #endif // no forwarding here due to aliasing/name mangling issues @@ -199,11 +199,11 @@ void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(a #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) - void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size); - void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size); - void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size); - void __libc_free(void* p) MI_FORWARD0(mi_free,p); - void __libc_cfree(void* p) MI_FORWARD0(mi_free,p); + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) + void __libc_free(void* p) MI_FORWARD0(mi_free,p) + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) void* __libc_valloc(size_t size) { return mi_valloc(size); } void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } diff --git a/test/test-stress.c b/test/test-stress.c index cf01dc40..d45e9899 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -37,6 +37,7 @@ static bool allow_large_objects = true; // allow very large objects? static size_t use_one_size = 0; // use single object size of `N * sizeof(uintptr_t)`? +// #define USE_STD_MALLOC #ifdef USE_STD_MALLOC #define custom_calloc(n,s) calloc(n,s) #define custom_realloc(p,s) realloc(p,s) @@ -250,10 +251,10 @@ int main(int argc, char** argv) { test_leak(); #endif -#ifndef NDEBUG - mi_collect(true); -#endif #ifndef USE_STD_MALLOC + #ifndef NDEBUG + mi_collect(true); + #endif mi_stats_print(NULL); #endif //bench_end_program(); From 728be93977152e0be222a54e6247aa60415e60bc Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 17 Jun 2021 19:38:51 -0700 Subject: [PATCH 035/134] fix for #414 making numa node count atomic --- include/mimalloc-internal.h | 7 ++++--- src/os.c | 20 +++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 4a803ff5..1e1a7966 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -667,13 +667,14 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { int _mi_os_numa_node_get(mi_os_tld_t* tld); size_t _mi_os_numa_node_count_get(void); -extern size_t _mi_numa_node_count; +extern _Atomic(size_t) _mi_numa_node_count; static inline int _mi_os_numa_node(mi_os_tld_t* tld) { - if (mi_likely(_mi_numa_node_count == 1)) return 0; + if (mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1)) return 0; else return _mi_os_numa_node_get(tld); } static inline size_t _mi_os_numa_node_count(void) { - if (mi_likely(_mi_numa_node_count>0)) return _mi_numa_node_count; + const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count); + if (mi_likely(count>0)) return count; else return _mi_os_numa_node_count_get(); } diff --git a/src/os.c b/src/os.c index 5fed6b93..85415232 100644 --- a/src/os.c +++ b/src/os.c @@ -1202,17 +1202,23 @@ static size_t mi_os_numa_node_countx(void) { } #endif -size_t _mi_numa_node_count = 0; // cache the node count +_Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count size_t _mi_os_numa_node_count_get(void) { - if (mi_unlikely(_mi_numa_node_count <= 0)) { + size_t count = mi_atomic_load_acquire(&_mi_numa_node_count); + if (count <= 0) { long ncount = mi_option_get(mi_option_use_numa_nodes); // given explicitly? - if (ncount <= 0) ncount = (long)mi_os_numa_node_countx(); // or detect dynamically - _mi_numa_node_count = (size_t)(ncount <= 0 ? 1 : ncount); - _mi_verbose_message("using %zd numa regions\n", _mi_numa_node_count); + if (ncount > 0) { + count = (size_t)ncount; + } + else { + count = mi_os_numa_node_countx(); // or detect dynamically + if (count == 0) count = 1; + } + mi_atomic_store_release(&_mi_numa_node_count, count); // save it + _mi_verbose_message("using %zd numa regions\n", count); } - mi_assert_internal(_mi_numa_node_count >= 1); - return _mi_numa_node_count; + return count; } int _mi_os_numa_node_get(mi_os_tld_t* tld) { From 752594e76423526e108413731518a26e3322b9ca Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 17 Jun 2021 19:47:41 -0700 Subject: [PATCH 036/134] add test for #414 --- test/main-override.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/main-override.cpp b/test/main-override.cpp index 890b79a6..5fe3f9a3 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -33,6 +33,7 @@ void padding_shrink(); // issue #209 void various_tests(); void test_mt_shutdown(); void fail_aslr(); // issue #372 +void tsan_numa_test(); // issue #414 int main() { mi_stats_reset(); // ignore earlier allocations @@ -41,6 +42,8 @@ int main() { heap_late_free(); padding_shrink(); various_tests(); + tsan_numa_test(); + //test_mt_shutdown(); //fail_aslr(); mi_stats_print(NULL); @@ -216,4 +219,16 @@ void fail_aslr() { void* p = malloc(sz); printf("pointer p: %p: area up to %p\n", p, (uint8_t*)p + sz); *(int*)0x5FFFFFFF000 = 0; // should segfault +} + +// issues #414 +void dummy_worker() { + void* p = mi_malloc(0); + mi_free(p); +} + +void tsan_numa_test() { + auto t1 = std::thread(dummy_worker); + dummy_worker(); + t1.join(); } \ No newline at end of file From b0441da7662d231041c7475c05ff9f9d98c303ec Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 17 Jun 2021 20:14:23 -0700 Subject: [PATCH 037/134] update readme for 1.7.2/2.0.2 --- readme.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index e3662368..0575c0a2 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the run-time systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.0.1` (beta, 2021-04-06). -Latest stable tag: `v1.7.1` (2021-04-06). +Latest release tag: `v2.0.2` (beta, 2021-06-17). +Latest stable tag: `v1.7.2` (2021-06-17). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -73,7 +73,10 @@ Enjoy! ### Releases -* 2021-04-06, `v1.7.1`, `v2.0.1` (beta): fix bug in arena allocation for huge pages, improved aslr on large allocations, improved M1 support (still experimental). +* 2021-06-17, `v1.7.2`, `v2.0.2` (beta): support M1, better installation layout on Linux, fix + thread_id on Android, prefer 2-6TiB area for aligned allocation to work better on pre-window 8, various small fixes. + +* 2021-04-06, `v1.7.1`, `v2.0.1` (beta): fix bug in arena allocation for huge pages, improved aslr on large allocations, initial M1 support (still experimental). * 2021-01-31, `v2.0.0`: beta release 2.0: new algorithm for managing internal mimalloc pages that tends to use reduce memory usage and fragmentation compared to mimalloc v1 (especially for large workloads). Should otherwise have similar performance From 076f815cece59e0a0ee08237c8fbba75465452b6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 17 Jun 2021 20:19:34 -0700 Subject: [PATCH 038/134] update readme --- readme.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/readme.md b/readme.md index 0575c0a2..cdb1b82a 100644 --- a/readme.md +++ b/readme.md @@ -73,15 +73,17 @@ Enjoy! ### Releases +Note: the `v2.x` beta has a new algorithm for managing internal mimalloc pages that tends to use reduce memory usage + and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance + (see [below](#performance)); please report if you observe any significant performance regression. + * 2021-06-17, `v1.7.2`, `v2.0.2` (beta): support M1, better installation layout on Linux, fix - thread_id on Android, prefer 2-6TiB area for aligned allocation to work better on pre-window 8, various small fixes. + thread_id on Android, prefer 2-6TiB area for aligned allocation to work better on pre-windows 8, various small fixes. * 2021-04-06, `v1.7.1`, `v2.0.1` (beta): fix bug in arena allocation for huge pages, improved aslr on large allocations, initial M1 support (still experimental). -* 2021-01-31, `v2.0.0`: beta release 2.0: new algorithm for managing internal mimalloc pages that tends to use reduce memory usage - and fragmentation compared to mimalloc v1 (especially for large workloads). Should otherwise have similar performance - (see [below](#performance)); please report if you observe any significant performance regression. - +* 2021-01-31, `v2.0.0`: beta release 2.0: new slice algorithm for managing internal mimalloc pages. + * 2021-01-31, `v1.7.0`: stable release 1.7: support explicit user provided memory regions, more precise statistics, improve macOS overriding, initial support for Apple M1, improved DragonFly support, faster memcpy on Windows, various small fixes. From a35a7d4f19b23787b67af8a49d17e18e2fc36df9 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 19 Jun 2021 09:14:43 +0000 Subject: [PATCH 039/134] haiku biuld fix proposal, warning suppression. --- src/random.c | 4 +++- src/stats.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/random.c b/src/random.c index 255bede4..723d3507 100644 --- a/src/random.c +++ b/src/random.c @@ -194,8 +194,10 @@ static bool os_random_buf(void* buf, size_t buf_len) { arc4random_buf(buf, buf_len); return true; } -#elif defined(__linux__) +#elif defined(__linux__) || defined(__HAIKU__) +#if defined(__linux__) #include +#endif #include #include #include diff --git a/src/stats.c b/src/stats.c index c94fbde9..74b783e9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -520,6 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec while (get_next_area_info(tid.team, &c, &mem) == B_OK) { *peak_rss += mem.ram_size; } + *page_faults = 0; #elif defined(__APPLE__) *peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; From 1c1571742d24b9cfbdbef4a3c537018f41756c64 Mon Sep 17 00:00:00 2001 From: hank Date: Mon, 21 Jun 2021 22:36:47 +0800 Subject: [PATCH 040/134] fix typo --- include/mimalloc-internal.h | 2 +- src/page.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..825b0ed0 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -427,7 +427,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) { return _mi_segment_page_of(_mi_ptr_segment(p), p); } -// Get the block size of a page (special cased for huge objects) +// Get the block size of a page (special case for huge objects) static inline size_t mi_page_block_size(const mi_page_t* page) { const size_t bsize = page->xblock_size; mi_assert_internal(bsize > 0); diff --git a/src/page.c b/src/page.c index c08be9c0..033976dc 100644 --- a/src/page.c +++ b/src/page.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- The core of the allocator. Every segment contains - pages of a {certain block size. The main function + pages of a certain block size. The main function exported is `mi_malloc_generic`. ----------------------------------------------------------- */ From 4369fe43239c56ad017911ef1b704b666a3e35e8 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Thu, 24 Jun 2021 17:29:06 +0800 Subject: [PATCH 041/134] Eliminate preprocessor warnings due to undefined "__GNUC__" with ClangCL When building some code against mimalloc with C inside Visual Studio with ClangCL, the compiler complains about __GNUC__ being undefined. Reported by Mojca Miklavec. Close #422 --- include/mimalloc-internal.h | 6 +++--- include/mimalloc.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..3be872af 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_noinline __declspec(noinline) #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) -#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) @@ -236,7 +236,7 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) } // Overflow detecting multiply -#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 +#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include // UINT_MAX, ULONG_MAX #if defined(_CLOCK_T) // for Illumos #undef _CLOCK_T @@ -903,7 +903,7 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) { // This is used for example in `mi_realloc`. // ------------------------------------------------------------------------------- -#if (__GNUC__ >= 4) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // On GCC/CLang we provide a hint that the pointers are word aligned. #include static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { diff --git a/include/mimalloc.h b/include/mimalloc.h index fe5aa8f3..7ebf3e60 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -26,7 +26,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__cplusplus) && (__cplusplus >= 201703) #define mi_decl_nodiscard [[nodiscard]] -#elif (__GNUC__ >= 4) || defined(__clang__) // includes clang, icc, and clang-cl +#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl #define mi_decl_nodiscard __attribute__((warn_unused_result)) #elif (_MSC_VER >= 1700) #define mi_decl_nodiscard _Check_return_ From edb0b93c6fdd2edbe4da520c1d632010e91dedcb Mon Sep 17 00:00:00 2001 From: Artur Sinila Date: Tue, 29 Jun 2021 21:07:13 +0300 Subject: [PATCH 042/134] Fix 'malloc-nomem1' test for 32-bit architectures --- test/test-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-api.c b/test/test-api.c index d3344928..739b1d18 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -83,7 +83,7 @@ int main() { void* p = mi_malloc(0); mi_free(p); }); CHECK_BODY("malloc-nomem1",{ - result = (mi_malloc(SIZE_MAX/2) == NULL); + result = (mi_malloc((size_t)PTRDIFF_MAX + (size_t)1) == NULL); }); CHECK_BODY("malloc-null",{ mi_free(NULL); From a3cf23c19fff6fd34492bee85788f2e52e41531b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 26 Jul 2021 19:10:21 -0700 Subject: [PATCH 043/134] add test for #445 --- test/main-override.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index 5fe3f9a3..d7663330 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -34,6 +34,7 @@ void various_tests(); void test_mt_shutdown(); void fail_aslr(); // issue #372 void tsan_numa_test(); // issue #414 +void strdup_test(); // issue #445 int main() { mi_stats_reset(); // ignore earlier allocations @@ -43,6 +44,7 @@ int main() { padding_shrink(); various_tests(); tsan_numa_test(); + strdup_test(); //test_mt_shutdown(); //fail_aslr(); @@ -74,18 +76,13 @@ void various_tests() { p1 = malloc(8); char* s = mi_strdup("hello\n"); - //char* s = _strdup("hello\n"); - //char* buf = NULL; - //size_t len; - //_dupenv_s(&buf,&len,"MIMALLOC_VERBOSE"); - //mi_free(buf); - mi_free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); free(p2); mi_free(s); + Test* t = new Test(42); delete t; t = new (std::nothrow) Test(42); @@ -125,7 +122,17 @@ bool test_stl_allocator2() { return vec.size() == 0; } - +// issue 445 +static void strdup_test() { +#ifdef _MSC_VER + char* s = _strdup("hello\n"); + char* buf = NULL; + size_t len; + _dupenv_s(&buf, &len, "MIMALLOC_VERBOSE"); + mi_free(buf); + mi_free(s); +#endif +} // Issue #202 void heap_no_delete_worker() { From 32c5e4774febc4f50348bd08af5b80cfc8745857 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 26 Jul 2021 19:14:29 -0700 Subject: [PATCH 044/134] update tests with static --- test/main-override.cpp | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index d7663330..e1795ecb 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -26,15 +26,15 @@ static void msleep(unsigned long msecs) { Sleep(msecs); } static void msleep(unsigned long msecs) { usleep(msecs * 1000UL); } #endif -void heap_thread_free_large(); // issue #221 -void heap_no_delete(); // issue #202 -void heap_late_free(); // issue #204 -void padding_shrink(); // issue #209 -void various_tests(); -void test_mt_shutdown(); -void fail_aslr(); // issue #372 -void tsan_numa_test(); // issue #414 -void strdup_test(); // issue #445 +static void heap_thread_free_large(); // issue #221 +static void heap_no_delete(); // issue #202 +static void heap_late_free(); // issue #204 +static void padding_shrink(); // issue #209 +static void various_tests(); +static void test_mt_shutdown(); +static void fail_aslr(); // issue #372 +static void tsan_numa_test(); // issue #414 +static void strdup_test(); // issue #445 int main() { mi_stats_reset(); // ignore earlier allocations @@ -68,7 +68,7 @@ public: }; -void various_tests() { +static void various_tests() { atexit(free_p); void* p1 = malloc(78); void* p2 = mi_malloc_aligned(16, 24); @@ -106,7 +106,7 @@ public: static Static s = Static(); -bool test_stl_allocator1() { +static bool test_stl_allocator1() { std::vector > vec; vec.push_back(1); vec.pop_back(); @@ -115,7 +115,7 @@ bool test_stl_allocator1() { struct some_struct { int i; int j; double z; }; -bool test_stl_allocator2() { +static bool test_stl_allocator2() { std::vector > vec; vec.push_back(some_struct()); vec.pop_back(); @@ -135,28 +135,28 @@ static void strdup_test() { } // Issue #202 -void heap_no_delete_worker() { +static void heap_no_delete_worker() { mi_heap_t* heap = mi_heap_new(); void* q = mi_heap_malloc(heap, 1024); // mi_heap_delete(heap); // uncomment to prevent assertion } -void heap_no_delete() { +static void heap_no_delete() { auto t1 = std::thread(heap_no_delete_worker); t1.join(); } // Issue #204 -volatile void* global_p; +static volatile void* global_p; -void t1main() { +static void t1main() { mi_heap_t* heap = mi_heap_new(); global_p = mi_heap_malloc(heap, 1024); mi_heap_delete(heap); } -void heap_late_free() { +static void heap_late_free() { auto t1 = std::thread(t1main); msleep(2000); @@ -173,7 +173,7 @@ static void alloc0(/* void* arg */) shared_p = mi_malloc(8); } -void padding_shrink(void) +static void padding_shrink(void) { auto t1 = std::thread(alloc0); t1.join(); @@ -182,11 +182,11 @@ void padding_shrink(void) // Issue #221 -void heap_thread_free_large_worker() { +static void heap_thread_free_large_worker() { mi_free(shared_p); } -void heap_thread_free_large() { +static void heap_thread_free_large() { for (int i = 0; i < 100; i++) { shared_p = mi_malloc_aligned(2*1024*1024 + 1, 8); auto t1 = std::thread(heap_thread_free_large_worker); @@ -196,7 +196,7 @@ void heap_thread_free_large() { -void test_mt_shutdown() +static void test_mt_shutdown() { const int threads = 5; std::vector< std::future< std::vector< char* > > > ts; @@ -221,7 +221,7 @@ void test_mt_shutdown() } // issue #372 -void fail_aslr() { +static void fail_aslr() { size_t sz = (4ULL << 40); // 4TiB void* p = malloc(sz); printf("pointer p: %p: area up to %p\n", p, (uint8_t*)p + sz); @@ -229,12 +229,12 @@ void fail_aslr() { } // issues #414 -void dummy_worker() { +static void dummy_worker() { void* p = mi_malloc(0); mi_free(p); } -void tsan_numa_test() { +static void tsan_numa_test() { auto t1 = std::thread(dummy_worker); dummy_worker(); t1.join(); From 30be78d97a4a1a25c81755fb2671428a5893a0d9 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:31:48 -0400 Subject: [PATCH 045/134] Fixed typo in headers --- include/mimalloc-types.h | 2 +- src/heap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index caf161d6..66b311c1 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -319,7 +319,7 @@ typedef struct mi_random_cxt_s { } mi_random_ctx_t; -// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows #if (MI_PADDING) typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) diff --git a/src/heap.c b/src/heap.c index bda10699..bbf4cef9 100644 --- a/src/heap.c +++ b/src/heap.c @@ -333,7 +333,7 @@ void mi_heap_destroy(mi_heap_t* heap) { Safe Heap delete ----------------------------------------------------------- */ -// Tranfer the pages from one heap to the other +// Transfer the pages from one heap to the other static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; From ee0b01c84dbaf8b65f505fbbb2debb64bfc489c6 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:46:58 -0400 Subject: [PATCH 046/134] Fixed stats in mi_os_mem_alloc_aligned on windows when re-allocate In the last try the previous mi_os_mem_free decrease stat and mi_win_virtual_alloc doesn't increase it --- src/os.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/os.c b/src/os.c index 85415232..89d166ef 100644 --- a/src/os.c +++ b/src/os.c @@ -593,6 +593,11 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); + if (p != NULL) + { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } + } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? mi_os_mem_free(p, size, commit, stats); From a2c3b0f8af5d43cea6398be8e7162a6fbce2ec95 Mon Sep 17 00:00:00 2001 From: paulip1792 Date: Tue, 10 Aug 2021 16:30:44 +0800 Subject: [PATCH 047/134] add option to reserve huge os pages at a specific numa node. --- doc/mimalloc-doc.h | 3 +++ include/mimalloc.h | 1 + readme.md | 2 ++ src/init.c | 7 ++++++- src/options.c | 1 + 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index b448f14a..e2287fe9 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -782,6 +782,7 @@ typedef enum mi_option_e { mi_option_eager_region_commit, ///< Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows) mi_option_large_os_pages, ///< Use large OS pages (2MiB in size) if possible mi_option_reserve_huge_os_pages, ///< The number of huge OS pages (1GiB in size) to reserve at the start of the program. + mi_option_reserve_huge_os_pages_at, ///< Reserve huge OS pages at node N. mi_option_segment_cache, ///< The number of segments per thread to keep cached. mi_option_page_reset, ///< Reset page memory after \a mi_option_reset_delay milliseconds when it becomes free. mi_option_segment_reset, ///< Experimental @@ -1053,6 +1054,8 @@ or via environment variables. `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. + (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the diff --git a/include/mimalloc.h b/include/mimalloc.h index fe5aa8f3..db5bff40 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -306,6 +306,7 @@ typedef enum mi_option_e { mi_option_reset_decommits, mi_option_large_os_pages, // implies eager commit mi_option_reserve_huge_os_pages, + mi_option_reserve_huge_os_pages_at, mi_option_reserve_os_memory, mi_option_segment_cache, mi_option_page_reset, diff --git a/readme.md b/readme.md index cdb1b82a..2c02f665 100644 --- a/readme.md +++ b/readme.md @@ -302,6 +302,8 @@ or via environment variables: `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. + (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the diff --git a/src/init.c b/src/init.c index c0f09b5e..587e6b1c 100644 --- a/src/init.c +++ b/src/init.c @@ -494,7 +494,12 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { size_t pages = mi_option_get(mi_option_reserve_huge_os_pages); - mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at); + if (reserve_at != -1) { + mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); + } else { + mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + } } if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); diff --git a/src/options.c b/src/options.c index 30025db2..aa4601fe 100644 --- a/src/options.c +++ b/src/options.c @@ -76,6 +76,7 @@ static mi_option_desc_t options[_mi_option_last] = #endif { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { -1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N { 0, UNINIT, MI_OPTION(reserve_os_memory) }, { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread { 1, UNINIT, MI_OPTION(page_reset) }, // reset page memory on free From 9ee780894a18c3892a1c64bb2950fa862a3b7588 Mon Sep 17 00:00:00 2001 From: Masashi Fujita Date: Thu, 16 Sep 2021 03:35:56 +0900 Subject: [PATCH 048/134] fix: Fix a typo --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..385d2071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ set(mi_sources src/init.c) # ----------------------------------------------------------------------------- -# Converience: set default build type depending on the build directory +# Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- if (NOT CMAKE_BUILD_TYPE) From 679aad065908ccf40fcf535705c6e2d4a1fbdb82 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 1 Oct 2021 15:05:01 -0700 Subject: [PATCH 049/134] update wasm support with emscripten compilation; now using sbrk instead of wasm_memory_grow --- include/mimalloc-internal.h | 10 +++++--- src/os.c | 49 ++++++++++++++++++++++++++++++++++--- src/random.c | 4 ++- src/stats.c | 2 +- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..ecdcf860 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -32,6 +32,10 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif +#if defined(__EMSCRIPTEN__) && !defined(__wasi__) +#define __wasi__ +#endif + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -242,10 +246,10 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) #undef _CLOCK_T #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #if (SIZE_MAX == UINT_MAX) - return __builtin_umul_overflow(count, size, total); - #elif (SIZE_MAX == ULONG_MAX) + #if (SIZE_MAX == ULONG_MAX) return __builtin_umull_overflow(count, size, total); + #elif (SIZE_MAX == UINT_MAX) + return __builtin_umul_overflow(count, size, total); #else return __builtin_umulll_overflow(count, size, total); #endif diff --git a/src/os.c b/src/os.c index 85415232..5b6b0c7e 100644 --- a/src/os.c +++ b/src/os.c @@ -26,11 +26,14 @@ terms of the MIT license. A copy of the license can be found in the file #pragma warning(disable:4996) // strerror #endif +#if defined(__wasi__) +#define MI_USE_SBRK +#endif #if defined(_WIN32) #include #elif defined(__wasi__) -// stdlib.h is all we need, and has already been included in mimalloc.h +#include // sbrk #else #include // mmap #include // sysconf @@ -99,11 +102,13 @@ size_t _mi_os_large_page_size() { return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); } +#if !defined(MI_USE_SBRK) && !defined(__wasi__) static bool use_large_os_page(size_t size, size_t alignment) { // if we have access, check the size and alignment requirements if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false; return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0); } +#endif // round to a good OS allocation size (bounded by max 12.5% waste) size_t _mi_os_good_alloc_size(size_t size) { @@ -236,8 +241,8 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats bool err = false; #if defined(_WIN32) err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); -#elif defined(__wasi__) - err = 0; // WebAssembly's heap cannot be shrunk +#elif defined(MI_USE_SBRK) + err = 0; // sbrk heap cannot be shrunk #else err = (munmap(addr, size) == -1); #endif @@ -252,7 +257,9 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats } } +#if !defined(MI_USE_SBRK) && !defined(__wasi__) static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); +#endif #ifdef _WIN32 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { @@ -318,7 +325,32 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, return p; } -#elif defined(__wasi__) +#elif defined(MI_USE_SBRK) +#define MI_SBRK_FAIL ((void*)(-1)) +static void* mi_sbrk_heap_grow(size_t size, size_t try_alignment) { + void* pbase0 = sbrk(0); + if (pbase0 == MI_SBRK_FAIL) { + _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); + errno = ENOMEM; + return NULL; + } + uintptr_t base = (uintptr_t)pbase0; + uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); + size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); + mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); + if (alloc_size < size) return NULL; + void* pbase1 = sbrk(alloc_size); + if (pbase1 == MI_SBRK_FAIL) { + _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); + errno = ENOMEM; + return NULL; + } + mi_assert(pbase0 == pbase1); + return (void*)aligned_base; +} + +#elif defined(__wasi__) + // currently unused as we use sbrk() on wasm static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size(); uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); @@ -326,11 +358,13 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); if (alloc_size < size) return NULL; if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) { + _mi_warning_message("unable to allocate wasm_memory_grow() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); errno = ENOMEM; return NULL; } return (void*)aligned_base; } + #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { @@ -506,6 +540,8 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint%try_alignment != 0) return NULL; return (void*)hint; } +#elif defined(__wasi__) || defined(MI_USE_SBRK) +// no need for mi_os_get_aligned_hint #else static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { UNUSED(try_alignment); UNUSED(size); @@ -536,7 +572,12 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo int flags = MEM_RESERVE; if (commit) flags |= MEM_COMMIT; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + #elif defined(MI_USE_SBRK) + KK_UNUSED(allow_large); + *is_large = false; + p = mi_sbrk_heap_grow(size, try_alignment); #elif defined(__wasi__) + KK_UNUSED(allow_large); *is_large = false; p = mi_wasm_heap_grow(size, try_alignment); #else diff --git a/src/random.c b/src/random.c index 255bede4..d5f75bf8 100644 --- a/src/random.c +++ b/src/random.c @@ -188,7 +188,7 @@ static bool os_random_buf(void* buf, size_t buf_len) { #elif defined(ANDROID) || defined(XP_DARWIN) || defined(__APPLE__) || defined(__DragonFly__) || \ defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__sun) || defined(__wasi__) + defined(__sun) // todo: what to use with __wasi__? #include static bool os_random_buf(void* buf, size_t buf_len) { arc4random_buf(buf, buf_len); @@ -280,7 +280,9 @@ void _mi_random_init(mi_random_ctx_t* ctx) { if (!os_random_buf(key, sizeof(key))) { // if we fail to get random data from the OS, we fall back to a // weak random source based on the current time + #if !defined(__wasi__) _mi_warning_message("unable to use secure randomness\n"); + #endif uintptr_t x = _os_random_weak(0); for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words. x = _mi_random_shuffle(x); diff --git a/src/stats.c b/src/stats.c index c94fbde9..ac728a1c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -479,7 +479,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec *page_faults = (size_t)info.PageFaultCount; } -#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) +#elif !defined(__wasi__) && (defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)) #include #include #include From c21b6df51e8ff558b852e088ec9d0a99a15bff94 Mon Sep 17 00:00:00 2001 From: Jeremy Lorelli Date: Tue, 5 Oct 2021 08:41:03 -0700 Subject: [PATCH 050/134] Fix missing parameter in mi_free error message --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..1154e70a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -465,7 +465,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #endif #if (MI_DEBUG>0 || MI_SECURE>=4) if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { - _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", p); + _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); } #endif return segment; From 3c058f07a98fe1bc35390099e9674509a74a8d3f Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 8 Oct 2021 23:59:35 -0700 Subject: [PATCH 051/134] Add an option to disable automatic use of `getenv` inside options.c --- src/options.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 30025db2..2dc7bfee 100644 --- a/src/options.c +++ b/src/options.c @@ -409,6 +409,14 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest[dest_size - 1] = 0; } +#ifdef MI_NO_GETENV +static bool mi_getenv(const char* name, char* result, size_t result_size) { + UNUSED(name); + UNUSED(result); + UNUSED(result_size); + return false; +} +#else static inline int mi_strnicmp(const char* s, const char* t, size_t n) { if (n==0) return 0; for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { @@ -416,7 +424,6 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) { } return (n==0 ? 0 : *s - *t); } - #if defined _WIN32 // On Windows use GetEnvironmentVariable instead of getenv to work // reliably even when this is invoked before the C runtime is initialized. @@ -485,6 +492,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { } } #endif +#endif static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 0fb61c9eaa36cc86ca58c99e9522334b9e5fbde6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 16:25:10 -0700 Subject: [PATCH 052/134] do not call exit if try handler fails but use abort instead --- src/alloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..f692f495 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -826,7 +826,10 @@ static std_new_handler_t mi_get_new_handler() { static bool mi_try_new_handler(bool nothrow) { std_new_handler_t h = mi_get_new_handler(); if (h==NULL) { - if (!nothrow) exit(ENOMEM); // cannot throw in plain C, use exit as we are out of memory anyway. + if (!nothrow) { + _mi_error_message(EFAULT, "out of memory in 'new' call"); // cannot throw in plain C, use EFAULT to abort + abort(); + } return false; } else { From bd8e3fd8e1aa00f8e2dd42f169cb72d59e8eccba Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 16:46:06 -0700 Subject: [PATCH 053/134] increase robustness of primitive windows allocation by always using a fallback to VirtualAlloc --- src/os.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index 5b6b0c7e..a818bf48 100644 --- a/src/os.c +++ b/src/os.c @@ -269,12 +269,16 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); if (p != NULL) return p; + // for robustness always fall through in case of an error + /* DWORD err = GetLastError(); if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) return NULL; } - // fall through + */ + _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); + // fall through on error } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) @@ -285,7 +289,10 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; param.Type = MemExtendedParameterAddressRequirements; param.Pointer = &reqs; - return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + if (p != NULL) return p; + _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), addr, try_alignment, flags); + // fall through on error } #endif // last resort From 6ef15943cc6ba50beb044daa6baae7baae55b615 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 16:59:19 -0700 Subject: [PATCH 054/134] fix comments --- src/os.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index a818bf48..889ec0c8 100644 --- a/src/os.c +++ b/src/os.c @@ -264,7 +264,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); #ifdef _WIN32 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) - // on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations + // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations void* hint; if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); @@ -377,7 +377,7 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { void* p = NULL; #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) - // on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations + // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations void* hint; if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { p = mmap(hint,size,protect_flags,flags,fd,0); @@ -509,7 +509,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif // On 64-bit systems, we can do efficient aligned allocation by using -// the 4TiB to 30TiB area to allocate them. +// the 2TiB to 30TiB area to allocate them. #if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; From d6bbc0811923cd8560f636224be71fbbd739b576 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 18 Oct 2021 18:24:59 -0700 Subject: [PATCH 055/134] prefer monotonic clock for stats (issue #457) --- src/stats.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stats.c b/src/stats.c index ac728a1c..662da53e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -412,10 +412,14 @@ mi_msecs_t _mi_clock_now(void) { } #else #include -#ifdef CLOCK_REALTIME +#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) mi_msecs_t _mi_clock_now(void) { struct timespec t; + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &t); + #else clock_gettime(CLOCK_REALTIME, &t); + #endif return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); } #else From bcce4d52bf1494746f4dd7f5569e542cbfdf5d36 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 20:39:39 -0700 Subject: [PATCH 056/134] fix bug in determination of block size in pre-reserved arena memory --- src/arena.c | 2 +- src/init.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index 0e6615a4..a4e473a3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -287,7 +287,7 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la is_committed = true; } - const size_t bcount = mi_block_count_of_size(size); + const size_t bcount = size / MI_ARENA_BLOCK_SIZE; const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS); const size_t bitmaps = (is_committed ? 2 : 3); const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t)); diff --git a/src/init.c b/src/init.c index c0f09b5e..d899f6ad 100644 --- a/src/init.c +++ b/src/init.c @@ -498,7 +498,9 @@ void mi_process_init(void) mi_attr_noexcept { } if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); - if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true); + if (ksize > 0) { + mi_reserve_os_memory(_mi_divide_up((size_t)ksize*KiB, MI_SEGMENT_SIZE), true, true); + } } } From 22c2fd82cc6e339ecdf34bf28accf1f6658f5509 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 20:44:19 -0700 Subject: [PATCH 057/134] ensure managed os memory is at least one arena block in size --- src/arena.c | 4 +++- src/init.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index a4e473a3..b786ffb4 100644 --- a/src/arena.c +++ b/src/arena.c @@ -282,6 +282,8 @@ static bool mi_arena_add(mi_arena_t* arena) { bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { + if (size < MI_ARENA_BLOCK_SIZE) return false; + if (is_large) { mi_assert_internal(is_committed); is_committed = true; @@ -321,7 +323,7 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la // Reserve a range of regular OS memory int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { - size = _mi_os_good_alloc_size(size); + size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block bool large = allow_large; void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &_mi_stats_main); if (start==NULL) return ENOMEM; diff --git a/src/init.c b/src/init.c index d899f6ad..c088cada 100644 --- a/src/init.c +++ b/src/init.c @@ -499,7 +499,7 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); if (ksize > 0) { - mi_reserve_os_memory(_mi_divide_up((size_t)ksize*KiB, MI_SEGMENT_SIZE), true, true); + mi_reserve_os_memory((size_t)ksize*KiB, true, true); } } } From 7c73e3996d5d729c1baec2e833985166a3f5db2e Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 10:38:57 +0200 Subject: [PATCH 058/134] Fix strict function prototype warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix warning ``warning: function declaration isn’t a prototype`` when building mimalloc with ``-Wstrict-prototypes`` flag. In C argumentless functions should be declared as ``func(void)``. Reproducer: ```shell $ cmake ../.. -DCMAKE_C_FLAGS="-Wstrict-prototypes" $ make VERBOSE=1 ``` Co-authored-by: Sam Gross Co-authored-by: Neil Schemenauer Signed-off-by: Christian Heimes --- include/mimalloc-internal.h | 2 +- src/alloc.c | 8 ++++---- src/options.c | 2 +- src/os.c | 2 +- src/region.c | 2 +- test/test-api.c | 10 +++++----- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..9b097b8e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -53,7 +53,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); -bool _mi_preloading(); // true while the C runtime is not ready +bool _mi_preloading(void); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); diff --git a/src/alloc.c b/src/alloc.c index 8acff783..d2ba5c68 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -747,7 +747,7 @@ mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char } #else #include // pathconf -static size_t mi_path_max() { +static size_t mi_path_max(void) { static size_t path_max = 0; if (path_max <= 0) { long m = pathconf("/",_PC_PATH_MAX); @@ -807,13 +807,13 @@ static bool mi_try_new_handler(bool nothrow) { } } #else -typedef void (*std_new_handler_t)(); +typedef void (*std_new_handler_t)(void); #if (defined(__GNUC__) || defined(__clang__)) -std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() { +std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv(void) { return NULL; } -static std_new_handler_t mi_get_new_handler() { +static std_new_handler_t mi_get_new_handler(void) { return _ZSt15get_new_handlerv(); } #else diff --git a/src/options.c b/src/options.c index 30025db2..4274af97 100644 --- a/src/options.c +++ b/src/options.c @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file static uintptr_t mi_max_error_count = 16; // stop outputting errors after this static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this -static void mi_add_stderr_output(); +static void mi_add_stderr_output(void); int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; diff --git a/src/os.c b/src/os.c index 85415232..16659b33 100644 --- a/src/os.c +++ b/src/os.c @@ -95,7 +95,7 @@ size_t _mi_os_page_size() { } // if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) -size_t _mi_os_large_page_size() { +size_t _mi_os_large_page_size(void) { return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); } diff --git a/src/region.c b/src/region.c index 79540730..2f68b140 100644 --- a/src/region.c +++ b/src/region.c @@ -40,7 +40,7 @@ Possible issues: #include "bitmap.h" // Internal raw OS interface -size_t _mi_os_large_page_size(); +size_t _mi_os_large_page_size(void); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); diff --git a/test/test-api.c b/test/test-api.c index d3344928..53e7f3d8 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -64,15 +64,15 @@ static int failed = 0; // --------------------------------------------------------------------------- // Test functions // --------------------------------------------------------------------------- -bool test_heap1(); -bool test_heap2(); -bool test_stl_allocator1(); -bool test_stl_allocator2(); +bool test_heap1(void); +bool test_heap2(void); +bool test_stl_allocator1(void); +bool test_stl_allocator2(void); // --------------------------------------------------------------------------- // Main testing // --------------------------------------------------------------------------- -int main() { +int main(void) { mi_option_disable(mi_option_verbose); // --------------------------------------------------- From afbcf20f24729e9bf2e87a72efadeb48ddcdd6f7 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 15:07:52 +0200 Subject: [PATCH 059/134] Define _DEFAULT_SOURCE for syscall and realpath Define ``_DEFAULT_SOURCE`` in ``random.c`` and ``alloc.c``. The macro is required for ``syscall()`` and ``realpath()``. Other files like ``os.c`` already define the macro. Signed-off-by: Christian Heimes --- src/alloc.c | 4 ++++ src/random.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..5e45e76f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for realpath() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" diff --git a/src/random.c b/src/random.c index 255bede4..0e2331bd 100644 --- a/src/random.c +++ b/src/random.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for syscall() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" From 60937b5bc8ac142ebc0ed2df5aca429fabd01c60 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:39:33 -0700 Subject: [PATCH 060/134] add -Wstrict-prototypes flag during compilation --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..ac4c0569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,7 +165,7 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -Wstrict-prototypes -fvisibility=hidden) if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() From f0f9aecfe49a1a14e6247770e6b677016902fab4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:52:20 -0700 Subject: [PATCH 061/134] add comment on #if ending --- src/options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/options.c b/src/options.c index 8d5f1556..c40a187b 100644 --- a/src/options.c +++ b/src/options.c @@ -491,8 +491,8 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { return false; } } -#endif -#endif +#endif // !MI_USE_ENVIRON +#endif // !MI_NO_GETENV static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 5b9409f4d6c1b010c79eaa81117ba9b7e57672e3 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:17:30 -0700 Subject: [PATCH 062/134] add space after _Atomic to prevent errors on msvc without /TP (see PR #452) --- include/mimalloc-atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index dc48f0a2..e07df84d 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -25,7 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_memory_order(name) std::memory_order_##name #elif defined(_MSC_VER) // Use MSVC C wrapper for C11 atomics -#define _Atomic(tp) tp +#define _Atomic(tp) tp #define ATOMIC_VAR_INIT(x) x #define mi_atomic(name) mi_atomic_##name #define mi_memory_order(name) mi_memory_order_##name @@ -173,7 +173,7 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt } static inline void mi_atomic_thread_fence(mi_memory_order mo) { (void)(mo); - _Atomic(uintptr_t)x = 0; + _Atomic(uintptr_t) x = 0; mi_atomic_exchange_explicit(&x, 1, mo); } static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { From adeef8d4035c91b34a97fb83acf1c6bde1f1e4c9 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:31:49 -0700 Subject: [PATCH 063/134] formatting --- src/os.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index d7d3d33d..ba6cc5e3 100644 --- a/src/os.c +++ b/src/os.c @@ -641,10 +641,9 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); - if (p != NULL) - { - _mi_stat_increase(&stats->reserved, size); - if (commit) { _mi_stat_increase(&stats->committed, size); } + if (p != NULL) { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? From 2d821003b07b4ccdfbf5259592f7831684f996ee Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:56:13 -0700 Subject: [PATCH 064/134] don't add prefix if build type is None (PR #427) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d1d9191..abab5a05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ else() endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel)$")) +if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$")) set(mi_basename "${mi_basename}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version endif() if(MI_BUILD_SHARED) From 9a724889ead0ab9ad04f66b91b3e22064ccd8d52 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 12:50:16 -0700 Subject: [PATCH 065/134] refine stat output with new SI units --- src/stats.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/stats.c b/src/stats.c index f85285e1..dda55cda 100644 --- a/src/stats.c +++ b/src/stats.c @@ -133,7 +133,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // unit == 0: count as decimal // unit < 0 : count in binary static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { - char buf[32]; + char buf[32]; buf[0] = 0; int len = 32; const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); @@ -141,17 +141,21 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t pos = (n < 0 ? -n : n); if (pos < base) { - snprintf(buf, len, "%d %s ", (int)n, suffix); + if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column + snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix)); + } } else { - int64_t divider = base; + int64_t divider = base; const char* magnitude = "K"; if (pos >= divider*base) { divider *= base; magnitude = "M"; } if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); + char unitdesc[16]; + snprintf(unitdesc, 16, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } @@ -221,7 +225,7 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* static void mi_print_header(mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); + _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); } #if MI_STAT>1 From bb00a5a32e53067e6e23215d883fbece5ec0a74f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 13:09:43 -0700 Subject: [PATCH 066/134] remove MADV_FREE_REUSABLE from mi_os_reset as it needs MADV_FREE_REUSE to reuse the memory again. --- src/os.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/os.c b/src/os.c index 08068c9d..bf1d4716 100644 --- a/src/os.c +++ b/src/os.c @@ -796,9 +796,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } - #if defined(MADV_FREE_REUSE) - while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } - #endif + //#if defined(MADV_FREE_REUSE) + // while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } + //#endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -860,17 +860,12 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - #if defined(MADV_FREE_REUSABLE) - #define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE - #else - #define KK_MADV_FREE_INITIAL MADV_FREE - #endif - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL); + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; - if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) { - // if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on + if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } From ae5aae7e105eaccef37be0cda601b005264d2d29 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 14:00:40 -0700 Subject: [PATCH 067/134] allow decommit in arena separate from initial commit --- src/arena.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/arena.c b/src/arena.c index fb25a89d..2f5a349f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -62,11 +62,11 @@ typedef struct mi_arena_s { size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) int numa_node; // associated NUMA node bool is_zero_init; // is the arena zero initialized? - bool is_committed; // is the memory fully committed? (if so, block_committed == NULL) + bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? - mi_bitmap_field_t* blocks_committed; // if `!is_committed`, are the blocks committed? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) } mi_arena_t; @@ -129,8 +129,8 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n *memid = mi_arena_id_create(arena_index, bitmap_index); *is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); *large = arena->is_large; - *is_pinned = (arena->is_large || arena->is_committed); - if (arena->is_committed) { + *is_pinned = (arena->is_large || !arena->allow_decommit); + if (arena->blocks_committed == NULL) { // always committed *commit = true; } @@ -245,12 +245,13 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s return; } // potentially decommit - if (arena->is_committed) { - mi_assert_internal(all_committed); + if (!arena->allow_decommit || arena->blocks_committed == NULL) { + mi_assert_internal(all_committed); // note: may be not true as we may "pretend" to be not committed (in segment.c) } else { mi_assert_internal(arena->blocks_committed != NULL); _mi_os_decommit(p, blocks * MI_ARENA_BLOCK_SIZE, stats); // ok if this fails + // todo: use reset instead of decommit on windows? _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); } // and make it available to others again @@ -300,12 +301,16 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->is_large = is_large; arena->is_zero_init = is_zero; - arena->is_committed = is_committed; + arena->allow_decommit = !is_large && !is_committed; // only allow decommit for initially uncommitted memory arena->search_idx = 0; arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap - arena->blocks_committed = (is_committed ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap + arena->blocks_committed = (!arena->allow_decommit ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap // the bitmaps are already zero initialized due to os_alloc - // just claim leftover blocks if needed + // initialize committed bitmap? + if (arena->blocks_committed != NULL && is_committed) { + memset(arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); + } + // and claim leftover blocks if needed (so we never allocate there) ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; mi_assert_internal(post >= 0); if (post > 0) { From 10c31f9b41f5559e459dc30cf463cc750bfa67ac Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 19 Oct 2021 15:13:01 -0700 Subject: [PATCH 068/134] fix warnings --- src/stats.c | 4 ++-- test/test-api.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/stats.c b/src/stats.c index dda55cda..115d938e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -153,8 +153,8 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - char unitdesc[16]; - snprintf(unitdesc, 16, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + char unitdesc[8]; + snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); diff --git a/test/test-api.c b/test/test-api.c index 55c80431..f72cfaf5 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -4,6 +4,7 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#pragma GCC diagnostic ignored "-Walloc-size-larger-than=" /* Testing allocators is difficult as bugs may only surface after particular From 5834751ca6184e0cdebc5477ddaf98e290962318 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Wed, 20 Oct 2021 16:17:13 +0100 Subject: [PATCH 069/134] api test conceal gcc pragma --- test/test-api.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test-api.c b/test/test-api.c index f72cfaf5..8ddbf7cf 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -4,7 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic ignored "-Walloc-size-larger-than=" +#endif /* Testing allocators is difficult as bugs may only surface after particular From 4d89176eb4584c16a2b3141a97c1970cc1664014 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 20 Oct 2021 09:35:58 -0700 Subject: [PATCH 070/134] add MI_USE_LIBATOMIC to link with -latomic on older systems (see also PR #429); rename MI_INTERPOSE to MI_OSX_INTERPOSE --- CMakeLists.txt | 13 +++++++++---- src/alloc-override-osx.c | 2 +- src/alloc-override.c | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index abab5a05..b872e375 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) +option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) @@ -23,6 +23,7 @@ option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems)" OFF) include("cmake/mimalloc-config-version.cmake") @@ -78,11 +79,11 @@ if(MI_OVERRIDE) list(APPEND mi_sources src/alloc-override-osx.c) list(APPEND mi_defines MI_OSX_ZONE=1) endif() - if(MI_INTERPOSE) + if(MI_OSX_INTERPOSE) # use interpose on macOS - message(STATUS " Use interpose to override malloc (MI_INTERPOSE=ON)") + message(STATUS " Use interpose to override malloc (MI_OSX_INTERPOSE=ON)") message(STATUS " WARNING: interpose does not seem to work reliably on the M1; use -DMI_OSX_ZONE=ON instead") - list(APPEND mi_defines MI_INTERPOSE) + list(APPEND mi_defines MI_OSX_INTERPOSE) endif() endif() endif() @@ -203,6 +204,10 @@ else() endif() endif() +if (MI_USE_LIBATOMIC) + list(APPEND mi_libraries atomic) +endif() + # ----------------------------------------------------------------------------- # Install and output names # ----------------------------------------------------------------------------- diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index f506d30a..a2e341bb 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -229,7 +229,7 @@ static malloc_zone_t mi_malloc_zone = { }; -#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) +#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) static malloc_zone_t *mi_malloc_default_zone(void) { return &mi_malloc_zone; diff --git a/src/alloc-override.c b/src/alloc-override.c index 6a87e7bd..f97b6e78 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" #endif -#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_INTERPOSE))) +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_OSX_INTERPOSE))) // ------------------------------------------------------ // Override system malloc @@ -40,7 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif -#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: struct mi_interpose_s { From e02f88a11c31f33b0bdde1f0a46086b7901087b7 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 20 Oct 2021 09:55:03 -0700 Subject: [PATCH 071/134] Fix warnings with g++-11 compilation --- CMakeLists.txt | 5 ++++- src/arena.c | 2 +- src/region.c | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b872e375..c46ffdac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,7 +166,10 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -Wstrict-prototypes -fvisibility=hidden) + list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + if(NOT MI_USE_CXX) + list(APPEND mi_cflags -Wstrict-prototypes) + endif() if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() diff --git a/src/arena.c b/src/arena.c index 9d717e9e..5e883b28 100644 --- a/src/arena.c +++ b/src/arena.c @@ -310,7 +310,7 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la // the bitmaps are already zero initialized due to os_alloc // initialize committed bitmap? if (arena->blocks_committed != NULL && is_committed) { - memset(arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); + memset((void*)arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); // cast to void* to avoid atomic warning } // and claim leftover blocks if needed (so we never allocate there) ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; diff --git a/src/region.c b/src/region.c index 2f68b140..d9232450 100644 --- a/src/region.c +++ b/src/region.c @@ -463,7 +463,7 @@ void _mi_mem_collect(mi_os_tld_t* tld) { uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ions[i].start); size_t arena_memid = mi_atomic_load_relaxed(®ions[i].arena_memid); uintptr_t commit = mi_atomic_load_relaxed(®ions[i].commit); - memset(®ions[i], 0, sizeof(mem_region_t)); + memset((void*)®ions[i], 0, sizeof(mem_region_t)); // cast to void* to avoid atomic warning // and release the whole region mi_atomic_store_release(®ion->info, (uintptr_t)0); if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { From d482555675fb80a60ebf194a605661062a390091 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 20 Oct 2021 09:55:21 -0700 Subject: [PATCH 072/134] Fix warnings on osx with g++ compilation --- src/alloc-override-osx.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index a2e341bb..723ef226 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -189,6 +189,10 @@ static malloc_zone_t* mi_get_default_zone() } } +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + static malloc_introspection_t mi_introspect = { .enumerator = &intro_enumerator, .good_size = &intro_good_size, @@ -199,23 +203,23 @@ static malloc_introspection_t mi_introspect = { .force_unlock = &intro_force_unlock, #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - .zone_locked = &intro_zone_locked, .statistics = &intro_statistics, + .zone_locked = &intro_zone_locked, #endif }; static malloc_zone_t mi_malloc_zone = { .size = &zone_size, - .zone_name = "mimalloc", - .introspect = &mi_introspect, .malloc = &zone_malloc, .calloc = &zone_calloc, .valloc = &zone_valloc, .free = &zone_free, .realloc = &zone_realloc, .destroy = &zone_destroy, + .zone_name = "mimalloc", .batch_malloc = &zone_batch_malloc, .batch_free = &zone_batch_free, + .introspect = &mi_introspect, #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 // switch to version 9 on OSX 10.6 to support memalign. @@ -243,16 +247,22 @@ __attribute__((used)) static struct { }; #endif -static void __attribute__((constructor(0))) _mi_macos_override_malloc() { + +#if defined(__clang__) +__attribute__((constructor(0))) +#else +__attribute__((constructor)) // seems not supported by g++-11 on the M1 +#endif +static void _mi_macos_override_malloc() { malloc_zone_t* purgeable_zone = NULL; -#if defined(MAC_OS_X_VERSION_10_6) && \ + #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 // force the purgeable zone to exist to avoid strange bugs if (malloc_default_purgeable_zone) { purgeable_zone = malloc_default_purgeable_zone(); } -#endif + #endif // Register our zone. // thomcc: I think this is still needed to put us in the zone list. From b5b52b8879a0574bdd5b8c2b416a26c48080ef41 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Wed, 20 Oct 2021 18:33:12 +0000 Subject: [PATCH 073/134] Haiku build update, since the beta3 few more posix functions are available e.g. madvise --- src/os.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/os.c b/src/os.c index bf1d4716..3066dad5 100644 --- a/src/os.c +++ b/src/os.c @@ -51,10 +51,6 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif -#if defined(__HAIKU__) -#define madvise posix_madvise -#define MADV_DONTNEED POSIX_MADV_DONTNEED -#endif #endif /* ----------------------------------------------------------- From 45321237b53435e75b1d8545b6b37c40cddef898 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Thu, 21 Oct 2021 21:15:08 +0800 Subject: [PATCH 074/134] Fix ARM64 MSVC linker problem Close https://github.com/microsoft/mimalloc/issues/426 --- src/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index c088cada..a712f9f1 100644 --- a/src/init.c +++ b/src/init.c @@ -571,7 +571,7 @@ static void mi_process_done(void) { return 0; } typedef int(*_crt_cb)(void); - #ifdef _M_X64 + #if defined(_M_X64) || defined(_M_ARM64) __pragma(comment(linker, "/include:" "_mi_msvc_initu")) #pragma section(".CRT$XIU", long, read) #else From 13de1920ae7bf39021d6c131222b6b71330ed0c2 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 21 Oct 2021 21:39:28 +0200 Subject: [PATCH 075/134] Rename _os_random_weak to _mi_os_random_weak The ``_os_random_weak`` function is the only non-static function besides ``_ZSt15get_new_handlerv`` that is not prefixed with ``mi`` or ``_mi``. The discrepancy was discovered by CPython's smelly script. The checker looks for exported symbols that don't have well-defined prefixes. Signed-off-by: Christian Heimes --- include/mimalloc-internal.h | 2 +- src/init.c | 2 +- src/random.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 5a29c54d..0563d3de 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -50,7 +50,7 @@ void _mi_random_init(mi_random_ctx_t* ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); uintptr_t _mi_random_next(mi_random_ctx_t* ctx); uintptr_t _mi_heap_random_next(mi_heap_t* heap); -uintptr_t _os_random_weak(uintptr_t extra_seed); +uintptr_t _mi_os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c diff --git a/src/init.c b/src/init.c index c088cada..a41bcdd1 100644 --- a/src/init.c +++ b/src/init.c @@ -141,7 +141,7 @@ mi_stats_t _mi_stats_main = { MI_STATS_NULL }; static void mi_heap_main_init(void) { if (_mi_heap_main.cookie == 0) { _mi_heap_main.thread_id = _mi_thread_id(); - _mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init); + _mi_heap_main.cookie = _mi_os_random_weak((uintptr_t)&mi_heap_main_init); _mi_random_init(&_mi_heap_main.random); _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); diff --git a/src/random.c b/src/random.c index ce6db7c6..05c5c99c 100644 --- a/src/random.c +++ b/src/random.c @@ -257,8 +257,8 @@ static bool os_random_buf(void* buf, size_t buf_len) { #include #endif -uintptr_t _os_random_weak(uintptr_t extra_seed) { - uintptr_t x = (uintptr_t)&_os_random_weak ^ extra_seed; // ASLR makes the address random +uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { + uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random #if defined(_WIN32) LARGE_INTEGER pcount; @@ -289,7 +289,7 @@ void _mi_random_init(mi_random_ctx_t* ctx) { #if !defined(__wasi__) _mi_warning_message("unable to use secure randomness\n"); #endif - uintptr_t x = _os_random_weak(0); + uintptr_t x = _mi_os_random_weak(0); for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words. x = _mi_random_shuffle(x); ((uint32_t*)key)[i] = (uint32_t)x; From 898a23ec2b4c7974d9381312f5ff01b78747a8d5 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 22 Oct 2021 08:12:14 +0100 Subject: [PATCH 076/134] fixes malloc_usable_size signature on FreeBSD. --- src/alloc-override.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index f97b6e78..af9035dd 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -173,7 +173,7 @@ extern "C" { void cfree(void* p) MI_FORWARD0(mi_free, p) void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) -#if !defined(__ANDROID__) +#if !defined(__ANDROID__) && !defined(__FreeBSD__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) From fc7777ee8c7538fa181ae52c621c09edc11d46b7 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 22 Oct 2021 10:08:16 +0100 Subject: [PATCH 077/134] NUMA base detection on FreeBSD. --- src/os.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/os.c b/src/os.c index 3066dad5..f15a2197 100644 --- a/src/os.c +++ b/src/os.c @@ -51,6 +51,14 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif +#if defined(__FreeBSD__) +#include +#if __FreeBSD_version >= 1200000 +#include +#include +#include +#endif +#endif #endif /* ----------------------------------------------------------- @@ -1236,6 +1244,29 @@ static size_t mi_os_numa_node_countx(void) { } return (node+1); } +#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 +static size_t mi_os_numa_nodex(void) { + domainset_t dom; + size_t node; + int policy; + + if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; + + for (node = 0; node < MAXMEMDOM; node++) { + if (DOMAINSET_ISSET(node, &dom)) return node; + } + + return 0ul; +} + +static size_t mi_os_numa_node_countx(void) { + size_t ndomains = 0; + size_t len = sizeof(ndomains); + + if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; + + return ndomains; +} #else static size_t mi_os_numa_nodex(void) { return 0; From 2d2d9af5c6e2473b8aa6b651b0dad7d67b8957f8 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 24 Oct 2021 10:57:40 +0100 Subject: [PATCH 078/134] while at it, doing dragonflybsd too --- src/os.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index f15a2197..a4b9602f 100644 --- a/src/os.c +++ b/src/os.c @@ -51,13 +51,13 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__DragonFly__) #include #if __FreeBSD_version >= 1200000 #include #include -#include #endif +#include #endif #endif @@ -1267,6 +1267,24 @@ static size_t mi_os_numa_node_countx(void) { return ndomains; } +#elif defined(__DragonFly__) +static size_t mi_os_numa_nodex(void) { + // TODO DragonFlyBSD does not seem to provide any userland mean to + // check this information, even less the possibility to control + // the allocation to a logical core level's granularity, only the kernel + // is fully NUMA aware at the moment. + return 0ul; +} + +static size_t mi_os_numa_node_countx(void) { + size_t ncpus = 0, nvirtcoresperphys = 0; + size_t len = sizeof(size_t); + + if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; + if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; + + return nvirtcoresperphys * ncpus; +} #else static size_t mi_os_numa_nodex(void) { return 0; From 00edd9201bad722eabc5066c9b8ee8622c0f0d8c Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 24 Oct 2021 19:49:11 +0100 Subject: [PATCH 079/134] proposal to not overcommit page unconditionally on Linux and FreeBSD, respecting userland settings. --- src/os.c | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 3066dad5..e40d57a9 100644 --- a/src/os.c +++ b/src/os.c @@ -39,6 +39,8 @@ terms of the MIT license. A copy of the license can be found in the file #include // sysconf #if defined(__linux__) #include +#include +#include #if defined(__GLIBC__) #include // linux mmap flags #else @@ -51,6 +53,9 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif +#if defined(__FreeBSD__) +#include +#endif #endif /* ----------------------------------------------------------- @@ -88,6 +93,9 @@ static size_t os_alloc_granularity = 4096; // if non-zero, use large page allocation static size_t large_os_page_size = 0; +// if non-zero, enable page overcommit +static int os_overcommit = 0; + // OS (small) page size size_t _mi_os_page_size() { return os_page_size; @@ -223,6 +231,31 @@ void _mi_os_init() { os_alloc_granularity = os_page_size; } large_os_page_size = 2*MiB; // TODO: can we query the OS for this? +#if defined(__linux__) + int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd != -1) { + char buf[1] = {0}; + if (read(fd, buf, sizeof(buf)) == sizeof(buf)) { + switch (buf[0]) { + case '0': // heuristic mode + case '1': // always + os_overcommit = 1; + break; + default: // never regardless + os_overcommit = 0; + } + } + close(fd); + } +#elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = val; + } +#elif defined(__NetBSD__) || defined(__HAIKU__) + os_overcommit = 1; +#endif } #endif @@ -398,8 +431,11 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #if !defined(MAP_NORESERVE) #define MAP_NORESERVE 0 #endif - int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + int flags = MAP_PRIVATE | MAP_ANONYMOUS; int fd = -1; + if (os_overcommit) { + flags |= MAP_NORESERVE; + } #if defined(MAP_ALIGNED) // BSD if (try_alignment > 0) { size_t n = mi_bsr(try_alignment); From ca9785d40e8c225f5176f2dfba502ab351da8d9c Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 10:06:34 -0700 Subject: [PATCH 080/134] fix compilation with MI_DEBUG>3, issue #480 --- src/page.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/page.c b/src/page.c index 033976dc..7efbcb8d 100644 --- a/src/page.c +++ b/src/page.c @@ -84,9 +84,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_list_is_valid(page,page->local_free)); #if MI_DEBUG>3 // generally too expensive to check this - if (page->flags.is_zero) { - for(mi_block_t* block = page->free; block != NULL; mi_block_next(page,block)) { - mi_assert_expensive(mi_mem_is_zero(block + 1, page->block_size - sizeof(mi_block_t))); + if (page->is_zero) { + const size_t ubsize = mi_page_usable_block_size(page); + for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); } } #endif From f38956568c07d0fe31111f141652954ccbb3635c Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:01:12 -0700 Subject: [PATCH 081/134] only set visibility attribute when compiling for a shared library --- include/mimalloc.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 7ebf3e60..fe0fd746 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -58,8 +58,12 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #define mi_attr_alloc_align(p) #elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif #define mi_cdecl // leads to warnings... __attribute__((cdecl)) - #define mi_decl_export __attribute__((visibility("default"))) #define mi_decl_restrict #define mi_attr_malloc __attribute__((malloc)) #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) From 6ccf7e87c0aa62154f63054a905842bb93d52e3f Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:06:10 -0700 Subject: [PATCH 082/134] minor edits --- src/os.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/os.c b/src/os.c index a4b9602f..73ef462e 100644 --- a/src/os.c +++ b/src/os.c @@ -1249,40 +1249,28 @@ static size_t mi_os_numa_nodex(void) { domainset_t dom; size_t node; int policy; - if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; - for (node = 0; node < MAXMEMDOM; node++) { - if (DOMAINSET_ISSET(node, &dom)) return node; + if (DOMAINSET_ISSET(node, &dom)) return node; } - return 0ul; } - static size_t mi_os_numa_node_countx(void) { size_t ndomains = 0; size_t len = sizeof(ndomains); - if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; - return ndomains; } #elif defined(__DragonFly__) static size_t mi_os_numa_nodex(void) { - // TODO DragonFlyBSD does not seem to provide any userland mean to - // check this information, even less the possibility to control - // the allocation to a logical core level's granularity, only the kernel - // is fully NUMA aware at the moment. + // TODO: DragonFly does not seem to provide any userland means to get this information. return 0ul; } - static size_t mi_os_numa_node_countx(void) { size_t ncpus = 0, nvirtcoresperphys = 0; size_t len = sizeof(size_t); - if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; - return nvirtcoresperphys * ncpus; } #else From fdcdc4cf2aadefce1f7e768cc933508ac88420c4 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:01:12 -0700 Subject: [PATCH 083/134] only set visibility attribute when compiling for a shared library (issue #475) --- include/mimalloc.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 7ebf3e60..fe0fd746 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -58,8 +58,12 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #define mi_attr_alloc_align(p) #elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif #define mi_cdecl // leads to warnings... __attribute__((cdecl)) - #define mi_decl_export __attribute__((visibility("default"))) #define mi_decl_restrict #define mi_attr_malloc __attribute__((malloc)) #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) From 5fd77aa1986305a30680f6b62a27b58940dfbfd0 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:39:11 -0700 Subject: [PATCH 084/134] refactor os_overcommit detection --- src/os.c | 62 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/src/os.c b/src/os.c index b8c3f603..507d9971 100644 --- a/src/os.c +++ b/src/os.c @@ -40,7 +40,6 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__linux__) #include #include -#include #if defined(__GLIBC__) #include // linux mmap flags #else @@ -98,8 +97,9 @@ static size_t os_alloc_granularity = 4096; // if non-zero, use large page allocation static size_t large_os_page_size = 0; -// if non-zero, enable page overcommit -static int os_overcommit = 0; +// is memory overcommit allowed? +// set dynamically in _mi_os_init (and if true we use MAP_NORESERVE) +static bool os_overcommit = true; // OS (small) page size size_t _mi_os_page_size() { @@ -189,7 +189,9 @@ static bool mi_win_enable_large_os_pages() return (ok!=0); } -void _mi_os_init(void) { +void _mi_os_init(void) +{ + os_overcommit = false; // get the page size SYSTEM_INFO si; GetSystemInfo(&si); @@ -224,10 +226,36 @@ void _mi_os_init(void) { } #elif defined(__wasi__) void _mi_os_init() { + os_overcommit = false; os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } + +#else // generic unix + +static void os_detect_overcommit(void) { +#if defined(__linux__) + int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd < 0) return; + char buf[128]; + ssize_t nread = read(fd, &buf, sizeof(buf)); + close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } +#elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } #else + // default: overcommit is true +#endif +} + void _mi_os_init() { // get the page size long result = sysconf(_SC_PAGESIZE); @@ -236,31 +264,7 @@ void _mi_os_init() { os_alloc_granularity = os_page_size; } large_os_page_size = 2*MiB; // TODO: can we query the OS for this? -#if defined(__linux__) - int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd != -1) { - char buf[1] = {0}; - if (read(fd, buf, sizeof(buf)) == sizeof(buf)) { - switch (buf[0]) { - case '0': // heuristic mode - case '1': // always - os_overcommit = 1; - break; - default: // never regardless - os_overcommit = 0; - } - } - close(fd); - } -#elif defined(__FreeBSD__) - int val = 0; - size_t olen = sizeof(val); - if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { - os_overcommit = val; - } -#elif defined(__NetBSD__) || defined(__HAIKU__) - os_overcommit = 1; -#endif + os_detect_overcommit(); } #endif From 1b1c9186a4a62c959140f437fa5e745dd159e854 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 27 Oct 2021 19:06:27 -0700 Subject: [PATCH 085/134] suppress warning on discard attribute on forwards --- src/alloc-override.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/alloc-override.c b/src/alloc-override.c index af9035dd..6c42ae00 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -22,6 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) + #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); #else #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); From 0ead911b6b1e1b5a531a156587c5d011f52f47e8 Mon Sep 17 00:00:00 2001 From: Yaroslav Syrytsia Date: Tue, 28 Sep 2021 20:34:56 +0300 Subject: [PATCH 086/134] cmake: removed hardcoded names for top level configuration --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..9ba14a38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clan option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") set(mi_sources @@ -208,9 +209,9 @@ endif() # ----------------------------------------------------------------------------- if (MI_INSTALL_TOPLEVEL) - set(mi_install_libdir "lib") - set(mi_install_incdir "include") - set(mi_install_cmakedir "cmake") + set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") + set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") + set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc") else() set(mi_install_libdir "lib/mimalloc-${mi_version}") set(mi_install_incdir "include/mimalloc-${mi_version}") From 5a5e4e303629d49bc09a2ef2614c1e0374c23dfb Mon Sep 17 00:00:00 2001 From: sblondon Date: Thu, 28 Oct 2021 15:01:36 +0200 Subject: [PATCH 087/134] Fix preload path in documentation example The .so files are usually in `/usr/lib`, not `/usr/bin`. The updated path is the same as below in the text. --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index cdb1b82a..cf984c6b 100644 --- a/readme.md +++ b/readme.md @@ -18,7 +18,7 @@ Latest stable tag: `v1.7.2` (2021-06-17). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: ``` -> LD_PRELOAD=/usr/bin/libmimalloc.so myprogram +> LD_PRELOAD=/usr/lib/libmimalloc.so myprogram ``` It also has an easy way to override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: From 5b0e73281fd4d98b6f211cdd1b6b29453c3e420e Mon Sep 17 00:00:00 2001 From: dc Date: Thu, 28 Oct 2021 22:43:21 +0100 Subject: [PATCH 088/134] fix spurious build warning with overflow builtins --- include/mimalloc-internal.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 0563d3de..982928cb 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -247,11 +247,11 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #if (SIZE_MAX == ULONG_MAX) - return __builtin_umull_overflow(count, size, total); + return __builtin_umull_overflow(count, size, (unsigned long *)total); #elif (SIZE_MAX == UINT_MAX) - return __builtin_umul_overflow(count, size, total); + return __builtin_umul_overflow(count, size, (unsigned int *)total); #else - return __builtin_umulll_overflow(count, size, total); + return __builtin_umulll_overflow(count, size, (unsigned long long *)total); #endif } #else /* __builtin_umul_overflow is unavailable */ From d0d47e6489dd5e9214091db9aed293310f7c8670 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:49:05 -0700 Subject: [PATCH 089/134] avoid accessing the thread_id after process initialized is set --- src/init.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index ef824e3a..ef44d273 100644 --- a/src/init.c +++ b/src/init.c @@ -102,6 +102,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { false }; + // the thread-local default heap for allocation mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; @@ -478,10 +479,11 @@ static void mi_detect_cpu_features(void) { void mi_process_init(void) mi_attr_noexcept { // ensure we are called once if (_mi_process_is_initialized) return; - _mi_process_is_initialized = true; mi_process_setup_auto_thread_done(); _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + _mi_process_is_initialized = true; + mi_detect_cpu_features(); _mi_os_init(); mi_heap_main_init(); From dbf8bafc3ae66ebb6e356b3e8a7485848e12b5ef Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:50:15 -0700 Subject: [PATCH 090/134] avoid accessing the thread_id after process initialized is set --- src/init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/init.c b/src/init.c index ef44d273..cb9d123f 100644 --- a/src/init.c +++ b/src/init.c @@ -479,10 +479,10 @@ static void mi_detect_cpu_features(void) { void mi_process_init(void) mi_attr_noexcept { // ensure we are called once if (_mi_process_is_initialized) return; - mi_process_setup_auto_thread_done(); - _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); _mi_process_is_initialized = true; + mi_process_setup_auto_thread_done(); + mi_detect_cpu_features(); _mi_os_init(); From 74efd6ddac0215c3b4d54c13563765a97b4e971e Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:50:54 -0700 Subject: [PATCH 091/134] use load relaxed in mi_free for better code on M1 --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 1af6650a..c91dcd92 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -486,7 +486,7 @@ void mi_free(void* p) mi_attr_noexcept mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; - if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks + if (mi_likely(tid == mi_atomic_load_relaxed(&segment->thread_id) && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned if (mi_unlikely(mi_check_is_double_free(page,block))) return; mi_check_padding(page, block); From 5360639748d8bfc7f7e97728e859a6380ac7db9f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:52:00 -0700 Subject: [PATCH 092/134] add mi_malloc_good_size --- include/mimalloc.h | 1 + src/alloc-posix.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/include/mimalloc.h b/include/mimalloc.h index fe0fd746..a77f1dc5 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -346,6 +346,7 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 43931e56..cff01b5c 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -33,13 +33,19 @@ terms of the MIT license. A copy of the license can be found in the file size_t mi_malloc_size(const void* p) mi_attr_noexcept { + //if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); } size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + //if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); } +size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { + return mi_good_size(size); +} + void mi_cfree(void* p) mi_attr_noexcept { if (mi_is_in_heap_region(p)) { mi_free(p); From 751a2249c809735629491d07f1994522e2bd6a84 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:53:20 -0700 Subject: [PATCH 093/134] add mi_decl_externc, nice layout --- include/mimalloc-internal.h | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 0563d3de..e8046528 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -36,6 +36,12 @@ terms of the MIT license. A copy of the license can be found in the file #define __wasi__ #endif +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#else +#define mi_decl_externc +#endif + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -299,6 +305,7 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #if defined(MI_MALLOC_OVERRIDE) #if defined(__APPLE__) // macOS #define MI_TLS_SLOT 89 // seems unused? +// #define MI_TLS_RECURSE_GUARD 1 // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) // see #elif defined(__OpenBSD__) @@ -336,10 +343,12 @@ extern pthread_key_t _mi_heap_default_key; // However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356). extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from + static inline mi_heap_t* mi_get_default_heap(void) { #if defined(MI_TLS_SLOT) mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT); - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); + if (mi_unlikely(heap == NULL)) { heap = (mi_heap_t*)&_mi_heap_empty; } //_mi_heap_empty_get(); } + return heap; #elif defined(MI_TLS_PTHREAD_SLOT_OFS) mi_heap_t* heap = *mi_tls_pthread_heap_slot(); return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); @@ -347,7 +356,7 @@ static inline mi_heap_t* mi_get_default_heap(void) { mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); #else - #if defined(MI_TLS_RECURSE_GUARD) + #if defined(MI_TLS_RECURSE_GUARD) if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); #endif return _mi_heap_default; @@ -716,12 +725,12 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { res = tcb[slot]; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); -#if defined(__APPLE__) // M1, issue #343 + #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits -#else + #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); -#endif + #endif res = tcb[slot]; #endif return res; @@ -744,12 +753,12 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { tcb[slot] = value; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); -#if defined(__APPLE__) // M1, issue #343 + #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits -#else + #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); -#endif + #endif tcb[slot] = value; #endif } From 32ee13f41eafb7de541438279100c28ed9df437f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:54:44 -0700 Subject: [PATCH 094/134] improve macOS M1 performance; use interpose in combination with zone's; add -fno-builtin-malloc flag in building with MI_OVERRIDE --- CMakeLists.txt | 14 ++- src/alloc-override-osx.c | 234 ++++++++++++++++++++++++++++++++------- src/alloc-override.c | 25 +++-- 3 files changed, 223 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c46ffdac..f07d892f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) +option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) @@ -78,12 +78,17 @@ if(MI_OVERRIDE) message(STATUS " Use malloc zone to override malloc (MI_OSX_ZONE=ON)") list(APPEND mi_sources src/alloc-override-osx.c) list(APPEND mi_defines MI_OSX_ZONE=1) + if (NOT MI_OSX_INTERPOSE) + message(STATUS " WARNING: zone overriding usually also needs interpose (use -DMI_OSX_INTERPOSE=ON)") + endif() endif() if(MI_OSX_INTERPOSE) # use interpose on macOS message(STATUS " Use interpose to override malloc (MI_OSX_INTERPOSE=ON)") - message(STATUS " WARNING: interpose does not seem to work reliably on the M1; use -DMI_OSX_ZONE=ON instead") list(APPEND mi_defines MI_OSX_INTERPOSE) + if (NOT MI_OSX_ZONE) + message(STATUS " WARNING: interpose usually also needs zone overriding (use -DMI_OSX_INTERPOSE=ON)") + endif() endif() endif() endif() @@ -188,6 +193,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM else() list(APPEND mi_cflags -ftls-model=initial-exec) endif() + if(MI_OVERRIDE) + list(APPEND -fno-builtin-malloc) + endif() endif() if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) @@ -364,7 +372,7 @@ if (MI_BUILD_TESTS) target_compile_definitions(mimalloc-test-api PRIVATE ${mi_defines}) target_compile_options(mimalloc-test-api PRIVATE ${mi_cflags}) target_include_directories(mimalloc-test-api PRIVATE include) - target_link_libraries(mimalloc-test-api PRIVATE mimalloc-static ${mi_libraries}) + target_link_libraries(mimalloc-test-api PRIVATE mimalloc ${mi_libraries}) add_executable(mimalloc-test-stress test/test-stress.c) target_compile_definitions(mimalloc-test-stress PRIVATE ${mi_defines}) diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 723ef226..0759f785 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -17,17 +17,20 @@ terms of the MIT license. A copy of the license can be found in the file /* ------------------------------------------------------ Override system malloc on macOS This is done through the malloc zone interface. - It seems we also need to interpose (see `alloc-override.c`) - or otherwise we get zone errors as there are usually - already allocations done by the time we take over the - zone. Unfortunately, that means we need to replace - the `free` with a checked free (`cfree`) impacting - performance. + It seems to be most robust in combination with interposing + though or otherwise we may get zone errors as there are could + be allocations done by the time we take over the + zone. ------------------------------------------------------ */ #include #include #include // memset +#include + +#ifdef __cplusplus +extern "C" { +#endif #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 @@ -41,9 +44,7 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im static size_t zone_size(malloc_zone_t* zone, const void* p) { UNUSED(zone); - if (!mi_is_in_heap_region(p)) - return 0; // not our pointer, bail out - + //if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out return mi_usable_size(p); } @@ -109,6 +110,11 @@ static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { zone_free(zone,p); } +static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { + UNUSED(zone); + return mi_is_in_heap_region(p); +} + /* ------------------------------------------------------ Introspection members @@ -174,21 +180,6 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) { At process start, override the default allocator ------------------------------------------------------ */ -static malloc_zone_t* mi_get_default_zone() -{ - // The first returned zone is the real default - malloc_zone_t** zones = NULL; - unsigned count = 0; - kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); - if (ret == KERN_SUCCESS && count > 0) { - return zones[0]; - } - else { - // fallback - return malloc_default_zone(); - } -} - #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #endif @@ -220,33 +211,197 @@ static malloc_zone_t mi_malloc_zone = { .batch_malloc = &zone_batch_malloc, .batch_free = &zone_batch_free, .introspect = &mi_introspect, -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - // switch to version 9 on OSX 10.6 to support memalign. - .version = 9, +#if defined(MAC_OS_X_VERSION_10_6) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 + // switch to version 9+ on OSX 10.6 to support memalign. .memalign = &zone_memalign, .free_definite_size = &zone_free_definite_size, .pressure_relief = &zone_pressure_relief, + #if defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7 + .claimed_address = &zone_claimed_address, + .version = 10 + #else + .version = 9 + #endif #else - .version = 4, + .version = 4 #endif }; +#ifdef __cplusplus +} +#endif -#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) -static malloc_zone_t *mi_malloc_default_zone(void) { +#if defined(MI_OSX_INTERPOSE) + +// ------------------------------------------------------ +// Override malloc_xxx and zone_xxx api's to use only +// our mimalloc zone. Since even the loader uses malloc +// on macOS, this ensures that all allocations go through +// mimalloc (as all calls are interposed). +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + static bool init; + if (mi_unlikely(!init)) { + init = true; + malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see ) + } return &mi_malloc_zone; } -// TODO: should use the macros in alloc-override but they aren't available here. -__attribute__((used)) static struct { - const void *replacement; - const void *target; -} replace_malloc_default_zone[] __attribute__((section("__DATA, __interpose"))) = { - { (const void*)mi_malloc_default_zone, (const void*)malloc_default_zone }, -}; -#endif +mi_decl_externc int malloc_jumpstart(uintptr_t cookie); +mi_decl_externc void _malloc_fork_prepare(void); +mi_decl_externc void _malloc_fork_parent(void); +mi_decl_externc void _malloc_fork_child(void); + + +static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { + UNUSED(size); UNUSED(flags); + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_zone (void) { + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { + return mi_get_default_zone(); +} + +static void mi_malloc_destroy_zone(malloc_zone_t* zone) { + UNUSED(zone); + // nothing. +} + +static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { + UNUSED(task); UNUSED(mr); + if (addresses != NULL) *addresses = NULL; + if (count != NULL) *count = 0; + return KERN_SUCCESS; +} + +static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { + return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name); +} + +static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { + UNUSED(zone); UNUSED(name); +} + +static int mi_malloc_jumpstart(uintptr_t cookie) { + UNUSED(cookie); + return 1; // or 0 for no error? +} + +static void mi__malloc_fork_prepare(void) { + // nothing +} +static void mi__malloc_fork_parent(void) { + // nothing +} +static void mi__malloc_fork_child(void) { + // nothing +} + +static void mi_malloc_printf(const char* fmt, ...) { + UNUSED(fmt); +} + +static bool zone_check(malloc_zone_t* zone) { + UNUSED(zone); + return true; +} + +static malloc_zone_t* zone_from_ptr(const void* p) { + UNUSED(p); + return mi_get_default_zone(); +} + +static void zone_log(malloc_zone_t* zone, void* p) { + UNUSED(zone); UNUSED(p); +} + +static void zone_print(malloc_zone_t* zone, bool b) { + UNUSED(zone); UNUSED(b); +} + +static void zone_print_ptr_info(void* p) { + UNUSED(p); +} + +static void zone_register(malloc_zone_t* zone) { + UNUSED(zone); +} + +static void zone_unregister(malloc_zone_t* zone) { + UNUSED(zone); +} + +// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` +// See: +struct mi_interpose_s { + const void* replacement; + const void* target; +}; +#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } +#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) +#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) +__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = +{ + + MI_INTERPOSE_MI(malloc_create_zone), + MI_INTERPOSE_MI(malloc_default_purgeable_zone), + MI_INTERPOSE_MI(malloc_default_zone), + MI_INTERPOSE_MI(malloc_destroy_zone), + MI_INTERPOSE_MI(malloc_get_all_zones), + MI_INTERPOSE_MI(malloc_get_zone_name), + MI_INTERPOSE_MI(malloc_jumpstart), + MI_INTERPOSE_MI(malloc_printf), + MI_INTERPOSE_MI(malloc_set_zone_name), + MI_INTERPOSE_MI(_malloc_fork_child), + MI_INTERPOSE_MI(_malloc_fork_parent), + MI_INTERPOSE_MI(_malloc_fork_prepare), + + MI_INTERPOSE_ZONE(zone_batch_free), + MI_INTERPOSE_ZONE(zone_batch_malloc), + MI_INTERPOSE_ZONE(zone_calloc), + MI_INTERPOSE_ZONE(zone_check), + MI_INTERPOSE_ZONE(zone_free), + MI_INTERPOSE_ZONE(zone_from_ptr), + MI_INTERPOSE_ZONE(zone_log), + MI_INTERPOSE_ZONE(zone_malloc), + MI_INTERPOSE_ZONE(zone_memalign), + MI_INTERPOSE_ZONE(zone_print), + MI_INTERPOSE_ZONE(zone_print_ptr_info), + MI_INTERPOSE_ZONE(zone_realloc), + MI_INTERPOSE_ZONE(zone_register), + MI_INTERPOSE_ZONE(zone_unregister), + MI_INTERPOSE_ZONE(zone_valloc) +}; + + +#else + +// ------------------------------------------------------ +// hook into the zone api's without interposing +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + // The first returned zone is the real default + malloc_zone_t** zones = NULL; + unsigned count = 0; + kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); + if (ret == KERN_SUCCESS && count > 0) { + return zones[0]; + } + else { + // fallback + return malloc_default_zone(); + } +} #if defined(__clang__) __attribute__((constructor(0))) @@ -287,5 +442,6 @@ static void _mi_macos_override_malloc() { } } +#endif // MI_OSX_INTERPOSE #endif // MI_MALLOC_OVERRIDE diff --git a/src/alloc-override.c b/src/alloc-override.c index 6c42ae00..123f5e07 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" #endif -#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_OSX_INTERPOSE))) +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // ------------------------------------------------------ // Override system malloc @@ -41,7 +41,12 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif -#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) + #include + mi_decl_externc void vfree(void* p); + mi_decl_externc size_t malloc_size(const void* p); + mi_decl_externc size_t malloc_good_size(size_t size); + // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: struct mi_interpose_s { @@ -50,6 +55,7 @@ terms of the MIT license. A copy of the license can be found in the file }; #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) + __attribute__((used)) static struct mi_interpose_s _mi_interposes[] __attribute__((section("__DATA, __interpose"))) = { MI_INTERPOSE_MI(malloc), @@ -61,21 +67,24 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(posix_memalign), MI_INTERPOSE_MI(reallocf), MI_INTERPOSE_MI(valloc), + MI_INTERPOSE_MI(malloc_size), + MI_INTERPOSE_MI(malloc_good_size), + MI_INTERPOSE_MI(aligned_alloc), #ifndef MI_OSX_ZONE - // some code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + MI_INTERPOSE_FUN(vfree,mi_cfree), #else - // We interpose malloc_default_zone in alloc-override-osx.c + // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely MI_INTERPOSE_MI(free), + MI_INTERPOSE_FUN(vfree,mi_free), #endif - // some code allocates from a zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) - MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us }; #elif defined(_MSC_VER) // cannot override malloc unless using a dll. // we just override new/delete which does work in a static library. #else - // On all other systems forward to our API + // On all other systems forward to our API void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) @@ -123,7 +132,7 @@ terms of the MIT license. A copy of the license can be found in the file void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } #endif -#elif (defined(__GNUC__) || defined(__clang__)) +#elif (defined(__GNUC__) || defined(__clang__)) && !defined(MI_OSX_ZONE) // ------------------------------------------------------ // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). From e853f530a0d90ae49caf17c994a61febbe80230a Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 4 Nov 2021 18:54:57 -0700 Subject: [PATCH 095/134] add noexcept attributes to improve mi_free codegen --- include/mimalloc-internal.h | 6 +++--- src/alloc.c | 2 +- src/page.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index e8046528..5c716a79 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -37,9 +37,9 @@ terms of the MIT license. A copy of the license can be found in the file #endif #if defined(__cplusplus) -#define mi_decl_externc extern "C" +#define mi_decl_externc extern "C" #else -#define mi_decl_externc +#define mi_decl_externc #endif // "options.c" @@ -100,7 +100,7 @@ void _mi_abandoned_await_readers(void); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc; -void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... diff --git a/src/alloc.c b/src/alloc.c index c91dcd92..d03207ce 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -435,7 +435,7 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p } -static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) { +static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) mi_attr_noexcept { mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); mi_stat_free(page, block); diff --git a/src/page.c b/src/page.c index 7efbcb8d..61f3245a 100644 --- a/src/page.c +++ b/src/page.c @@ -386,7 +386,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // Note: called from `mi_free` and benchmarks often // trigger this due to freeing everything and then // allocating again so careful when changing this. -void _mi_page_retire(mi_page_t* page) { +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_assert_internal(page != NULL); mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_internal(mi_page_all_free(page)); From c47de7eb9090cf8c0a5f307a74437cf927a361be Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 4 Nov 2021 18:55:12 -0700 Subject: [PATCH 096/134] improve macOS overriding --- CMakeLists.txt | 11 ++- src/alloc-override-osx.c | 9 ++- src/alloc-override.c | 145 +++++++++++++++++++++++++-------------- 3 files changed, 108 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f07d892f..ed5fb951 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ set(mi_sources src/options.c src/init.c) + # ----------------------------------------------------------------------------- # Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- @@ -62,6 +63,7 @@ if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") set(MI_SECURE "ON") endif() + # ----------------------------------------------------------------------------- # Process options # ----------------------------------------------------------------------------- @@ -85,11 +87,14 @@ if(MI_OVERRIDE) if(MI_OSX_INTERPOSE) # use interpose on macOS message(STATUS " Use interpose to override malloc (MI_OSX_INTERPOSE=ON)") - list(APPEND mi_defines MI_OSX_INTERPOSE) + list(APPEND mi_defines MI_OSX_INTERPOSE=1) if (NOT MI_OSX_ZONE) message(STATUS " WARNING: interpose usually also needs zone overriding (use -DMI_OSX_INTERPOSE=ON)") endif() endif() + if((NOT MI_USE_CXX) AND MI_OVERRIDE) + message(STATUS " WARNING: if overriding C++ new/delete, it is best to build mimalloc with a C++ compiler (use -DMI_USE_CXX=ON)") + endif() endif() endif() @@ -260,9 +265,9 @@ message(STATUS "") message(STATUS "Library base name: ${mi_basename}") message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") if(MI_USE_CXX) - message(STATUS "Compiler : ${CMAKE_CXX_COMPILER}") + message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() - message(STATUS "Compiler : ${CMAKE_C_COMPILER}") + message(STATUS "C Compiler : ${CMAKE_C_COMPILER}") endif() message(STATUS "Version : ${mi_version}") message(STATUS "Build targets : ${mi_build_targets}") diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 0759f785..35b95a24 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -232,13 +232,16 @@ static malloc_zone_t mi_malloc_zone = { #endif -#if defined(MI_OSX_INTERPOSE) +#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT) // ------------------------------------------------------ -// Override malloc_xxx and zone_xxx api's to use only +// Override malloc_xxx and malloc_zone_xxx api's to use only // our mimalloc zone. Since even the loader uses malloc // on macOS, this ensures that all allocations go through // mimalloc (as all calls are interposed). +// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, +// Here, we also override macOS specific API's like +// `malloc_zone_calloc` etc. see // ------------------------------------------------------ static inline malloc_zone_t* mi_get_default_zone(void) @@ -386,6 +389,8 @@ __attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] // ------------------------------------------------------ // hook into the zone api's without interposing +// This is the official way of adding an allocator but +// it seems less robust than using interpose. // ------------------------------------------------------ static inline malloc_zone_t* mi_get_default_zone(void) diff --git a/src/alloc-override.c b/src/alloc-override.c index 123f5e07..9581a185 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -15,12 +15,21 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) +#if defined(__APPLE__) +mi_decl_externc void vfree(void* p); +mi_decl_externc size_t malloc_size(const void* p); +mi_decl_externc size_t malloc_good_size(size_t size); +#endif + +// helper definition for C override of C++ new +typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; + // ------------------------------------------------------ // Override system malloc // ------------------------------------------------------ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) - // use aliasing to alias the exported function to one of our `mi_` functions +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !defined(MI_VALGRIND) + // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); @@ -33,7 +42,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD0(fun,x) MI_FORWARD(fun) #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) #else - // use forwarding by calling our `mi_` function + // otherwise use forwarding by calling our `mi_` function #define MI_FORWARD1(fun,x) { return fun(x); } #define MI_FORWARD2(fun,x,y) { return fun(x,y); } #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } @@ -41,11 +50,10 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif -#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) - #include - mi_decl_externc void vfree(void* p); - mi_decl_externc size_t malloc_size(const void* p); - mi_decl_externc size_t malloc_good_size(size_t size); +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) + // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for + // functions that are interposed (or the interposing does not work) + #define MI_OSX_IS_INTERPOSED // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: @@ -70,16 +78,41 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(malloc_size), MI_INTERPOSE_MI(malloc_good_size), MI_INTERPOSE_MI(aligned_alloc), - #ifndef MI_OSX_ZONE - // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) - MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us - MI_INTERPOSE_FUN(vfree,mi_cfree), - #else + #ifdef MI_OSX_ZONE // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely MI_INTERPOSE_MI(free), MI_INTERPOSE_FUN(vfree,mi_free), + #else + // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + MI_INTERPOSE_FUN(vfree,mi_cfree), #endif }; + + #ifdef __cplusplus + extern "C" { + void _ZdlPv(void* p); // delete + void _ZdaPv(void* p); // delete[] + void _ZdlPvm(void* p, size_t n); // delete + void _ZdaPvm(void* p, size_t n); // delete[] + void* _Znwm(size_t n); // new + void* _Znam(size_t n); // new[] + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow + } + __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) = + { + MI_INTERPOSE_FUN(_ZdlPv,mi_free), + MI_INTERPOSE_FUN(_ZdaPv,mi_free), + MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size), + MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size), + MI_INTERPOSE_FUN(_Znwm,mi_new), + MI_INTERPOSE_FUN(_Znam,mi_new), + MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow), + MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow), + }; + #endif // __cplusplus + #elif defined(_MSC_VER) // cannot override malloc unless using a dll. // we just override new/delete which does work in a static library. @@ -106,18 +139,21 @@ terms of the MIT license. A copy of the license can be found in the file // see // ------------------------------------------------------ #include - void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) - void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + #ifndef MI_OSX_IS_INTERPOSED + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) - void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } - void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) - #if (__cplusplus >= 201402L || _MSC_VER >= 1916) - void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) - void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + #endif #endif #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) @@ -132,12 +168,13 @@ terms of the MIT license. A copy of the license can be found in the file void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } #endif -#elif (defined(__GNUC__) || defined(__clang__)) && !defined(MI_OSX_ZONE) +#elif (defined(__GNUC__) || defined(__clang__)) // ------------------------------------------------------ // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). // See // ------------------------------------------------------ + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) @@ -146,67 +183,71 @@ terms of the MIT license. A copy of the license can be found in the file void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } - - typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; + #if (MI_INTPTR_SIZE==8) void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #else - #error "define overloads for new/delete for this platform (just for performance, can be skipped)" + #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif #endif // __cplusplus +// ------------------------------------------------------ +// Further Posix & Unix functions definitions +// ------------------------------------------------------ #ifdef __cplusplus extern "C" { #endif -// ------------------------------------------------------ -// Posix & Unix functions definitions -// ------------------------------------------------------ +#ifndef MI_OSX_IS_INTERPOSED + // Forward Posix/Unix calls as well + void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) + size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_good_size(size_t size) MI_FORWARD1(mi_malloc_good_size,size) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) + size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) + #else + size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + #endif -void cfree(void* p) MI_FORWARD0(mi_free, p) -void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) -size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) -#if !defined(__ANDROID__) && !defined(__FreeBSD__) -size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) -#else -size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + // No forwarding here due to aliasing/name mangling issues + void* valloc(size_t size) { return mi_valloc(size); } + void vfree(void* p) { mi_free(p); } + int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + + // `aligned_alloc` is only available when __USE_ISOC11 is defined. + // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot + // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. + // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it + // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. + #if __USE_ISOC11 + void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } + #endif #endif // no forwarding here due to aliasing/name mangling issues -void* valloc(size_t size) { return mi_valloc(size); } +void cfree(void* p) { mi_free(p); } void* pvalloc(size_t size) { return mi_pvalloc(size); } void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -// `aligned_alloc` is only available when __USE_ISOC11 is defined. -// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot -// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. -// Fortunately, in the case where `aligned_alloc` is declared as `static inline` it -// uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. -#if __USE_ISOC11 -void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -#endif - - #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) From f2e3cca2136a422024df4c525902cdad33bbcc4c Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 4 Nov 2021 19:10:15 -0700 Subject: [PATCH 097/134] fix build for missing malloc_good_size --- src/alloc-override.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index 9581a185..fa04b460 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -219,7 +219,6 @@ extern "C" { // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) - size_t malloc_good_size(size_t size) MI_FORWARD1(mi_malloc_good_size,size) #if !defined(__ANDROID__) && !defined(__FreeBSD__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else @@ -227,9 +226,11 @@ extern "C" { #endif // No forwarding here due to aliasing/name mangling issues - void* valloc(size_t size) { return mi_valloc(size); } - void vfree(void* p) { mi_free(p); } - int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + void* valloc(size_t size) { return mi_valloc(size); } + void vfree(void* p) { mi_free(p); } + size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } + int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + // `aligned_alloc` is only available when __USE_ISOC11 is defined. // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot @@ -242,23 +243,23 @@ extern "C" { #endif // no forwarding here due to aliasing/name mangling issues -void cfree(void* p) { mi_free(p); } -void* pvalloc(size_t size) { return mi_pvalloc(size); } -void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } -void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } +void cfree(void* p) { mi_free(p); } +void* pvalloc(size_t size) { return mi_pvalloc(size); } +void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) - void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) - void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) - void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) - void __libc_free(void* p) MI_FORWARD0(mi_free,p) - void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) + void __libc_free(void* p) MI_FORWARD0(mi_free,p) + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) - void* __libc_valloc(size_t size) { return mi_valloc(size); } - void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } - void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } + void* __libc_valloc(size_t size) { return mi_valloc(size); } + void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } #endif From e96614961f1a6b2a3ab01e4b21fc600c23d0ba1b Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 6 Nov 2021 14:19:14 -0700 Subject: [PATCH 098/134] fix printf format type mismatches (issue #486) --- src/os.c | 6 +++--- src/stats.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index 507d9971..26b8bcd2 100644 --- a/src/os.c +++ b/src/os.c @@ -846,7 +846,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ if (err != 0) { err = errno; } #endif if (err != 0) { - _mi_warning_message("%s error: start: %p, csize: 0x%x, err: %i\n", commit ? "commit" : "decommit", start, csize, err); + _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err); mi_mprotect_hint(err); } mi_assert_internal(err == 0); @@ -916,7 +916,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) int err = madvise(start, csize, MADV_DONTNEED); #endif if (err != 0) { - _mi_warning_message("madvise reset error: start: %p, csize: 0x%x, errno: %i\n", start, csize, errno); + _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, errno); } //mi_assert(err == 0); if (err != 0) return false; @@ -975,7 +975,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { if (err != 0) { err = errno; } #endif if (err != 0) { - _mi_warning_message("mprotect error: start: %p, csize: 0x%x, err: %i\n", start, csize, err); + _mi_warning_message("mprotect error: start: %p, csize: 0x%zx, err: %i\n", start, csize, err); mi_mprotect_hint(err); } return (err == 0); diff --git a/src/stats.c b/src/stats.c index 115d938e..6d486f42 100644 --- a/src/stats.c +++ b/src/stats.c @@ -327,7 +327,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); - _mi_fprintf(out, arg, "%10s: %7i\n", "numa nodes", _mi_os_numa_node_count()); + _mi_fprintf(out, arg, "%10s: %7zu\n", "numa nodes", _mi_os_numa_node_count()); mi_msecs_t elapsed; mi_msecs_t user_time; From 8c9ccea2f576394b69d0489b6f8883b95c4ddc1a Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 10:46:06 -0800 Subject: [PATCH 099/134] fix huge page madvise in case mmap failed --- src/os.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/os.c b/src/os.c index 26b8bcd2..77e8216e 100644 --- a/src/os.c +++ b/src/os.c @@ -510,7 +510,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (uintptr_t)8); // on error, don't try again for the next N allocations } } } @@ -518,29 +518,30 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro if (p == NULL) { *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); - #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (allow_large && use_large_os_page(size, try_alignment)) { - if (madvise(p, size, MADV_HUGEPAGE) == 0) { - *is_large = true; // possibly - }; - } - #endif - #if defined(__sun) - if (allow_large && use_large_os_page(size, try_alignment)) { - struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = large_os_page_size; - cmd.mha_cmd = MHA_MAPSIZE_VA; - if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - *is_large = true; + if (p != NULL) { + #if defined(MADV_HUGEPAGE) + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (allow_large && use_large_os_page(size, try_alignment)) { + if (madvise(p, size, MADV_HUGEPAGE) == 0) { + *is_large = true; // possibly + }; } + #elif defined(__sun) + if (allow_large && use_large_os_page(size, try_alignment)) { + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = large_os_page_size; + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + *is_large = true; + } + } + #endif } - #endif } if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); From 89abbe75d81ed4ed4858a3319a3ed938bc751ca1 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:23:11 -0800 Subject: [PATCH 100/134] improve aligned support on BSD and MAP_ALIGN systems --- src/os.c | 56 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/os.c b/src/os.c index 77e8216e..8334771c 100644 --- a/src/os.c +++ b/src/os.c @@ -413,23 +413,41 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + UNUSED(try_alignment); void* p = NULL; + #if defined(MAP_ALIGNED) // BSD + if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + size_t n = mi_bsr(try_alignment); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + flags |= MAP_ALIGNED(n); + p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #elif defined(MAP_ALIGN) // Solaris + if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + #endif #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations - void* hint; - if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { - p = mmap(hint,size,protect_flags,flags,fd,0); - if (p==MAP_FAILED) p = NULL; // fall back to regular mmap + if (addr == NULL) { + void* hint = mi_os_get_aligned_hint(try_alignment, size); + if (hint != NULL) { + p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } } - #else - UNUSED(try_alignment); - UNUSED(mi_os_get_aligned_hint); #endif - if (p==NULL) { - p = mmap(addr,size,protect_flags,flags,fd,0); - if (p==MAP_FAILED) p = NULL; - } - return p; + // regular mmap + p = mmap(addr, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // failed to allocate + return NULL; } static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { @@ -444,24 +462,17 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro int fd = -1; if (os_overcommit) { flags |= MAP_NORESERVE; - } - #if defined(MAP_ALIGNED) // BSD - if (try_alignment > 0) { - size_t n = mi_bsr(try_alignment); - if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB - flags |= MAP_ALIGNED(n); - } - } - #endif + } #if defined(PROT_MAX) protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif #if defined(VM_MAKE_TAG) // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) int os_tag = (int)mi_option_get(mi_option_os_tag); - if (os_tag < 100 || os_tag > 255) os_tag = 100; + if (os_tag < 100 || os_tag > 255) { os_tag = 100; } fd = VM_MAKE_TAG(os_tag); #endif + // huge page allocation if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { static _Atomic(uintptr_t) large_page_try_ok; // = 0; uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); @@ -515,6 +526,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro } } } + // regular allocation if (p == NULL) { *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); From fd61997cef6af8eab86fc40da5636f95c6b6fb47 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:26:36 -0800 Subject: [PATCH 101/134] improve aligned support on BSD and MAP_ALIGN systems --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 8334771c..d43f9afd 100644 --- a/src/os.c +++ b/src/os.c @@ -564,7 +564,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // On 64-bit systems, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate them. -#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) +#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || defined(MI_OS_USE_MMAP)) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // Return a 4MiB aligned address that is probably available. From f72e5688f5b6bbbe6a61289f0ed73541518de08e Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:58:04 -0800 Subject: [PATCH 102/134] remove assign in while condition --- src/os.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/os.c b/src/os.c index d43f9afd..426ead4a 100644 --- a/src/os.c +++ b/src/os.c @@ -303,20 +303,21 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations - void* hint; - if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { - void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); - if (p != NULL) return p; - // for robustness always fall through in case of an error - /* - DWORD err = GetLastError(); - if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) - err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) - return NULL; + if (addr == NULL) { + void* hint = mi_os_get_aligned_hint(try_alignment,size); + if (hint != NULL) { + void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); + if (p != NULL) return p; + // for robustness always fall through in case of an error + /* + DWORD err = GetLastError(); + if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) + err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) + return NULL; + } + */ + _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); } - */ - _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); - // fall through on error } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) @@ -414,20 +415,19 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { UNUSED(try_alignment); - void* p = NULL; #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); - p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { - p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); + void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } @@ -437,14 +437,14 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr if (addr == NULL) { void* hint = mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { - p = mmap(hint, size, protect_flags, flags, fd, 0); + void* p = mmap(hint, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } } #endif // regular mmap - p = mmap(addr, size, protect_flags, flags, fd, 0); + void* p = mmap(addr, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // failed to allocate return NULL; From d2de66b8fabc95fcba2dfd5e4f8a9c3004eb78c7 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 12 Nov 2021 18:44:33 -0800 Subject: [PATCH 103/134] do not delay eager commit for the main thread --- src/segment.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index bdf97019..4c2fa32e 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,8 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && !_mi_is_main_thread() && + tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); bool pages_still_good = false; From f9ac60a90ffcaacf576920508431d6fa7f3c92e4 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 12 Nov 2021 19:03:20 -0800 Subject: [PATCH 104/134] suppres eager commit delay for the first 4 threads --- include/mimalloc-internal.h | 1 + src/init.c | 8 ++++++++ src/segment.c | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 5c716a79..a2549f7e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -63,6 +63,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); +size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not ready // os.c diff --git a/src/init.c b/src/init.c index cb9d123f..db3d24ba 100644 --- a/src/init.c +++ b/src/init.c @@ -332,6 +332,12 @@ bool _mi_is_main_thread(void) { return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); } +static _Atomic(uintptr_t) thread_count = ATOMIC_VAR_INIT(1); + +size_t _mi_current_thread_count(void) { + return mi_atomic_load_relaxed(&thread_count); +} + // This is called from the `mi_malloc_generic` void mi_thread_init(void) mi_attr_noexcept { @@ -344,6 +350,7 @@ void mi_thread_init(void) mi_attr_noexcept if (_mi_heap_init()) return; // returns true if already initialized _mi_stat_increase(&_mi_stats_main.threads, 1); + mi_atomic_increment_relaxed(&thread_count); //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); } @@ -352,6 +359,7 @@ void mi_thread_done(void) mi_attr_noexcept { } static void _mi_thread_done(mi_heap_t* heap) { + mi_atomic_decrement_relaxed(&thread_count); _mi_stat_decrease(&_mi_stats_main.threads, 1); // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... diff --git a/src/segment.c b/src/segment.c index 4c2fa32e..1fffd745 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && !_mi_is_main_thread() && + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 4 && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From d52b559b571a14bb190a7cd7de15c2d23f0804a3 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 12 Nov 2021 19:59:59 -0800 Subject: [PATCH 105/134] use eager delay after 2 threads --- src/segment.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index 1fffd745..64b4720c 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 4 && + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 2 && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From c56be7ac5ae682733ddeb14288fd432a68951d5c Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 13:30:03 -0800 Subject: [PATCH 106/134] show cflags in cmake; fix -fno-builtin-malloc flag --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed5fb951..65549e47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,7 @@ set(mi_sources # Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- +message(STATUS "") if (NOT CMAKE_BUILD_TYPE) if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$" OR MI_DEBUG_FULL) message(STATUS "No build type selected, default to: Debug") @@ -199,7 +200,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM list(APPEND mi_cflags -ftls-model=initial-exec) endif() if(MI_OVERRIDE) - list(APPEND -fno-builtin-malloc) + list(APPEND mi_cflags -fno-builtin-malloc) endif() endif() @@ -263,13 +264,14 @@ endif() message(STATUS "") message(STATUS "Library base name: ${mi_basename}") +message(STATUS "Version : ${mi_version}") message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") if(MI_USE_CXX) message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() message(STATUS "C Compiler : ${CMAKE_C_COMPILER}") endif() -message(STATUS "Version : ${mi_version}") +message(STATUS "Compiler flags : ${mi_cflags}") message(STATUS "Build targets : ${mi_build_targets}") message(STATUS "") From 959845540d8263ed08fea321f78313d9e89090ae Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 14:13:03 -0800 Subject: [PATCH 107/134] use W4 for msvc compilation --- ide/vs2019/mimalloc.vcxproj | 6 +++--- src/options.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 6c7e276c..c12955c3 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -92,7 +92,7 @@ - Level3 + Level4 Disabled true true @@ -138,7 +138,7 @@ - Level3 + Level4 MaxSpeed true true @@ -166,7 +166,7 @@ - Level3 + Level4 MaxSpeed true true diff --git a/src/options.c b/src/options.c index c40a187b..e3d6c8c9 100644 --- a/src/options.c +++ b/src/options.c @@ -113,7 +113,7 @@ void _mi_options_init(void) { mi_max_warning_count = mi_option_get(mi_option_max_warnings); } -long mi_option_get(mi_option_t option) { +mi_decl_nodiscard long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); mi_option_desc_t* desc = &options[option]; mi_assert(desc->option == option); // index should match the option @@ -139,7 +139,7 @@ void mi_option_set_default(mi_option_t option, long value) { } } -bool mi_option_is_enabled(mi_option_t option) { +mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) { return (mi_option_get(option) != 0); } From 28896e5b19c5fd947626fdb924acde3de6425913 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 14:46:03 -0800 Subject: [PATCH 108/134] prefix UNUSED,KiB,MiB,GiB; add mi_threadid_t type; add mi_ssize_t --- include/mimalloc-internal.h | 38 ++++++++++----------- include/mimalloc-types.h | 64 +++++++++++++++++++++++++--------- src/alloc-override-osx.c | 68 ++++++++++++++++++------------------- src/alloc-override.c | 20 +++++------ src/alloc.c | 34 +++++++++---------- src/arena.c | 2 +- src/bitmap.c | 8 ++--- src/heap.c | 34 +++++++++---------- src/init.c | 8 ++--- src/options.c | 20 +++++------ src/os.c | 52 ++++++++++++++-------------- src/page.c | 6 ++-- src/region.c | 8 ++--- src/segment.c | 2 +- 14 files changed, 198 insertions(+), 166 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index a2549f7e..23078330 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -187,11 +187,11 @@ bool _mi_page_is_valid(mi_page_t* page); /* ----------------------------------------------------------- Inlined definitions ----------------------------------------------------------- */ -#define UNUSED(x) (void)(x) +#define MI_UNUSED(x) (void)(x) #if (MI_DEBUG>0) -#define UNUSED_RELEASE(x) +#define MI_UNUSED_RELEASE(x) #else -#define UNUSED_RELEASE(x) UNUSED(x) +#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) #endif #define MI_INIT4(x) x(),x(),x(),x() @@ -413,11 +413,11 @@ static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { } // used internally -static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { +static inline size_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; mi_assert_internal(diff >= 0 && (size_t)diff < MI_SEGMENT_SIZE); - uintptr_t idx = (uintptr_t)diff >> segment->page_shift; + size_t idx = (size_t)diff >> segment->page_shift; mi_assert_internal(idx < segment->capacity); mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0); return idx; @@ -425,7 +425,7 @@ static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, con // Get the page containing the pointer static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { - uintptr_t idx = _mi_segment_page_idx_of(segment, p); + size_t idx = _mi_segment_page_idx_of(segment, p); return &((mi_segment_t*)segment)->pages[idx]; } @@ -583,8 +583,8 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { mi_segment_t* segmentp = _mi_ptr_segment(p); mi_segment_t* segmentq = _mi_ptr_segment(q); if (segmentp != segmentq) return false; - uintptr_t idxp = _mi_segment_page_idx_of(segmentp, p); - uintptr_t idxq = _mi_segment_page_idx_of(segmentq, q); + size_t idxp = _mi_segment_page_idx_of(segmentp, p); + size_t idxq = _mi_segment_page_idx_of(segmentq, q); return (idxp == idxq); } @@ -611,7 +611,7 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl #ifdef MI_ENCODE_FREELIST return (mi_block_t*)mi_ptr_decode(null, block->next, keys); #else - UNUSED(keys); UNUSED(null); + MI_UNUSED(keys); MI_UNUSED(null); return (mi_block_t*)block->next; #endif } @@ -620,7 +620,7 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const #ifdef MI_ENCODE_FREELIST block->next = mi_ptr_encode(null, next, keys); #else - UNUSED(keys); UNUSED(null); + MI_UNUSED(keys); MI_UNUSED(null); block->next = (mi_encoded_t)next; #endif } @@ -636,7 +636,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* } return next; #else - UNUSED(page); + MI_UNUSED(page); return mi_block_nextx(page,block,NULL); #endif } @@ -645,7 +645,7 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c #ifdef MI_ENCODE_FREELIST mi_block_set_nextx(page,block,next, page->keys); #else - UNUSED(page); + MI_UNUSED(page); mi_block_set_nextx(page,block,next,NULL); #endif } @@ -700,7 +700,7 @@ static inline size_t _mi_os_numa_node_count(void) { #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { +static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { // Windows: works on Intel and ARM in both 32- and 64-bit return (uintptr_t)NtCurrentTeb(); } @@ -721,11 +721,11 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { #elif defined(__x86_64__) __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); res = tcb[slot]; #elif defined(__aarch64__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits @@ -749,11 +749,11 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { #elif defined(__x86_64__) __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); tcb[slot] = value; #elif defined(__aarch64__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits @@ -764,7 +764,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { #endif } -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { +static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { #if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) // on Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); @@ -775,7 +775,7 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { } #else // otherwise use standard C -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { +static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { return (uintptr_t)&_mi_heap_default; } #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 3ddd4e8a..c9f399df 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -17,7 +17,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Minimal alignment necessary. On most platforms 16 bytes are needed -// due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` +// due to SSE registers for example. This must be at least `sizeof(void*)` #ifndef MI_MAX_ALIGN_SIZE #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif @@ -67,6 +67,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif + // ------------------------------------------------------ // Platform specific values // ------------------------------------------------------ @@ -83,20 +84,43 @@ terms of the MIT license. A copy of the license can be found in the file // or otherwise one might define an intptr_t type that is larger than a pointer... // ------------------------------------------------------ -#if INTPTR_MAX == 9223372036854775807LL +#if INTPTR_MAX > INT64_MAX +# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) +#elif INTPTR_MAX == INT64_MAX # define MI_INTPTR_SHIFT (3) -#elif INTPTR_MAX == 2147483647LL +#elif INTPTR_MAX == INT32_MAX # define MI_INTPTR_SHIFT (2) #else -#error platform must be 32 or 64 bits +#error platform pointers must be 32, 64, or 128 bits +#endif + +#if SIZE_MAX == UINT64_MAX +# define MI_SIZE_SHIFT (3) +typedef int64_t mi_ssize_t; +#elif SIZE_MAX == UINT32_MAX +# define MI_SIZE_SHIFT (2) +typedef int32_t mi_ssize_t; +#else +#error platform objects must be 32 or 64 bits +#endif + +#if (SIZE_MAX/2) > LONG_MAX +# define MI_ZU(x) x##ULL +# define MI_ZI(x) x##LL +#else +# define MI_ZU(x) x##UL +# define MI_ZI(x) x##L #endif #define MI_INTPTR_SIZE (1<cookie` // layout like this to optimize access in `mi_free` - size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). - _Atomic(uintptr_t) thread_id; // unique id of the thread owning this segment + size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). + _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment mi_page_kind_t page_kind; // kind of pages: small, large, or huge mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages } mi_segment_t; @@ -341,7 +373,7 @@ struct mi_heap_s { mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") _Atomic(mi_block_t*) thread_delayed_free; - uintptr_t thread_id; // thread this heap belongs too + mi_threadid_t thread_id; // thread this heap belongs too uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list mi_random_ctx_t random; // random number context used for secure allocation diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 35b95a24..63297c4c 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -43,43 +43,43 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im ------------------------------------------------------ */ static size_t zone_size(malloc_zone_t* zone, const void* p) { - UNUSED(zone); + MI_UNUSED(zone); //if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out return mi_usable_size(p); } static void* zone_malloc(malloc_zone_t* zone, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_malloc(size); } static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_calloc(count, size); } static void* zone_valloc(malloc_zone_t* zone, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_malloc_aligned(size, _mi_os_page_size()); } static void zone_free(malloc_zone_t* zone, void* p) { - UNUSED(zone); + MI_UNUSED(zone); mi_free(p); } static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { - UNUSED(zone); + MI_UNUSED(zone); return mi_realloc(p, newsize); } static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_malloc_aligned(size,alignment); } static void zone_destroy(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // todo: ignore for now? } @@ -100,18 +100,18 @@ static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { } static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { - UNUSED(zone); UNUSED(size); + MI_UNUSED(zone); MI_UNUSED(size); mi_collect(false); return 0; } static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { - UNUSED(size); + MI_UNUSED(size); zone_free(zone,p); } static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { - UNUSED(zone); + MI_UNUSED(zone); return mi_is_in_heap_region(p); } @@ -126,43 +126,43 @@ static kern_return_t intro_enumerator(task_t task, void* p, vm_range_recorder_t recorder) { // todo: enumerate all memory - UNUSED(task); UNUSED(p); UNUSED(type_mask); UNUSED(zone_address); - UNUSED(reader); UNUSED(recorder); + MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address); + MI_UNUSED(reader); MI_UNUSED(recorder); return KERN_SUCCESS; } static size_t intro_good_size(malloc_zone_t* zone, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_good_size(size); } static boolean_t intro_check(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); return true; } static void intro_print(malloc_zone_t* zone, boolean_t verbose) { - UNUSED(zone); UNUSED(verbose); + MI_UNUSED(zone); MI_UNUSED(verbose); mi_stats_print(NULL); } static void intro_log(malloc_zone_t* zone, void* p) { - UNUSED(zone); UNUSED(p); + MI_UNUSED(zone); MI_UNUSED(p); // todo? } static void intro_force_lock(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // todo? } static void intro_force_unlock(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // todo? } static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { - UNUSED(zone); + MI_UNUSED(zone); // todo... stats->blocks_in_use = 0; stats->size_in_use = 0; @@ -171,7 +171,7 @@ static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { } static boolean_t intro_zone_locked(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); return false; } @@ -261,7 +261,7 @@ mi_decl_externc void _malloc_fork_child(void); static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { - UNUSED(size); UNUSED(flags); + MI_UNUSED(size); MI_UNUSED(flags); return mi_get_default_zone(); } @@ -274,12 +274,12 @@ static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { } static void mi_malloc_destroy_zone(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // nothing. } static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { - UNUSED(task); UNUSED(mr); + MI_UNUSED(task); MI_UNUSED(mr); if (addresses != NULL) *addresses = NULL; if (count != NULL) *count = 0; return KERN_SUCCESS; @@ -290,11 +290,11 @@ static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { } static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { - UNUSED(zone); UNUSED(name); + MI_UNUSED(zone); MI_UNUSED(name); } static int mi_malloc_jumpstart(uintptr_t cookie) { - UNUSED(cookie); + MI_UNUSED(cookie); return 1; // or 0 for no error? } @@ -309,37 +309,37 @@ static void mi__malloc_fork_child(void) { } static void mi_malloc_printf(const char* fmt, ...) { - UNUSED(fmt); + MI_UNUSED(fmt); } static bool zone_check(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); return true; } static malloc_zone_t* zone_from_ptr(const void* p) { - UNUSED(p); + MI_UNUSED(p); return mi_get_default_zone(); } static void zone_log(malloc_zone_t* zone, void* p) { - UNUSED(zone); UNUSED(p); + MI_UNUSED(zone); MI_UNUSED(p); } static void zone_print(malloc_zone_t* zone, bool b) { - UNUSED(zone); UNUSED(b); + MI_UNUSED(zone); MI_UNUSED(b); } static void zone_print_ptr_info(void* p) { - UNUSED(p); + MI_UNUSED(p); } static void zone_register(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); } static void zone_unregister(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); } // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` diff --git a/src/alloc-override.c b/src/alloc-override.c index fa04b460..42fecbb3 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -147,8 +147,8 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) - void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } - void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } #if (__cplusplus >= 201402L || _MSC_VER >= 1916) void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) @@ -187,21 +187,21 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit - void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit - void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #else #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif diff --git a/src/alloc.c b/src/alloc.c index d03207ce..1dbd31a7 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -123,7 +123,7 @@ extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) { // note: we need to initialize the whole usable block size to zero, not just the requested size, // or the recalloc/rezalloc functions cannot safely expand in place (see issue #63) - UNUSED(size); + MI_UNUSED(size); mi_assert_internal(p != NULL); mi_assert_internal(mi_usable_size(p) >= size); // size can be zero mi_assert_internal(_mi_ptr_page(p)==page); @@ -205,8 +205,8 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block } #else static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { - UNUSED(page); - UNUSED(block); + MI_UNUSED(page); + MI_UNUSED(block); return false; } #endif @@ -278,19 +278,19 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co } #else static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - UNUSED(page); - UNUSED(block); + MI_UNUSED(page); + MI_UNUSED(block); } static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { - UNUSED(block); + MI_UNUSED(block); return mi_page_usable_block_size(page); } static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { - UNUSED(page); - UNUSED(block); - UNUSED(min_size); + MI_UNUSED(page); + MI_UNUSED(block); + MI_UNUSED(min_size); } #endif @@ -298,7 +298,7 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co #if (MI_STAT>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { #if (MI_STAT < 2) - UNUSED(block); + MI_UNUSED(block); #endif mi_heap_t* const heap = mi_heap_get_default(); const size_t bsize = mi_page_usable_block_size(page); @@ -315,7 +315,7 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { } #else static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { - UNUSED(page); UNUSED(block); + MI_UNUSED(page); MI_UNUSED(block); } #endif @@ -333,7 +333,7 @@ static void mi_stat_huge_free(const mi_page_t* page) { } #else static void mi_stat_huge_free(const mi_page_t* page) { - UNUSED(page); + MI_UNUSED(page); } #endif @@ -447,7 +447,7 @@ static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool l // (and secure mode) if this was a valid pointer. static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg) { - UNUSED(msg); + MI_UNUSED(msg); #if (MI_DEBUG>0) if (mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); @@ -482,7 +482,7 @@ void mi_free(void* p) mi_attr_noexcept const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; - const uintptr_t tid = _mi_thread_id(); + const mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; @@ -574,19 +574,19 @@ void* _mi_externs[] = { // ------------------------------------------------------ void mi_free_size(void* p, size_t size) mi_attr_noexcept { - UNUSED_RELEASE(size); + MI_UNUSED_RELEASE(size); mi_assert(p == NULL || size <= _mi_usable_size(p,"mi_free_size")); mi_free(p); } void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - UNUSED_RELEASE(alignment); + MI_UNUSED_RELEASE(alignment); mi_assert(((uintptr_t)p % alignment) == 0); mi_free_size(p,size); } void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { - UNUSED_RELEASE(alignment); + MI_UNUSED_RELEASE(alignment); mi_assert(((uintptr_t)p % alignment) == 0); mi_free(p); } diff --git a/src/arena.c b/src/arena.c index 5e883b28..a87a53a3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -396,7 +396,7 @@ int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t } int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept { - UNUSED(max_secs); + MI_UNUSED(max_secs); _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n"); if (pages_reserved != NULL) *pages_reserved = 0; int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0)); diff --git a/src/bitmap.c b/src/bitmap.c index 3b5c8199..7936e07f 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -122,7 +122,7 @@ bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, m const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); const uintptr_t mask = mi_bitmap_mask_(count, bitidx); - mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); // mi_assert_internal((bitmap[idx] & mask) == mask); uintptr_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); return ((prev & mask) == mask); @@ -135,7 +135,7 @@ bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); const uintptr_t mask = mi_bitmap_mask_(count, bitidx); - mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); uintptr_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); if (any_zero != NULL) *any_zero = ((prev & mask) != mask); @@ -147,7 +147,7 @@ static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); const uintptr_t mask = mi_bitmap_mask_(count, bitidx); - mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); uintptr_t field = mi_atomic_load_relaxed(&bitmap[idx]); if (any_ones != NULL) *any_ones = ((field & mask) != 0); return ((field & mask) == mask); @@ -281,7 +281,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm // Helper for masks across fields; returns the mid count, post_mask may be 0 static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, uintptr_t* pre_mask, uintptr_t* mid_mask, uintptr_t* post_mask) { - UNUSED_RELEASE(bitmap_fields); + MI_UNUSED_RELEASE(bitmap_fields); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) { *pre_mask = mi_bitmap_mask_(count, bitidx); diff --git a/src/heap.c b/src/heap.c index bbf4cef9..d560fbc6 100644 --- a/src/heap.c +++ b/src/heap.c @@ -50,9 +50,9 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void #if MI_DEBUG>=2 static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - UNUSED(arg1); - UNUSED(arg2); - UNUSED(pq); + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(pq); mi_assert_internal(mi_page_heap(page) == heap); mi_segment_t* segment = _mi_page_segment(page); mi_assert_internal(segment->thread_id == heap->thread_id); @@ -86,8 +86,8 @@ typedef enum mi_collect_e { static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { - UNUSED(arg2); - UNUSED(heap); + MI_UNUSED(arg2); + MI_UNUSED(heap); mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page, collect >= MI_FORCE); @@ -104,10 +104,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t } static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - UNUSED(arg1); - UNUSED(arg2); - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(heap); + MI_UNUSED(pq); _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); return true; // don't break } @@ -262,10 +262,10 @@ static void mi_heap_free(mi_heap_t* heap) { ----------------------------------------------------------- */ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - UNUSED(arg1); - UNUSED(arg2); - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(heap); + MI_UNUSED(pq); // ensure no more thread_delayed_free will be added _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); @@ -422,8 +422,8 @@ bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) { - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(heap); + MI_UNUSED(pq); bool* found = (bool*)vfound; mi_segment_t* segment = _mi_page_segment(page); void* start = _mi_page_start(segment, page, NULL); @@ -521,8 +521,8 @@ typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(heap); + MI_UNUSED(pq); mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; mi_heap_area_ex_t xarea; const size_t bsize = mi_page_block_size(page); diff --git a/src/init.c b/src/init.c index db3d24ba..77b0c07f 100644 --- a/src/init.c +++ b/src/init.c @@ -450,7 +450,7 @@ static void mi_process_load(void) { mi_heap_main_init(); #if defined(MI_TLS_RECURSE_GUARD) volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; - UNUSED(dummy); + MI_UNUSED(dummy); #endif os_preloading = false; atexit(&mi_process_done); @@ -509,7 +509,7 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); if (ksize > 0) { - mi_reserve_os_memory((size_t)ksize*KiB, true, true); + mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } } @@ -548,8 +548,8 @@ static void mi_process_done(void) { #if defined(_WIN32) && defined(MI_SHARED_LIB) // Windows DLL: easy to hook into process_init and thread_done __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { - UNUSED(reserved); - UNUSED(inst); + MI_UNUSED(reserved); + MI_UNUSED(inst); if (reason==DLL_PROCESS_ATTACH) { mi_process_load(); } diff --git a/src/options.c b/src/options.c index e3d6c8c9..66da8372 100644 --- a/src/options.c +++ b/src/options.c @@ -103,7 +103,7 @@ void _mi_options_init(void) { mi_add_stderr_output(); // now it safe to use stderr for output for(int i = 0; i < _mi_option_last; i++ ) { mi_option_t option = (mi_option_t)i; - long l = mi_option_get(option); UNUSED(l); // initialize + long l = mi_option_get(option); MI_UNUSED(l); // initialize if (option != mi_option_verbose) { mi_option_desc_t* desc = &options[option]; _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value); @@ -161,7 +161,7 @@ void mi_option_disable(mi_option_t option) { static void mi_out_stderr(const char* msg, void* arg) { - UNUSED(arg); + MI_UNUSED(arg); #ifdef _WIN32 // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. @@ -182,7 +182,7 @@ static char out_buf[MI_MAX_DELAY_OUTPUT+1]; static _Atomic(uintptr_t) out_len; static void mi_out_buf(const char* msg, void* arg) { - UNUSED(arg); + MI_UNUSED(arg); if (msg==NULL) return; if (mi_atomic_load_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; size_t n = strlen(msg); @@ -353,7 +353,7 @@ static mi_error_fun* volatile mi_error_handler; // = NULL static _Atomic(void*) mi_error_arg; // = NULL static void mi_error_default(int err) { - UNUSED(err); + MI_UNUSED(err); #if (MI_DEBUG>0) if (err==EFAULT) { #ifdef _MSC_VER @@ -411,9 +411,9 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { #ifdef MI_NO_GETENV static bool mi_getenv(const char* name, char* result, size_t result_size) { - UNUSED(name); - UNUSED(result); - UNUSED(result_size); + MI_UNUSED(name); + MI_UNUSED(result); + MI_UNUSED(result_size); return false; } #else @@ -521,9 +521,9 @@ static void mi_option_init(mi_option_desc_t* desc) { if (desc->option == mi_option_reserve_os_memory) { // this option is interpreted in KiB to prevent overflow of `long` if (*end == 'K') { end++; } - else if (*end == 'M') { value *= KiB; end++; } - else if (*end == 'G') { value *= MiB; end++; } - else { value = (value + KiB - 1) / KiB; } + else if (*end == 'M') { value *= MI_KiB; end++; } + else if (*end == 'G') { value *= MI_MiB; end++; } + else { value = (value + MI_KiB - 1) / MI_KiB; } if (*end == 'B') { end++; } } if (*end == 0) { diff --git a/src/os.c b/src/os.c index 426ead4a..b4853689 100644 --- a/src/os.c +++ b/src/os.c @@ -122,11 +122,11 @@ static bool use_large_os_page(size_t size, size_t alignment) { // round to a good OS allocation size (bounded by max 12.5% waste) size_t _mi_os_good_alloc_size(size_t size) { size_t align_size; - if (size < 512*KiB) align_size = _mi_os_page_size(); - else if (size < 2*MiB) align_size = 64*KiB; - else if (size < 8*MiB) align_size = 256*KiB; - else if (size < 32*MiB) align_size = 1*MiB; - else align_size = 4*MiB; + if (size < 512*MI_KiB) align_size = _mi_os_page_size(); + else if (size < 2*MI_MiB) align_size = 64*MI_KiB; + else if (size < 8*MI_MiB) align_size = 256*MI_KiB; + else if (size < 32*MI_MiB) align_size = 1*MI_MiB; + else align_size = 4*MI_MiB; if (mi_unlikely(size >= (SIZE_MAX - align_size))) return size; // possible overflow? return _mi_align_up(size, align_size); } @@ -263,7 +263,7 @@ void _mi_os_init() { os_page_size = (size_t)result; os_alloc_granularity = os_page_size; } - large_os_page_size = 2*MiB; // TODO: can we query the OS for this? + large_os_page_size = 2*MI_MiB; // TODO: can we query the OS for this? os_detect_overcommit(); } #endif @@ -414,7 +414,7 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { - UNUSED(try_alignment); + MI_UNUSED(try_alignment); #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); @@ -494,7 +494,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif #ifdef MAP_HUGE_1GB static bool mi_huge_pages_available = true; - if ((size % GiB) == 0 && mi_huge_pages_available) { + if ((size % MI_GiB) == 0 && mi_huge_pages_available) { lflags |= MAP_HUGE_1GB; } else @@ -582,7 +582,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if ((size%MI_SEGMENT_SIZE) != 0) return NULL; - if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. + if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. #endif @@ -605,7 +605,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) // no need for mi_os_get_aligned_hint #else static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { - UNUSED(try_alignment); UNUSED(size); + MI_UNUSED(try_alignment); MI_UNUSED(size); return NULL; } #endif @@ -732,7 +732,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, ----------------------------------------------------------- */ void* _mi_os_alloc(size_t size, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); @@ -741,7 +741,7 @@ void* _mi_os_alloc(size_t size, mi_stats_t* tld_stats) { } void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (size == 0 || p == NULL) return; size = _mi_os_good_alloc_size(size); @@ -754,7 +754,7 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); @@ -805,7 +805,7 @@ static void mi_mprotect_hint(int err) { " > sudo sysctl -w vm.max_map_count=262144\n"); } #else - UNUSED(err); + MI_UNUSED(err); #endif } @@ -867,13 +867,13 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ } bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; return mi_os_commitx(addr, size, true, false /* liberal */, is_zero, stats); } bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; bool is_zero; return mi_os_commitx(addr, size, false, true /* conservative */, &is_zero, stats); @@ -942,7 +942,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) // pages and reduce swapping while keeping the memory committed. // We page align to a conservative area inside the range to reset. bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (mi_option_is_enabled(mi_option_reset_decommits)) { return _mi_os_decommit(addr, size, stats); @@ -953,7 +953,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats) { } bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (mi_option_is_enabled(mi_option_reset_decommits)) { return mi_os_commit_unreset(addr, size, is_zero, stats); // re-commit it (conservatively!) @@ -1029,12 +1029,12 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { Support for allocating huge OS pages (1Gib) that are reserved up-front and possibly associated with a specific NUMA node. (use `numa_node>=0`) -----------------------------------------------------------------------------*/ -#define MI_HUGE_OS_PAGE_SIZE (GiB) +#define MI_HUGE_OS_PAGE_SIZE (MI_GiB) #if defined(_WIN32) && (MI_INTPTR_SIZE >= 8) static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - mi_assert_internal(size%GiB == 0); + mi_assert_internal(size%MI_GiB == 0); mi_assert_internal(addr != NULL); const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; @@ -1075,7 +1075,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); } #else - UNUSED(numa_node); + MI_UNUSED(numa_node); #endif // otherwise use regular virtual alloc on older windows return VirtualAlloc(addr, size, flags, PAGE_READWRITE); @@ -1092,12 +1092,12 @@ static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, cons } #else static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { - UNUSED(start); UNUSED(len); UNUSED(mode); UNUSED(nmask); UNUSED(maxnode); UNUSED(flags); + MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags); return 0; } #endif static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - mi_assert_internal(size%GiB == 0); + mi_assert_internal(size%MI_GiB == 0); bool is_large = true; void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); if (p == NULL) return NULL; @@ -1115,7 +1115,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) } #else static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - UNUSED(addr); UNUSED(size); UNUSED(numa_node); + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); return NULL; } #endif @@ -1151,7 +1151,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { } #else static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { - UNUSED(pages); + MI_UNUSED(pages); if (total_size != NULL) *total_size = 0; return NULL; } @@ -1352,7 +1352,7 @@ size_t _mi_os_numa_node_count_get(void) { } int _mi_os_numa_node_get(mi_os_tld_t* tld) { - UNUSED(tld); + MI_UNUSED(tld); size_t numa_count = _mi_os_numa_node_count(); if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 // never more than the node count and >= 0 diff --git a/src/page.c b/src/page.c index 61f3245a..c0659395 100644 --- a/src/page.c +++ b/src/page.c @@ -30,7 +30,7 @@ terms of the MIT license. A copy of the license can be found in the file // Index a block in a page static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) { - UNUSED(page); + MI_UNUSED(page); mi_assert_internal(page != NULL); mi_assert_internal(i <= page->reserved); return (mi_block_t*)((uint8_t*)page_start + (i * block_size)); @@ -459,7 +459,7 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) { #define MI_MIN_SLICES (2) static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) { - UNUSED(stats); + MI_UNUSED(stats); #if (MI_SECURE<=2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); @@ -517,7 +517,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) { - UNUSED(stats); + MI_UNUSED(stats); #if (MI_SECURE <= 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); diff --git a/src/region.c b/src/region.c index d9232450..d99b74af 100644 --- a/src/region.c +++ b/src/region.c @@ -57,9 +57,9 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo // Constants #if (MI_INTPTR_SIZE==8) -#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 64KiB for the region map +#define MI_HEAP_REGION_MAX_SIZE (256 * MI_GiB) // 64KiB for the region map #elif (MI_INTPTR_SIZE==4) -#define MI_HEAP_REGION_MAX_SIZE (3 * GiB) // ~ KiB for the region map +#define MI_HEAP_REGION_MAX_SIZE (3 * MI_GiB) // ~ KiB for the region map #else #error "define the maximum heap space allowed for regions on this platform" #endif @@ -190,7 +190,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, if (idx >= MI_REGION_MAX) { mi_atomic_decrement_acq_rel(®ions_count); _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats); - _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, GiB)); + _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, MI_GiB)); return false; } @@ -441,7 +441,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re // and unclaim bool all_unclaimed = mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx); - mi_assert_internal(all_unclaimed); UNUSED(all_unclaimed); + mi_assert_internal(all_unclaimed); MI_UNUSED(all_unclaimed); } } diff --git a/src/segment.c b/src/segment.c index 64b4720c..a853bf41 100644 --- a/src/segment.c +++ b/src/segment.c @@ -696,7 +696,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, } static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { - UNUSED(force); + MI_UNUSED(force); mi_assert(segment != NULL); // note: don't reset pages even on abandon as the whole segment is freed? (and ready for reuse) bool force_reset = (force && mi_option_is_enabled(mi_option_abandoned_page_reset)); From 09e59e06104f800d2a154df4ba4aa908e5f26945 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 15:13:16 -0800 Subject: [PATCH 109/134] use size_t instead of uintptr_t where appropiate --- src/arena.c | 8 ++--- src/bitmap.c | 82 +++++++++++++++++++++++----------------------- src/bitmap.h | 10 +++--- src/init.c | 2 +- src/options.c | 14 ++++---- src/os.c | 16 ++++----- src/region.c | 24 +++++++------- src/segment.c | 14 ++++---- test/test-stress.c | 2 +- 9 files changed, 86 insertions(+), 86 deletions(-) diff --git a/src/arena.c b/src/arena.c index a87a53a3..133f1546 100644 --- a/src/arena.c +++ b/src/arena.c @@ -20,7 +20,7 @@ which is sometimes needed for embedded devices or shared memory for example. The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. The current implementation of the bitmap can -only do this within a field (`uintptr_t`) so we can allocate at most +only do this within a field (`size_t`) so we can allocate at most blocks of 2GiB (64*32MiB) and no object can cross the boundary. This can lead to fragmentation but fortunately most objects will be regions of 256MiB in practice. @@ -64,7 +64,7 @@ typedef struct mi_arena_s { bool is_zero_init; // is the arena zero initialized? bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) - _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks + _Atomic(size_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) @@ -73,7 +73,7 @@ typedef struct mi_arena_s { // The available arenas static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS]; -static mi_decl_cache_align _Atomic(uintptr_t) mi_arena_count; // = 0 +static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 /* ----------------------------------------------------------- @@ -272,7 +272,7 @@ static bool mi_arena_add(mi_arena_t* arena) { mi_assert_internal((uintptr_t)mi_atomic_load_ptr_relaxed(uint8_t,&arena->start) % MI_SEGMENT_ALIGN == 0); mi_assert_internal(arena->block_count > 0); - uintptr_t i = mi_atomic_increment_acq_rel(&mi_arena_count); + size_t i = mi_atomic_increment_acq_rel(&mi_arena_count); if (i >= MI_MAX_ARENAS) { mi_atomic_decrement_acq_rel(&mi_arena_count); return false; diff --git a/src/bitmap.c b/src/bitmap.c index 7936e07f..51926bbd 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`uintptr_t`) +represeted as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -26,12 +26,12 @@ between the fields. (This is used in arena allocation) ----------------------------------------------------------- */ // The bit mask for a given number of blocks at a specified bit index. -static inline uintptr_t mi_bitmap_mask_(size_t count, size_t bitidx) { +static inline size_t mi_bitmap_mask_(size_t count, size_t bitidx) { mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS); mi_assert_internal(count > 0); if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL; if (count == 0) return 0; - return ((((uintptr_t)1 << count) - 1) << bitidx); + return ((((size_t)1 << count) - 1) << bitidx); } @@ -46,27 +46,27 @@ bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_ { mi_assert_internal(bitmap_idx != NULL); mi_assert_internal(count <= MI_BITMAP_FIELD_BITS); - _Atomic(uintptr_t)* field = &bitmap[idx]; - uintptr_t map = mi_atomic_load_relaxed(field); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t map = mi_atomic_load_relaxed(field); if (map==MI_BITMAP_FIELD_FULL) return false; // short cut // search for 0-bit sequence of length count - const uintptr_t mask = mi_bitmap_mask_(count, 0); - const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; + const size_t mask = mi_bitmap_mask_(count, 0); + const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; #ifdef MI_HAVE_FAST_BITSCAN size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible #else size_t bitidx = 0; // otherwise start at 0 #endif - uintptr_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx + size_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx // scan linearly for a free range of zero bits while (bitidx <= bitidx_max) { - const uintptr_t mapm = map & m; + const size_t mapm = map & m; if (mapm == 0) { // are the mask bits free at bitidx? mi_assert_internal((m >> bitidx) == mask); // no overflow? - const uintptr_t newmap = map | m; + const size_t newmap = map | m; mi_assert_internal((newmap^map) >> bitidx == mask); if (!mi_atomic_cas_weak_acq_rel(field, &map, newmap)) { // TODO: use strong cas here? // no success, another thread claimed concurrently.. keep going (with updated `map`) @@ -121,10 +121,10 @@ bool _mi_bitmap_try_find_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, c bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + const size_t mask = mi_bitmap_mask_(count, bitidx); mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); // mi_assert_internal((bitmap[idx] & mask) == mask); - uintptr_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); + size_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); return ((prev & mask) == mask); } @@ -134,10 +134,10 @@ bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, m bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) { const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + const size_t mask = mi_bitmap_mask_(count, bitidx); mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); - uintptr_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); + size_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); if (any_zero != NULL) *any_zero = ((prev & mask) != mask); return ((prev & mask) == 0); } @@ -146,9 +146,9 @@ bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) { const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + const size_t mask = mi_bitmap_mask_(count, bitidx); mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); - uintptr_t field = mi_atomic_load_relaxed(&bitmap[idx]); + size_t field = mi_atomic_load_relaxed(&bitmap[idx]); if (any_ones != NULL) *any_ones = ((field & mask) != 0); return ((field & mask) == mask); } @@ -176,8 +176,8 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit mi_assert_internal(bitmap_idx != NULL); // check initial trailing zeros - _Atomic(uintptr_t)* field = &bitmap[idx]; - uintptr_t map = mi_atomic_load_relaxed(field); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t map = mi_atomic_load_relaxed(field); const size_t initial = mi_clz(map); // count of initial zeros starting at idx mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS); if (initial == 0) return false; @@ -186,11 +186,11 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit // scan ahead size_t found = initial; - uintptr_t mask = 0; // mask bits for the final field + size_t mask = 0; // mask bits for the final field while(found < count) { field++; map = mi_atomic_load_relaxed(field); - const uintptr_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); + const size_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); mask = mi_bitmap_mask_(mask_bits, 0); if ((map & mask) != 0) return false; found += mask_bits; @@ -199,13 +199,13 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit // found range of zeros up to the final field; mask contains mask in the final field // now claim it atomically - _Atomic(uintptr_t)* const final_field = field; - const uintptr_t final_mask = mask; - _Atomic(uintptr_t)* const initial_field = &bitmap[idx]; - const uintptr_t initial_mask = mi_bitmap_mask_(initial, MI_BITMAP_FIELD_BITS - initial); + mi_bitmap_field_t* const final_field = field; + const size_t final_mask = mask; + mi_bitmap_field_t* const initial_field = &bitmap[idx]; + const size_t initial_mask = mi_bitmap_mask_(initial, MI_BITMAP_FIELD_BITS - initial); // initial field - uintptr_t newmap; + size_t newmap; field = initial_field; map = mi_atomic_load_relaxed(field); do { @@ -280,7 +280,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm } // Helper for masks across fields; returns the mid count, post_mask may be 0 -static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, uintptr_t* pre_mask, uintptr_t* mid_mask, uintptr_t* post_mask) { +static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, size_t* pre_mask, size_t* mid_mask, size_t* post_mask) { MI_UNUSED_RELEASE(bitmap_fields); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) { @@ -308,13 +308,13 @@ static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_ // Returns `true` if all `count` bits were 1 previously. bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { size_t idx = mi_bitmap_index_field(bitmap_idx); - uintptr_t pre_mask; - uintptr_t mid_mask; - uintptr_t post_mask; + size_t pre_mask; + size_t mid_mask; + size_t post_mask; size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); bool all_one = true; - _Atomic(uintptr_t)*field = &bitmap[idx]; - uintptr_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); if ((prev & pre_mask) != pre_mask) all_one = false; while(mid_count-- > 0) { prev = mi_atomic_and_acq_rel(field++, ~mid_mask); @@ -331,14 +331,14 @@ bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t // Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero) { size_t idx = mi_bitmap_index_field(bitmap_idx); - uintptr_t pre_mask; - uintptr_t mid_mask; - uintptr_t post_mask; + size_t pre_mask; + size_t mid_mask; + size_t post_mask; size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); bool all_zero = true; bool any_zero = false; - _Atomic(uintptr_t)*field = &bitmap[idx]; - uintptr_t prev = mi_atomic_or_acq_rel(field++, pre_mask); + _Atomic(size_t)*field = &bitmap[idx]; + size_t prev = mi_atomic_or_acq_rel(field++, pre_mask); if ((prev & pre_mask) != 0) all_zero = false; if ((prev & pre_mask) != pre_mask) any_zero = true; while (mid_count-- > 0) { @@ -360,14 +360,14 @@ bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t co // `any_ones` is `true` if there was at least one bit set to one. static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones) { size_t idx = mi_bitmap_index_field(bitmap_idx); - uintptr_t pre_mask; - uintptr_t mid_mask; - uintptr_t post_mask; + size_t pre_mask; + size_t mid_mask; + size_t post_mask; size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); bool all_ones = true; bool any_ones = false; - _Atomic(uintptr_t)* field = &bitmap[idx]; - uintptr_t prev = mi_atomic_load_relaxed(field++); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t prev = mi_atomic_load_relaxed(field++); if ((prev & pre_mask) != pre_mask) all_ones = false; if ((prev & pre_mask) != 0) any_ones = true; while (mid_count-- > 0) { diff --git a/src/bitmap.h b/src/bitmap.h index 21fd4e13..39ca55b2 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`uintptr_t`) +represeted as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -24,11 +24,11 @@ between the fields. (This is used in arena allocation) Bitmap definition ----------------------------------------------------------- */ -#define MI_BITMAP_FIELD_BITS (8*MI_INTPTR_SIZE) -#define MI_BITMAP_FIELD_FULL (~((uintptr_t)0)) // all bits set +#define MI_BITMAP_FIELD_BITS (8*MI_SIZE_SIZE) +#define MI_BITMAP_FIELD_FULL (~((size_t)0)) // all bits set -// An atomic bitmap of `uintptr_t` fields -typedef _Atomic(uintptr_t) mi_bitmap_field_t; +// An atomic bitmap of `size_t` fields +typedef _Atomic(size_t) mi_bitmap_field_t; typedef mi_bitmap_field_t* mi_bitmap_t; // A bitmap index is the index of the bit in a bitmap. diff --git a/src/init.c b/src/init.c index 77b0c07f..f9c73a3f 100644 --- a/src/init.c +++ b/src/init.c @@ -332,7 +332,7 @@ bool _mi_is_main_thread(void) { return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); } -static _Atomic(uintptr_t) thread_count = ATOMIC_VAR_INIT(1); +static _Atomic(size_t) thread_count = ATOMIC_VAR_INIT(1); size_t _mi_current_thread_count(void) { return mi_atomic_load_relaxed(&thread_count); diff --git a/src/options.c b/src/options.c index 66da8372..9549e3ad 100644 --- a/src/options.c +++ b/src/options.c @@ -19,8 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif -static uintptr_t mi_max_error_count = 16; // stop outputting errors after this -static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this +static size_t mi_max_error_count = 16; // stop outputting errors after this +static size_t mi_max_warning_count = 16; // stop outputting warnings after this static void mi_add_stderr_output(void); @@ -176,10 +176,10 @@ static void mi_out_stderr(const char* msg, void* arg) { // an output function is registered it is called immediately with // the output up to that point. #ifndef MI_MAX_DELAY_OUTPUT -#define MI_MAX_DELAY_OUTPUT ((uintptr_t)(32*1024)) +#define MI_MAX_DELAY_OUTPUT ((size_t)(32*1024)) #endif static char out_buf[MI_MAX_DELAY_OUTPUT+1]; -static _Atomic(uintptr_t) out_len; +static _Atomic(size_t) out_len; static void mi_out_buf(const char* msg, void* arg) { MI_UNUSED(arg); @@ -188,7 +188,7 @@ static void mi_out_buf(const char* msg, void* arg) { size_t n = strlen(msg); if (n==0) return; // claim space - uintptr_t start = mi_atomic_add_acq_rel(&out_len, n); + size_t start = mi_atomic_add_acq_rel(&out_len, n); if (start >= MI_MAX_DELAY_OUTPUT) return; // check bound if (start+n >= MI_MAX_DELAY_OUTPUT) { @@ -251,8 +251,8 @@ static void mi_add_stderr_output() { // -------------------------------------------------------- // Messages, all end up calling `_mi_fputs`. // -------------------------------------------------------- -static _Atomic(uintptr_t) error_count; // = 0; // when >= max_error_count stop emitting errors -static _Atomic(uintptr_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings +static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors +static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings // When overriding malloc, we may recurse into mi_vfprintf if an allocation // inside the C runtime causes another message. diff --git a/src/os.c b/src/os.c index b4853689..67b6c6db 100644 --- a/src/os.c +++ b/src/os.c @@ -340,11 +340,11 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { mi_assert_internal(!(large_only && !allow_large)); - static _Atomic(uintptr_t) large_page_try_ok; // = 0; + static _Atomic(size_t) large_page_try_ok; // = 0; void* p = NULL; if ((large_only || use_large_os_page(size, try_alignment)) && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { - uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. @@ -474,8 +474,8 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif // huge page allocation if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { - static _Atomic(uintptr_t) large_page_try_ok; // = 0; - uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + static _Atomic(size_t) large_page_try_ok; // = 0; + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { // If the OS is not configured for large OS pages, or the user does not have // enough permission, the `mmap` will always fail (but it might also fail for other reasons). @@ -521,7 +521,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, (uintptr_t)8); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations } } } @@ -914,13 +914,13 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); + static _Atomic(size_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on - mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); + mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } #elif defined(__wasi__) @@ -1102,7 +1102,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); if (p == NULL) return NULL; if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes - uintptr_t numa_mask = (1UL << numa_node); + unsigned long numa_mask = (1UL << numa_node); // TODO: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: diff --git a/src/region.c b/src/region.c index d99b74af..f864f73b 100644 --- a/src/region.c +++ b/src/region.c @@ -74,7 +74,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo // Region info typedef union mi_region_info_u { - uintptr_t value; + size_t value; struct { bool valid; // initialized? bool is_large:1; // allocated in fixed large/huge OS pages @@ -87,21 +87,21 @@ typedef union mi_region_info_u { // A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with // a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block. typedef struct mem_region_s { - _Atomic(uintptr_t) info; // mi_region_info_t.value + _Atomic(size_t) info; // mi_region_info_t.value _Atomic(void*) start; // start of the memory area mi_bitmap_field_t in_use; // bit per in-use block mi_bitmap_field_t dirty; // track if non-zero per block mi_bitmap_field_t commit; // track if committed per block mi_bitmap_field_t reset; // track if reset per block - _Atomic(uintptr_t) arena_memid; // if allocated from a (huge page) arena - uintptr_t padding; // round to 8 fields + _Atomic(size_t) arena_memid; // if allocated from a (huge page) arena + size_t padding; // round to 8 fields } mem_region_t; // The region map static mem_region_t regions[MI_REGION_MAX]; // Allocated regions -static _Atomic(uintptr_t) regions_count; // = 0; +static _Atomic(size_t) regions_count; // = 0; /* ---------------------------------------------------------------------------- @@ -186,7 +186,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, mi_assert_internal(!region_large || region_commit); // claim a fresh slot - const uintptr_t idx = mi_atomic_increment_acq_rel(®ions_count); + const size_t idx = mi_atomic_increment_acq_rel(®ions_count); if (idx >= MI_REGION_MAX) { mi_atomic_decrement_acq_rel(®ions_count); _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats); @@ -197,10 +197,10 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, // allocated, initialize and claim the initial blocks mem_region_t* r = ®ions[idx]; r->arena_memid = arena_memid; - mi_atomic_store_release(&r->in_use, (uintptr_t)0); + mi_atomic_store_release(&r->in_use, (size_t)0); mi_atomic_store_release(&r->dirty, (is_zero ? 0 : MI_BITMAP_FIELD_FULL)); mi_atomic_store_release(&r->commit, (region_commit ? MI_BITMAP_FIELD_FULL : 0)); - mi_atomic_store_release(&r->reset, (uintptr_t)0); + mi_atomic_store_release(&r->reset, (size_t)0); *bit_idx = 0; _mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL); mi_atomic_store_ptr_release(void,&r->start, start); @@ -451,21 +451,21 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re -----------------------------------------------------------------------------*/ void _mi_mem_collect(mi_os_tld_t* tld) { // free every region that has no segments in use. - uintptr_t rcount = mi_atomic_load_relaxed(®ions_count); + size_t rcount = mi_atomic_load_relaxed(®ions_count); for (size_t i = 0; i < rcount; i++) { mem_region_t* region = ®ions[i]; if (mi_atomic_load_relaxed(®ion->info) != 0) { // if no segments used, try to claim the whole region - uintptr_t m = mi_atomic_load_relaxed(®ion->in_use); + size_t m = mi_atomic_load_relaxed(®ion->in_use); while (m == 0 && !mi_atomic_cas_weak_release(®ion->in_use, &m, MI_BITMAP_FIELD_FULL)) { /* nothing */ }; if (m == 0) { // on success, free the whole region uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ions[i].start); size_t arena_memid = mi_atomic_load_relaxed(®ions[i].arena_memid); - uintptr_t commit = mi_atomic_load_relaxed(®ions[i].commit); + size_t commit = mi_atomic_load_relaxed(®ions[i].commit); memset((void*)®ions[i], 0, sizeof(mem_region_t)); // cast to void* to avoid atomic warning // and release the whole region - mi_atomic_store_release(®ion->info, (uintptr_t)0); + mi_atomic_store_release(®ion->info, (size_t)0); if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { _mi_abandoned_await_readers(); // ensure no pending reads _mi_arena_free(start, MI_REGION_SIZE, arena_memid, (~commit == 0), tld->stats); diff --git a/src/segment.c b/src/segment.c index a853bf41..6de3a8ae 100644 --- a/src/segment.c +++ b/src/segment.c @@ -897,13 +897,13 @@ static mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = static mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL // Maintain these for debug purposes (these counts may be a bit off) -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_count; -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_visited_count; +static mi_decl_cache_align _Atomic(size_t) abandoned_count; +static mi_decl_cache_align _Atomic(size_t) abandoned_visited_count; // We also maintain a count of current readers of the abandoned list // in order to prevent resetting/decommitting segment memory if it might // still be read. -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_readers; // = 0 +static mi_decl_cache_align _Atomic(size_t) abandoned_readers; // = 0 // Push on the visited list static void mi_abandoned_visited_push(mi_segment_t* segment) { @@ -932,7 +932,7 @@ static bool mi_abandoned_visited_revisit(void) mi_tagged_segment_t afirst; mi_tagged_segment_t ts = mi_atomic_load_relaxed(&abandoned); if (mi_tagged_segment_ptr(ts)==NULL) { - uintptr_t count = mi_atomic_load_relaxed(&abandoned_visited_count); + size_t count = mi_atomic_load_relaxed(&abandoned_visited_count); afirst = mi_tagged_segment(first, ts); if (mi_atomic_cas_strong_acq_rel(&abandoned, &ts, afirst)) { mi_atomic_add_relaxed(&abandoned_count, count); @@ -951,7 +951,7 @@ static bool mi_abandoned_visited_revisit(void) // and atomically prepend to the abandoned list // (no need to increase the readers as we don't access the abandoned segments) mi_tagged_segment_t anext = mi_atomic_load_relaxed(&abandoned); - uintptr_t count; + size_t count; do { count = mi_atomic_load_relaxed(&abandoned_visited_count); mi_atomic_store_ptr_release(mi_segment_t, &last->abandoned_next, mi_tagged_segment_ptr(anext)); @@ -979,7 +979,7 @@ static void mi_abandoned_push(mi_segment_t* segment) { // Wait until there are no more pending reads on segments that used to be in the abandoned list void _mi_abandoned_await_readers(void) { - uintptr_t n; + size_t n; do { n = mi_atomic_load_acquire(&abandoned_readers); if (n != 0) mi_atomic_yield(); @@ -1327,7 +1327,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block // claim it and free mi_heap_t* heap = mi_heap_get_default(); // issue #221; don't use the internal get_default_heap as we need to ensure the thread is initialized. // paranoia: if this it the last reference, the cas should always succeed - uintptr_t expected_tid = 0; + size_t expected_tid = 0; if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected_tid, heap->thread_id)) { mi_block_set_next(page, block, page->free); page->free = block; diff --git a/test/test-stress.c b/test/test-stress.c index d45e9899..498b7ec6 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -25,7 +25,7 @@ terms of the MIT license. // // argument defaults static int THREADS = 32; // more repeatable if THREADS <= #processors -static int SCALE = 10; // scaling factor +static int SCALE = 25; // scaling factor static int ITER = 50; // N full iterations destructing and re-creating all threads // static int THREADS = 8; // more repeatable if THREADS <= #processors From 4b0cda2012ba0088639afc899d024f4e5e0514e4 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Nov 2021 11:23:11 -0800 Subject: [PATCH 110/134] take overcommit into account for eager commit delay --- include/mimalloc-internal.h | 1 + src/os.c | 6 +++++- src/segment.c | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 23078330..058acf80 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -72,6 +72,7 @@ void _mi_os_init(void); // called fro void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); // memory.c void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld); diff --git a/src/os.c b/src/os.c index 67b6c6db..ee6e92c0 100644 --- a/src/os.c +++ b/src/os.c @@ -101,6 +101,10 @@ static size_t large_os_page_size = 0; // set dynamically in _mi_os_init (and if true we use MAP_NORESERVE) static bool os_overcommit = true; +bool _mi_os_has_overcommit(void) { + return os_overcommit; +} + // OS (small) page size size_t _mi_os_page_size() { return os_page_size; @@ -460,7 +464,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif int flags = MAP_PRIVATE | MAP_ANONYMOUS; int fd = -1; - if (os_overcommit) { + if (_mi_os_has_overcommit()) { flags |= MAP_NORESERVE; } #if defined(PROT_MAX) diff --git a/src/segment.c b/src/segment.c index 6de3a8ae..424495ca 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,9 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 2 && + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && // don't delay for large objects + !_mi_os_has_overcommit() && // never delay on overcommit systems + _mi_current_thread_count() > 2 && // do not delay for the first N threads tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From 4ce6821c09506d29feabdb8cfc4ac1d94b6574b6 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:32:21 -0800 Subject: [PATCH 111/134] update cmake install for MI_INSTALL_TOPLEVEL --- cmake/mimalloc-config.cmake | 6 +++--- test/CMakeLists.txt | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 024c97d9..9b0a56d7 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -4,8 +4,8 @@ if (MIMALLOC_SHARE_DIR MATCHES "/share/") string(REPLACE "/share/" "/lib/" MIMALLOC_LIBRARY_DIR ${MIMALLOC_SHARE_DIR}) string(REPLACE "/share/" "/include/" MIMALLOC_INCLUDE_DIR ${MIMALLOC_SHARE_DIR}) else() - # if MI_INSTALL_TOPLEVEL==ON - set(MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}/lib") - set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}/include") + # installed with -DMI_INSTALL_TOPLEVEL=ON + string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}") + string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}") endif() set(MIMALLOC_TARGET_DIR "${MIMALLOC_LIBRARY_DIR}") # legacy diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7986d2da..13bf32de 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,9 @@ cmake_minimum_required(VERSION 3.0) project(mimalloc-test C CXX) +set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) + # Set default build type if (NOT CMAKE_BUILD_TYPE) if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$") From d67ff1ca9fbffee067f5bbfa8b0c6e6e90437ec2 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:32:43 -0800 Subject: [PATCH 112/134] add include cstdef for std::size_t in C++ --- include/mimalloc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/mimalloc.h b/include/mimalloc.h index a77f1dc5..0a4ae689 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -388,6 +388,7 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, // --------------------------------------------------------------------------------------------- #ifdef __cplusplus +#include // std::size_t #include // PTRDIFF_MAX #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 #include // std::true_type From 07253fb44c3c220c93dbb64c6dd12c5d7608fb4a Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:33:04 -0800 Subject: [PATCH 113/134] emit error message if out-of-memory in C++ --- src/alloc.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 1dbd31a7..d9b6dd60 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -802,7 +802,10 @@ static bool mi_try_new_handler(bool nothrow) { std::set_new_handler(h); #endif if (h==NULL) { - if (!nothrow) throw std::bad_alloc(); + _mi_error_message(ENOMEM, "out of memory in 'new'"); + if (!nothrow) { + throw std::bad_alloc(); + } return false; } else { @@ -830,9 +833,9 @@ static std_new_handler_t mi_get_new_handler() { static bool mi_try_new_handler(bool nothrow) { std_new_handler_t h = mi_get_new_handler(); if (h==NULL) { + _mi_error_message(ENOMEM, "out of memory in 'new'"); if (!nothrow) { - _mi_error_message(EFAULT, "out of memory in 'new' call"); // cannot throw in plain C, use EFAULT to abort - abort(); + abort(); // cannot throw in plain C, use abort } return false; } From 4a590b1447e9d27e00dcd5bc9ac62eab6d7e51bb Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:33:56 -0800 Subject: [PATCH 114/134] bump version number to 1.7.3 --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 0a4ae689..4467173e 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 171 // major + 2 digits minor +#define MI_MALLOC_VERSION 173 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes From 6ca454a04a272a2a2f1309db3b8a050049734a14 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Nov 2021 16:41:24 -0800 Subject: [PATCH 115/134] update readme --- readme.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/readme.md b/readme.md index cdb1b82a..2369b9c1 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the run-time systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.0.2` (beta, 2021-06-17). -Latest stable tag: `v1.7.2` (2021-06-17). +Latest release tag: `v2.0.3` (beta, 2021-11-14). +Latest stable tag: `v1.7.3` (2021-11-14). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -77,6 +77,10 @@ Note: the `v2.x` beta has a new algorithm for managing internal mimalloc pages t and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. +* 2021-11-14, `v1.7.3`, `v2.0.3` (beta): improved WASM support, improved macOS support and performance (including + M1), improved performance for v2 for large objects, Python integration improvements, more standard + installation directories, various small fixes. + * 2021-06-17, `v1.7.2`, `v2.0.2` (beta): support M1, better installation layout on Linux, fix thread_id on Android, prefer 2-6TiB area for aligned allocation to work better on pre-windows 8, various small fixes. @@ -142,7 +146,7 @@ mimalloc is used in various large scale low-latency services and programs, for e ## Windows -Open `ide/vs2019/mimalloc.sln` in Visual Studio 2019 and build (or `ide/vs2017/mimalloc.sln`). +Open `ide/vs2019/mimalloc.sln` in Visual Studio 2019 and build. The `mimalloc` project builds a static library (in `out/msvc-x64`), while the `mimalloc-override` project builds a DLL for overriding malloc in the entire program. @@ -191,6 +195,11 @@ Notes: 2. Install CCMake: `sudo apt-get install cmake-curses-gui` +## Single source + +You can also directly build the single `src/static.c` file as part of your project without +needing `cmake` at all. Make sure to also add the mimalloc `include` directory to the include path. + # Using the library @@ -337,9 +346,9 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to - Corrupted free-lists and some forms of use-after-free are detected. -# Overriding Malloc +# Overriding Standard Malloc -Overriding the standard `malloc` can be done either _dynamically_ or _statically_. +Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. ## Dynamic override @@ -370,13 +379,12 @@ On macOS we can also preload the mimalloc shared library so all calls to the standard `malloc` interface are resolved to the _mimalloc_ library. ``` -> env DYLD_FORCE_FLAT_NAMESPACE=1 DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram +> env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram ``` Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -(Note: macOS support for dynamic overriding is recent, please report any issues.) ### Override on Windows @@ -386,7 +394,7 @@ the (dynamic) C runtime allocator, including those from other DLL's or libraries The overriding on Windows requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be available +Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be put in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in `mimalloc-override.dll`). From 0560fc27c08d28d523b7f741a42deb26cd01c0c6 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Nov 2021 16:47:50 -0800 Subject: [PATCH 116/134] update readme --- readme.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 1586bddc..dfef82d5 100644 --- a/readme.md +++ b/readme.md @@ -311,8 +311,9 @@ or via environment variables: `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). -- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. - (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) + The huge pages are usually allocated evenly among NUMA nodes. + We can use `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N` where `N` is the numa node (starting at 0) to allocate all + the huge pages at a specific numa node instead. Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the From 09c658dd401a9a4a7a2c807f6bd274d0420e109d Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 15 Nov 2021 10:10:47 -0800 Subject: [PATCH 117/134] try to allocate within our hint space (partially addresses issue #360) --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index ee6e92c0..90773fc6 100644 --- a/src/os.c +++ b/src/os.c @@ -585,7 +585,7 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; - if ((size%MI_SEGMENT_SIZE) != 0) return NULL; + size = _mi_align_up(size, MI_SEGMENT_SIZE); if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. From 1c22650719cc09cdeba10b9add7af4b1e9b2ef2d Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 15 Nov 2021 10:52:09 -0800 Subject: [PATCH 118/134] remove no-invalid-memory-model supression --- CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 87f4cddd..7c2c6cfc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -181,10 +181,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) if(NOT MI_USE_CXX) list(APPEND mi_cflags -Wstrict-prototypes) - endif() - if(CMAKE_C_COMPILER_ID MATCHES "GNU") - list(APPEND mi_cflags -Wno-invalid-memory-model) - endif() + endif() if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wpedantic -Wno-static-in-inline) endif() From 8b60a5ab70b342ca1ff404a4ee6c5340ccfb47a6 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Nov 2021 17:59:27 -0800 Subject: [PATCH 119/134] add mi_unsafe_free_with_threadid and mi_get_current_threadid() --- include/mimalloc-internal.h | 3 +++ include/mimalloc.h | 2 ++ src/alloc.c | 25 +++++++++++++++++++++---- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf5b6783..1333c80f 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -20,14 +20,17 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) #define mi_decl_noinline __declspec(noinline) +#define mi_decl_always_inline __forceinline #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_always_inline __attribute__((always_inline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) #else #define mi_decl_noinline +#define mi_decl_always_inline inline #define mi_decl_thread __thread // hope for the best :-) #define mi_decl_cache_align #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index 756e516c..f90af4a3 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -271,6 +271,8 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; +mi_decl_export size_t mi_get_current_threadid(void) mi_attr_noexcept; +mi_decl_export void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept; // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; diff --git a/src/alloc.c b/src/alloc.c index d9b6dd60..4d0d9e59 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -475,14 +475,12 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms return segment; } - -// Free a block -void mi_free(void* p) mi_attr_noexcept +// Free a block with a known threadid +static mi_decl_always_inline void _mi_free_with_threadid(void* p, mi_threadid_t tid) mi_attr_noexcept { const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; - const mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; @@ -507,6 +505,25 @@ void mi_free(void* p) mi_attr_noexcept } } +// Get the current thread id +size_t mi_get_current_threadid(void) mi_attr_noexcept { + return _mi_thread_id(); +} + +// Free a block passing the current thread id explicitly +void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept +{ + mi_assert(current_tid == _mi_thread_id()); + _mi_free_with_threadid(p,current_tid); +} + + +// Free a block +void mi_free(void* p) mi_attr_noexcept { + _mi_free_with_threadid(p, _mi_thread_id()); +} + + bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); From 3548d8d716569f96967af4da0fdc57d2f09e7c38 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Nov 2021 18:39:03 -0800 Subject: [PATCH 120/134] add noinline to avoid warnings --- src/alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 4d0d9e59..e9167848 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -511,7 +511,7 @@ size_t mi_get_current_threadid(void) mi_attr_noexcept { } // Free a block passing the current thread id explicitly -void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept +void mi_decl_noinline mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept { mi_assert(current_tid == _mi_thread_id()); _mi_free_with_threadid(p,current_tid); @@ -519,7 +519,7 @@ void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept // Free a block -void mi_free(void* p) mi_attr_noexcept { +void mi_decl_noinline mi_free(void* p) mi_attr_noexcept { _mi_free_with_threadid(p, _mi_thread_id()); } From 9183b1eec005be9863e58d51ba0f96b97721d48c Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Nov 2021 19:04:41 -0800 Subject: [PATCH 121/134] remove experiment with unsafe_free_with_threadid --- include/mimalloc-internal.h | 3 --- include/mimalloc.h | 3 --- src/alloc.c | 24 +++--------------------- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1333c80f..cf5b6783 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -20,17 +20,14 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) #define mi_decl_noinline __declspec(noinline) -#define mi_decl_always_inline __forceinline #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) -#define mi_decl_always_inline __attribute__((always_inline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) #else #define mi_decl_noinline -#define mi_decl_always_inline inline #define mi_decl_thread __thread // hope for the best :-) #define mi_decl_cache_align #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index f90af4a3..06a16703 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -271,9 +271,6 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; -mi_decl_export size_t mi_get_current_threadid(void) mi_attr_noexcept; -mi_decl_export void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept; - // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; diff --git a/src/alloc.c b/src/alloc.c index e9167848..ca32caba 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -475,12 +475,13 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms return segment; } -// Free a block with a known threadid -static mi_decl_always_inline void _mi_free_with_threadid(void* p, mi_threadid_t tid) mi_attr_noexcept +// Free a block +void mi_free(void* p) mi_attr_noexcept { const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; + mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; @@ -505,25 +506,6 @@ static mi_decl_always_inline void _mi_free_with_threadid(void* p, mi_threadid_t } } -// Get the current thread id -size_t mi_get_current_threadid(void) mi_attr_noexcept { - return _mi_thread_id(); -} - -// Free a block passing the current thread id explicitly -void mi_decl_noinline mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept -{ - mi_assert(current_tid == _mi_thread_id()); - _mi_free_with_threadid(p,current_tid); -} - - -// Free a block -void mi_decl_noinline mi_free(void* p) mi_attr_noexcept { - _mi_free_with_threadid(p, _mi_thread_id()); -} - - bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); From 0be71a2cac17062bd8913cbd272c472a44331b7f Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 24 Nov 2021 12:54:54 -0800 Subject: [PATCH 122/134] fix prefix --- src/os.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/os.c b/src/os.c index 90773fc6..603ec04a 100644 --- a/src/os.c +++ b/src/os.c @@ -578,29 +578,29 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) -#define KK_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start -#define KK_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) -#define KK_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) +#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start +#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) +#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; size = _mi_align_up(size, MI_SEGMENT_SIZE); - if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. + if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. #endif uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); - if (hint == 0 || hint > KK_HINT_MAX) { // wrap or initialize - uintptr_t init = KK_HINT_BASE; + if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize + uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % KK_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > KK_HINT_MAX but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all } if (hint%try_alignment != 0) return NULL; return (void*)hint; @@ -638,11 +638,11 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo if (commit) flags |= MEM_COMMIT; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); #elif defined(MI_USE_SBRK) - KK_UNUSED(allow_large); + MI_UNUSED(allow_large); *is_large = false; p = mi_sbrk_heap_grow(size, try_alignment); #elif defined(__wasi__) - KK_UNUSED(allow_large); + MI_UNUSED(allow_large); *is_large = false; p = mi_wasm_heap_grow(size, try_alignment); #else From acc64ee5cd58277490a89d4c4f97e3f3839beff5 Mon Sep 17 00:00:00 2001 From: Igor Vlasenko Date: Sun, 28 Nov 2021 19:14:31 +0200 Subject: [PATCH 123/134] added SOVERSION to mimalloc shared lib (issue #490) This is a portability improvement. A cross-platform library needs SOVERSION field for Unix platforms. With SOVERSION field cmake itself will do proper management of libmimalloc.so.SOVERSION -> libmimalloc.so.VERSION symlink on Unix, so a piece of code that tried to emulate this behavior manually is no more needed and is removed here too. --- CMakeLists.txt | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c2c6cfc..8eb5c958 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -280,7 +280,7 @@ message(STATUS "") # shared library if(MI_BUILD_SHARED) add_library(mimalloc SHARED ${mi_sources}) - set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} OUTPUT_NAME ${mi_basename} ) + set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} SOVERSION ${mi_version_major} OUTPUT_NAME ${mi_basename} ) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) target_link_libraries(mimalloc PUBLIC ${mi_libraries}) @@ -336,16 +336,6 @@ install(FILES include/mimalloc-new-delete.h DESTINATION ${mi_install_incdir}) install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_cmakedir}) install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_cmakedir}) -if(NOT WIN32 AND MI_BUILD_SHARED AND NOT MI_INSTALL_TOPLEVEL) - # install a symlink in the /usr/local/lib to the versioned library - # note: use delayed prefix expansion as \${CMAKE_INSTALL_PREFIX} - set(mi_symlink "${CMAKE_SHARED_MODULE_PREFIX}${mi_basename}${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(mi_soname "mimalloc-${mi_version}/${mi_symlink}.${mi_version}") - install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") - install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink} -> ${mi_soname}\")") - install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink}.${mi_version} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") - install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink}.${mi_version} -> ${mi_soname}\")") -endif() # single object file for more predictable static overriding if (MI_BUILD_OBJECT) From f82e13ac91716ff60d4e656e1f825ce727b7fd7d Mon Sep 17 00:00:00 2001 From: Igor Vlasenko Date: Sun, 28 Nov 2021 19:20:12 +0200 Subject: [PATCH 124/134] let the library VERSION = the project's one (issue #490) This is a cross-platform usability improvement. On Unix platforms it is customary for library to have VERSION and SOVERSION, where SOVERSION changes on major API changes and VERSION is the same as project's version, so library users always know what vesion this library belongs to just by name. With this patch we have a proper libmimalloc.so.VERSION on Unix. --- cmake/mimalloc-config-version.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index ed95c19e..3768d7de 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,7 @@ set(mi_version_major 1) set(mi_version_minor 7) -set(mi_version ${mi_version_major}.${mi_version_minor}) +set(mi_version_patch 3) +set(mi_version ${mi_version_major}.${mi_version_minor}.${mi_version_patch}) set(PACKAGE_VERSION ${mi_version}) if(PACKAGE_FIND_VERSION_MAJOR) From 36edfbc70ae4610d06b9fce245f35d92e6be92a5 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 9 Dec 2021 16:18:17 -0800 Subject: [PATCH 125/134] use rtlgenrandom by default on windows --- src/random.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/random.c b/src/random.c index 05c5c99c..1220401a 100644 --- a/src/random.c +++ b/src/random.c @@ -167,8 +167,9 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim #if defined(_WIN32) -#if !defined(MI_USE_RTLGENRANDOM) -// We prefer BCryptGenRandom over RtlGenRandom +#if defined(MI_USE_BCRYPTGENRANDOM) +// We would like to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock +// under the VS debugger when using dynamic overriding. #pragma comment (lib,"bcrypt.lib") #include static bool os_random_buf(void* buf, size_t buf_len) { From 3a212d1895c8722a31a78d354793fbd6b2f3192f Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 12 Dec 2021 10:35:13 -0800 Subject: [PATCH 126/134] fix assembly for mi_tls_slot_set on x32 and x64. Issue #488 --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf5b6783..4713a75e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -746,9 +746,9 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { #elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI + __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI #elif defined(__x86_64__) - __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS + __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); From d575aacfde579c241580d61297ec2430c4994109 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 13 Dec 2021 13:10:33 -0800 Subject: [PATCH 127/134] use find_library for pthread (issue #496) --- CMakeLists.txt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c2c6cfc..47b43ce5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,13 +210,14 @@ endif() if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() - if(NOT ${CMAKE_C_COMPILER} MATCHES "android") - list(APPEND mi_libraries pthread) - find_library(LIBRT rt) - if(LIBRT) - list(APPEND mi_libraries ${LIBRT}) - endif() + find_library(LIBPTHREAD pthread) + if (LIBPTHREAD) + list(APPEND mi_libraries ${LIBPTHREAD}) endif() + find_library(LIBRT rt) + if(LIBRT) + list(APPEND mi_libraries ${LIBRT}) + endif() endif() if (MI_USE_LIBATOMIC) @@ -270,6 +271,8 @@ else() message(STATUS "C Compiler : ${CMAKE_C_COMPILER}") endif() message(STATUS "Compiler flags : ${mi_cflags}") +message(STATUS "Compiler defines : ${mi_defines}") +message(STATUS "Link libraries : ${mi_libraries}") message(STATUS "Build targets : ${mi_build_targets}") message(STATUS "") From 69b6b246880d04e911416ffce8e8ccf6539ae2d7 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 14 Dec 2021 18:29:14 -0800 Subject: [PATCH 128/134] further improvements to installation directories --- CMakeLists.txt | 19 +++++++++++-------- cmake/mimalloc-config-version.cmake | 4 ++-- cmake/mimalloc-config.cmake | 19 +++++++++++-------- include/mimalloc.h | 2 +- test/CMakeLists.txt | 2 +- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3790b09..0ceadffd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,14 +228,17 @@ endif() # Install and output names # ----------------------------------------------------------------------------- +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") # for dynamic/shared library and symlinks + +# install at top level or use versioned directories for side-by-side installation? if (MI_INSTALL_TOPLEVEL) - set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") - set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") - set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc") + set(mi_install_objdir "${CMAKE_INSTALL_LIBDIR}") + set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") + set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc") else() - set(mi_install_libdir "lib/mimalloc-${mi_version}") - set(mi_install_incdir "include/mimalloc-${mi_version}") - set(mi_install_cmakedir "share/mimalloc-${mi_version}/cmake") + set(mi_install_objdir "${CMAKE_INSTALL_LIBDIR}/mimalloc-${mi_version}") # for static library and object files + set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}/mimalloc-${mi_version}") # for includes + set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc-${mi_version}") # for cmake package info endif() if(MI_SECURE) @@ -329,7 +332,7 @@ if (MI_BUILD_STATIC) set_target_properties(mimalloc-static PROPERTIES OUTPUT_NAME ${mi_basename}) endif() - install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) + install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_objdir} LIBRARY) endif() # install include files @@ -357,7 +360,7 @@ if (MI_BUILD_OBJECT) # the FILES expression can also be: $ # but that fails cmake versions less than 3.10 so we leave it as is for now install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION} - DESTINATION ${mi_install_libdir} + DESTINATION ${mi_install_objdir} RENAME ${mi_basename}${CMAKE_C_OUTPUT_EXTENSION} ) endif() diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 3768d7de..7bbd7313 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,7 +1,7 @@ set(mi_version_major 1) set(mi_version_minor 7) -set(mi_version_patch 3) -set(mi_version ${mi_version_major}.${mi_version_minor}.${mi_version_patch}) +set(mi_version_patch 4) +set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) if(PACKAGE_FIND_VERSION_MAJOR) diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 9b0a56d7..45d9692f 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -1,11 +1,14 @@ include(${CMAKE_CURRENT_LIST_DIR}/mimalloc.cmake) -get_filename_component(MIMALLOC_SHARE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/share/mimalloc-2.0 -if (MIMALLOC_SHARE_DIR MATCHES "/share/") - string(REPLACE "/share/" "/lib/" MIMALLOC_LIBRARY_DIR ${MIMALLOC_SHARE_DIR}) - string(REPLACE "/share/" "/include/" MIMALLOC_INCLUDE_DIR ${MIMALLOC_SHARE_DIR}) -else() - # installed with -DMI_INSTALL_TOPLEVEL=ON - string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}") - string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}") +get_filename_component(MIMALLOC_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/lib/cmake/mimalloc-2.0 +get_filename_component(MIMALLOC_VER_DIR "${CMAKE_CURRENT_LIST_DIR}" NAME) +string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_CMAKE_DIR}") +if("${MIMALLOC_VER_DIR}" EQUAL "mimalloc") + # top level install + string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_CMAKE_DIR}") + set(MIMALLOC_OBJECT_DIR "${MIMALLOC_LIBRARY_DIR}") +else() + # versioned + string(REPLACE "/lib/cmake/" "/include/" MIMALLOC_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}") + string(REPLACE "/lib/cmake/" "/lib/" MIMALLOC_OBJECT_DIR "${CMAKE_CURRENT_LIST_DIR}") endif() set(MIMALLOC_TARGET_DIR "${MIMALLOC_LIBRARY_DIR}") # legacy diff --git a/include/mimalloc.h b/include/mimalloc.h index 06a16703..a955e35c 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 173 // major + 2 digits minor +#define MI_MALLOC_VERSION 174 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 13bf32de..407dbee9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,7 @@ target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) # overriding with a static object file works reliable as the symbols in the # object file have priority over those in library files -add_executable(static-override-obj main-override.c ${MIMALLOC_LIBRARY_DIR}/mimalloc.o) +add_executable(static-override-obj main-override.c ${MIMALLOC_OBJECT_DIR}/mimalloc.o) target_include_directories(static-override-obj PUBLIC ${MIMALLOC_INCLUDE_DIR}) target_link_libraries(static-override-obj PUBLIC pthread) From 73ced777dd11feeafc4664e8fe1814fa942481af Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 14 Dec 2021 18:42:10 -0800 Subject: [PATCH 129/134] upgrade macos to latest --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aeb2908b..7261fc13 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,7 +115,7 @@ jobs: displayName: macOS pool: vmImage: - macOS-10.14 + macOS-latest strategy: matrix: Debug: From 6503ad7a76dacfae142ed4f8f23d1039a18dede8 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 14 Dec 2021 18:45:44 -0800 Subject: [PATCH 130/134] check if using bcryptgenrandom fixes windows pipeline --- src/random.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/random.c b/src/random.c index 1220401a..fabd6c6e 100644 --- a/src/random.c +++ b/src/random.c @@ -167,8 +167,8 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim #if defined(_WIN32) -#if defined(MI_USE_BCRYPTGENRANDOM) -// We would like to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock +#if !defined(MI_USE_RTLGENRANDOM) +// We prefer to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock // under the VS debugger when using dynamic overriding. #pragma comment (lib,"bcrypt.lib") #include From 2d9b8aa6b56241ff3cc88ed5d5e3d862d91fd096 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:33:14 -0800 Subject: [PATCH 131/134] rename VER_DIR to VERSION_DIR --- cmake/mimalloc-config.cmake | 4 ++-- test/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 45d9692f..8a28e37e 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -1,8 +1,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/mimalloc.cmake) get_filename_component(MIMALLOC_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/lib/cmake/mimalloc-2.0 -get_filename_component(MIMALLOC_VER_DIR "${CMAKE_CURRENT_LIST_DIR}" NAME) +get_filename_component(MIMALLOC_VERSION_DIR "${CMAKE_CURRENT_LIST_DIR}" NAME) string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_CMAKE_DIR}") -if("${MIMALLOC_VER_DIR}" EQUAL "mimalloc") +if("${MIMALLOC_VERSION_DIR}" EQUAL "mimalloc") # top level install string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_CMAKE_DIR}") set(MIMALLOC_OBJECT_DIR "${MIMALLOC_LIBRARY_DIR}") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 407dbee9..054d9409 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,7 +17,7 @@ endif() # Import mimalloc (if installed) find_package(mimalloc 1.7 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) -message(STATUS "Found mimalloc installed at: ${MIMALLOC_LIBRARY_DIR}") +message(STATUS "Found mimalloc installed at: ${MIMALLOC_LIBRARY_DIR} (${MIMALLOC_VERSION_DIR})") # overriding with a dynamic library add_executable(dynamic-override main-override.c) From b7d33c2c33aa4475014f80602a09e5d348d5821e Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:41:52 -0800 Subject: [PATCH 132/134] fix MI_ prefix for libraries --- CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ceadffd..48ecd315 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,13 +210,13 @@ endif() if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() - find_library(LIBPTHREAD pthread) - if (LIBPTHREAD) - list(APPEND mi_libraries ${LIBPTHREAD}) + find_library(MI_LIBPTHREAD pthread) + if (MI_LIBPTHREAD) + list(APPEND mi_libraries ${MI_LIBPTHREAD}) endif() - find_library(LIBRT rt) - if(LIBRT) - list(APPEND mi_libraries ${LIBRT}) + find_library(MI_LIBRT rt) + if(MI_LIBRT) + list(APPEND mi_libraries ${MI_LIBRT}) endif() endif() @@ -228,9 +228,11 @@ endif() # Install and output names # ----------------------------------------------------------------------------- -set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") # for dynamic/shared library and symlinks +# dynamic/shared library and symlinks always go to /usr/local/lib equivalent +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") -# install at top level or use versioned directories for side-by-side installation? +# static libraries and object files, includes, and cmake config files +# are either installed at top level, or use versioned directories for side-by-side installation (default) if (MI_INSTALL_TOPLEVEL) set(mi_install_objdir "${CMAKE_INSTALL_LIBDIR}") set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") From 144b4a2d3e1535634b5e8a5d12fa9e64e44dadbd Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:42:58 -0800 Subject: [PATCH 133/134] link with libatomic automatically if found --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 48ecd315..6b2e9b83 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,10 +218,10 @@ else() if(MI_LIBRT) list(APPEND mi_libraries ${MI_LIBRT}) endif() -endif() - -if (MI_USE_LIBATOMIC) - list(APPEND mi_libraries atomic) + find_library(MI_LIBATOMIC atomic) + if (MI_LIBATOMIC OR MI_USE_LIBATOMIC) + list(APPEND mi_libraries atomic) + endif() endif() # ----------------------------------------------------------------------------- From 523e6d5c9b251bc8d969a685b2b6feda6fd1f8ab Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 16:05:07 -0800 Subject: [PATCH 134/134] fix thread slot on arm32 (issue #495) --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 4713a75e..324ad4a3 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -766,8 +766,8 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) - // on Android, slot 1 is the thread ID (pointer to pthread internal struct) +#if defined(__arm__) || (defined(__BIONIC__) && defined(__aarch64__)) + // issue #384, #495: on arm32 and arm32/arm64 Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); #else // in all our other targets, slot 0 is the pointer to the thread control block