From cb45e3c6b198ed60a86428350d69b7a3e781fb77 Mon Sep 17 00:00:00 2001 From: Vasya B Date: Mon, 19 Oct 2020 21:00:16 +0000 Subject: [PATCH 01/10] fix for x32 builds --- include/mimalloc-internal.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 145f6f92..c0fa1d66 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -699,6 +699,8 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS +#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI #elif defined(__x86_64__) __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) @@ -720,6 +722,8 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS +#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI #elif defined(__x86_64__) __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) From e6c2fd44fcd892a6109e4a80d2bbb1a475c9cef9 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Thu, 22 Oct 2020 11:15:37 +0100 Subject: [PATCH 02/10] DragonFly support fix (for 5.8.x and forward). The pthread slot approach is somewhat buggy (pretty visible with the stress unit test which segfault more or less randomly, but the stats never show up). Using the default approach instead, the test passes eventough it s relatively slow (e.g 1.5 sec on FreeBSD vs 4.5 on DragonFly with same machine). --- include/mimalloc-internal.h | 6 +++--- test/test-stress.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 3c0210b3..b5517e4b 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -286,7 +286,7 @@ We try to circumvent this in an efficient way: - macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the loader itself calls `malloc` even before the modules are initialized. - OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). -- DragonFly: not yet working. +- DragonFly: the uniqueid use is buggy but kept for reference. ------------------------------------------------------------------------------------------- */ extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap @@ -304,7 +304,7 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) #elif defined(__DragonFly__) #warning "mimalloc is not working correctly on DragonFly yet." -#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +//#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) #endif #endif @@ -316,7 +316,7 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { pthread_t self = pthread_self(); #if defined(__DragonFly__) if (self==NULL) { - static mi_heap_t* pheap_main = _mi_heap_main_get(); + mi_heap_t* pheap_main = _mi_heap_main_get(); return &pheap_main; } #endif diff --git a/test/test-stress.c b/test/test-stress.c index cba56310..ead88e37 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -306,7 +306,7 @@ static void run_os_threads(size_t nthreads, void (*fun)(intptr_t)) { pthread_t* threads = (pthread_t*)custom_calloc(nthreads,sizeof(pthread_t)); memset(threads, 0, sizeof(pthread_t) * nthreads); //pthread_setconcurrency(nthreads); - for (uintptr_t i = 0; i < nthreads; i++) { + for (size_t i = 0; i < nthreads; i++) { pthread_create(&threads[i], NULL, &thread_entry, (void*)i); } for (size_t i = 0; i < nthreads; i++) { From 1deea03bf18119bab2b010ac5fdc2d303ac5403d Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 31 Oct 2020 22:22:04 +0000 Subject: [PATCH 03/10] On Darwin, using MADV_FREE_REUSABLE/MADV_FREE_REUSE. The former to notify the pages are available for other processes, the latter is needed for proper counting in case those pages where tagged as reusable previously otherwise is a no-op, all for better RSS reporting for task_info apps. --- src/os.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index e8d6b74a..dfe654f5 100644 --- a/src/os.c +++ b/src/os.c @@ -435,7 +435,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // though since properly aligned allocations will already use large pages if available // in that case -- in particular for our large regions (in `memory.c`). // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvice` anyways. + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. if (allow_large && use_large_os_page(size, try_alignment)) { if (madvise(p, size, MADV_HUGEPAGE) == 0) { *is_large = true; // possibly @@ -721,6 +721,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } + #if defined(MADV_FREE_REUSE) + while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) {} + #endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -782,9 +785,17 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) + #if defined(MADV_FREE_REUSABLE) + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE_REUSABLE); + int oadvice = (int)mi_atomic_load_relaxed(&advice); + int err; + while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) {} + if (err != 0 && errno == EINVAL && advice == MADV_FREE_REUSABLE) { + #else static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int err = madvise(start, csize, (int)mi_atomic_load_relaxed(&advice)); if (err != 0 && errno == EINVAL && advice == MADV_FREE) { + #endif // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); From 9c45221243b48ae5a1eea0027a88759cfe020dda Mon Sep 17 00:00:00 2001 From: Anton Korobeynikov Date: Sun, 1 Nov 2020 23:57:42 +0300 Subject: [PATCH 04/10] Do not use the same counter for warnings and errors. Warnings happen normally and could be safely ignored in the most cases, however errors, if enabled, should not be ignored. Currently since warnings and errors share the same counter we effectively stop showing errors after 16 warnings (which happen all the time). Use different counters for errors and warnings. --- include/mimalloc.h | 1 + src/options.c | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index d4fbeec5..03535f0c 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -317,6 +317,7 @@ typedef enum mi_option_e { mi_option_limit_os_alloc, mi_option_os_tag, mi_option_max_errors, + mi_option_max_warnings, _mi_option_last } mi_option_t; diff --git a/src/options.c b/src/options.c index 2dbc8b03..f3e8f30b 100644 --- a/src/options.c +++ b/src/options.c @@ -19,7 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif -static uintptr_t mi_max_error_count = 16; // stop outputting errors after this +static uintptr_t mi_max_error_count = -1ULL; // stop outputting errors after this +static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this static void mi_add_stderr_output(); @@ -89,7 +90,8 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output + { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output + { -1ULL, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output }; static void mi_option_init(mi_option_desc_t* desc); @@ -107,6 +109,7 @@ void _mi_options_init(void) { } } mi_max_error_count = mi_option_get(mi_option_max_errors); + mi_max_warning_count = mi_option_get(mi_option_max_warnings); } long mi_option_get(mi_option_t option) { @@ -325,7 +328,7 @@ static void mi_show_error_message(const char* fmt, va_list args) { void _mi_warning_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; - if (mi_atomic_increment_acq_rel(&error_count) > mi_max_error_count) return; + if (mi_atomic_increment_acq_rel(&error_count) > mi_max_warning_count) return; va_list args; va_start(args,fmt); mi_vfprintf(NULL, NULL, "mimalloc: warning: ", fmt, args); From 39bcf8a6b0af0c07379aea9c1a2330db49d3d5a0 Mon Sep 17 00:00:00 2001 From: Anton Korobeynikov Date: Mon, 2 Nov 2020 00:14:02 +0300 Subject: [PATCH 05/10] Honour MI_STAT in couple more places. --- include/mimalloc-types.h | 1 + src/heap.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 8ffcd2fc..b94bbf6f 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -447,6 +447,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); #define mi_stat_counter_increase(stat,amount) (void)0 #endif +#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) diff --git a/src/heap.c b/src/heap.c index 4a91786b..1dd6343f 100644 --- a/src/heap.c +++ b/src/heap.c @@ -274,10 +274,10 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_ const size_t bsize = mi_page_block_size(page); if (bsize > MI_LARGE_OBJ_SIZE_MAX) { if (bsize > MI_HUGE_OBJ_SIZE_MAX) { - _mi_stat_decrease(&heap->tld->stats.giant, bsize); + mi_heap_stat_decrease(heap, giant, bsize); } else { - _mi_stat_decrease(&heap->tld->stats.huge, bsize); + mi_heap_stat_decrease(heap, huge, bsize); } } #if (MI_STAT>1) From bbdf470715d2f8c0adae4f21e79e73386d1b3da5 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Wed, 18 Nov 2020 17:21:51 +0000 Subject: [PATCH 06/10] mi_bitmap_try_find_claim_field_across number of leading zeros is unsigned. --- src/bitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitmap.c b/src/bitmap.c index a94e15cf..68ae3b2e 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -179,7 +179,7 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit _Atomic(uintptr_t)* field = &bitmap[idx]; uintptr_t map = mi_atomic_load_relaxed(field); const size_t initial = mi_clz(map); // count of initial zeros starting at idx - mi_assert_internal(initial >= 0 && initial <= MI_BITMAP_FIELD_BITS); + mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS); if (initial == 0) return false; if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries From cbc14a9287847599e2b4fb1e2b1f7a5287f5a401 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 10 Dec 2020 10:28:54 -0800 Subject: [PATCH 07/10] count warnings and errors separately --- src/options.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/options.c b/src/options.c index f3e8f30b..56ba6bdd 100644 --- a/src/options.c +++ b/src/options.c @@ -19,8 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif -static uintptr_t mi_max_error_count = -1ULL; // stop outputting errors after this -static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this +static uintptr_t mi_max_error_count = 64; // stop outputting errors after this +static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this static void mi_add_stderr_output(); @@ -90,8 +90,9 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output - { -1ULL, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output + { 64, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 16, UNINIT, MI_OPTION(max_warnings) } // maximum warnings that are output + }; static void mi_option_init(mi_option_desc_t* desc); @@ -250,7 +251,8 @@ static void mi_add_stderr_output() { // -------------------------------------------------------- // Messages, all end up calling `_mi_fputs`. // -------------------------------------------------------- -static _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings +static _Atomic(uintptr_t) error_count; // = 0; // when >= max_error_count stop emitting errors +static _Atomic(uintptr_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings // When overriding malloc, we may recurse into mi_vfprintf if an allocation // inside the C runtime causes another message. @@ -328,7 +330,7 @@ static void mi_show_error_message(const char* fmt, va_list args) { void _mi_warning_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; - if (mi_atomic_increment_acq_rel(&error_count) > mi_max_warning_count) return; + if (mi_atomic_increment_acq_rel(&warning_count) > mi_max_warning_count) return; va_list args; va_start(args,fmt); mi_vfprintf(NULL, NULL, "mimalloc: warning: ", fmt, args); From f37a3db37c43509dbe9f76ec49cdd2a1b5dbadaa Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 10 Dec 2020 11:51:58 -0800 Subject: [PATCH 08/10] cleanup madv_resuable --- src/options.c | 4 ++-- src/os.c | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/options.c b/src/options.c index 56ba6bdd..d4f3f29a 100644 --- a/src/options.c +++ b/src/options.c @@ -19,7 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif -static uintptr_t mi_max_error_count = 64; // stop outputting errors after this +static uintptr_t mi_max_error_count = 16; // stop outputting errors after this static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this static void mi_add_stderr_output(); @@ -90,7 +90,7 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose - { 64, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output + { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output { 16, UNINIT, MI_OPTION(max_warnings) } // maximum warnings that are output }; diff --git a/src/os.c b/src/os.c index dfe654f5..22e736f7 100644 --- a/src/os.c +++ b/src/os.c @@ -721,9 +721,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } - #if defined(MADV_FREE_REUSE) - while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) {} - #endif + #if defined(MADV_FREE_REUSE) + while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } + #endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -786,17 +786,16 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) #else #if defined(MADV_FREE) #if defined(MADV_FREE_REUSABLE) - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE_REUSABLE); + #define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE + #else + #define KK_MADV_FREE_INITIAL MADV_FREE + #endif + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; - while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) {} - if (err != 0 && errno == EINVAL && advice == MADV_FREE_REUSABLE) { - #else - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); - int err = madvise(start, csize, (int)mi_atomic_load_relaxed(&advice)); - if (err != 0 && errno == EINVAL && advice == MADV_FREE) { - #endif - // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on + while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; + if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) { + // if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } From d1d06b67eb4b67dfba8fccfb4f90e4c1fd37d752 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 10 Dec 2020 12:13:06 -0800 Subject: [PATCH 09/10] fix type warning (issue #337) --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 22e736f7..e2c93d72 100644 --- a/src/os.c +++ b/src/os.c @@ -421,7 +421,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, 10UL); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations } } } From 745cf1e2f57f2f9207a9c45beda3ec1b20fd04ba Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 10 Dec 2020 12:33:35 -0800 Subject: [PATCH 10/10] fix build on ghc4.8 (issue #330) --- src/alloc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 6bb9e30a..3aa97158 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -778,7 +778,12 @@ but we call `exit` instead (i.e. not returning). #ifdef __cplusplus #include static bool mi_try_new_handler(bool nothrow) { - std::new_handler h = std::get_new_handler(); + #if defined(_MSC_VER) || (__cplusplus >= 201103L) + std::new_handler h = std::get_new_handler(); + #else + std::new_handler h = std::set_new_handler(); + std::set_new_handler(h); + #endif if (h==NULL) { if (!nothrow) throw std::bad_alloc(); return false;