diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bc8936c..b93ea094 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,9 +21,11 @@ option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) + +# deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) -option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) -option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems)" OFF) +option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version (deprecated)" OFF) +option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 445ca241..999e2293 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,7 +115,7 @@ jobs: displayName: macOS pool: vmImage: - macOS-latest + macOS-latest strategy: matrix: Debug: diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index aa8011fe..e63ade79 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -43,6 +43,11 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_externc #endif +#if !defined(_WIN32) && !defined(__wasi__) +#define MI_USE_PTHREADS +#include +#endif + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -318,7 +323,7 @@ We try to circumvent this in an efficient way: - macOSX : we use an unused TLS slot from the OS allocated slots (MI_TLS_SLOT). On OSX, the loader itself calls `malloc` even before the modules are initialized. - OpenBSD: we use an unused slot from the pthread block (MI_TLS_PTHREAD_SLOT_OFS). -- DragonFly: the uniqueid use is buggy but kept for reference. +- DragonFly: defaults are working but seem slow compared to freeBSD (see PR #323) ------------------------------------------------------------------------------------------- */ extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap @@ -335,16 +340,18 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) // see #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) -#elif defined(__DragonFly__) -#warning "mimalloc is not working correctly on DragonFly yet." -//#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +// #elif defined(__DragonFly__) +// #warning "mimalloc is not working correctly on DragonFly yet." +// #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +#elif defined(__ANDROID__) +// See issue #381 +#define MI_TLS_PTHREAD #endif #endif #if defined(MI_TLS_SLOT) static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept; // forward declaration #elif defined(MI_TLS_PTHREAD_SLOT_OFS) -#include static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { pthread_t self = pthread_self(); #if defined(__DragonFly__) @@ -356,7 +363,6 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); } #elif defined(MI_TLS_PTHREAD) -#include extern pthread_key_t _mi_heap_default_key; #endif @@ -854,8 +860,8 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) - // on Android, slot 1 is the thread ID (pointer to pthread internal struct) +#if defined(__arm__) || (defined(__ANDROID__) && defined(__aarch64__)) + // issue #384, #495: on arm32 and arm32/arm64 Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); #else // in all our other targets, slot 0 is the pointer to the thread control block diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index 7d9f3e7d..c63b0b91 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -48,6 +48,7 @@ not accidentally mix pointers from different allocators). #define valloc(n) mi_valloc(n) #define pvalloc(n) mi_pvalloc(n) #define reallocarray(p,s,n) mi_reallocarray(p,s,n) +#define reallocarr(p,s,n) mi_reallocarr(p,s,n) #define memalign(a,n) mi_memalign(a,n) #define aligned_alloc(a,n) mi_aligned_alloc(a,n) #define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) diff --git a/include/mimalloc.h b/include/mimalloc.h index f37bd630..0f6f2353 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -360,6 +360,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; diff --git a/src/alloc-override.c b/src/alloc-override.c index 42fecbb3..6bbe4aac 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -231,7 +231,6 @@ extern "C" { size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } - // `aligned_alloc` is only available when __USE_ISOC11 is defined. // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. @@ -246,6 +245,7 @@ extern "C" { void cfree(void* p) { mi_free(p); } void* pvalloc(size_t size) { return mi_pvalloc(size); } void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } diff --git a/src/alloc-posix.c b/src/alloc-posix.c index cff01b5c..2eaede07 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -92,13 +92,23 @@ mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_n void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD void* newp = mi_reallocn(p,count,size); - if (newp==NULL) errno = ENOMEM; + if (newp==NULL) { errno = ENOMEM; } return newp; } +int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD + mi_assert(p != NULL); + if (p == NULL) return EINVAL; // should we set errno as well? + void** op = (void**)p; + void* newp = mi_reallocarray(*op, count, size); + if (mi_unlikely(newp == NULL)) return errno; + *op = newp; + return 0; +} + void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft void* res = mi_expand(p, newsize); - if (res == NULL) errno = ENOMEM; + if (res == NULL) { errno = ENOMEM; } return res; } diff --git a/src/init.c b/src/init.c index 678db8b9..9b6c6c51 100644 --- a/src/init.c +++ b/src/init.c @@ -302,12 +302,6 @@ static bool _mi_heap_done(mi_heap_t* heap) { static void _mi_thread_done(mi_heap_t* default_heap); -#ifdef __wasi__ -// no pthreads in the WebAssembly Standard Interface -#elif !defined(_WIN32) -#define MI_USE_PTHREADS -#endif - #if defined(_WIN32) && defined(MI_SHARED_LIB) // nothing to do as it is done in DllMain #elif defined(_WIN32) && !defined(MI_SHARED_LIB) @@ -327,7 +321,6 @@ static void _mi_thread_done(mi_heap_t* default_heap); #elif defined(MI_USE_PTHREADS) // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default heap) - #include pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); static void mi_pthread_done(void* value) { if (value!=NULL) _mi_thread_done((mi_heap_t*)value); diff --git a/src/os.c b/src/os.c index 42bde63b..edb749a6 100644 --- a/src/os.c +++ b/src/os.c @@ -220,7 +220,7 @@ void _mi_os_init(void) #elif defined(__wasi__) void _mi_os_init() { os_overcommit = false; - os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB + os_page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } @@ -272,7 +272,7 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats bool err = false; #if defined(_WIN32) err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); -#elif defined(MI_USE_SBRK) +#elif defined(MI_USE_SBRK) || defined(__wasi__) err = 0; // sbrk heap cannot be shrunk #else err = (munmap(addr, size) == -1); @@ -367,47 +367,85 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, return p; } -#elif defined(MI_USE_SBRK) -#define MI_SBRK_FAIL ((void*)(-1)) -static void* mi_sbrk_heap_grow(size_t size, size_t try_alignment) { - void* pbase0 = sbrk(0); - if (pbase0 == MI_SBRK_FAIL) { - _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); - errno = ENOMEM; - return NULL; +#elif defined(MI_USE_SBRK) || defined(__wasi__) +#if defined(MI_USE_SBRK) +// unfortunately sbrk is usually not safe to call from multiple threads +#if defined(MI_USE_PTHREADS) + static pthread_mutex_t mi_sbrk_mutex = PTHREAD_MUTEX_INITIALIZER; + static void* mi_sbrk(size_t size) { + pthread_mutex_lock(&mi_sbrk_mutex); + void* p = sbrk(size); + pthread_mutex_unlock(&mi_sbrk_mutex); + return p; } + #else + static void* mi_sbrk(size_t size) { + return sbrk(size); + } + #endif + static void* mi_memory_grow( size_t size ) { + void* p = mi_sbrk(size); + if (p == (void*)(-1)) { + _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); + errno = ENOMEM; + return NULL; + } + if (size > 0) { memset(p,0,size); } + return p; + } +#elif defined(__wasi__) + static void* mi_memory_grow( size_t size ) { + size_t base; + if (size > 0) { + base = __builtin_wasm_memory_grow( 0, _mi_divide_up(size, _mi_os_page_size()) ); + } + else { + base = __builtin_wasm_memory_size(0); + } + if (base == SIZE_MAX) { + _mi_warning_message("unable to allocate wasm_memory_grow OS memory (%zu bytes)\n", size); + errno = ENOMEM; + return NULL; + } + return (void*)(base * _mi_os_page_size()); + } +#endif + +static void* mi_heap_grow(size_t size, size_t try_alignment) { + if (try_alignment == 0) { try_alignment = _mi_os_page_size(); }; + void* pbase0 = mi_memory_grow(0); + if (pbase0 == NULL) { return NULL; } uintptr_t base = (uintptr_t)pbase0; - uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); + uintptr_t aligned_base = _mi_align_up(base, try_alignment); size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); if (alloc_size < size) return NULL; - void* pbase1 = sbrk(alloc_size); - if (pbase1 == MI_SBRK_FAIL) { - _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); - errno = ENOMEM; - return NULL; - } - mi_assert(pbase0 == pbase1); + void* pbase1 = mi_memory_grow(alloc_size); + if (pbase1 == NULL) { return NULL; } + if (pbase0 != pbase1) { + // another thread allocated in-between; now we may not be able to align correctly + base = (uintptr_t)pbase1; + aligned_base = _mi_align_up(base, try_alignment); + if (aligned_base + size > base + alloc_size) { + // we do not have enough space after alignment; since we cannot shrink safely, + // we waste the space :-( and allocate fresh with guaranteed enough overallocation + alloc_size = _mi_align_up( size + try_alignment, _mi_os_page_size() ); + errno = 0; + void* pbase2 = mi_memory_grow( alloc_size ); + if (pbase2 == NULL) { return NULL; } + base = (uintptr_t)pbase2; + aligned_base = _mi_align_up(base, try_alignment); + } + else { + // it still fits + mi_assert_internal(aligned_base + size <= base + alloc_size); + } + } + mi_assert_internal(aligned_base + size <= base + alloc_size); return (void*)aligned_base; } -#elif defined(__wasi__) - // currently unused as we use sbrk() on wasm -static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { - uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size(); - uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); - size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); - mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); - if (alloc_size < size) return NULL; - if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) { - _mi_warning_message("unable to allocate wasm_memory_grow() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); - errno = ENOMEM; - return NULL; - } - return (void*)aligned_base; -} - -#else +#else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { MI_UNUSED(try_alignment); @@ -634,14 +672,10 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo int flags = MEM_RESERVE; if (commit) flags |= MEM_COMMIT; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); - #elif defined(MI_USE_SBRK) + #elif defined(MI_USE_SBRK) || defined(__wasi__) MI_UNUSED(allow_large); *is_large = false; - p = mi_sbrk_heap_grow(size, try_alignment); - #elif defined(__wasi__) - MI_UNUSED(allow_large); - *is_large = false; - p = mi_wasm_heap_grow(size, try_alignment); + p = mi_heap_grow(size, try_alignment); #else int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); diff --git a/src/random.c b/src/random.c index fabd6c6e..e47946a6 100644 --- a/src/random.c +++ b/src/random.c @@ -160,7 +160,8 @@ uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { /* ---------------------------------------------------------------------------- To initialize a fresh random context we rely on the OS: - Windows : BCryptGenRandom (or RtlGenRandom) -- osX,bsd,wasi: arc4random_buf +- macOS : CCRandomGenerateBytes, arc4random_buf +- bsd,wasi : arc4random_buf - Linux : getrandom,/dev/urandom If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ @@ -191,7 +192,24 @@ static bool os_random_buf(void* buf, size_t buf_len) { } #endif -#elif defined(ANDROID) || defined(XP_DARWIN) || defined(__APPLE__) || defined(__DragonFly__) || \ +#elif defined(__APPLE__) +#include +#if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 +#include +#endif +static bool os_random_buf(void* buf, size_t buf_len) { + #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 + // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // may fail silently on macOS. See PR #390, and + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); + #else + // fall back on older macOS + arc4random_buf(buf, buf_len); + return true; + #endif +} + +#elif defined(__ANDROID__) || defined(__DragonFly__) || \ defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ defined(__sun) // todo: what to use with __wasi__? #include diff --git a/test/main-override.c b/test/main-override.c index 1bec1179..284fdd20 100644 --- a/test/main-override.c +++ b/test/main-override.c @@ -3,7 +3,7 @@ #include #include -#include +#include int main() { mi_version(); // ensure mimalloc library is linked @@ -25,6 +25,12 @@ int main() { //free(p1); //p2 = malloc(32); //mi_free(p2); + p1 = malloc(24); + p2 = reallocarray(p1, 16, 16); + free(p2); + p1 = malloc(24); + assert(reallocarr(&p1, 16, 16) == 0); + free(p1); mi_stats_print(NULL); return 0; }