From 7d098836ebafacacae55088c906305b93e683340 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 13 Feb 2020 00:43:34 -0800 Subject: [PATCH] wip: basic tracking of allocation locations is working --- include/mimalloc-internal.h | 125 +++++++---- include/mimalloc-new-delete.h | 26 +-- include/mimalloc-override.h | 56 ++--- include/mimalloc.h | 402 +++++++++++++++++++--------------- src/alloc-aligned.c | 50 ++--- src/alloc-override.c | 126 +++++------ src/alloc-posix.c | 131 +++++++---- src/alloc.c | 133 +++++------ src/init.c | 4 +- src/options.c | 8 +- src/page.c | 4 +- src/static.c | 1 + test/main-override.cpp | 5 +- 13 files changed, 598 insertions(+), 473 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 52c4884c..52cf3029 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -90,7 +90,7 @@ void _mi_abandoned_await_readers(void); // "page.c" -void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -123,13 +123,13 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); // "alloc.c" -void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_PARAM) mi_attr_noexcept; // called from `_mi_malloc_generic` +void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept; // called from `_mi_malloc_generic` mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); bool _mi_free_delayed_block(mi_block_t* block); void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size); -mi_decl_allocator void* _mi_heap_source_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_PARAM); -mi_decl_allocator void* _mi_heap_source_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_PARAM); +mi_decl_restrict void* _mi_base_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_XPARAM); +mi_decl_restrict void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM); #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); @@ -739,56 +739,103 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { #endif +// ------------------------------------------------------------------------------------------------------------- +// When defining API entry points we generally need 5 declarations: +// 1. mi_malloc(size) : normal entry +// 2. mi_heap_malloc(heap,size) : takes explicit heap +// 3. dbg_mi_malloc(size,source) : debug entry that takes a source location too +// 4. dbg_mi_heap_malloc(heap,size,source) : debug entry that takes a source location too +// 5. mi_base_malloc(heap,size[,source]) : actual implementation, only has the source parameter in debug mode +// +// The base version declares the source parameter as MI_SOURCE_XPARAM and can pass it through as MI_SOURCE_XARG +// +// The big preprocessor macros that follow emit the 5 declarations with default +// implementations for the first 4 versions so only the 5th needs to be implemented. +// ------------------------------------------------------------------------------------------------------------- +#ifdef NDEBUG +#define MI_DEBUG_ONLY(x) +#else +#define MI_DEBUG_ONLY(x) x +#endif + #define MI_ALLOC_API1(tp,name,tp0,arg0,tp1,arg1) \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1 MI_SOURCE_PARAM) mi_attr_noexcept; \ - mi_decl_allocator tp mi_source_##name(tp1 arg1 MI_SOURCE_PARAM) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1 MI_SOURCE_ARG); } \ - mi_decl_allocator tp mi_heap_##name(mi_heap_t* heap, tp1 arg1) mi_attr_noexcept { return mi_heap_source_##name(heap, arg1 MI_SOURCE_RET()); } \ - mi_decl_allocator tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1 MI_SOURCE_RET()); } \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + mi_decl_restrict tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XRET()); } \ + mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1) mi_attr_noexcept { return mi_base_##name(heap, arg1 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API2(tp,name,tp0,arg0,tp1,arg1,tp2,arg2) \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_PARAM) mi_attr_noexcept; \ - mi_decl_allocator tp mi_source_##name(tp1 arg1, tp2 arg2 MI_SOURCE_PARAM) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_ARG); } \ - mi_decl_allocator tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_heap_source_##name(heap, arg1, arg2 MI_SOURCE_RET()); } \ - mi_decl_allocator tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_RET()); } \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XRET()); } \ + mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API3(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_PARAM) mi_attr_noexcept; \ - mi_decl_allocator tp mi_source_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_PARAM) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_ARG); } \ - mi_decl_allocator tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_heap_source_##name(heap, arg1, arg2, arg3 MI_SOURCE_RET()); } \ - mi_decl_allocator tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_RET()); } \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API4(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3,tp4,arg4) \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_PARAM) mi_attr_noexcept; \ - mi_decl_allocator tp mi_source_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_PARAM) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_ARG); } \ - mi_decl_allocator tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_heap_source_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_RET()); } \ - mi_decl_allocator tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_RET()); } \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ + mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API5(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3,tp4,arg4,tp5,arg5) \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_PARAM) mi_attr_noexcept; \ - mi_decl_allocator tp mi_source_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_PARAM) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_ARG); } \ - mi_decl_allocator tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_heap_source_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_RET()); } \ - mi_decl_allocator tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_heap_source_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_RET()); } \ - static inline mi_decl_allocator tp mi_heap_source_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ + mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); } ) \ + static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept + + +#define MI_NEW_API1(tp,name,tp1,arg1) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ + mi_decl_restrict tp mi_##name(tp1 arg1) { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) + +#define MI_NEW_API2(tp,name,tp1,arg1,tp2,arg2) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2) { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) + +#define MI_NEW_API3(tp,name,tp1,arg1,tp2,arg2,tp3,arg3) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) + + #define MI_SOURCE_API1(tp,name,tp1,arg1) \ - tp mi_source_##name(tp1 arg1 MI_SOURCE_PARAM) mi_attr_noexcept; \ - tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_source_##name(arg1 MI_SOURCE_RET()); } \ - tp mi_source_##name(tp1 arg1 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ + mi_decl_restrict tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) #define MI_SOURCE_API2(tp,name,tp1,arg1,tp2,arg2) \ - tp mi_source_##name(tp1 arg1, tp2 arg2 MI_SOURCE_PARAM) mi_attr_noexcept; \ - tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_source_##name(arg1, arg2 MI_SOURCE_RET()); } \ - tp mi_source_##name(tp1 arg1, tp2 arg2 MI_SOURCE_PARAM) mi_attr_noexcept + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) #define MI_SOURCE_API3(tp,name,tp1,arg1,tp2,arg2,tp3,arg3) \ - tp mi_source_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_PARAM) mi_attr_noexcept; \ - tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_source_##name(arg1, arg2, arg3 MI_SOURCE_RET()); } \ - tp mi_source_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_PARAM) mi_attr_noexcept - + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ + mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) #endif diff --git a/include/mimalloc-new-delete.h b/include/mimalloc-new-delete.h index 274f7fa0..16e856c9 100644 --- a/include/mimalloc-new-delete.h +++ b/include/mimalloc-new-delete.h @@ -25,11 +25,11 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete(void* p) noexcept { mi_free(p); }; void operator delete[](void* p) noexcept { mi_free(p); }; - void* operator new(std::size_t n) noexcept(false) { return mi_source_new(n MI_SOURCE_RET()); } - void* operator new[](std::size_t n) noexcept(false) { return mi_source_new(n MI_SOURCE_RET()); } + void* operator new(std::size_t n) noexcept(false) { return MI_SOURCE_RET(mi_new, n); } + void* operator new[](std::size_t n) noexcept(false) { return MI_SOURCE_RET(mi_new, n); } - void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_source_new_nothrow(n MI_SOURCE_RET()); } - void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_source_new_nothrow(n MI_SOURCE_RET()); } + void* operator new (std::size_t n, const std::nothrow_t& ) noexcept { return MI_SOURCE_RET(mi_new_nothrow, n); } + void* operator new[](std::size_t n, const std::nothrow_t& ) noexcept { return MI_SOURCE_RET(mi_new_nothrow, n); } #if (__cplusplus >= 201402L || _MSC_VER >= 1916) void operator delete (void* p, std::size_t n) noexcept { mi_free_size(p,n); }; @@ -42,19 +42,19 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; - void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_source_new_aligned(n, static_cast(al) MI_SOURCE_RET()); } - void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_source_new_aligned(n, static_cast(al) MI_SOURCE_RET()); } - void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_source_new_aligned_nothrow(n, static_cast(al) MI_SOURCE_RET()); } - void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_source_new_aligned_nothrow(n, static_cast(al) MI_SOURCE_RET()); } + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return MI_SOURCE_RET(mi_new_aligned, n, static_cast(al)); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return MI_SOURCE_RET(mi_new_aligned, n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return MI_SOURCE_RET(mi_new_aligned_nothrow, n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return MI_SOURCE_RET(mi_new_aligned_nothrow, n, static_cast(al)); } #endif + #if !defined(NDEBUG) // Instances for debug override of the new operator - #ifndef NDEBUG - void* operator new(std::size_t n MI_SOURCE_PARAM) noexcept(false) { return mi_source_new(n MI_SOURCE_ARG); } - void* operator new[](std::size_t n MI_SOURCE_PARAM) noexcept(false) { return mi_source_new(n MI_SOURCE_ARG); } + void* operator new(std::size_t n, mi_source_t __mi_source) noexcept(false) { (void)(__mi_source); return MI_SOURCE_ARG(mi_new, n); } + void* operator new[](std::size_t n, mi_source_t __mi_source) noexcept(false) { (void)(__mi_source); return MI_SOURCE_ARG(mi_new, n); } - void operator delete(void* p MI_SOURCE_PARAM) noexcept { mi_free(p); }; - void operator delete[](void* p MI_SOURCE_PARAM) noexcept { mi_free(p); }; + void operator delete(void* p, mi_source_t ) noexcept { mi_free(p); }; + void operator delete[](void* p, mi_source_t ) noexcept { mi_free(p); }; #endif #endif diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index b9cb72c8..55041d8f 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -18,55 +18,55 @@ not accidentally mix pointers from different allocators). #include // Standard C allocation -#define malloc(n) mi_source_malloc(n MI_SOURCE_LOC()) -#define calloc(n,c) mi_source_calloc(n,c MI_SOURCE_LOC()) -#define realloc(p,n) mi_source_realloc(p,n MI_SOURCE_LOC()) +#define malloc(n) mi_malloc(n) +#define calloc(n,c) mi_calloc(n,c) +#define realloc(p,n) mi_realloc(p,n) #define free(p) mi_free(p) -#define strdup(s) mi_source_strdup(s MI_SOURCE_LOC()) -#define strndup(s) mi_source_strndup(s MI_SOURCE_LOC()) -#define realpath(f,n) mi_source_realpath(f,n MI_SOURCE_LOC()) +#define strdup(s) mi_strdup(s) +#define strndup(s) mi_strndup(s) +#define realpath(f,n) mi_realpath(f,n) // Microsoft extensions #define _expand(p,n) mi_expand(p,n) #define _msize(p) mi_usable_size(p) -#define _recalloc(p,n,c) mi_source_recalloc(p,n,c MI_SOURCE_LOC()) +#define _recalloc(p,n,c) mi_recalloc(p,n,c) -#define _strdup(s) mi_source_strdup(s MI_SOURCE_LOC()) -#define _strndup(s) mi_source_strndup(s MI_SOURCE_LOC()) -#define _wcsdup(s) (wchar_t*)mi_source_wcsdup((const unsigned short*)(s) MI_SOURCE_LOC()) -#define _mbsdup(s) mi_source_mbsdup(s MI_SOURCE_LOC()) -#define _dupenv_s(b,n,v) mi_source_dupenv_s(b,n,v MI_SOURCE_LOC()) -#define _wdupenv_s(b,n,v) mi_source_wdupenv_s((unsigned short*)(b),n,(const unsigned short*)(v) MI_SOURCE_LOC()) +#define _strdup(s) mi_strdup(s) +#define _strndup(s) mi_strndup(s) +#define _wcsdup(s) (wchar_t*)mi_wcsdup((const unsigned short*)(s)) +#define _mbsdup(s) mi_mbsdup(s) +#define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v) +#define _wdupenv_s(b,n,v) mi_wdupenv_s((unsigned short*)(b),n,(const unsigned short*)(v)) // Various Posix and Unix variants -#define reallocf(p,n) mi_source_reallocf(p,n MI_SOURCE_LOC()) +#define reallocf(p,n) mi_reallocf(p,n) #define malloc_size(p) mi_usable_size(p) #define malloc_usable_size(p) mi_usable_size(p) #define cfree(p) mi_free(p) -#define valloc(n) mi_source_valloc(n MI_SOURCE_LOC()) -#define pvalloc(n) mi_source_pvalloc(n MI_SOURCE_LOC()) -#define reallocarray(p,s,n) mi_source_reallocarray(p,s,n MI_SOURCE_LOC()) -#define memalign(a,n) mi_source_memalign(a,n MI_SOURCE_LOC()) -#define aligned_alloc(a,n) mi_source_aligned_alloc(a,n MI_SOURCE_LOC()) -#define posix_memalign(p,a,n) mi_source_posix_memalign(p,a,n MI_SOURCE_LOC()) -#define _posix_memalign(p,a,n) mi_source_posix_memalign(p,a,n MI_SOURCE_LOC()) +#define valloc(n) mi_valloc(n) +#define pvalloc(n) mi_pvalloc(n) +#define reallocarray(p,s,n) mi_reallocarray(p,s,n) +#define memalign(a,n) mi_memalign(a,n) +#define aligned_alloc(a,n) mi_aligned_alloc(a,n) +#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) +#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n) // Microsoft aligned variants -#define _aligned_malloc(n,a) mi_source_malloc_aligned(n,a MI_SOURCE_LOC()) -#define _aligned_realloc(p,n,a) mi_source_realloc_aligned(p,n,a MI_SOURCE_LOC()) -#define _aligned_recalloc(p,s,n,a) mi_source_recalloc_aligned(p,s,n,a MI_SOURCE_LOC()) +#define _aligned_malloc(n,a) mi_malloc_aligned(n,a) +#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a) +#define _aligned_recalloc(p,s,n,a) mi_aligned_recalloc(p,s,n,a) #define _aligned_msize(p,a,o) mi_usable_size(p) #define _aligned_free(p) mi_free(p) -#define _aligned_offset_malloc(n,a,o) mi_source_malloc_aligned_at(n,a,o MI_SOURCE_LOC()) -#define _aligned_offset_realloc(p,n,a,o) mi_source_realloc_aligned_at(p,n,a,o MI_SOURCE_LOC()) -#define _aligned_offset_recalloc(p,s,n,a,o) mi_source_recalloc_aligned_at(p,s,n,a,o MI_SOURCE_LOC()) +#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o) +#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o) +#define _aligned_offset_recalloc(p,s,n,a,o) mi_recalloc_aligned_at(p,s,n,a,o) // Overload new operators // This requires including somewhere! // See also -#if !defined(NDEBUG) && defined(__cplusplus) && !defined(MI_NO_NEW_OVERRIDE) +#if !defined(NDEBUG) && defined(__cplusplus) && !defined(MI_NO_SOURCE_DEBUG) #define new new(mi_source_loc(__FILE__,__LINE__)) #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index b3e1c1ef..e78eae75 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -43,9 +43,9 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_export __declspec(dllimport) #endif #if (_MSC_VER >= 1900) && !defined(__EDG__) - #define mi_decl_allocator __declspec(allocator) __declspec(restrict) + #define mi_decl_restrict __declspec(allocator) __declspec(restrict) #else - #define mi_decl_allocator __declspec(restrict) + #define mi_decl_restrict __declspec(restrict) #endif #define mi_cdecl __cdecl #define mi_attr_malloc @@ -55,7 +55,7 @@ terms of the MIT license. A copy of the license can be found in the file #elif defined(__GNUC__) // includes clang and icc #define mi_cdecl // leads to warnings... __attribute__((cdecl)) #define mi_decl_export __attribute__((visibility("default"))) - #define mi_decl_allocator + #define mi_decl_restrict #define mi_attr_malloc __attribute__((malloc)) #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) #define mi_attr_alloc_size(s) @@ -73,7 +73,7 @@ terms of the MIT license. A copy of the license can be found in the file #else #define mi_cdecl #define mi_decl_export - #define mi_decl_allocator + #define mi_decl_restrict #define mi_attr_malloc #define mi_attr_alloc_size(s) #define mi_attr_alloc_size2(s1,s2) @@ -91,19 +91,52 @@ terms of the MIT license. A copy of the license can be found in the file extern "C" { #endif + +// ------------------------------------------------------ +// Debugging +// ------------------------------------------------------ + +#ifdef NDEBUG + +#define mi_decl_alloc(tp,name,...) \ + mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) mi_attr_noexcept + +#define mi_decl_new(tp,name,...) \ + mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) + +#else +typedef struct mi_source_s { + long long src; // packed encoding of the source location. +} mi_source_t; + +mi_decl_export mi_source_t mi_source_ret(void* return_address); +mi_decl_export mi_source_t mi_source_loc(const char* fname, int lineno); +mi_decl_export void* mi_source_unpack(mi_source_t source, const char** fname, int* lineno); + +#define mi_decl_alloc(tp,name,...) \ + mi_decl_nodiscard mi_decl_export mi_decl_restrict tp dbg_##name( __VA_ARGS__, mi_source_t dbg_source) mi_attr_noexcept; \ + mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) mi_attr_noexcept + +#define mi_decl_new(tp,name,...) \ + mi_decl_nodiscard mi_decl_export mi_decl_restrict tp dbg_##name( __VA_ARGS__, mi_source_t dbg_source); \ + mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) +#endif + // ------------------------------------------------------ // Standard malloc interface // ------------------------------------------------------ -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_export mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_export void mi_free(void* p) mi_attr_noexcept; + +mi_decl_alloc(void*, mi_malloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_calloc, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); +mi_decl_alloc(void*, mi_realloc, void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_expand, void* p, size_t newsize) mi_attr_alloc_size(2); + +mi_decl_alloc(char*, mi_strdup, const char* s) mi_attr_malloc; +mi_decl_alloc(char*, mi_strndup, const char* s, size_t n) mi_attr_malloc; +mi_decl_alloc(char*, mi_realpath,const char* fname, char* resolved_name); -mi_decl_export void mi_free(void* p) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_strdup(const char* s) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_strndup(const char* s, size_t n) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept; // ------------------------------------------------------ // Extended functionality @@ -111,13 +144,13 @@ mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_realpath(const char* #define MI_SMALL_WSIZE_MAX (128) #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_malloc_small, size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_zalloc_small, size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_zalloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_mallocn, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_alloc(void*, mi_reallocn, void* p, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2,3); +mi_decl_alloc(void*, mi_reallocf, void* p, size_t newsize) mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; @@ -155,14 +188,15 @@ mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_ // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. // ------------------------------------------------------------------------------------- -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_malloc_aligned, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_alloc(void*, mi_zalloc_aligned, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_alloc(void*, mi_calloc_aligned, size_t count, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); +mi_decl_alloc(void*, mi_realloc_aligned, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); + +mi_decl_alloc(void*, mi_malloc_aligned_at, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_zalloc_aligned_at, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_calloc_aligned_at, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size2(1, 2); +mi_decl_alloc(void*, mi_realloc_aligned_at, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(2); // ------------------------------------------------------------------------------------- @@ -180,28 +214,30 @@ mi_decl_export mi_heap_t* mi_heap_get_default(void); mi_decl_export mi_heap_t* mi_heap_get_backing(void); mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_heap_malloc, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_heap_zalloc, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_heap_calloc, mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_alloc(void*, mi_heap_mallocn, mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_alloc(void*, mi_heap_malloc_small, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_heap_zalloc_small, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc; -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); +mi_decl_alloc(void*, mi_heap_realloc, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); +mi_decl_alloc(void*, mi_heap_reallocn, mi_heap_t* heap, void* p, size_t count, size_t size); +mi_decl_alloc(void*, mi_heap_reallocf, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept; +mi_decl_alloc(char*, mi_heap_strdup, mi_heap_t* heap, const char* s) mi_attr_malloc; +mi_decl_alloc(char*, mi_heap_strndup, mi_heap_t* heap, const char* s, size_t n) mi_attr_malloc; +mi_decl_alloc(char*, mi_heap_realpath, mi_heap_t* heap, const char* fname, char* resolved_name); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3) mi_attr_alloc_align(4); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); +mi_decl_alloc(void*, mi_heap_malloc_aligned, mi_heap_t* heap, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_alloc(void*, mi_heap_zalloc_aligned, mi_heap_t* heap, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_alloc(void*, mi_heap_calloc_aligned, mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_alloc(void*, mi_heap_realloc_aligned,mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(3) mi_attr_alloc_align(4); + +mi_decl_alloc(void*, mi_heap_malloc_aligned_at, mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_heap_zalloc_aligned_at, mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_heap_calloc_aligned_at, mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size2(2, 3); +mi_decl_alloc(void*, mi_heap_realloc_aligned_at,mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(3); // -------------------------------------------------------------------------------- @@ -211,21 +247,21 @@ mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned // see // -------------------------------------------------------------------------------- -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); +mi_decl_alloc(void*, mi_rezalloc, void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_recalloc, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2,3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); +mi_decl_alloc(void*, mi_rezalloc_aligned, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_alloc(void*, mi_recalloc_aligned, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); +mi_decl_alloc(void*, mi_rezalloc_aligned_at, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(2); +mi_decl_alloc(void*, mi_recalloc_aligned_at, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(2,3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(3,4); +mi_decl_alloc(void*, mi_heap_rezalloc, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); +mi_decl_alloc(void*, mi_heap_recalloc, mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(3,4); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3) mi_attr_alloc_align(4); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(3,4); +mi_decl_alloc(void*, mi_heap_rezalloc_aligned, mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_alloc(void*, mi_heap_recalloc_aligned, mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); +mi_decl_alloc(void*, mi_heap_rezalloc_aligned_at, mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(3); +mi_decl_alloc(void*, mi_heap_recalloc_aligned_at, mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(3, 4); // ------------------------------------------------------ @@ -329,19 +365,21 @@ mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_ mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; -mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); -mi_decl_nodiscard mi_decl_export void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); -mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); +mi_decl_alloc(void*, mi_memalign, size_t alignment, size_t size) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_alloc(void*, mi_valloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; +mi_decl_alloc(void*, mi_pvalloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_alloc(void*, mi_aligned_alloc, size_t alignment, size_t size) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_alloc(void*, mi_reallocarray, void* p, size_t count, size_t size) mi_attr_alloc_size2(2,3); + +mi_decl_alloc(void*, mi_aligned_recalloc, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_alloc(void*, mi_aligned_offset_recalloc, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) ; + +mi_decl_alloc(unsigned short*, mi_wcsdup, const unsigned short* s); +mi_decl_alloc(unsigned char*, mi_mbsdup, const unsigned char* s); -mi_decl_nodiscard mi_decl_export unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept; mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; @@ -349,104 +387,21 @@ mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; +#ifndef NDEBUG +mi_decl_export int dbg_mi_posix_memalign(void** p, size_t alignment, size_t size, mi_source_t) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(2); +mi_decl_export int dbg_mi_dupenv_s(char** buf, size_t* size, const char* name, mi_source_t source) mi_attr_noexcept; +mi_decl_export int dbg_mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name, mi_source_t source) mi_attr_noexcept; +#endif + // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new_nothrow(size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new_realloc(void* p, size_t newsize) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); - - -// --------------------------------------------------------------------------------------------- -// Experimental: Debugging API that tracks the source location of an allocation -// --------------------------------------------------------------------------------------------- - -typedef struct mi_source_s { - long long src; // packed encoding of the source location -} mi_source_t; - -mi_decl_export mi_source_t mi_source_ret(void* return_address); -mi_decl_export mi_source_t mi_source_loc(const char* fname, int lineno); -mi_decl_export void* mi_source_unpack(mi_source_t source, const char** fname, int* lineno); - -#ifdef NDEBUG -#define MI_SOURCE_PARAM -#define MI_SOURCE_ARG -#define MI_SOURCE_RET() -#define MI_SOURCE_LOC() -#else -#ifdef _MSC_VER -#include -#define mi_return_address() _ReturnAddress() -#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc -#define mi_return_address() __builtin_return_address(0) -#else -#define mi_return_address() NULL -#endif -#define MI_SOURCE_PARAM , mi_source_t __mi_source -#define MI_SOURCE_ARG , __mi_source -#define MI_SOURCE_RET() , mi_source_ret(mi_return_address()) -#define MI_SOURCE_LOC() , mi_source_loc(__FILE__,__LINE__) -#endif - -// malloc -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_malloc(size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_calloc(size_t count, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_realloc(void* p, size_t newsize MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); - -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_source_strdup(const char* s MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_source_strndup(const char* s, size_t n MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator char* mi_source_realpath(const char* fname, char* resolved_name MI_SOURCE_PARAM) mi_attr_noexcept; - -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_malloc_small(size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_zalloc_small(size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_zalloc(size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); - -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_mallocn(size_t count, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_reallocn(void* p, size_t count, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_reallocf(void* p, size_t newsize MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); - -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_malloc_aligned(size_t size, size_t alignment MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_malloc_aligned_at(size_t size, size_t alignment, size_t offset MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_zalloc_aligned(size_t size, size_t alignment MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_zalloc_aligned_at(size_t size, size_t alignment, size_t offset MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_calloc_aligned(size_t count, size_t size, size_t alignment MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_realloc_aligned(void* p, size_t newsize, size_t alignment MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); - -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_rezalloc(void* p, size_t newsize MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_recalloc(void* p, size_t newcount, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); - - -// new delete -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new(size_t size MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new_aligned(size_t size, size_t alignment MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new_nothrow(size_t size MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new_aligned_nothrow(size_t size, size_t alignment MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new_n(size_t count, size_t size MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new_realloc(void* p, size_t newsize MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export mi_decl_allocator void* mi_source_new_reallocn(void* p, size_t newcount, size_t size MI_SOURCE_PARAM) mi_attr_malloc mi_attr_alloc_size2(2, 3); - -// posix -mi_decl_export int mi_source_posix_memalign(void** p, size_t alignment, size_t size MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export void* mi_source_memalign(size_t alignment, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); -mi_decl_nodiscard mi_decl_export void* mi_source_valloc(size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); - -mi_decl_nodiscard mi_decl_export void* mi_source_pvalloc(size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_nodiscard mi_decl_export void* mi_source_aligned_alloc(size_t alignment, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); -mi_decl_nodiscard mi_decl_export void* mi_source_reallocarray(void* p, size_t count, size_t size MI_SOURCE_PARAM) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); - -mi_decl_nodiscard mi_decl_export void* mi_source_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export void* mi_sourcealigned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset MI_SOURCE_PARAM) mi_attr_noexcept; - -mi_decl_nodiscard mi_decl_export unsigned short* mi_source_wcsdup(const unsigned short* s MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export unsigned char* mi_source_mbsdup(const unsigned char* s MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_export int mi_source_dupenv_s(char** buf, size_t* size, const char* name MI_SOURCE_PARAM) mi_attr_noexcept; -mi_decl_export int mi_source_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name MI_SOURCE_PARAM) mi_attr_noexcept; +mi_decl_new(void*, mi_new, size_t size) mi_attr_malloc mi_attr_alloc_size(1) ; +mi_decl_new(void*, mi_new_aligned, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2) ; +mi_decl_new(void*, mi_new_nothrow, size_t size) mi_attr_malloc mi_attr_alloc_size(1) ; +mi_decl_new(void*, mi_new_aligned_nothrow, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2) ; +mi_decl_new(void*, mi_new_n, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2) ; +mi_decl_new(void*, mi_new_realloc, void* p, size_t newsize) mi_attr_alloc_size(2) ; +mi_decl_new(void*, mi_new_reallocn, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3) ; // ---------------------------------------------------------------------- @@ -472,9 +427,9 @@ template struct mi_stl_allocator { typedef T value_type; typedef std::size_t size_type; typedef std::ptrdiff_t difference_type; - typedef value_type& reference; + typedef value_type& reference; typedef value_type const& const_reference; - typedef value_type* pointer; + typedef value_type* pointer; typedef value_type const* const_pointer; template struct rebind { typedef mi_stl_allocator other; }; @@ -484,35 +439,128 @@ template struct mi_stl_allocator { mi_stl_allocator select_on_container_copy_construction() const { return *this; } void deallocate(T* p, size_type) { mi_free(p); } - #if (__cplusplus >= 201703L) // C++17 +#if (__cplusplus >= 201703L) // C++17 mi_decl_nodiscard T* allocate(size_type count) { return static_cast(mi_new_n(count, sizeof(T))); } mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } - #else +#else mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast(mi_new_n(count, sizeof(value_type))); } - #endif +#endif - #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 +#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 using propagate_on_container_copy_assignment = std::true_type; using propagate_on_container_move_assignment = std::true_type; - using propagate_on_container_swap = std::true_type; - using is_always_equal = std::true_type; + using propagate_on_container_swap = std::true_type; + using is_always_equal = std::true_type; template void construct(U* p, Args&& ...args) { ::new(p) U(std::forward(args)...); } template void destroy(U* p) mi_attr_noexcept { p->~U(); } - #else +#else void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } void destroy(pointer p) { p->~value_type(); } - #endif +#endif size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } - pointer address(reference x) const { return &x; } - const_pointer address(const_reference x) const { return &x; } + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } }; -template bool operator==(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return true; } -template bool operator!=(const mi_stl_allocator& , const mi_stl_allocator& ) mi_attr_noexcept { return false; } +template bool operator==(const mi_stl_allocator&, const mi_stl_allocator&) mi_attr_noexcept { return true; } +template bool operator!=(const mi_stl_allocator&, const mi_stl_allocator&) mi_attr_noexcept { return false; } #endif // __cplusplus +// --------------------------------------------------------------------------------------------- +// Debugging API that tracks the source location of an allocation +// --------------------------------------------------------------------------------------------- + +#ifdef NDEBUG +#define MI_SOURCE_XPARAM +#define MI_SOURCE_XARG +#define MI_SOURCE_XRET() +#define MI_SOURCE_XLOC() +#define MI_SOURCE_ARG(fun,...) fun(__VA_ARGS__) +#define MI_SOURCE_RET(fun,...) fun(__VA_ARGS__) +#define MI_SOURCE_LOC(fun,...) fun(__VA_ARGS__) + +#else +#ifdef _MSC_VER +#include +#define mi_return_address() _ReturnAddress() +#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc +#define mi_return_address() __builtin_return_address(0) +#else +#define mi_return_address() NULL +#endif +#define MI_SOURCE_XPARAM , mi_source_t __mi_source +#define MI_SOURCE_XARG , __mi_source +#define MI_SOURCE_XRET() , mi_source_ret(mi_return_address()) +#define MI_SOURCE_XLOC() , mi_source_loc(__FILE__,__LINE__) +#define MI_SOURCE_ARG(fun,...) dbg_##fun(__VA_ARGS__ MI_SOURCE_XARG) +#define MI_SOURCE_RET(fun,...) dbg_##fun(__VA_ARGS__ MI_SOURCE_XRET()) +#define MI_SOURCE_LOC(fun,...) dbg_##fun(__VA_ARGS__ MI_SOURCE_XLOC()) +#endif + +#if !defined(NDEBUG) && !defined(MI_NO_SOURCE_DEBUG) + +#define mi_malloc(n) MI_SOURCE_LOC(mi_malloc,n) +#define mi_calloc(c,n) MI_SOURCE_LOC(mi_calloc,c,n) +#define mi_realloc(p,n) MI_SOURCE_LOC(mi_realloc,p,n) +#define mi_strdup(s) MI_SOURCE_LOC(mi_strdup,s) +#define mi_strndup(s,n) MI_SOURCE_LOC(mi_strndup,s,n) +#define mi_realpath(f,n) MI_SOURCE_LOC(mi_realpath,f,n) +#define mi_malloc_small(n) MI_SOURCE_LOC(mi_malloc_small,n) +#define mi_zalloc_small(n) MI_SOURCE_LOC(mi_zalloc_small,n) +#define mi_zalloc(n) MI_SOURCE_LOC(mi_zalloc,n) +#define mi_mallocn(c,n) MI_SOURCE_LOC(mi_mallocn,c,n) +#define mi_reallocn(p,c,n) MI_SOURCE_LOC(mi_reallocn,p,c,n) +#define mi_reallocf(p,n) MI_SOURCE_LOC(mi_reallocf,p,n) + +#define mi_malloc_aligned(n,a) MI_SOURCE_LOC(mi_malloc_aligned,n,a) +#define mi_zalloc_aligned(n,a) MI_SOURCE_LOC(mi_zalloc_aligned,n,a) +#define mi_calloc_aligned(c,n,a) MI_SOURCE_LOC(mi_calloc_aligned,c,n,a) +#define mi_realloc_aligned(p,n,a) MI_SOURCE_LOC(mi_realloc_aligned,p,n,a) +#define mi_malloc_aligned_at(n,a,o) MI_SOURCE_LOC(mi_malloc_aligned_at,n,a,o) +#define mi_zalloc_aligned_at(n,a,o) MI_SOURCE_LOC(mi_zalloc_aligned_at,n,a,o) +#define mi_calloc_aligned_at(c,n,a,o) MI_SOURCE_LOC(mi_calloc_aligned_at,c,n,a,o) +#define mi_realloc_aligned_at(p,n,a,o) MI_SOURCE_LOC(mi_realloc_aligned_at,p,n,a,o) + +#define mi_rezalloc(p,n) MI_SOURCE_LOC(mi_rezalloc,p,n) +#define mi_recalloc(p,c,n) MI_SOURCE_LOC(mi_recalloc,p,c,n) +#define mi_rezalloc_aligned(p,n,a) MI_SOURCE_LOC(mi_rezalloc_aligned,p,n,a) +#define mi_recalloc_aligned(p,c,n,a) MI_SOURCE_LOC(mi_recalloc_aligned,p,c,n,a) +#define mi_rezalloc_aligned_at(p,n,a,o) MI_SOURCE_LOC(mi_rezalloc_aligned_at,p,n,a,o) +#define mi_recalloc_aligned_at(p,c,n,a,o) MI_SOURCE_LOC(mi_recalloc_aligned_at,p,c,n,a,o) + +#define mi_heap_malloc(h,n) MI_SOURCE_LOC(mi_heap_malloc,h,n) +#define mi_heap_calloc(h,c,n) MI_SOURCE_LOC(mi_heap_calloc,h,c,n) +#define mi_heap_realloc(h,p,n) MI_SOURCE_LOC(mi_heap_realloc,h,p,n) +#define mi_heap_strdup(h,s) MI_SOURCE_LOC(mi_heap_strdup,h,s) +#define mi_heap_strndup(h,s,n) MI_SOURCE_LOC(mi_heap_strndup,h,s,n) +#define mi_heap_realpath(h,f,n) MI_SOURCE_LOC(mi_heap_realpath,h,f,n) +#define mi_heap_malloc_small(h,n) MI_SOURCE_LOC(mi_heap_malloc_small,h,n) +#define mi_heap_zalloc_small(h,n) MI_SOURCE_LOC(mi_heap_zalloc_small,h,n) +#define mi_heap_zalloc(h,n) MI_SOURCE_LOC(mi_heap_zalloc,h,n) +#define mi_heap_mallocn(h,c,n) MI_SOURCE_LOC(mi_heap_mallocn,h,c,n) +#define mi_heap_reallocn(h,p,c,n) MI_SOURCE_LOC(mi_heap_reallocn,h,p,c,n) +#define mi_heap_reallocf(h,p,n) MI_SOURCE_LOC(mi_heap_reallocf,h,p,n) + +#define mi_heap_malloc_aligned(h,n,a) MI_SOURCE_LOC(mi_heap_malloc_aligned,h,n,a) +#define mi_heap_zalloc_aligned(h,n,a) MI_SOURCE_LOC(mi_heap_zalloc_aligned,h,n,a) +#define mi_heap_calloc_aligned(h,c,n,a) MI_SOURCE_LOC(mi_heap_calloc_aligned,h,c,n,a) +#define mi_heap_realloc_aligned(h,p,n,a) MI_SOURCE_LOC(mi_heap_realloc_aligned,h,p,n,a) +#define mi_heap_malloc_aligned_at(h,n,a,o) MI_SOURCE_LOC(mi_heap_malloc_aligned_at,h,n,a,o) +#define mi_heap_zalloc_aligned_at(h,n,a,o) MI_SOURCE_LOC(mi_heap_zalloc_aligned_at,h,n,a,o) +#define mi_heap_calloc_aligned_at(h,c,n,a,o) MI_SOURCE_LOC(mi_heap_calloc_aligned_at,h,c,n,a,o) +#define mi_heap_realloc_aligned_at(h,p,n,a,o) MI_SOURCE_LOC(mi_heap_realloc_aligned_at,h,p,n,a,o) + +#define mi_heap_rezalloc(h,p,n) MI_SOURCE_LOC(mi_heap_rezalloc,h,p,n) +#define mi_heap_recalloc(h,p,c,n) MI_SOURCE_LOC(mi_heap_recalloc,h,p,c,n) +#define mi_heap_rezalloc_aligned(h,p,n,a) MI_SOURCE_LOC(mi_heap_rezalloc_aligned,h,p,n,a) +#define mi_heap_recalloc_aligned(h,p,c,n,a) MI_SOURCE_LOC(mi_heap_recalloc_aligned,h,p,c,n,a) +#define mi_heap_rezalloc_aligned_at(h,p,n,a,o) MI_SOURCE_LOC(mi_heap_rezalloc_aligned_at,h,p,n,a,o) +#define mi_heap_recalloc_aligned_at(h,p,c,n,a,o) MI_SOURCE_LOC(mi_heap_recalloc_aligned_at,h,p,c,n,a,o) + +#endif + #endif // MIMALLOC_H diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index e9ef0916..bb5d0143 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -4,7 +4,7 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ - +#define MI_NO_SOURCE_DEBUG #include "mimalloc.h" #include "mimalloc-internal.h" @@ -14,14 +14,14 @@ terms of the MIT license. A copy of the license can be found in the file // Aligned Allocation // ------------------------------------------------------ -static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero MI_SOURCE_PARAM) mi_attr_noexcept { +static void* mi_base_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero MI_SOURCE_XPARAM) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that // the address at offset is aligned regardless of the allocated size. mi_assert(alignment > 0 && alignment % sizeof(void*) == 0); if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see ) if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see ) - if (alignment <= MI_MAX_ALIGN_SIZE && offset==0) return _mi_heap_source_malloc_zero(heap, size, zero MI_SOURCE_ARG); + if (alignment <= MI_MAX_ALIGN_SIZE && offset==0) return _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG); const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` // try if there is a small block available with just the right alignment @@ -34,7 +34,7 @@ static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const #if MI_STAT>1 mi_heap_stat_increase( heap, malloc, size); #endif - void* p = _mi_page_malloc(heap,page,padsize MI_SOURCE_ARG); // TODO: inline _mi_page_malloc + void* p = _mi_page_malloc(heap,page,padsize MI_SOURCE_XARG); // TODO: inline _mi_page_malloc mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); if (zero) _mi_block_zero_init(page,p,size); @@ -44,13 +44,13 @@ static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const // use regular allocation if it is guaranteed to fit the alignment constraints if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) { - void* p = _mi_heap_source_malloc_zero(heap, size, zero MI_SOURCE_ARG); + void* p = _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); return p; } // otherwise over-allocate - void* p = _mi_heap_source_malloc_zero(heap, size + alignment - 1, zero MI_SOURCE_ARG); + void* p = _mi_base_malloc_zero(heap, size + alignment - 1, zero MI_SOURCE_XARG); if (p == NULL) return NULL; // .. and align within the allocation @@ -66,51 +66,51 @@ static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const MI_ALLOC_API3(void*, malloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset) { - return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, offset, false MI_SOURCE_ARG); + return mi_base_malloc_zero_aligned_at(heap, size, alignment, offset, false MI_SOURCE_XARG); } MI_ALLOC_API2(void*, malloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment) { - return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, 0, false MI_SOURCE_ARG); + return mi_base_malloc_zero_aligned_at(heap, size, alignment, 0, false MI_SOURCE_XARG); } MI_ALLOC_API3(void*, zalloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset) { - return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, offset, true MI_SOURCE_ARG); + return mi_base_malloc_zero_aligned_at(heap, size, alignment, offset, true MI_SOURCE_XARG); } MI_ALLOC_API2(void*, zalloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment) { - return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, 0, true MI_SOURCE_ARG); + return mi_base_malloc_zero_aligned_at(heap, size, alignment, 0, true MI_SOURCE_XARG); } MI_ALLOC_API4(void*, calloc_aligned_at, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment, size_t, offset) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_heap_source_malloc_zero_aligned_at(heap, total, alignment, offset, true MI_SOURCE_ARG); + return mi_base_malloc_zero_aligned_at(heap, total, alignment, offset, true MI_SOURCE_XARG); } MI_ALLOC_API3(void*, calloc_aligned, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_heap_source_malloc_zero_aligned_at(heap, total, alignment, 0, true MI_SOURCE_ARG); + return mi_base_malloc_zero_aligned_at(heap, total, alignment, 0, true MI_SOURCE_XARG); } -static void* mi_heap_source_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero MI_SOURCE_PARAM) mi_attr_noexcept { +static void* mi_base_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_heap_source_realloc_zero(heap,p,newsize,zero MI_SOURCE_ARG); - if (p == NULL) return mi_heap_source_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero MI_SOURCE_ARG); + if (alignment <= sizeof(uintptr_t)) return _mi_base_realloc_zero(heap,p,newsize,zero MI_SOURCE_XARG); + if (p == NULL) return mi_base_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero MI_SOURCE_XARG); size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) % alignment) == 0) { return p; // reallocation still fits, is aligned and not more than 50% waste } else { - void* newp = mi_heap_source_malloc_aligned_at(heap,newsize,alignment,offset MI_SOURCE_ARG); + void* newp = mi_base_malloc_aligned_at(heap,newsize,alignment,offset MI_SOURCE_XARG); if (newp != NULL) { if (zero && newsize > size) { const mi_page_t* page = _mi_ptr_page(newp); @@ -131,43 +131,43 @@ static void* mi_heap_source_realloc_zero_aligned_at(mi_heap_t* heap, void* p, si } } -static void* mi_heap_source_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero MI_SOURCE_PARAM) mi_attr_noexcept { +static void* mi_base_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_heap_source_realloc_zero(heap,p,newsize,zero MI_SOURCE_ARG); + if (alignment <= sizeof(uintptr_t)) return _mi_base_realloc_zero(heap,p,newsize,zero MI_SOURCE_XARG); size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL) - return mi_heap_source_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero MI_SOURCE_XARG); } MI_ALLOC_API4(void*, realloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment, size_t, offset) { - return mi_heap_source_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false MI_SOURCE_XARG); } MI_ALLOC_API3(void*, realloc_aligned, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment) { - return mi_heap_source_realloc_zero_aligned(heap,p,newsize,alignment,false MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned(heap,p,newsize,alignment,false MI_SOURCE_XARG); } MI_ALLOC_API4(void*, rezalloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment, size_t, offset) { - return mi_heap_source_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true MI_SOURCE_XARG); } MI_ALLOC_API3(void*, rezalloc_aligned, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment) { - return mi_heap_source_realloc_zero_aligned(heap, p, newsize, alignment, true MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned(heap, p, newsize, alignment, true MI_SOURCE_XARG); } MI_ALLOC_API5(void*, recalloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newcount, size_t, size, size_t, alignment, size_t, offset) { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; - return mi_heap_source_realloc_zero_aligned_at(heap, p, total, alignment, offset, true MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned_at(heap, p, total, alignment, offset, true MI_SOURCE_XARG); } MI_ALLOC_API4(void*, recalloc_aligned, mi_heap_t*, heap, void*, p, size_t, newcount, size_t, size, size_t, alignment) { size_t total; if (mi_count_size_overflow(newcount, size, &total)) return NULL; - return mi_heap_source_realloc_zero_aligned_at(heap, p, total, alignment, 0, true MI_SOURCE_ARG); + return mi_base_realloc_zero_aligned_at(heap, p, total, alignment, 0, true MI_SOURCE_XARG); } diff --git a/src/alloc-override.c b/src/alloc-override.c index f91d02ec..36e15187 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -26,18 +26,18 @@ terms of the MIT license. A copy of the license can be found in the file #else #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))) #endif - #define MI_FORWARD1(fun,x) MI_FORWARD(mi_##fun) - #define MI_FORWARD2(fun,x,y) MI_FORWARD(mi_##fun) - #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(mi_##fun) - #define MI_FORWARD0(fun,x) MI_FORWARD(mi_##fun) - #define MI_FORWARD02(fun,x,y) MI_FORWARD(mi_##fun) + #define MI_FORWARD1(fun,x) MI_FORWARD(fun) + #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) + #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) + #define MI_FORWARD0(fun,x) MI_FORWARD(fun) + #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) #else // use forwarding by calling our `mi_` function - #define MI_FORWARD1(fun,x) { return mi_source_##fun(x MI_SOURCE_RET()); } - #define MI_FORWARD2(fun,x,y) { return mi_source_##fun(x,y MI_SOURCE_RET()); } - #define MI_FORWARD3(fun,x,y,z) { return mi_source_##fun(x,y,z MI_SOURCE_RET()); } - #define MI_FORWARD0(fun,x) { mi_##fun(x); } - #define MI_FORWARD02(fun,x,y) { mi_##fun(x,y); } + #define MI_FORWARD1(fun,x) { return MI_SOURCE_RET(fun, x); } + #define MI_FORWARD2(fun,x,y) { return MI_SOURCE_RET(fun, x, y); } + #define MI_FORWARD3(fun,x,y,z) { return MI_SOURCE_RET(fun, x, y, z); } + #define MI_FORWARD0(fun,x) { fun(x); } + #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif #if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) @@ -68,10 +68,10 @@ terms of the MIT license. A copy of the license can be found in the file // we just override new/delete which does work in a static library. #else // On all other systems forward to our API - void* malloc(size_t size) MI_FORWARD1(malloc, size); - void* calloc(size_t size, size_t n) MI_FORWARD2(calloc, size, n); - void* realloc(void* p, size_t newsize) MI_FORWARD2(realloc, p, newsize); - void free(void* p) MI_FORWARD0(free, p); + void* malloc(size_t size) MI_FORWARD1(mi_malloc, size); + void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n); + void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize); + void free(void* p) MI_FORWARD0(mi_free, p); #endif #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) @@ -89,18 +89,18 @@ terms of the MIT license. A copy of the license can be found in the file // see // ------------------------------------------------------ #include - void operator delete(void* p) noexcept MI_FORWARD0(free,p); - void operator delete[](void* p) noexcept MI_FORWARD0(free,p); + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(new,n); - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(new,n); + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); - void* operator new (std::size_t n, const std::nothrow_t& ) noexcept { return mi_source_new_nothrow(n MI_SOURCE_RET()); } - void* operator new[](std::size_t n, const std::nothrow_t& ) noexcept { return mi_source_new_nothrow(n MI_SOURCE_RET()); } + void* operator new (std::size_t n, const std::nothrow_t& ) noexcept { return MI_SOURCE_RET(mi_new_nothrow, n); } + void* operator new[](std::size_t n, const std::nothrow_t& ) noexcept { return MI_SOURCE_RET(mi_new_nothrow, n); } #if (__cplusplus >= 201402L || _MSC_VER >= 1916) - void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(free_size,p,n); - void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(free_size,p,n); + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); #endif #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) @@ -109,10 +109,10 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; - void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_source_new_aligned(n, static_cast(al) MI_SOURCE_RET()); } - void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_source_new_aligned(n, static_cast(al) MI_SOURCE_RET()); } - void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_source_new_aligned_nothrow(n, static_cast(al) MI_SOURCE_RET()); } - void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_source_new_aligned_nothrow(n, static_cast(al) MI_SOURCE_RET()); } + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return MI_SOURCE_RET(mi_new_aligned, n, static_cast(al)); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return MI_SOURCE_RET(mi_new_aligned, n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return MI_SOURCE_RET(mi_new_aligned_nothrow, n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return MI_SOURCE_RET(mi_new_aligned_nothrow, n, static_cast(al)); } #endif #elif (defined(__GNUC__) || defined(__clang__)) @@ -121,10 +121,10 @@ terms of the MIT license. A copy of the license can be found in the file // used by GCC and CLang). // See // ------------------------------------------------------ - void _ZdlPv(void* p) MI_FORWARD0(free,p); // delete - void _ZdaPv(void* p) MI_FORWARD0(free,p); // delete[] - void _ZdlPvm(void* p, size_t n) MI_FORWARD02(free_size,p,n); - void _ZdaPvm(void* p, size_t n) MI_FORWARD02(free_size,p,n); + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[] + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } @@ -132,23 +132,23 @@ terms of the MIT license. A copy of the license can be found in the file typedef struct mi_nothrow_s { } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) - void* _Znwm(size_t n) MI_FORWARD1(new,n); // new 64-bit - void* _Znam(size_t n) MI_FORWARD1(new,n); // new[] 64-bit - void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al); - void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al); - void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_source_new_nothrow(n MI_SOURCE_RET()); } - void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_source_new_nothrow(n MI_SOURCE_RET()); } - void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al MI_SOURCE_RET()); } - void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al MI_SOURCE_RET()); } + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit + void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow, n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow, n); } + void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow, n, al); } + void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow, n, al); } #elif (MI_INTPTR_SIZE==4) - void* _Znwj(size_t n) MI_FORWARD1(new,n); // new 64-bit - void* _Znaj(size_t n) MI_FORWARD1(new,n); // new[] 64-bit - void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al); - void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al); - void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow,n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow,n); } + void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow,n,al); } + void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow,n,al); } #else #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif @@ -163,31 +163,31 @@ extern "C" { // Posix & Unix functions definitions // ------------------------------------------------------ -void* reallocf(void* p, size_t newsize) MI_FORWARD2(reallocf,p,newsize); -void cfree(void* p) MI_FORWARD0(free, p); +void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); +void cfree(void* p) MI_FORWARD0(mi_free, p); size_t malloc_size(void* p) { return mi_usable_size(p); } size_t malloc_usable_size(void* p) { return mi_usable_size(p); } // no forwarding here due to aliasing/name mangling issues -void* valloc(size_t size) { return mi_source_valloc(size MI_SOURCE_RET()); } -void* pvalloc(size_t size) { return mi_source_pvalloc(size MI_SOURCE_RET()); } -void* reallocarray(void* p, size_t count, size_t size) { return mi_source_reallocarray(p, count, size MI_SOURCE_RET()); } -void* memalign(size_t alignment, size_t size) { return mi_source_memalign(alignment, size MI_SOURCE_RET()); } -void* aligned_alloc(size_t alignment, size_t size) { return mi_source_aligned_alloc(alignment, size MI_SOURCE_RET()); } -int posix_memalign(void** p, size_t alignment, size_t size) { return mi_source_posix_memalign(p, alignment, size MI_SOURCE_RET()); } +void* valloc(size_t size) { return MI_SOURCE_RET(mi_valloc, size); } +void* pvalloc(size_t size) { return MI_SOURCE_RET(mi_pvalloc, size); } +void* reallocarray(void* p, size_t count, size_t size) { return MI_SOURCE_RET(mi_reallocarray, p, count, size); } +void* memalign(size_t alignment, size_t size) { return MI_SOURCE_RET(mi_memalign, alignment, size); } +void* aligned_alloc(size_t alignment, size_t size) { return MI_SOURCE_RET(mi_aligned_alloc, alignment, size); } +int posix_memalign(void** p, size_t alignment, size_t size) { return MI_SOURCE_RET(mi_posix_memalign, p, alignment, size); } #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) - void* __libc_malloc(size_t size) MI_FORWARD1(malloc,size); - void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(calloc,count,size); - void* __libc_realloc(void* p, size_t size) MI_FORWARD2(realloc,p,size); - void __libc_free(void* p) MI_FORWARD0(free,p); - void __libc_cfree(void* p) MI_FORWARD0(free,p); + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size); + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size); + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size); + void __libc_free(void* p) MI_FORWARD0(mi_free,p); + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p); - void* __libc_valloc(size_t size) { return mi_source_valloc(size MI_SOURCE_RET()); } - void* __libc_pvalloc(size_t size) { return mi_source_pvalloc(size MI_SOURCE_RET()); } - void* __libc_memalign(size_t alignment, size_t size) { return mi_source_memalign(alignment, size MI_SOURCE_RET()); } - int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_source_posix_memalign(p, alignment, size MI_SOURCE_RET()); } + void* __libc_valloc(size_t size) { return MI_SOURCE_RET(mi_valloc, size); } + void* __libc_pvalloc(size_t size) { return MI_SOURCE_RET(mi_pvalloc, size); } + void* __libc_memalign(size_t alignment, size_t size) { return MI_SOURCE_RET(mi_memalign, alignment, size); } + int __posix_memalign(void** p, size_t alignment, size_t size) { return MI_SOURCE_RET(mi_posix_memalign, p, alignment, size); } #endif #ifdef __cplusplus diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 3d05e2da..d3aa750c 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file // mi prefixed publi definitions of various Posix, Unix, and C++ functions // for convenience and used when overriding these functions. // ------------------------------------------------------------------------ +#define MI_NO_SOURCE_DEBUG #include "mimalloc.h" #include "mimalloc-internal.h" @@ -42,34 +43,27 @@ void mi_cfree(void* p) mi_attr_noexcept { } } -MI_SOURCE_API3(int, posix_memalign, void**, p, size_t, alignment, size_t, size) +void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft + void* res = mi_expand(p, newsize); + if (res == NULL) errno = ENOMEM; + return res; +} + +MI_SOURCE_API3(void*, reallocarray, void*, p, size_t, count, size_t, size) { - // Note: The spec dictates we should not modify `*p` on an error. (issue#27) - // - if (p == NULL) return EINVAL; - if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment - if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 - void* q; - if (alignment <= MI_MAX_ALIGN_SIZE) { - q = mi_source_malloc(size MI_SOURCE_ARG); - } - else { - q = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG); - } - if (q==NULL && size != 0) return ENOMEM; - mi_assert_internal(((uintptr_t)q % alignment) == 0); - *p = q; - return 0; + void* newp = MI_SOURCE_ARG(mi_reallocn, p, count, size); + if (newp==NULL) errno = ENOMEM; + return newp; } MI_SOURCE_API2(void*, memalign, size_t, alignment, size_t, size) { void* p; if (alignment <= MI_MAX_ALIGN_SIZE) { - p = mi_source_malloc(size MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc, size); } else { - p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment); } mi_assert_internal(((uintptr_t)p % alignment) == 0); return p; @@ -77,15 +71,15 @@ MI_SOURCE_API2(void*, memalign, size_t, alignment, size_t, size) MI_SOURCE_API1(void*, valloc, size_t, size) { - return mi_source_malloc_aligned(size, _mi_os_page_size() MI_SOURCE_ARG); + return MI_SOURCE_ARG(mi_malloc_aligned, size, _mi_os_page_size()); } MI_SOURCE_API1(void*, pvalloc, size_t, size) { size_t psize = _mi_os_page_size(); if (size >= SIZE_MAX - psize) return NULL; // overflow - size_t asize = ((size + psize - 1) / psize) * psize; - return mi_source_malloc_aligned(asize, psize MI_SOURCE_ARG); + size_t asize = _mi_align_up(size, psize); + return MI_SOURCE_ARG(mi_malloc_aligned, asize, psize); } MI_SOURCE_API2(void*, aligned_alloc, size_t, alignment, size_t, size) @@ -94,27 +88,46 @@ MI_SOURCE_API2(void*, aligned_alloc, size_t, alignment, size_t, size) if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see void* p; if (alignment <= MI_MAX_ALIGN_SIZE) { - p = mi_source_malloc(size MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc, size); } else { - p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment); } mi_assert_internal(((uintptr_t)p % alignment) == 0); return p; } -MI_SOURCE_API3(void*, reallocarray, void*, p, size_t, count, size_t, size ) +static int mi_base_posix_memalign(void** p, size_t alignment, size_t size MI_SOURCE_XPARAM) { - void* newp = mi_source_reallocn(p, count, size MI_SOURCE_ARG); - if (newp==NULL) errno = ENOMEM; - return newp; + // Note: The spec dictates we should not modify `*p` on an error. (issue#27) + // + if (p == NULL) return EINVAL; + if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment + if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 + void* q; + if (alignment <= MI_MAX_ALIGN_SIZE) { + q = MI_SOURCE_ARG(mi_malloc, size); + } + else { + q = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment); + } + if (q==NULL && size != 0) return ENOMEM; + mi_assert_internal(((uintptr_t)q % alignment) == 0); + *p = q; + return 0; } -void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft - void* res = mi_expand(p, newsize); - if (res == NULL) errno = ENOMEM; - return res; +#ifndef NDEBUG +int dbg_mi_posix_memalign(void** p, size_t alignment, size_t size, mi_source_t __mi_source) mi_attr_noexcept { + UNUSED(__mi_source); + return mi_base_posix_memalign(p, alignment, size MI_SOURCE_XARG); } +#endif + +int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { + return mi_base_posix_memalign(p, alignment, size MI_SOURCE_XRET()); +} + MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s) { @@ -122,7 +135,7 @@ MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s) size_t len; for(len = 0; s[len] != 0; len++) { } size_t size = (len+1)*sizeof(unsigned short); - unsigned short* p = (unsigned short*)mi_source_malloc(size MI_SOURCE_ARG); + unsigned short* p = (unsigned short*)MI_SOURCE_ARG(mi_malloc, size); if (p != NULL) { memcpy(p,s,size); } @@ -131,10 +144,10 @@ MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s) MI_SOURCE_API1(unsigned char*, mbsdup, const unsigned char*, s) { - return (unsigned char*)mi_source_strdup((const char*)s MI_SOURCE_ARG); + return (unsigned char*)MI_SOURCE_ARG(mi_strdup,(const char*)s); } -MI_SOURCE_API3(int, dupenv_s, char**, buf, size_t*, size, const char*, name) +static int mi_base_dupenv_s(char** buf, size_t* size, const char* name MI_SOURCE_XPARAM) { if (buf==NULL || name==NULL) return EINVAL; if (size != NULL) *size = 0; @@ -144,14 +157,26 @@ MI_SOURCE_API3(int, dupenv_s, char**, buf, size_t*, size, const char*, name) *buf = NULL; } else { - *buf = mi_source_strdup(p MI_SOURCE_ARG); + *buf = MI_SOURCE_ARG(mi_strdup, p); if (*buf==NULL) return ENOMEM; if (size != NULL) *size = strlen(p); } return 0; } -MI_SOURCE_API3(int, wdupenv_s, unsigned short**, buf, size_t*, size, const unsigned short*, name) +#ifndef NDEBUG +int dbg_mi_dupenv_s(char** buf, size_t* size, const char* name, mi_source_t __mi_source) mi_attr_noexcept { + UNUSED(__mi_source); + return mi_base_dupenv_s(buf, size, name MI_SOURCE_XARG); +} +#endif + +int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { + return mi_base_dupenv_s(buf, size, name MI_SOURCE_XRET()); +} + + +static int mi_base_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name MI_SOURCE_XPARAM) { if (buf==NULL || name==NULL) return EINVAL; if (size != NULL) *size = 0; @@ -169,7 +194,7 @@ MI_SOURCE_API3(int, wdupenv_s, unsigned short**, buf, size_t*, size, const unsig *buf = NULL; } else { - *buf = mi_source_wcsdup(p MI_SOURCE_ARG); + *buf = MI_SOURCE_ARG(mi_wcsdup, p); if (*buf==NULL) return ENOMEM; if (size != NULL) *size = wcslen((const wchar_t*)p); } @@ -177,10 +202,32 @@ MI_SOURCE_API3(int, wdupenv_s, unsigned short**, buf, size_t*, size, const unsig #endif } -void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft - return mi_recalloc_aligned_at(p, newcount, size, alignment, offset); +#ifndef NDEBUG +int dbg_mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name, mi_source_t __mi_source) mi_attr_noexcept { + UNUSED(__mi_source); + return mi_base_wdupenv_s(buf, size, name MI_SOURCE_XARG); +} +#endif + +int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept { + return mi_base_wdupenv_s(buf, size, name MI_SOURCE_XRET()); } -void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft - return mi_recalloc_aligned(p, newcount, size, alignment); + +#ifndef NDEBUG +mi_decl_restrict void* dbg_mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft + return dbg_mi_recalloc_aligned_at(p, newcount, size, alignment, offset, __mi_source); +} + +mi_decl_restrict void* dbg_mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft + return dbg_mi_recalloc_aligned(p, newcount, size, alignment, __mi_source); +} +#endif + +mi_decl_restrict void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft + return MI_SOURCE_RET(mi_recalloc_aligned_at,p, newcount, size, alignment, offset); +} + +mi_decl_restrict void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft + return MI_SOURCE_RET(mi_recalloc_aligned,p, newcount, size, alignment); } diff --git a/src/alloc.c b/src/alloc.c index 6283c478..07c25fc0 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,6 +4,7 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#define MI_NO_SOURCE_DEBUG #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" @@ -22,11 +23,11 @@ terms of the MIT license. A copy of the license can be found in the file // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. -extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_PARAM) mi_attr_noexcept { +extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size); mi_block_t* block = page->free; if (mi_unlikely(block == NULL)) { - return _mi_malloc_generic(heap, size MI_SOURCE_ARG); // slow path + return _mi_malloc_generic(heap, size MI_SOURCE_XARG); // slow path } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); // pop from the free list @@ -60,13 +61,13 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } // allocate a small block -MI_ALLOC_API1(void*,malloc_small, mi_heap_t*,heap, size_t,size) +MI_ALLOC_API1(void*, malloc_small, mi_heap_t*, heap, size_t, size) { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local mi_assert(size <= MI_SMALL_SIZE_MAX); mi_page_t* page = _mi_heap_get_free_small_page(heap,size + MI_PADDING_SIZE); - void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE MI_SOURCE_ARG); + void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE MI_SOURCE_XARG); mi_assert_internal(p==NULL || mi_usable_size(p) >= size); #if MI_STAT>1 if (p != NULL) { @@ -79,15 +80,15 @@ MI_ALLOC_API1(void*,malloc_small, mi_heap_t*,heap, size_t,size) // The main allocation function -MI_ALLOC_API1(void*, malloc, mi_heap_t*,heap, size_t,size) +MI_ALLOC_API1(void*, malloc, mi_heap_t*, heap, size_t, size) { if (mi_likely(size <= MI_SMALL_SIZE_MAX)) { - return mi_heap_source_malloc_small(heap, size MI_SOURCE_ARG); + return mi_base_malloc_small(heap, size MI_SOURCE_XARG); } else { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local - void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE MI_SOURCE_ARG); + void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE MI_SOURCE_XARG); mi_assert_internal(p == NULL || mi_usable_size(p) >= size); #if MI_STAT>1 if (p != NULL) { @@ -121,15 +122,15 @@ void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) { // zero initialized small block MI_ALLOC_API1(void*, zalloc_small, mi_heap_t*, heap, size_t, size) { - void* p = mi_heap_source_malloc_small(heap, size MI_SOURCE_ARG); + void* p = mi_base_malloc_small(heap, size MI_SOURCE_XARG); if (p != NULL) { _mi_block_zero_init(_mi_ptr_page(p), p, size); // todo: can we avoid getting the page again? } return p; } -mi_decl_allocator void* _mi_heap_source_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_PARAM) { - void* p = mi_heap_source_malloc(heap,size MI_SOURCE_ARG); +mi_decl_restrict void* _mi_base_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_XPARAM) { + void* p = mi_base_malloc(heap, size MI_SOURCE_XARG); if (zero && p != NULL) { _mi_block_zero_init(_mi_ptr_page(p),p,size); // todo: can we avoid getting the page again? } @@ -138,7 +139,7 @@ mi_decl_allocator void* _mi_heap_source_malloc_zero(mi_heap_t* heap, size_t size MI_ALLOC_API1(void*, zalloc, mi_heap_t*,heap, size_t,size) { - return _mi_heap_source_malloc_zero(heap, size, true MI_SOURCE_ARG); + return _mi_base_malloc_zero(heap, size, true MI_SOURCE_XARG); } @@ -518,37 +519,37 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { mi_free(p); } -MI_ALLOC_API2(void*, calloc, mi_heap_t*,heap, size_t,count, size_t,size) +MI_ALLOC_API2(void*, calloc, mi_heap_t*, heap, size_t, count, size_t, size) { size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; - return mi_heap_source_zalloc(heap, total MI_SOURCE_ARG); + return mi_base_zalloc(heap, total MI_SOURCE_XARG); } // Uninitialized `calloc` -MI_ALLOC_API2(void*, mallocn, mi_heap_t*,heap, size_t,count, size_t,size) +MI_ALLOC_API2(void*, mallocn, mi_heap_t*, heap, size_t, count, size_t, size) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_heap_source_malloc(heap, total MI_SOURCE_ARG); + return mi_base_malloc(heap, total MI_SOURCE_XARG); } // Expand in place or fail -mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_restrict void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { if (p == NULL) return NULL; size_t size = mi_usable_size(p); if (newsize > size) return NULL; return p; // it fits } -mi_decl_allocator void* _mi_heap_source_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_PARAM) { - if (p == NULL) return _mi_heap_source_malloc_zero(heap,newsize,zero MI_SOURCE_ARG); +mi_decl_restrict void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM) { + if (p == NULL) return _mi_base_malloc_zero(heap,newsize,zero MI_SOURCE_XARG); size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size / 2)) { return p; // reallocation still fits and not more than 50% waste } - void* newp = mi_heap_source_malloc(heap,newsize MI_SOURCE_ARG); + void* newp = mi_base_malloc(heap, newsize MI_SOURCE_XARG); if (mi_likely(newp != NULL)) { if (zero && newsize > size) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized @@ -561,36 +562,36 @@ mi_decl_allocator void* _mi_heap_source_realloc_zero(mi_heap_t* heap, void* p, s return newp; } -MI_ALLOC_API2(void*, realloc, mi_heap_t*,heap, void*,p, size_t,newsize) +MI_ALLOC_API2(void*, realloc, mi_heap_t*, heap, void*, p, size_t, newsize) { - return _mi_heap_source_realloc_zero(heap, p, newsize, false MI_SOURCE_ARG); + return _mi_base_realloc_zero(heap, p, newsize, false MI_SOURCE_XARG); } -MI_ALLOC_API3(void*, reallocn, mi_heap_t*,heap, void*,p, size_t,count, size_t,size) +MI_ALLOC_API3(void*, reallocn, mi_heap_t*, heap, void*, p, size_t, count, size_t, size) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return mi_heap_source_realloc(heap, p, total MI_SOURCE_ARG); + return mi_base_realloc(heap, p, total MI_SOURCE_XARG); } // Reallocate but free `p` on errors MI_ALLOC_API2(void*, reallocf, mi_heap_t*, heap, void*, p, size_t, newsize) { - void* newp = mi_heap_source_realloc(heap, p, newsize MI_SOURCE_ARG); + void* newp = mi_base_realloc(heap, p, newsize MI_SOURCE_XARG); if (newp==NULL && p!=NULL) mi_free(p); return newp; } MI_ALLOC_API2(void*, rezalloc, mi_heap_t*, heap, void*, p, size_t, newsize) { - return _mi_heap_source_realloc_zero(heap, p, newsize, true MI_SOURCE_ARG); + return _mi_base_realloc_zero(heap, p, newsize, true MI_SOURCE_XARG); } MI_ALLOC_API3(void*, recalloc, mi_heap_t*, heap, void*, p, size_t, count, size_t, size) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; - return _mi_heap_source_realloc_zero(heap, p, total, true MI_SOURCE_ARG); + return _mi_base_realloc_zero(heap, p, total, true MI_SOURCE_XARG); } @@ -604,7 +605,7 @@ MI_ALLOC_API1(char*, strdup, mi_heap_t*,heap, const char*,s) { if (s == NULL) return NULL; size_t n = strlen(s); - char* t = (char*)mi_heap_source_malloc(heap, n+1 MI_SOURCE_ARG); + char* t = (char*)mi_base_malloc(heap, n+1 MI_SOURCE_XARG); if (t != NULL) memcpy(t, s, n + 1); return t; } @@ -616,7 +617,7 @@ MI_ALLOC_API2(char*, strndup, mi_heap_t*, heap, const char*, s, size_t, n) if (s == NULL) return NULL; size_t m = strlen(s); if (n > m) n = m; - char* t = (char*)mi_heap_source_malloc(heap, n+1 MI_SOURCE_ARG); + char* t = (char*)mi_base_malloc(heap, n+1 MI_SOURCE_XARG); if (t == NULL) return NULL; memcpy(t, s, n); t[n] = 0; @@ -646,7 +647,7 @@ MI_ALLOC_API2(char*, realpath, mi_heap_t*, heap, const char*, fname, char*, reso return resolved_name; } else { - return mi_heap_source_strndup(heap, buf, PATH_MAX MI_SOURCE_ARG); + return mi_base_strndup(heap, buf, PATH_MAX MI_SOURCE_XARG); } } #else @@ -672,7 +673,7 @@ MI_ALLOC_API2(char*, realpath, mi_heap_t*, heap, const char*, fname, char*, reso char* buf = (char*)mi_malloc(n+1); if (buf==NULL) return NULL; char* rname = realpath(fname,buf); - char* result = mi_heap_source_strndup(heap, rname, n MI_SOURCE_ARG); // ok if `rname==NULL` + char* result = mi_base_strndup(heap, rname, n MI_SOURCE_XARG); // ok if `rname==NULL` mi_free(buf); return result; } @@ -733,90 +734,72 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_source_try_new(size_t size, bool nothrow MI_SOURCE_PARAM) { +static mi_decl_noinline void* mi_base_try_new(size_t size, bool nothrow MI_SOURCE_XPARAM) { void* p = NULL; while(p == NULL && mi_try_new_handler(nothrow)) { - p = mi_source_malloc(size MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc, size); } return p; } -inline mi_decl_allocator void* mi_source_new(size_t size MI_SOURCE_PARAM) { - void* p = mi_source_malloc(size MI_SOURCE_ARG); - if (mi_unlikely(p == NULL)) return mi_source_try_new(size, false MI_SOURCE_ARG); +MI_NEW_API1(void*, new, size_t, size) +{ + void* p = MI_SOURCE_ARG(mi_malloc, size); + if (mi_unlikely(p == NULL)) return mi_base_try_new(size, false MI_SOURCE_XARG); return p; } -mi_decl_allocator void* mi_new(size_t size) { - return mi_source_new(size MI_SOURCE_RET()); -} - -mi_decl_allocator void* mi_source_new_nothrow(size_t size MI_SOURCE_PARAM) { - void* p = mi_source_malloc(size MI_SOURCE_ARG); - if (mi_unlikely(p == NULL)) return mi_source_try_new(size, true MI_SOURCE_ARG); +MI_NEW_API1(void*, new_nothrow, size_t, size) +{ + void* p = MI_SOURCE_ARG(mi_malloc, size); + if (mi_unlikely(p == NULL)) return mi_base_try_new(size, true MI_SOURCE_XARG); return p; } -mi_decl_allocator void* mi_new_nothrow(size_t size) { - return mi_source_new_nothrow(size MI_SOURCE_RET()); -} - -mi_decl_allocator void* mi_source_new_aligned(size_t size, size_t alignment MI_SOURCE_PARAM) { +MI_NEW_API2(void*, new_aligned, size_t, size, size_t, alignment) +{ void* p; do { - p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment); } while(p == NULL && mi_try_new_handler(false)); return p; } -mi_decl_allocator void* mi_new_aligned(size_t size, size_t alignment) { - return mi_source_new_aligned(size, alignment MI_SOURCE_RET()); -} - -mi_decl_allocator void* mi_source_new_aligned_nothrow(size_t size, size_t alignment MI_SOURCE_PARAM) { +MI_NEW_API2(void*, new_aligned_nothrow, size_t, size, size_t, alignment) +{ void* p; do { - p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG); + p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment); } while(p == NULL && mi_try_new_handler(true)); return p; } -mi_decl_allocator void* mi_new_aligned_nothrow(size_t size, size_t alignment) { - return mi_source_new_aligned_nothrow(size, alignment MI_SOURCE_RET()); -} - - -mi_decl_allocator void* mi_source_new_n(size_t count, size_t size MI_SOURCE_PARAM) { +MI_NEW_API2(void*, new_n, size_t, count, size_t, size) +{ size_t total; if (mi_unlikely(mi_count_size_overflow(count, size, &total))) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc return NULL; } else { - return mi_source_new(total MI_SOURCE_ARG); + return MI_SOURCE_ARG(mi_new, total); } } -mi_decl_allocator void* mi_new_n(size_t count, size_t size) { - return mi_source_new_n(count, size MI_SOURCE_RET()); -} - -mi_decl_allocator void* mi_source_new_realloc(void* p, size_t newsize MI_SOURCE_PARAM) { +MI_NEW_API2(void*, new_realloc, void*, p, size_t, newsize) +{ void* q; do { - q = mi_source_realloc(p, newsize MI_SOURCE_ARG); + q = MI_SOURCE_ARG(mi_realloc, p, newsize); } while (q == NULL && mi_try_new_handler(false)); return q; } -mi_decl_allocator void* mi_new_realloc(void* p, size_t newsize) { - return mi_source_new_realloc(p, newsize MI_SOURCE_RET()); -} - -mi_decl_allocator void* mi_source_new_reallocn(void* p, size_t newcount, size_t size MI_SOURCE_PARAM) { +MI_NEW_API3(void*, new_reallocn, void*, p, size_t, newcount, size_t, size) +{ size_t total; if (mi_unlikely(mi_count_size_overflow(newcount, size, &total))) { mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc @@ -824,11 +807,7 @@ mi_decl_allocator void* mi_source_new_reallocn(void* p, size_t newcount, size_t } void* q; do { - q = mi_source_realloc(p, total MI_SOURCE_ARG); + q = MI_SOURCE_ARG(mi_realloc, p, total); } while (q == NULL && mi_try_new_handler(false)); return q; } - -mi_decl_allocator void* mi_new_reallocn(void* p, size_t newcount, size_t size) { - return mi_source_new_reallocn(p, newcount, size MI_SOURCE_RET()); -} \ No newline at end of file diff --git a/src/init.c b/src/init.c index 2f5ca224..945f487f 100644 --- a/src/init.c +++ b/src/init.c @@ -33,9 +33,9 @@ const mi_page_t _mi_page_empty = { #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) #if defined(MI_PADDING) && (MI_INTPTR_SIZE >= 8) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } -#elif defined(MI_PADDING) #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#elif defined(MI_PADDING) +#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #else #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #endif diff --git a/src/options.c b/src/options.c index 79ef47a1..1dc03826 100644 --- a/src/options.c +++ b/src/options.c @@ -386,6 +386,7 @@ void _mi_error_message(int err, const char* fmt, ...) { // lsb=1: bit 63-19: relative file name char* (to `mi_fname_base`), bit 18-1: line number // lsb=0: bit 63-01: return address // ----------------------------------------------------------------------------------------------- +#ifndef NDEBUG static const char* mi_debug_fname_base = "mimalloc_fname_base"; #define MI_FNAME_SHIFT 16 @@ -393,14 +394,14 @@ static const char* mi_debug_fname_base = "mimalloc_fname_base"; #define MI_LINE_MASK ((1L << MI_LINE_SHIFT) - 1) mi_source_t mi_source_ret(void* return_address) { - mi_source_t source = { ((intptr_t)return_address << 1) }; + mi_source_t source = { ((long long)return_address << 1) }; return source; } mi_source_t mi_source_loc(const char* fname, int lineno ) { - ptrdiff_t delta = fname - mi_debug_fname_base; + long long delta = fname - mi_debug_fname_base; mi_assert_internal(((delta << MI_FNAME_SHIFT) >> MI_FNAME_SHIFT) == delta); - mi_source_t source = { ((((intptr_t)delta) << MI_FNAME_SHIFT) | ((lineno << 1) & MI_LINE_MASK) | 1) }; + mi_source_t source = { ((((long long)delta) << MI_FNAME_SHIFT) | ((lineno << 1) & MI_LINE_MASK) | 1) }; return source; } @@ -419,6 +420,7 @@ void* mi_source_unpack(mi_source_t source, const char** fname, int* lineno) { return ((void*)(source.src >> 1)); } } +#endif // ----------------------------------------------------------------------------------------------- // Error message for a specific heap block diff --git a/src/page.c b/src/page.c index b5f66df3..88e24c1e 100644 --- a/src/page.c +++ b/src/page.c @@ -774,7 +774,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { // Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. -void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_PARAM) mi_attr_noexcept +void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert_internal(heap != NULL); @@ -815,5 +815,5 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_PARAM) mi_attr_n mi_assert_internal(mi_page_block_size(page) >= size); // and try again, this time succeeding! (i.e. this should never recurse) - return _mi_page_malloc(heap, page, size MI_SOURCE_ARG); + return _mi_page_malloc(heap, page, size MI_SOURCE_XARG); } diff --git a/src/static.c b/src/static.c index ec9370eb..1a99d243 100644 --- a/src/static.c +++ b/src/static.c @@ -6,6 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #define _DEFAULT_SOURCE +#define MI_NO_SOURCE_DEBUG #include "mimalloc.h" #include "mimalloc-internal.h" diff --git a/test/main-override.cpp b/test/main-override.cpp index 271af6d7..8af8d38e 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -8,7 +8,7 @@ #include #include #include -// #include +#include static void* p = malloc(8); @@ -49,6 +49,7 @@ int main() { free(p1); free(p2); mi_free(s); + s[0] = 0; Test* t = new Test(42); delete t; // t = new(std::nothrow) Test(42); // does not work with overriding :-( @@ -61,7 +62,7 @@ int main() { static void dangling_ptr_write() { for (int i = 0; i < 1000; i++) { uint8_t* p; - if ((i & 1) == 1) { + if ((i & 1) == 0) { // do ==0 or ==1 to get either malloc or new allocation p = (uint8_t*)malloc(16); free(p); }