diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 52cf3029..5c989092 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -128,8 +128,8 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p bool _mi_free_delayed_block(mi_block_t* block); void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size); -mi_decl_restrict void* _mi_base_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_XPARAM); -mi_decl_restrict void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM); +mi_decl_restrict void* _mi_base_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_XPARAM) mi_attr_malloc mi_attr_alloc_size(2); +void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM) mi_attr_alloc_size(2); #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); @@ -759,83 +759,83 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { #endif #define MI_ALLOC_API1(tp,name,tp0,arg0,tp1,arg1) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept; \ - mi_decl_restrict tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XRET()); } \ - mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1) mi_attr_noexcept { return mi_base_##name(heap, arg1 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XARG); }) \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept + static inline tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XRET()); } \ + tp mi_heap_##name(mi_heap_t* heap, tp1 arg1) mi_attr_noexcept { return mi_base_##name(heap, arg1 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API2(tp,name,tp0,arg0,tp1,arg1,tp2,arg2) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept; \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XRET()); } \ - mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XARG); }) \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XRET()); } \ + tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API3(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept; \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XRET()); } \ - mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API4(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3,tp4,arg4) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept; \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ - mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ + tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API5(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3,tp4,arg4,tp5,arg5) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept; \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ - mi_decl_restrict tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); }) \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); } ) \ - static inline mi_decl_restrict tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ + tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); }) \ + MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); } ) \ + static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_NEW_API1(tp,name,tp1,arg1) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ - mi_decl_restrict tp mi_##name(tp1 arg1) { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) + static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ + tp mi_##name(tp1 arg1) { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) #define MI_NEW_API2(tp,name,tp1,arg1,tp2,arg2) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2) { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) + static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ + tp mi_##name(tp1 arg1, tp2 arg2) { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) #define MI_NEW_API3(tp,name,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) + static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ + tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) #define MI_SOURCE_API1(tp,name,tp1,arg1) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ - mi_decl_restrict tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) + static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ + tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) #define MI_SOURCE_API2(tp,name,tp1,arg1,tp2,arg2) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) + static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ + tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) #define MI_SOURCE_API3(tp,name,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ - mi_decl_restrict tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ - MI_DEBUG_ONLY(mi_decl_restrict tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - static inline mi_decl_restrict tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) + static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ + tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ + MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ + static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 7cf0786e..a2b8d74b 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -332,6 +332,7 @@ struct mi_heap_s { uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. + mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages }; @@ -472,6 +473,7 @@ struct mi_tld_s { unsigned long long heartbeat; // monotonic heartbeat count bool recurse; // true if deferred was called; used to prevent infinite recursion. mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) + mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) mi_segments_tld_t segments; // segment tld mi_os_tld_t os; // os tld mi_stats_t stats; // statistics diff --git a/include/mimalloc.h b/include/mimalloc.h index e78eae75..9ec8fc23 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -98,11 +98,7 @@ extern "C" { #ifdef NDEBUG -#define mi_decl_alloc(tp,name,...) \ - mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) mi_attr_noexcept - -#define mi_decl_new(tp,name,...) \ - mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) +#define mi_declx(tp,name,attrs,...) mi_decl_export tp name(__VA_ARGS__) attrs #else typedef struct mi_source_s { @@ -111,31 +107,35 @@ typedef struct mi_source_s { mi_decl_export mi_source_t mi_source_ret(void* return_address); mi_decl_export mi_source_t mi_source_loc(const char* fname, int lineno); -mi_decl_export void* mi_source_unpack(mi_source_t source, const char** fname, int* lineno); +mi_decl_export void* mi_source_unpack(mi_source_t source, const char** fname, int* lineno); -#define mi_decl_alloc(tp,name,...) \ - mi_decl_nodiscard mi_decl_export mi_decl_restrict tp dbg_##name( __VA_ARGS__, mi_source_t dbg_source) mi_attr_noexcept; \ - mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) mi_attr_noexcept +#define mi_declx(tp,name,attrs,...) \ + mi_decl_export tp dbg_##name( __VA_ARGS__, mi_source_t dbg_source) attrs; \ + mi_decl_export tp name(__VA_ARGS__) attrs -#define mi_decl_new(tp,name,...) \ - mi_decl_nodiscard mi_decl_export mi_decl_restrict tp dbg_##name( __VA_ARGS__, mi_source_t dbg_source); \ - mi_decl_nodiscard mi_decl_export mi_decl_restrict tp name(__VA_ARGS__) #endif +#define mi_decl_malloc(tp,name,...) mi_declx(mi_decl_nodiscard mi_decl_restrict tp, name, mi_attr_noexcept mi_attr_malloc, __VA_ARGS__) +#define mi_decl_new(tp,name,...) mi_declx(mi_decl_nodiscard mi_decl_restrict tp, name, mi_attr_malloc, __VA_ARGS__) +#define mi_decl_realloc(tp,name,...) mi_declx(mi_decl_nodiscard tp, name, mi_attr_noexcept, __VA_ARGS__) +#define mi_decl_noexcpt(tp,name,...) mi_declx(tp, name, mi_attr_noexcept, __VA_ARGS__) + + // ------------------------------------------------------ // Standard malloc interface // ------------------------------------------------------ + mi_decl_export void mi_free(void* p) mi_attr_noexcept; -mi_decl_alloc(void*, mi_malloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_calloc, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_alloc(void*, mi_realloc, void* p, size_t newsize) mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_expand, void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_malloc, size_t size) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_calloc, size_t count, size_t size) mi_attr_alloc_size2(1, 2); +mi_decl_realloc(void*, mi_realloc, void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_noexcpt(void*, mi_expand, void* p, size_t newsize) mi_attr_alloc_size(2); -mi_decl_alloc(char*, mi_strdup, const char* s) mi_attr_malloc; -mi_decl_alloc(char*, mi_strndup, const char* s, size_t n) mi_attr_malloc; -mi_decl_alloc(char*, mi_realpath,const char* fname, char* resolved_name); +mi_decl_malloc( char*, mi_strdup, const char* s); +mi_decl_malloc( char*, mi_strndup, const char* s, size_t n); +mi_decl_malloc( char*, mi_realpath,const char* fname, char* resolved_name); // ------------------------------------------------------ @@ -144,16 +144,16 @@ mi_decl_alloc(char*, mi_realpath,const char* fname, char* resolved_name); #define MI_SMALL_WSIZE_MAX (128) #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) -mi_decl_alloc(void*, mi_malloc_small, size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_zalloc_small, size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_zalloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_malloc_small, size_t size) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_zalloc_small, size_t size) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_zalloc, size_t size) mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_mallocn, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1,2); -mi_decl_alloc(void*, mi_reallocn, void* p, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2,3); -mi_decl_alloc(void*, mi_reallocf, void* p, size_t newsize) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_mallocn, size_t count, size_t size) mi_attr_alloc_size2(1,2); +mi_decl_realloc(void*, mi_reallocn, void* p, size_t count, size_t size) mi_attr_alloc_size2(2,3); +mi_decl_realloc(void*, mi_reallocf, void* p, size_t newsize) mi_attr_alloc_size(2); -mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; +mi_decl_export mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept; +mi_decl_export mi_decl_nodiscard size_t mi_good_size(size_t size) mi_attr_noexcept; // ------------------------------------------------------ @@ -188,15 +188,15 @@ mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_ // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. // ------------------------------------------------------------------------------------- -mi_decl_alloc(void*, mi_malloc_aligned, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_alloc(void*, mi_zalloc_aligned, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); -mi_decl_alloc(void*, mi_calloc_aligned, size_t count, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); -mi_decl_alloc(void*, mi_realloc_aligned, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_malloc( void*, mi_malloc_aligned, size_t size, size_t alignment) mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_malloc( void*, mi_zalloc_aligned, size_t size, size_t alignment) mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_malloc( void*, mi_calloc_aligned, size_t count, size_t size, size_t alignment) mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); +mi_decl_realloc(void*, mi_realloc_aligned, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_alloc(void*, mi_malloc_aligned_at, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_zalloc_aligned_at, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_calloc_aligned_at, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size2(1, 2); -mi_decl_alloc(void*, mi_realloc_aligned_at, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_malloc_aligned_at, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_zalloc_aligned_at, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_calloc_aligned_at, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(1, 2); +mi_decl_realloc(void*, mi_realloc_aligned_at, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(2); // ------------------------------------------------------------------------------------- @@ -206,7 +206,7 @@ mi_decl_alloc(void*, mi_realloc_aligned_at, void* p, size_t newsize, size_t alig struct mi_heap_s; typedef struct mi_heap_s mi_heap_t; -mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); +mi_decl_export mi_decl_nodiscard mi_heap_t* mi_heap_new(void); mi_decl_export void mi_heap_delete(mi_heap_t* heap); mi_decl_export void mi_heap_destroy(mi_heap_t* heap); mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap); @@ -214,30 +214,30 @@ mi_decl_export mi_heap_t* mi_heap_get_default(void); mi_decl_export mi_heap_t* mi_heap_get_backing(void); mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; -mi_decl_alloc(void*, mi_heap_malloc, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_heap_zalloc, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_heap_calloc, mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_alloc(void*, mi_heap_mallocn, mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_alloc(void*, mi_heap_malloc_small, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_heap_zalloc_small, mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_heap_malloc, mi_heap_t* heap, size_t size) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_heap_zalloc, mi_heap_t* heap, size_t size) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_heap_calloc, mi_heap_t* heap, size_t count, size_t size) mi_attr_alloc_size2(2, 3); +mi_decl_malloc( void*, mi_heap_mallocn, mi_heap_t* heap, size_t count, size_t size) mi_attr_alloc_size2(2, 3); +mi_decl_malloc( void*, mi_heap_malloc_small, mi_heap_t* heap, size_t size) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_heap_zalloc_small, mi_heap_t* heap, size_t size) mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_heap_realloc, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); -mi_decl_alloc(void*, mi_heap_reallocn, mi_heap_t* heap, void* p, size_t count, size_t size); -mi_decl_alloc(void*, mi_heap_reallocf, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); +mi_decl_realloc(void*, mi_heap_realloc, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); +mi_decl_realloc(void*, mi_heap_reallocn, mi_heap_t* heap, void* p, size_t count, size_t size); +mi_decl_realloc(void*, mi_heap_reallocf, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); -mi_decl_alloc(char*, mi_heap_strdup, mi_heap_t* heap, const char* s) mi_attr_malloc; -mi_decl_alloc(char*, mi_heap_strndup, mi_heap_t* heap, const char* s, size_t n) mi_attr_malloc; -mi_decl_alloc(char*, mi_heap_realpath, mi_heap_t* heap, const char* fname, char* resolved_name); +mi_decl_malloc( char*, mi_heap_strdup, mi_heap_t* heap, const char* s); +mi_decl_malloc( char*, mi_heap_strndup, mi_heap_t* heap, const char* s, size_t n); +mi_decl_malloc( char*, mi_heap_realpath, mi_heap_t* heap, const char* fname, char* resolved_name); -mi_decl_alloc(void*, mi_heap_malloc_aligned, mi_heap_t* heap, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_alloc(void*, mi_heap_zalloc_aligned, mi_heap_t* heap, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_alloc(void*, mi_heap_calloc_aligned, mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); -mi_decl_alloc(void*, mi_heap_realloc_aligned,mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_malloc( void*, mi_heap_malloc_aligned, mi_heap_t* heap, size_t size, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_malloc( void*, mi_heap_zalloc_aligned, mi_heap_t* heap, size_t size, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_malloc( void*, mi_heap_calloc_aligned, mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_realloc(void*, mi_heap_realloc_aligned,mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(3) mi_attr_alloc_align(4); -mi_decl_alloc(void*, mi_heap_malloc_aligned_at, mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_heap_zalloc_aligned_at, mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_heap_calloc_aligned_at, mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_malloc mi_attr_alloc_size2(2, 3); -mi_decl_alloc(void*, mi_heap_realloc_aligned_at,mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(3); +mi_decl_malloc( void*, mi_heap_malloc_aligned_at, mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_heap_zalloc_aligned_at, mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size(2); +mi_decl_malloc( void*, mi_heap_calloc_aligned_at, mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(2, 3); +mi_decl_realloc(void*, mi_heap_realloc_aligned_at,mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(3); // -------------------------------------------------------------------------------- @@ -247,30 +247,28 @@ mi_decl_alloc(void*, mi_heap_realloc_aligned_at,mi_heap_t* heap, void* p, size_t // see // -------------------------------------------------------------------------------- -mi_decl_alloc(void*, mi_rezalloc, void* p, size_t newsize) mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_recalloc, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2,3); +mi_decl_realloc(void*, mi_rezalloc, void* p, size_t newsize) mi_attr_alloc_size(2); +mi_decl_realloc(void*, mi_recalloc, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2,3); +mi_decl_realloc(void*, mi_rezalloc_aligned, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_realloc(void*, mi_recalloc_aligned, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); +mi_decl_realloc(void*, mi_rezalloc_aligned_at, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(2); +mi_decl_realloc(void*, mi_recalloc_aligned_at, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(2,3); -mi_decl_alloc(void*, mi_rezalloc_aligned, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(2) mi_attr_alloc_align(3); -mi_decl_alloc(void*, mi_recalloc_aligned, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); -mi_decl_alloc(void*, mi_rezalloc_aligned_at, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(2); -mi_decl_alloc(void*, mi_recalloc_aligned_at, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(2,3); - -mi_decl_alloc(void*, mi_heap_rezalloc, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); -mi_decl_alloc(void*, mi_heap_recalloc, mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(3,4); - -mi_decl_alloc(void*, mi_heap_rezalloc_aligned, mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(3) mi_attr_alloc_align(4); -mi_decl_alloc(void*, mi_heap_recalloc_aligned, mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); -mi_decl_alloc(void*, mi_heap_rezalloc_aligned_at, mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(3); -mi_decl_alloc(void*, mi_heap_recalloc_aligned_at, mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(3, 4); +mi_decl_realloc(void*, mi_heap_rezalloc, mi_heap_t* heap, void* p, size_t newsize) mi_attr_alloc_size(3); +mi_decl_realloc(void*, mi_heap_recalloc, mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(3,4); +mi_decl_realloc(void*, mi_heap_rezalloc_aligned, mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_alloc_size(3) mi_attr_alloc_align(4); +mi_decl_realloc(void*, mi_heap_recalloc_aligned, mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); +mi_decl_realloc(void*, mi_heap_rezalloc_aligned_at, mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_alloc_size(3); +mi_decl_realloc(void*, mi_heap_recalloc_aligned_at, mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(3, 4); // ------------------------------------------------------ // Analysis // ------------------------------------------------------ -mi_decl_nodiscard mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p); -mi_decl_nodiscard mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p); -mi_decl_nodiscard mi_decl_export bool mi_check_owned(const void* p); +mi_decl_export mi_decl_nodiscard bool mi_heap_contains_block(mi_heap_t* heap, const void* p); +mi_decl_export mi_decl_nodiscard bool mi_heap_check_owned(mi_heap_t* heap, const void* p); +mi_decl_export mi_decl_nodiscard bool mi_check_owned(const void* p); // An area of heap space contains blocks of a single size. typedef struct mi_heap_area_s { @@ -286,8 +284,8 @@ typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_ mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); // Experimental -mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export bool mi_is_redirected() mi_attr_noexcept; +mi_decl_export mi_decl_nodiscard bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; +mi_decl_export mi_decl_nodiscard bool mi_is_redirected() mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; @@ -344,13 +342,13 @@ typedef enum mi_option_e { } mi_option_t; -mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); +mi_decl_export mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option); mi_decl_export void mi_option_enable(mi_option_t option); mi_decl_export void mi_option_disable(mi_option_t option); mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); -mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); +mi_decl_export mi_decl_nodiscard long mi_option_get(mi_option_t option); mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); @@ -358,55 +356,48 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); // ------------------------------------------------------------------------------------------------------- // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) +// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing. // ------------------------------------------------------------------------------------------------------- -mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; -mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; -mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; - -mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(2); - -mi_decl_alloc(void*, mi_memalign, size_t alignment, size_t size) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); -mi_decl_alloc(void*, mi_valloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); - -mi_decl_alloc(void*, mi_pvalloc, size_t size) mi_attr_malloc mi_attr_alloc_size(1); -mi_decl_alloc(void*, mi_aligned_alloc, size_t alignment, size_t size) mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); -mi_decl_alloc(void*, mi_reallocarray, void* p, size_t count, size_t size) mi_attr_alloc_size2(2,3); - -mi_decl_alloc(void*, mi_aligned_recalloc, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); -mi_decl_alloc(void*, mi_aligned_offset_recalloc, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) ; - -mi_decl_alloc(unsigned short*, mi_wcsdup, const unsigned short* s); -mi_decl_alloc(unsigned char*, mi_mbsdup, const unsigned char* s); - -mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; -mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; +mi_decl_export mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_export mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; +mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; -#ifndef NDEBUG -mi_decl_export int dbg_mi_posix_memalign(void** p, size_t alignment, size_t size, mi_source_t) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(2); -mi_decl_export int dbg_mi_dupenv_s(char** buf, size_t* size, const char* name, mi_source_t source) mi_attr_noexcept; -mi_decl_export int dbg_mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name, mi_source_t source) mi_attr_noexcept; -#endif +mi_decl_noexcpt(int, mi_posix_memalign, void** p, size_t alignment, size_t size) mi_attr_alloc_size(3) mi_attr_alloc_align(2); +mi_decl_malloc( void*, mi_memalign, size_t alignment, size_t size) mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_malloc( void*, mi_valloc, size_t size) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_pvalloc, size_t size) mi_attr_alloc_size(1); +mi_decl_malloc( void*, mi_aligned_alloc, size_t alignment, size_t size) mi_attr_alloc_size(2) mi_attr_alloc_align(1); +mi_decl_realloc(void*, mi_reallocarray, void* p, size_t count, size_t size) mi_attr_alloc_size2(2,3); +mi_decl_realloc(void*, mi_aligned_recalloc, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); +mi_decl_realloc(void*, mi_aligned_offset_recalloc, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_alloc_size2(2, 3); + +mi_decl_malloc( unsigned short*, mi_wcsdup, const unsigned short* s); +mi_decl_malloc( unsigned char*, mi_mbsdup, const unsigned char* s); +mi_decl_noexcpt(int, mi_dupenv_s, char** buf, size_t* size, const char* name); +mi_decl_noexcpt(int, mi_wdupenv_s, unsigned short** buf, size_t* size, const unsigned short* name); +mi_decl_noexcpt(void*, mi__expand, void* p, size_t newsize); + // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). -mi_decl_new(void*, mi_new, size_t size) mi_attr_malloc mi_attr_alloc_size(1) ; -mi_decl_new(void*, mi_new_aligned, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2) ; -mi_decl_new(void*, mi_new_nothrow, size_t size) mi_attr_malloc mi_attr_alloc_size(1) ; -mi_decl_new(void*, mi_new_aligned_nothrow, size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2) ; -mi_decl_new(void*, mi_new_n, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2) ; -mi_decl_new(void*, mi_new_realloc, void* p, size_t newsize) mi_attr_alloc_size(2) ; -mi_decl_new(void*, mi_new_reallocn, void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3) ; +mi_decl_new(void*, mi_new, size_t size) mi_attr_alloc_size(1) ; +mi_decl_new(void*, mi_new_aligned, size_t size, size_t alignment) mi_attr_alloc_size(1) mi_attr_alloc_align(2) ; +mi_decl_new(void*, mi_new_n, size_t count, size_t size) mi_attr_alloc_size2(1, 2); +mi_decl_malloc(void*, mi_new_nothrow, size_t size) mi_attr_alloc_size(1); +mi_decl_malloc(void*, mi_new_aligned_nothrow, size_t size, size_t alignment) mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_declx(mi_decl_nodiscard void*, mi_new_realloc, , void* p, size_t newsize) mi_attr_alloc_size(2) ; +mi_declx(mi_decl_nodiscard void*, mi_new_reallocn, , void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3) ; // ---------------------------------------------------------------------- // end of extern "C" // ---------------------------------------------------------------------- + #ifdef __cplusplus } #endif @@ -504,6 +495,7 @@ template bool operator!=(const mi_stl_allocator&, const #define mi_malloc(n) MI_SOURCE_LOC(mi_malloc,n) #define mi_calloc(c,n) MI_SOURCE_LOC(mi_calloc,c,n) #define mi_realloc(p,n) MI_SOURCE_LOC(mi_realloc,p,n) +#define mi_expand(p,n) MI_SOURCE_LOC(mi_expand,p,n) #define mi_strdup(s) MI_SOURCE_LOC(mi_strdup,s) #define mi_strndup(s,n) MI_SOURCE_LOC(mi_strndup,s,n) #define mi_realpath(f,n) MI_SOURCE_LOC(mi_realpath,f,n) @@ -559,6 +551,21 @@ template bool operator!=(const mi_stl_allocator&, const #define mi_heap_rezalloc_aligned_at(h,p,n,a,o) MI_SOURCE_LOC(mi_heap_rezalloc_aligned_at,h,p,n,a,o) #define mi_heap_recalloc_aligned_at(h,p,c,n,a,o) MI_SOURCE_LOC(mi_heap_recalloc_aligned_at,h,p,c,n,a,o) +#define mi_wcsdup(s) MI_SOURCE_LOC(mi_wcsdup,s) +#define mi_mbsdup(s) MI_SOURCE_LOC(mi_mbsdup,s) +#define mi_dupenv_s(b,s,n) MI_SOURCE_LOC(mi_dupenv_s,b,s,n) +#define mi_wdupenv_s(b,s,n) MI_SOURCE_LOC(mi_wdupenv_s,b,s,n) + +#define mi_posix_memalign(p,a,s) MI_SOURCE_LOC(mi_posix_memalign,p,a,s) +#define mi_memalign(a,s) MI_SOURCE_LOC(mi_memalign,a,s) +#define mi_valloc(s) MI_SOURCE_LOC(mi_valloc,s) +#define mi_pvalloc(s) MI_SOURCE_LOC(mi_pvalloc,s) +#define mi_aligned_alloc(a,s) MI_SOURCE_LOC(mi_aligned_alloc,a,s) +#define mi_reallocarray(p,c,s) MI_SOURCE_LOC(mi_reallocarray,p,c,s) + +#define mi_aligned_recalloc(p,c,s,a) MI_SOURCE_LOC(mi_aligned_recalloc,p,c,s,a) +#define mi_aligned_offset_recalloc(p,c,s,a,o) MI_SOURCE_LOC(mi_aligned_offset_recalloc,p,c,s,a,o) + #endif diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index bb5d0143..9fcb4882 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file // Aligned Allocation // ------------------------------------------------------ -static void* mi_base_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero MI_SOURCE_XPARAM) mi_attr_noexcept { +static mi_decl_restrict void* mi_base_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero MI_SOURCE_XPARAM) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that // the address at offset is aligned regardless of the allocated size. mi_assert(alignment > 0 && alignment % sizeof(void*) == 0); @@ -64,34 +64,34 @@ static void* mi_base_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t } -MI_ALLOC_API3(void*, malloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset) +MI_ALLOC_API3(mi_decl_restrict void*, malloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset) { return mi_base_malloc_zero_aligned_at(heap, size, alignment, offset, false MI_SOURCE_XARG); } -MI_ALLOC_API2(void*, malloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment) +MI_ALLOC_API2(mi_decl_restrict void*, malloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment) { return mi_base_malloc_zero_aligned_at(heap, size, alignment, 0, false MI_SOURCE_XARG); } -MI_ALLOC_API3(void*, zalloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset) +MI_ALLOC_API3(mi_decl_restrict void*, zalloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset) { return mi_base_malloc_zero_aligned_at(heap, size, alignment, offset, true MI_SOURCE_XARG); } -MI_ALLOC_API2(void*, zalloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment) +MI_ALLOC_API2(mi_decl_restrict void*, zalloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment) { return mi_base_malloc_zero_aligned_at(heap, size, alignment, 0, true MI_SOURCE_XARG); } -MI_ALLOC_API4(void*, calloc_aligned_at, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment, size_t, offset) +MI_ALLOC_API4(mi_decl_restrict void*, calloc_aligned_at, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment, size_t, offset) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; return mi_base_malloc_zero_aligned_at(heap, total, alignment, offset, true MI_SOURCE_XARG); } -MI_ALLOC_API3(void*, calloc_aligned, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment) +MI_ALLOC_API3(mi_decl_restrict void*, calloc_aligned, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; @@ -99,7 +99,6 @@ MI_ALLOC_API3(void*, calloc_aligned, mi_heap_t*, heap, size_t, count, size_t, si } - static void* mi_base_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert(alignment > 0); if (alignment <= sizeof(uintptr_t)) return _mi_base_realloc_zero(heap,p,newsize,zero MI_SOURCE_XARG); @@ -138,6 +137,7 @@ static void* mi_base_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsi return mi_base_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero MI_SOURCE_XARG); } + MI_ALLOC_API4(void*, realloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment, size_t, offset) { return mi_base_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false MI_SOURCE_XARG); @@ -165,6 +165,7 @@ MI_ALLOC_API5(void*, recalloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, ne return mi_base_realloc_zero_aligned_at(heap, p, total, alignment, offset, true MI_SOURCE_XARG); } + MI_ALLOC_API4(void*, recalloc_aligned, mi_heap_t*, heap, void*, p, size_t, newcount, size_t, size, size_t, alignment) { size_t total; diff --git a/src/alloc-posix.c b/src/alloc-posix.c index d3aa750c..09a0c977 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -56,7 +56,7 @@ MI_SOURCE_API3(void*, reallocarray, void*, p, size_t, count, size_t, size) return newp; } -MI_SOURCE_API2(void*, memalign, size_t, alignment, size_t, size) +MI_SOURCE_API2(mi_decl_restrict void*, memalign, size_t, alignment, size_t, size) { void* p; if (alignment <= MI_MAX_ALIGN_SIZE) { @@ -69,12 +69,12 @@ MI_SOURCE_API2(void*, memalign, size_t, alignment, size_t, size) return p; } -MI_SOURCE_API1(void*, valloc, size_t, size) +MI_SOURCE_API1(mi_decl_restrict void*, valloc, size_t, size) { return MI_SOURCE_ARG(mi_malloc_aligned, size, _mi_os_page_size()); } -MI_SOURCE_API1(void*, pvalloc, size_t, size) +MI_SOURCE_API1(mi_decl_restrict void*, pvalloc, size_t, size) { size_t psize = _mi_os_page_size(); if (size >= SIZE_MAX - psize) return NULL; // overflow @@ -82,7 +82,7 @@ MI_SOURCE_API1(void*, pvalloc, size_t, size) return MI_SOURCE_ARG(mi_malloc_aligned, asize, psize); } -MI_SOURCE_API2(void*, aligned_alloc, size_t, alignment, size_t, size) +MI_SOURCE_API2(mi_decl_restrict void*, aligned_alloc, size_t, alignment, size_t, size) { if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL; if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see @@ -128,8 +128,7 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept return mi_base_posix_memalign(p, alignment, size MI_SOURCE_XRET()); } - -MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s) +MI_SOURCE_API1(mi_decl_restrict unsigned short*, wcsdup, const unsigned short*, s) { if (s==NULL) return NULL; size_t len; @@ -142,7 +141,7 @@ MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s) return p; } -MI_SOURCE_API1(unsigned char*, mbsdup, const unsigned char*, s) +MI_SOURCE_API1(mi_decl_restrict unsigned char*, mbsdup, const unsigned char*, s) { return (unsigned char*)MI_SOURCE_ARG(mi_strdup,(const char*)s); } @@ -215,19 +214,19 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) #ifndef NDEBUG -mi_decl_restrict void* dbg_mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft +void* dbg_mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft return dbg_mi_recalloc_aligned_at(p, newcount, size, alignment, offset, __mi_source); } -mi_decl_restrict void* dbg_mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft +void* dbg_mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft return dbg_mi_recalloc_aligned(p, newcount, size, alignment, __mi_source); } #endif -mi_decl_restrict void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft +void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft return MI_SOURCE_RET(mi_recalloc_aligned_at,p, newcount, size, alignment, offset); } -mi_decl_restrict void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft +void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft return MI_SOURCE_RET(mi_recalloc_aligned,p, newcount, size, alignment); } diff --git a/src/alloc.c b/src/alloc.c index 07c25fc0..f83fd1ae 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -61,7 +61,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } // allocate a small block -MI_ALLOC_API1(void*, malloc_small, mi_heap_t*, heap, size_t, size) +MI_ALLOC_API1(mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, size_t, size) { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local @@ -80,7 +80,7 @@ MI_ALLOC_API1(void*, malloc_small, mi_heap_t*, heap, size_t, size) // The main allocation function -MI_ALLOC_API1(void*, malloc, mi_heap_t*, heap, size_t, size) +MI_ALLOC_API1(mi_decl_restrict void*, malloc, mi_heap_t*, heap, size_t, size) { if (mi_likely(size <= MI_SMALL_SIZE_MAX)) { return mi_base_malloc_small(heap, size MI_SOURCE_XARG); @@ -120,7 +120,7 @@ void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) { } // zero initialized small block -MI_ALLOC_API1(void*, zalloc_small, mi_heap_t*, heap, size_t, size) +MI_ALLOC_API1(mi_decl_restrict void*, zalloc_small, mi_heap_t*, heap, size_t, size) { void* p = mi_base_malloc_small(heap, size MI_SOURCE_XARG); if (p != NULL) { @@ -137,7 +137,7 @@ mi_decl_restrict void* _mi_base_malloc_zero(mi_heap_t* heap, size_t size, bool z return p; } -MI_ALLOC_API1(void*, zalloc, mi_heap_t*,heap, size_t,size) +MI_ALLOC_API1(mi_decl_restrict void*, zalloc, mi_heap_t*,heap, size_t,size) { return _mi_base_malloc_zero(heap, size, true MI_SOURCE_XARG); } @@ -519,7 +519,7 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { mi_free(p); } -MI_ALLOC_API2(void*, calloc, mi_heap_t*, heap, size_t, count, size_t, size) +MI_ALLOC_API2(mi_decl_restrict void*, calloc, mi_heap_t*, heap, size_t, count, size_t, size) { size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; @@ -527,7 +527,7 @@ MI_ALLOC_API2(void*, calloc, mi_heap_t*, heap, size_t, count, size_t, size) } // Uninitialized `calloc` -MI_ALLOC_API2(void*, mallocn, mi_heap_t*, heap, size_t, count, size_t, size) +MI_ALLOC_API2(mi_decl_restrict void*, mallocn, mi_heap_t*, heap, size_t, count, size_t, size) { size_t total; if (mi_count_size_overflow(count, size, &total)) return NULL; @@ -536,14 +536,19 @@ MI_ALLOC_API2(void*, mallocn, mi_heap_t*, heap, size_t, count, size_t, size) // Expand in place or fail -mi_decl_restrict void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { +MI_ALLOC_API2(void*, expand, mi_heap_t*, heap, void*, p, size_t, newsize) +{ + UNUSED(heap); +#ifndef NDEBUG + UNUSED(__mi_source); +#endif if (p == NULL) return NULL; size_t size = mi_usable_size(p); if (newsize > size) return NULL; return p; // it fits } -mi_decl_restrict void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM) { +void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM) { if (p == NULL) return _mi_base_malloc_zero(heap,newsize,zero MI_SOURCE_XARG); size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size / 2)) { @@ -601,7 +606,7 @@ MI_ALLOC_API3(void*, recalloc, mi_heap_t*, heap, void*, p, size_t, count, size_t // ------------------------------------------------------ // `strdup` using mi_malloc -MI_ALLOC_API1(char*, strdup, mi_heap_t*,heap, const char*,s) +MI_ALLOC_API1(mi_decl_restrict char*, strdup, mi_heap_t*,heap, const char*,s) { if (s == NULL) return NULL; size_t n = strlen(s); @@ -610,9 +615,8 @@ MI_ALLOC_API1(char*, strdup, mi_heap_t*,heap, const char*,s) return t; } - // `strndup` using mi_malloc -MI_ALLOC_API2(char*, strndup, mi_heap_t*, heap, const char*, s, size_t, n) +MI_ALLOC_API2(mi_decl_restrict char*, strndup, mi_heap_t*, heap, const char*, s, size_t, n) { if (s == NULL) return NULL; size_t m = strlen(s); @@ -632,7 +636,7 @@ MI_ALLOC_API2(char*, strndup, mi_heap_t*, heap, const char*, s, size_t, n) #define PATH_MAX MAX_PATH #endif #include -MI_ALLOC_API2(char*, realpath, mi_heap_t*, heap, const char*, fname, char*, resolved_name) +MI_ALLOC_API2(mi_decl_restrict char*, realpath, mi_heap_t*, heap, const char*, fname, char*, resolved_name) { // todo: use GetFullPathNameW to allow longer file names char buf[PATH_MAX]; @@ -663,7 +667,7 @@ static size_t mi_path_max() { return path_max; } -MI_ALLOC_API2(char*, realpath, mi_heap_t*, heap, const char*, fname, char*, resolved_name) +MI_ALLOC_API2(mi_decl_restrict char*, realpath, mi_heap_t*, heap, const char*, fname, char*, resolved_name) { if (resolved_name != NULL) { return realpath(fname,resolved_name); @@ -734,7 +738,7 @@ static bool mi_try_new_handler(bool nothrow) { } #endif -static mi_decl_noinline void* mi_base_try_new(size_t size, bool nothrow MI_SOURCE_XPARAM) { +static mi_decl_noinline mi_decl_restrict void* mi_base_try_new(size_t size, bool nothrow MI_SOURCE_XPARAM) { void* p = NULL; while(p == NULL && mi_try_new_handler(nothrow)) { p = MI_SOURCE_ARG(mi_malloc, size); @@ -742,14 +746,14 @@ static mi_decl_noinline void* mi_base_try_new(size_t size, bool nothrow MI_SOU return p; } -MI_NEW_API1(void*, new, size_t, size) +MI_NEW_API1(mi_decl_restrict void*, new, size_t, size) { void* p = MI_SOURCE_ARG(mi_malloc, size); if (mi_unlikely(p == NULL)) return mi_base_try_new(size, false MI_SOURCE_XARG); return p; } -MI_NEW_API1(void*, new_nothrow, size_t, size) +MI_SOURCE_API1(mi_decl_restrict void*, new_nothrow, size_t, size) { void* p = MI_SOURCE_ARG(mi_malloc, size); if (mi_unlikely(p == NULL)) return mi_base_try_new(size, true MI_SOURCE_XARG); @@ -757,7 +761,7 @@ MI_NEW_API1(void*, new_nothrow, size_t, size) } -MI_NEW_API2(void*, new_aligned, size_t, size, size_t, alignment) +MI_NEW_API2(mi_decl_restrict void*, new_aligned, size_t, size, size_t, alignment) { void* p; do { @@ -767,7 +771,7 @@ MI_NEW_API2(void*, new_aligned, size_t, size, size_t, alignment) return p; } -MI_NEW_API2(void*, new_aligned_nothrow, size_t, size, size_t, alignment) +MI_SOURCE_API2(mi_decl_restrict void*, new_aligned_nothrow, size_t, size, size_t, alignment) { void* p; do { @@ -777,7 +781,7 @@ MI_NEW_API2(void*, new_aligned_nothrow, size_t, size, size_t, alignment) return p; } -MI_NEW_API2(void*, new_n, size_t, count, size_t, size) +MI_NEW_API2(mi_decl_restrict void*, new_n, size_t, count, size_t, size) { size_t total; if (mi_unlikely(mi_count_size_overflow(count, size, &total))) { diff --git a/src/heap.c b/src/heap.c index 93275747..5d0d4b8a 100644 --- a/src/heap.c +++ b/src/heap.c @@ -191,7 +191,7 @@ mi_heap_t* mi_heap_get_backing(void) { mi_heap_t* mi_heap_new(void) { mi_heap_t* bheap = mi_heap_get_backing(); - mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); + mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap==NULL) return NULL; memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); heap->tld = bheap->tld; @@ -201,6 +201,9 @@ mi_heap_t* mi_heap_new(void) { heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe + // push on the thread local heaps list + heap->next = heap->tld->heaps; + heap->tld->heaps = heap; return heap; } @@ -223,6 +226,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) { // called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources. static void mi_heap_free(mi_heap_t* heap) { + mi_assert(heap != NULL); mi_assert_internal(mi_heap_is_initialized(heap)); if (mi_heap_is_backing(heap)) return; // dont free the backing heap @@ -230,6 +234,22 @@ static void mi_heap_free(mi_heap_t* heap) { if (mi_heap_is_default(heap)) { _mi_heap_set_default_direct(heap->tld->heap_backing); } + + // remove ourselves from the thread local heaps list + // linear search but we expect the number of heaps to be relatively small + mi_heap_t* prev = NULL; + mi_heap_t* curr = heap->tld->heaps; + while (curr != heap && curr != NULL) { + prev = curr; + curr = curr->next; + } + mi_assert_internal(curr == heap); + if (curr == heap) { + if (prev != NULL) { prev->next = heap->next; } + else { heap->tld->heaps = heap->next; } + } + mi_assert_internal(heap->tld->heaps != NULL); + // and free the used memory mi_free(heap); } @@ -286,6 +306,7 @@ void _mi_heap_destroy_pages(mi_heap_t* heap) { } void mi_heap_destroy(mi_heap_t* heap) { + mi_assert(heap != NULL); mi_assert(mi_heap_is_initialized(heap)); mi_assert(heap->no_reclaim); mi_assert_expensive(mi_heap_is_valid(heap)); @@ -312,38 +333,37 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; - // unfull all full pages in the `from` heap - mi_page_t* page = from->pages[MI_BIN_FULL].first; - while (page != NULL) { - mi_page_t* next = page->next; - _mi_page_unfull(page); - page = next; - } - mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL); - - // free outstanding thread delayed free blocks + // reduce the size of the delayed frees _mi_heap_delayed_free(from); - - // transfer all pages by appending the queues; this will set - // a new heap field which is ok as all pages are unfull'd and thus - // other threads won't access this field anymore (see `mi_free_block_mt`) - for (size_t i = 0; i < MI_BIN_FULL; i++) { + + // transfer all pages by appending the queues; this will set a new heap field + // so threads may do delayed frees in either heap for a while. + // note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state + // so after this only the new heap will get delayed frees + for (size_t i = 0; i <= MI_BIN_FULL; i++) { mi_page_queue_t* pq = &heap->pages[i]; mi_page_queue_t* append = &from->pages[i]; size_t pcount = _mi_page_queue_append(heap, pq, append); heap->page_count += pcount; from->page_count -= pcount; } - mi_assert_internal(from->thread_delayed_free == NULL); mi_assert_internal(from->page_count == 0); + // and do outstanding delayed frees in the `from` heap + // note: be careful here as the `heap` field in all those pages no longer point to `from`, + // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a + // the regular `_mi_free_delayed_block` which is safe. + _mi_heap_delayed_free(from); + mi_assert_internal(from->thread_delayed_free == NULL); + // and reset the `from` heap - mi_heap_reset_pages(from); + mi_heap_reset_pages(from); } // Safe delete a heap without freeing any still allocated blocks in that heap. void mi_heap_delete(mi_heap_t* heap) { + mi_assert(heap != NULL); mi_assert(mi_heap_is_initialized(heap)); mi_assert_expensive(mi_heap_is_valid(heap)); if (!mi_heap_is_initialized(heap)) return; diff --git a/src/init.c b/src/init.c index 945f487f..ce797add 100644 --- a/src/init.c +++ b/src/init.c @@ -97,6 +97,7 @@ const mi_heap_t _mi_heap_empty = { { 0, 0 }, // keys { {0}, {0}, 0 }, 0, // page count + NULL, // next false }; @@ -111,7 +112,7 @@ extern mi_heap_t _mi_heap_main; static mi_tld_t tld_main = { 0, false, - &_mi_heap_main, + &_mi_heap_main, &_mi_heap_main, { { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0}, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats, tld_main_os @@ -130,6 +131,7 @@ mi_heap_t _mi_heap_main = { { 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!) { {0x846ca68b}, {0}, 0 }, // random 0, // page count + NULL, // next heap false // can reclaim }; @@ -192,6 +194,7 @@ static bool _mi_heap_init(void) { heap->keys[1] = _mi_heap_random_next(heap); heap->tld = tld; tld->heap_backing = heap; + tld->heaps = heap; tld->segments.stats = &tld->stats; tld->segments.os = &tld->os; tld->os.stats = &tld->stats; @@ -207,12 +210,24 @@ static bool _mi_heap_done(mi_heap_t* heap) { // reset default heap _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); - // todo: delete all non-backing heaps? - - // switch to backing heap and free it + // switch to backing heap heap = heap->tld->heap_backing; if (!mi_heap_is_initialized(heap)) return false; + + // delete all non-backing heaps in this thread + mi_heap_t* curr = heap->tld->heaps; + while (curr != NULL) { + mi_heap_t* next = curr->next; // save `next` as `curr` will be freed + if (curr != heap) { + mi_assert_internal(!mi_heap_is_backing(curr)); + mi_heap_delete(curr); + } + curr = next; + } + mi_assert_internal(heap->tld->heaps == heap && heap->next == NULL); + mi_assert_internal(mi_heap_is_backing(heap)); + // collect if not the main thread if (heap != &_mi_heap_main) { _mi_heap_collect_abandon(heap); diff --git a/src/page-queue.c b/src/page-queue.c index 68e2aaa4..b2687c92 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -329,6 +329,7 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro mi_page_set_in_full(page, mi_page_queue_is_full(to)); } +// Only called from `mi_heap_absorb`. size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) { mi_assert_internal(mi_heap_contains_queue(heap,pq)); mi_assert_internal(pq->block_size == append->block_size); @@ -339,6 +340,10 @@ size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue size_t count = 0; for (mi_page_t* page = append->first; page != NULL; page = page->next) { mi_page_set_heap(page,heap); + // set it to delayed free (not overriding NEVER_DELAYED_FREE) which has as a + // side effect that it spins until any DELAYED_FREEING is finished. This ensures + // that after appending only the new heap will be used for delayed free operations. + _mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, false); count++; } diff --git a/src/page.c b/src/page.c index 88e24c1e..6f528d50 100644 --- a/src/page.c +++ b/src/page.c @@ -105,7 +105,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) { bool _mi_page_is_valid(mi_page_t* page) { mi_assert_internal(mi_page_is_valid_init(page)); #if MI_SECURE - mi_assert_internal(page->key != 0); + mi_assert_internal(page->keys[0] != 0); #endif if (mi_page_heap(page)!=NULL) { mi_segment_t* segment = _mi_page_segment(page); diff --git a/test/main-override.cpp b/test/main-override.cpp index 8af8d38e..5ed04e0d 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -3,13 +3,35 @@ #include #include #include - -#include -#include #include #include +#include +#include +#include #include + +#ifdef _WIN32 +#include +static void msleep(unsigned long msecs) { Sleep(msecs); } +#else +#include +static void msleep(unsigned long msecs) { usleep(msecs * 1000UL); } +#endif + +void heap_no_delete(); +void heap_late_free(); +void various_tests(); + +int main() { + mi_stats_reset(); // ignore earlier allocations + // heap_no_delete(); // issue #202 + // heap_late_free(); // issue #204 + various_tests(); + mi_stats_print(NULL); + return 0; +} + static void* p = malloc(8); void free_p() { @@ -27,8 +49,7 @@ public: void dangling_ptr_write(); -int main() { - mi_stats_reset(); // ignore earlier allocations +void various_tests() { atexit(free_p); //dangling_ptr_write(); void* p1 = malloc(78); @@ -36,13 +57,13 @@ int main() { free(p1); p1 = malloc(8); char* s = _strdup("hello\n"); - /* - char* s = _strdup("hello\n"); - char* buf = NULL; - size_t len; - _dupenv_s(&buf,&len,"MIMALLOC_VERBOSE"); - mi_free(buf); - */ + + //char* s = _strdup("hello\n"); + //char* buf = NULL; + //size_t len; + //_dupenv_s(&buf,&len,"MIMALLOC_VERBOSE"); + //mi_free(buf); + mi_free(p2); p2 = malloc(16); p1 = realloc(p1, 32); @@ -55,8 +76,6 @@ int main() { // t = new(std::nothrow) Test(42); // does not work with overriding :-( t = new Test(42); delete t; - mi_stats_print(NULL); - return 0; } static void dangling_ptr_write() { @@ -106,4 +125,38 @@ bool test_stl_allocator2() { vec.push_back(some_struct()); vec.pop_back(); return vec.size() == 0; +} + + + +// Issue #202 +void heap_no_delete_worker() { + mi_heap_t* heap = mi_heap_new(); + void* q = mi_heap_malloc(heap,1024); + // mi_heap_delete(heap); // uncomment to prevent assertion +} + +void heap_no_delete() { + auto t1 = std::thread(heap_no_delete_worker); + t1.join(); +} + + +// Issue #204 +volatile void* global_p; + +void t1main() { + mi_heap_t* heap = mi_heap_new(); + global_p = mi_heap_malloc(heap, 1024); + mi_heap_delete(heap); +} + +void heap_late_free() { + auto t1 = std::thread(t1main); + + msleep(2000); + assert(global_p); + mi_free((void*)global_p); + + t1.join(); } \ No newline at end of file