kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-08-25 16:54:47 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip: basic tracking of allocation locations is working

Browse source
This commit is contained in:
daan 2020-02-13 00:43:34 -08:00
parent 2fbe0e8842
commit 7d098836eb
13 changed files with 598 additions and 473 deletions
Download patch file Download diff file Expand all files Collapse all files

50
src/alloc-aligned.c
View file

@ -4,7 +4,7 @@ This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
#define MI_NO_SOURCE_DEBUG
#include "mimalloc.h"
#include "mimalloc-internal.h"
@ -14,14 +14,14 @@ terms of the MIT license. A copy of the license can be found in the file
// Aligned Allocation
// ------------------------------------------------------
static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero MI_SOURCE_PARAM) mi_attr_noexcept {
static void* mi_base_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero MI_SOURCE_XPARAM) mi_attr_noexcept {
// note: we don't require `size > offset`, we just guarantee that
// the address at offset is aligned regardless of the allocated size.
mi_assert(alignment > 0 && alignment % sizeof(void*) == 0);
if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
if (alignment <= MI_MAX_ALIGN_SIZE && offset==0) return _mi_heap_source_malloc_zero(heap, size, zero MI_SOURCE_ARG);
if (alignment <= MI_MAX_ALIGN_SIZE && offset==0) return _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG);
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
// try if there is a small block available with just the right alignment
@ -34,7 +34,7 @@ static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const
#if MI_STAT>1
mi_heap_stat_increase( heap, malloc, size);
#endif
void* p = _mi_page_malloc(heap,page,padsize MI_SOURCE_ARG); // TODO: inline _mi_page_malloc
void* p = _mi_page_malloc(heap,page,padsize MI_SOURCE_XARG); // TODO: inline _mi_page_malloc
mi_assert_internal(p != NULL);
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
if (zero) _mi_block_zero_init(page,p,size);
@ -44,13 +44,13 @@ static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const
// use regular allocation if it is guaranteed to fit the alignment constraints
if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) {
void* p = _mi_heap_source_malloc_zero(heap, size, zero MI_SOURCE_ARG);
void* p = _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG);
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
return p;
}
// otherwise over-allocate
void* p = _mi_heap_source_malloc_zero(heap, size + alignment - 1, zero MI_SOURCE_ARG);
void* p = _mi_base_malloc_zero(heap, size + alignment - 1, zero MI_SOURCE_XARG);
if (p == NULL) return NULL;
// .. and align within the allocation
@ -66,51 +66,51 @@ static void* mi_heap_source_malloc_zero_aligned_at(mi_heap_t* const heap, const
MI_ALLOC_API3(void*, malloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset)
{
return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, offset, false MI_SOURCE_ARG);
return mi_base_malloc_zero_aligned_at(heap, size, alignment, offset, false MI_SOURCE_XARG);
}
MI_ALLOC_API2(void*, malloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment)
{
return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, 0, false MI_SOURCE_ARG);
return mi_base_malloc_zero_aligned_at(heap, size, alignment, 0, false MI_SOURCE_XARG);
}
MI_ALLOC_API3(void*, zalloc_aligned_at, mi_heap_t*, heap, size_t, size, size_t, alignment, size_t, offset)
{
return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, offset, true MI_SOURCE_ARG);
return mi_base_malloc_zero_aligned_at(heap, size, alignment, offset, true MI_SOURCE_XARG);
}
MI_ALLOC_API2(void*, zalloc_aligned, mi_heap_t*,heap, size_t, size, size_t, alignment)
{
return mi_heap_source_malloc_zero_aligned_at(heap, size, alignment, 0, true MI_SOURCE_ARG);
return mi_base_malloc_zero_aligned_at(heap, size, alignment, 0, true MI_SOURCE_XARG);
}
MI_ALLOC_API4(void*, calloc_aligned_at, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment, size_t, offset)
{
size_t total;
if (mi_count_size_overflow(count, size, &total)) return NULL;
return mi_heap_source_malloc_zero_aligned_at(heap, total, alignment, offset, true MI_SOURCE_ARG);
return mi_base_malloc_zero_aligned_at(heap, total, alignment, offset, true MI_SOURCE_XARG);
}
MI_ALLOC_API3(void*, calloc_aligned, mi_heap_t*, heap, size_t, count, size_t, size, size_t, alignment)
{
size_t total;
if (mi_count_size_overflow(count, size, &total)) return NULL;
return mi_heap_source_malloc_zero_aligned_at(heap, total, alignment, 0, true MI_SOURCE_ARG);
return mi_base_malloc_zero_aligned_at(heap, total, alignment, 0, true MI_SOURCE_XARG);
}
static void* mi_heap_source_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero MI_SOURCE_PARAM) mi_attr_noexcept {
static void* mi_base_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero MI_SOURCE_XPARAM) mi_attr_noexcept {
mi_assert(alignment > 0);
if (alignment <= sizeof(uintptr_t)) return _mi_heap_source_realloc_zero(heap,p,newsize,zero MI_SOURCE_ARG);
if (p == NULL) return mi_heap_source_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero MI_SOURCE_ARG);
if (alignment <= sizeof(uintptr_t)) return _mi_base_realloc_zero(heap,p,newsize,zero MI_SOURCE_XARG);
if (p == NULL) return mi_base_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero MI_SOURCE_XARG);
size_t size = mi_usable_size(p);
if (newsize <= size && newsize >= (size - (size / 2))
&& (((uintptr_t)p + offset) % alignment) == 0) {
return p; // reallocation still fits, is aligned and not more than 50% waste
}
else {
void* newp = mi_heap_source_malloc_aligned_at(heap,newsize,alignment,offset MI_SOURCE_ARG);
void* newp = mi_base_malloc_aligned_at(heap,newsize,alignment,offset MI_SOURCE_XARG);
if (newp != NULL) {
if (zero && newsize > size) {
const mi_page_t* page = _mi_ptr_page(newp);
@ -131,43 +131,43 @@ static void* mi_heap_source_realloc_zero_aligned_at(mi_heap_t* heap, void* p, si
}
}
static void* mi_heap_source_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero MI_SOURCE_PARAM) mi_attr_noexcept {
static void* mi_base_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero MI_SOURCE_XPARAM) mi_attr_noexcept {
mi_assert(alignment > 0);
if (alignment <= sizeof(uintptr_t)) return _mi_heap_source_realloc_zero(heap,p,newsize,zero MI_SOURCE_ARG);
if (alignment <= sizeof(uintptr_t)) return _mi_base_realloc_zero(heap,p,newsize,zero MI_SOURCE_XARG);
size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL)
return mi_heap_source_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero MI_SOURCE_XARG);
}
MI_ALLOC_API4(void*, realloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment, size_t, offset)
{
return mi_heap_source_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false MI_SOURCE_XARG);
}
MI_ALLOC_API3(void*, realloc_aligned, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment)
{
return mi_heap_source_realloc_zero_aligned(heap,p,newsize,alignment,false MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned(heap,p,newsize,alignment,false MI_SOURCE_XARG);
}
MI_ALLOC_API4(void*, rezalloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment, size_t, offset)
{
return mi_heap_source_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true MI_SOURCE_XARG);
}
MI_ALLOC_API3(void*, rezalloc_aligned, mi_heap_t*, heap, void*, p, size_t, newsize, size_t, alignment)
{
return mi_heap_source_realloc_zero_aligned(heap, p, newsize, alignment, true MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned(heap, p, newsize, alignment, true MI_SOURCE_XARG);
}
MI_ALLOC_API5(void*, recalloc_aligned_at, mi_heap_t*, heap, void*, p, size_t, newcount, size_t, size, size_t, alignment, size_t, offset)
{
size_t total;
if (mi_count_size_overflow(newcount, size, &total)) return NULL;
return mi_heap_source_realloc_zero_aligned_at(heap, p, total, alignment, offset, true MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned_at(heap, p, total, alignment, offset, true MI_SOURCE_XARG);
}
MI_ALLOC_API4(void*, recalloc_aligned, mi_heap_t*, heap, void*, p, size_t, newcount, size_t, size, size_t, alignment)
{
size_t total;
if (mi_count_size_overflow(newcount, size, &total)) return NULL;
return mi_heap_source_realloc_zero_aligned_at(heap, p, total, alignment, 0, true MI_SOURCE_ARG);
return mi_base_realloc_zero_aligned_at(heap, p, total, alignment, 0, true MI_SOURCE_XARG);
}

126
src/alloc-override.c
View file

@ -26,18 +26,18 @@ terms of the MIT license. A copy of the license can be found in the file
#else
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default")))
#endif
#define MI_FORWARD1(fun,x) MI_FORWARD(mi_##fun)
#define MI_FORWARD2(fun,x,y) MI_FORWARD(mi_##fun)
#define MI_FORWARD3(fun,x,y,z) MI_FORWARD(mi_##fun)
#define MI_FORWARD0(fun,x) MI_FORWARD(mi_##fun)
#define MI_FORWARD02(fun,x,y) MI_FORWARD(mi_##fun)
#define MI_FORWARD1(fun,x) MI_FORWARD(fun)
#define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)
#define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun)
#define MI_FORWARD0(fun,x) MI_FORWARD(fun)
#define MI_FORWARD02(fun,x,y) MI_FORWARD(fun)
#else
// use forwarding by calling our `mi_` function
#define MI_FORWARD1(fun,x) { return mi_source_##fun(x MI_SOURCE_RET()); }
#define MI_FORWARD2(fun,x,y) { return mi_source_##fun(x,y MI_SOURCE_RET()); }
#define MI_FORWARD3(fun,x,y,z) { return mi_source_##fun(x,y,z MI_SOURCE_RET()); }
#define MI_FORWARD0(fun,x) { mi_##fun(x); }
#define MI_FORWARD02(fun,x,y) { mi_##fun(x,y); }
#define MI_FORWARD1(fun,x) { return MI_SOURCE_RET(fun, x); }
#define MI_FORWARD2(fun,x,y) { return MI_SOURCE_RET(fun, x, y); }
#define MI_FORWARD3(fun,x,y,z) { return MI_SOURCE_RET(fun, x, y, z); }
#define MI_FORWARD0(fun,x) { fun(x); }
#define MI_FORWARD02(fun,x,y) { fun(x,y); }
#endif
#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE)
@ -68,10 +68,10 @@ terms of the MIT license. A copy of the license can be found in the file
// we just override new/delete which does work in a static library.
#else
// On all other systems forward to our API
void* malloc(size_t size) MI_FORWARD1(malloc, size);
void* calloc(size_t size, size_t n) MI_FORWARD2(calloc, size, n);
void* realloc(void* p, size_t newsize) MI_FORWARD2(realloc, p, newsize);
void free(void* p) MI_FORWARD0(free, p);
void* malloc(size_t size) MI_FORWARD1(mi_malloc, size);
void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n);
void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize);
void free(void* p) MI_FORWARD0(mi_free, p);
#endif
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
@ -89,18 +89,18 @@ terms of the MIT license. A copy of the license can be found in the file
// see <https://en.cppreference.com/w/cpp/memory/new/operator_new>
// ------------------------------------------------------
#include <new>
void operator delete(void* p) noexcept MI_FORWARD0(free,p);
void operator delete[](void* p) noexcept MI_FORWARD0(free,p);
void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p);
void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p);
void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(new,n);
void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(new,n);
void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n);
void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n);
void* operator new (std::size_t n, const std::nothrow_t& ) noexcept { return mi_source_new_nothrow(n MI_SOURCE_RET()); }
void* operator new[](std::size_t n, const std::nothrow_t& ) noexcept { return mi_source_new_nothrow(n MI_SOURCE_RET()); }
void* operator new (std::size_t n, const std::nothrow_t& ) noexcept { return MI_SOURCE_RET(mi_new_nothrow, n); }
void* operator new[](std::size_t n, const std::nothrow_t& ) noexcept { return MI_SOURCE_RET(mi_new_nothrow, n); }
#if (__cplusplus >= 201402L || _MSC_VER >= 1916)
void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(free_size,p,n);
void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(free_size,p,n);
void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n);
void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n);
#endif
#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5))
@ -109,10 +109,10 @@ terms of the MIT license. A copy of the license can be found in the file
void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast<size_t>(al)); };
void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_source_new_aligned(n, static_cast<size_t>(al) MI_SOURCE_RET()); }
void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_source_new_aligned(n, static_cast<size_t>(al) MI_SOURCE_RET()); }
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_source_new_aligned_nothrow(n, static_cast<size_t>(al) MI_SOURCE_RET()); }
void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_source_new_aligned_nothrow(n, static_cast<size_t>(al) MI_SOURCE_RET()); }
void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return MI_SOURCE_RET(mi_new_aligned, n, static_cast<size_t>(al)); }
void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return MI_SOURCE_RET(mi_new_aligned, n, static_cast<size_t>(al)); }
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return MI_SOURCE_RET(mi_new_aligned_nothrow, n, static_cast<size_t>(al)); }
void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return MI_SOURCE_RET(mi_new_aligned_nothrow, n, static_cast<size_t>(al)); }
#endif
#elif (defined(__GNUC__) || defined(__clang__))
@ -121,10 +121,10 @@ terms of the MIT license. A copy of the license can be found in the file
// used by GCC and CLang).
// See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
// ------------------------------------------------------
void _ZdlPv(void* p) MI_FORWARD0(free,p); // delete
void _ZdaPv(void* p) MI_FORWARD0(free,p); // delete[]
void _ZdlPvm(void* p, size_t n) MI_FORWARD02(free_size,p,n);
void _ZdaPvm(void* p, size_t n) MI_FORWARD02(free_size,p,n);
void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete
void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[]
void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n);
void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n);
void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }
void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); }
void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); }
@ -132,23 +132,23 @@ terms of the MIT license. A copy of the license can be found in the file
typedef struct mi_nothrow_s { } mi_nothrow_t;
#if (MI_INTPTR_SIZE==8)
void* _Znwm(size_t n) MI_FORWARD1(new,n); // new 64-bit
void* _Znam(size_t n) MI_FORWARD1(new,n); // new[] 64-bit
void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al);
void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al);
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_source_new_nothrow(n MI_SOURCE_RET()); }
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_source_new_nothrow(n MI_SOURCE_RET()); }
void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al MI_SOURCE_RET()); }
void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al MI_SOURCE_RET()); }
void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit
void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit
void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow, n); }
void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow, n); }
void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow, n, al); }
void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow, n, al); }
#elif (MI_INTPTR_SIZE==4)
void* _Znwj(size_t n) MI_FORWARD1(new,n); // new 64-bit
void* _Znaj(size_t n) MI_FORWARD1(new,n); // new[] 64-bit
void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al);
void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(new_aligned, n, al);
void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); }
void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); }
void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit
void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit
void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al);
void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow,n); }
void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_nothrow,n); }
void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow,n,al); }
void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return MI_SOURCE_RET(mi_new_aligned_nothrow,n,al); }
#else
#error "define overloads for new/delete for this platform (just for performance, can be skipped)"
#endif
@ -163,31 +163,31 @@ extern "C" {
// Posix & Unix functions definitions
// ------------------------------------------------------
void* reallocf(void* p, size_t newsize) MI_FORWARD2(reallocf,p,newsize);
void cfree(void* p) MI_FORWARD0(free, p);
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize);
void cfree(void* p) MI_FORWARD0(mi_free, p);
size_t malloc_size(void* p) { return mi_usable_size(p); }
size_t malloc_usable_size(void* p) { return mi_usable_size(p); }
// no forwarding here due to aliasing/name mangling issues
void* valloc(size_t size) { return mi_source_valloc(size MI_SOURCE_RET()); }
void* pvalloc(size_t size) { return mi_source_pvalloc(size MI_SOURCE_RET()); }
void* reallocarray(void* p, size_t count, size_t size) { return mi_source_reallocarray(p, count, size MI_SOURCE_RET()); }
void* memalign(size_t alignment, size_t size) { return mi_source_memalign(alignment, size MI_SOURCE_RET()); }
void* aligned_alloc(size_t alignment, size_t size) { return mi_source_aligned_alloc(alignment, size MI_SOURCE_RET()); }
int posix_memalign(void** p, size_t alignment, size_t size) { return mi_source_posix_memalign(p, alignment, size MI_SOURCE_RET()); }
void* valloc(size_t size) { return MI_SOURCE_RET(mi_valloc, size); }
void* pvalloc(size_t size) { return MI_SOURCE_RET(mi_pvalloc, size); }
void* reallocarray(void* p, size_t count, size_t size) { return MI_SOURCE_RET(mi_reallocarray, p, count, size); }
void* memalign(size_t alignment, size_t size) { return MI_SOURCE_RET(mi_memalign, alignment, size); }
void* aligned_alloc(size_t alignment, size_t size) { return MI_SOURCE_RET(mi_aligned_alloc, alignment, size); }
int posix_memalign(void** p, size_t alignment, size_t size) { return MI_SOURCE_RET(mi_posix_memalign, p, alignment, size); }
#if defined(__GLIBC__) && defined(__linux__)
// forward __libc interface (needed for glibc-based Linux distributions)
void* __libc_malloc(size_t size) MI_FORWARD1(malloc,size);
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(calloc,count,size);
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(realloc,p,size);
void __libc_free(void* p) MI_FORWARD0(free,p);
void __libc_cfree(void* p) MI_FORWARD0(free,p);
void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size);
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size);
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size);
void __libc_free(void* p) MI_FORWARD0(mi_free,p);
void __libc_cfree(void* p) MI_FORWARD0(mi_free,p);
void* __libc_valloc(size_t size) { return mi_source_valloc(size MI_SOURCE_RET()); }
void* __libc_pvalloc(size_t size) { return mi_source_pvalloc(size MI_SOURCE_RET()); }
void* __libc_memalign(size_t alignment, size_t size) { return mi_source_memalign(alignment, size MI_SOURCE_RET()); }
int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_source_posix_memalign(p, alignment, size MI_SOURCE_RET()); }
void* __libc_valloc(size_t size) { return MI_SOURCE_RET(mi_valloc, size); }
void* __libc_pvalloc(size_t size) { return MI_SOURCE_RET(mi_pvalloc, size); }
void* __libc_memalign(size_t alignment, size_t size) { return MI_SOURCE_RET(mi_memalign, alignment, size); }
int __posix_memalign(void** p, size_t alignment, size_t size) { return MI_SOURCE_RET(mi_posix_memalign, p, alignment, size); }
#endif
#ifdef __cplusplus

131
src/alloc-posix.c
View file

@ -9,6 +9,7 @@ terms of the MIT license. A copy of the license can be found in the file
// mi prefixed publi definitions of various Posix, Unix, and C++ functions
// for convenience and used when overriding these functions.
// ------------------------------------------------------------------------
#define MI_NO_SOURCE_DEBUG
#include "mimalloc.h"
#include "mimalloc-internal.h"
@ -42,34 +43,27 @@ void mi_cfree(void* p) mi_attr_noexcept {
}
}
MI_SOURCE_API3(int, posix_memalign, void**, p, size_t, alignment, size_t, size)
void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
void* res = mi_expand(p, newsize);
if (res == NULL) errno = ENOMEM;
return res;
}
MI_SOURCE_API3(void*, reallocarray, void*, p, size_t, count, size_t, size)
{
// Note: The spec dictates we should not modify `*p` on an error. (issue#27)
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
if (p == NULL) return EINVAL;
if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment
if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
void* q;
if (alignment <= MI_MAX_ALIGN_SIZE) {
q = mi_source_malloc(size MI_SOURCE_ARG);
}
else {
q = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG);
}
if (q==NULL && size != 0) return ENOMEM;
mi_assert_internal(((uintptr_t)q % alignment) == 0);
*p = q;
return 0;
void* newp = MI_SOURCE_ARG(mi_reallocn, p, count, size);
if (newp==NULL) errno = ENOMEM;
return newp;
}
MI_SOURCE_API2(void*, memalign, size_t, alignment, size_t, size)
{
void* p;
if (alignment <= MI_MAX_ALIGN_SIZE) {
p = mi_source_malloc(size MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc, size);
}
else {
p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment);
}
mi_assert_internal(((uintptr_t)p % alignment) == 0);
return p;
@ -77,15 +71,15 @@ MI_SOURCE_API2(void*, memalign, size_t, alignment, size_t, size)
MI_SOURCE_API1(void*, valloc, size_t, size)
{
return mi_source_malloc_aligned(size, _mi_os_page_size() MI_SOURCE_ARG);
return MI_SOURCE_ARG(mi_malloc_aligned, size, _mi_os_page_size());
}
MI_SOURCE_API1(void*, pvalloc, size_t, size)
{
size_t psize = _mi_os_page_size();
if (size >= SIZE_MAX - psize) return NULL; // overflow
size_t asize = ((size + psize - 1) / psize) * psize;
return mi_source_malloc_aligned(asize, psize MI_SOURCE_ARG);
size_t asize = _mi_align_up(size, psize);
return MI_SOURCE_ARG(mi_malloc_aligned, asize, psize);
}
MI_SOURCE_API2(void*, aligned_alloc, size_t, alignment, size_t, size)
@ -94,27 +88,46 @@ MI_SOURCE_API2(void*, aligned_alloc, size_t, alignment, size_t, size)
if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
void* p;
if (alignment <= MI_MAX_ALIGN_SIZE) {
p = mi_source_malloc(size MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc, size);
}
else {
p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment);
}
mi_assert_internal(((uintptr_t)p % alignment) == 0);
return p;
}
MI_SOURCE_API3(void*, reallocarray, void*, p, size_t, count, size_t, size )
static int mi_base_posix_memalign(void** p, size_t alignment, size_t size MI_SOURCE_XPARAM)
{
void* newp = mi_source_reallocn(p, count, size MI_SOURCE_ARG);
if (newp==NULL) errno = ENOMEM;
return newp;
// Note: The spec dictates we should not modify `*p` on an error. (issue#27)
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
if (p == NULL) return EINVAL;
if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment
if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
void* q;
if (alignment <= MI_MAX_ALIGN_SIZE) {
q = MI_SOURCE_ARG(mi_malloc, size);
}
else {
q = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment);
}
if (q==NULL && size != 0) return ENOMEM;
mi_assert_internal(((uintptr_t)q % alignment) == 0);
*p = q;
return 0;
}
void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
void* res = mi_expand(p, newsize);
if (res == NULL) errno = ENOMEM;
return res;
#ifndef NDEBUG
int dbg_mi_posix_memalign(void** p, size_t alignment, size_t size, mi_source_t __mi_source) mi_attr_noexcept {
UNUSED(__mi_source);
return mi_base_posix_memalign(p, alignment, size MI_SOURCE_XARG);
}
#endif
int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
return mi_base_posix_memalign(p, alignment, size MI_SOURCE_XRET());
}
MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s)
{
@ -122,7 +135,7 @@ MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s)
size_t len;
for(len = 0; s[len] != 0; len++) { }
size_t size = (len+1)*sizeof(unsigned short);
unsigned short* p = (unsigned short*)mi_source_malloc(size MI_SOURCE_ARG);
unsigned short* p = (unsigned short*)MI_SOURCE_ARG(mi_malloc, size);
if (p != NULL) {
memcpy(p,s,size);
}
@ -131,10 +144,10 @@ MI_SOURCE_API1(unsigned short*, wcsdup, const unsigned short*, s)
MI_SOURCE_API1(unsigned char*, mbsdup, const unsigned char*, s)
{
return (unsigned char*)mi_source_strdup((const char*)s MI_SOURCE_ARG);
return (unsigned char*)MI_SOURCE_ARG(mi_strdup,(const char*)s);
}
MI_SOURCE_API3(int, dupenv_s, char**, buf, size_t*, size, const char*, name)
static int mi_base_dupenv_s(char** buf, size_t* size, const char* name MI_SOURCE_XPARAM)
{
if (buf==NULL || name==NULL) return EINVAL;
if (size != NULL) *size = 0;
@ -144,14 +157,26 @@ MI_SOURCE_API3(int, dupenv_s, char**, buf, size_t*, size, const char*, name)
*buf = NULL;
}
else {
*buf = mi_source_strdup(p MI_SOURCE_ARG);
*buf = MI_SOURCE_ARG(mi_strdup, p);
if (*buf==NULL) return ENOMEM;
if (size != NULL) *size = strlen(p);
}
return 0;
}
MI_SOURCE_API3(int, wdupenv_s, unsigned short**, buf, size_t*, size, const unsigned short*, name)
#ifndef NDEBUG
int dbg_mi_dupenv_s(char** buf, size_t* size, const char* name, mi_source_t __mi_source) mi_attr_noexcept {
UNUSED(__mi_source);
return mi_base_dupenv_s(buf, size, name MI_SOURCE_XARG);
}
#endif
int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept {
return mi_base_dupenv_s(buf, size, name MI_SOURCE_XRET());
}
static int mi_base_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name MI_SOURCE_XPARAM)
{
if (buf==NULL || name==NULL) return EINVAL;
if (size != NULL) *size = 0;
@ -169,7 +194,7 @@ MI_SOURCE_API3(int, wdupenv_s, unsigned short**, buf, size_t*, size, const unsig
*buf = NULL;
}
else {
*buf = mi_source_wcsdup(p MI_SOURCE_ARG);
*buf = MI_SOURCE_ARG(mi_wcsdup, p);
if (*buf==NULL) return ENOMEM;
if (size != NULL) *size = wcslen((const wchar_t*)p);
}
@ -177,10 +202,32 @@ MI_SOURCE_API3(int, wdupenv_s, unsigned short**, buf, size_t*, size, const unsig
#endif
}
void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
#ifndef NDEBUG
int dbg_mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name, mi_source_t __mi_source) mi_attr_noexcept {
UNUSED(__mi_source);
return mi_base_wdupenv_s(buf, size, name MI_SOURCE_XARG);
}
#endif
int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept {
return mi_base_wdupenv_s(buf, size, name MI_SOURCE_XRET());
}
void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
return mi_recalloc_aligned(p, newcount, size, alignment);
#ifndef NDEBUG
mi_decl_restrict void* dbg_mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft
return dbg_mi_recalloc_aligned_at(p, newcount, size, alignment, offset, __mi_source);
}
mi_decl_restrict void* dbg_mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment, mi_source_t __mi_source) mi_attr_noexcept { // Microsoft
return dbg_mi_recalloc_aligned(p, newcount, size, alignment, __mi_source);
}
#endif
mi_decl_restrict void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
return MI_SOURCE_RET(mi_recalloc_aligned_at,p, newcount, size, alignment, offset);
}
mi_decl_restrict void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
return MI_SOURCE_RET(mi_recalloc_aligned,p, newcount, size, alignment);
}

133
src/alloc.c
View file

@ -4,6 +4,7 @@ This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
#define MI_NO_SOURCE_DEBUG
#include "mimalloc.h"
#include "mimalloc-internal.h"
#include "mimalloc-atomic.h"
@ -22,11 +23,11 @@ terms of the MIT license. A copy of the license can be found in the file
// Fast allocation in a page: just pop from the free list.
// Fall back to generic allocation only if the list is empty.
extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_PARAM) mi_attr_noexcept {
extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept {
mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size);
mi_block_t* block = page->free;
if (mi_unlikely(block == NULL)) {
return _mi_malloc_generic(heap, size MI_SOURCE_ARG); // slow path
return _mi_malloc_generic(heap, size MI_SOURCE_XARG); // slow path
}
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
// pop from the free list
@ -60,13 +61,13 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
}
// allocate a small block
MI_ALLOC_API1(void*,malloc_small, mi_heap_t*,heap, size_t,size)
MI_ALLOC_API1(void*, malloc_small, mi_heap_t*, heap, size_t, size)
{
mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
mi_assert(size <= MI_SMALL_SIZE_MAX);
mi_page_t* page = _mi_heap_get_free_small_page(heap,size + MI_PADDING_SIZE);
void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE MI_SOURCE_ARG);
void* p = _mi_page_malloc(heap, page, size + MI_PADDING_SIZE MI_SOURCE_XARG);
mi_assert_internal(p==NULL || mi_usable_size(p) >= size);
#if MI_STAT>1
if (p != NULL) {
@ -79,15 +80,15 @@ MI_ALLOC_API1(void*,malloc_small, mi_heap_t*,heap, size_t,size)
// The main allocation function
MI_ALLOC_API1(void*, malloc, mi_heap_t*,heap, size_t,size)
MI_ALLOC_API1(void*, malloc, mi_heap_t*, heap, size_t, size)
{
if (mi_likely(size <= MI_SMALL_SIZE_MAX)) {
return mi_heap_source_malloc_small(heap, size MI_SOURCE_ARG);
return mi_base_malloc_small(heap, size MI_SOURCE_XARG);
}
else {
mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE MI_SOURCE_ARG);
void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE MI_SOURCE_XARG);
mi_assert_internal(p == NULL || mi_usable_size(p) >= size);
#if MI_STAT>1
if (p != NULL) {
@ -121,15 +122,15 @@ void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) {
// zero initialized small block
MI_ALLOC_API1(void*, zalloc_small, mi_heap_t*, heap, size_t, size)
{
void* p = mi_heap_source_malloc_small(heap, size MI_SOURCE_ARG);
void* p = mi_base_malloc_small(heap, size MI_SOURCE_XARG);
if (p != NULL) {
_mi_block_zero_init(_mi_ptr_page(p), p, size); // todo: can we avoid getting the page again?
}
return p;
}
mi_decl_allocator void* _mi_heap_source_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_PARAM) {
void* p = mi_heap_source_malloc(heap,size MI_SOURCE_ARG);
mi_decl_restrict void* _mi_base_malloc_zero(mi_heap_t* heap, size_t size, bool zero MI_SOURCE_XPARAM) {
void* p = mi_base_malloc(heap, size MI_SOURCE_XARG);
if (zero && p != NULL) {
_mi_block_zero_init(_mi_ptr_page(p),p,size); // todo: can we avoid getting the page again?
}
@ -138,7 +139,7 @@ mi_decl_allocator void* _mi_heap_source_malloc_zero(mi_heap_t* heap, size_t size
MI_ALLOC_API1(void*, zalloc, mi_heap_t*,heap, size_t,size)
{
return _mi_heap_source_malloc_zero(heap, size, true MI_SOURCE_ARG);
return _mi_base_malloc_zero(heap, size, true MI_SOURCE_XARG);
}
@ -518,37 +519,37 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
mi_free(p);
}
MI_ALLOC_API2(void*, calloc, mi_heap_t*,heap, size_t,count, size_t,size)
MI_ALLOC_API2(void*, calloc, mi_heap_t*, heap, size_t, count, size_t, size)
{
size_t total;
if (mi_count_size_overflow(count,size,&total)) return NULL;
return mi_heap_source_zalloc(heap, total MI_SOURCE_ARG);
return mi_base_zalloc(heap, total MI_SOURCE_XARG);
}
// Uninitialized `calloc`
MI_ALLOC_API2(void*, mallocn, mi_heap_t*,heap, size_t,count, size_t,size)
MI_ALLOC_API2(void*, mallocn, mi_heap_t*, heap, size_t, count, size_t, size)
{
size_t total;
if (mi_count_size_overflow(count, size, &total)) return NULL;
return mi_heap_source_malloc(heap, total MI_SOURCE_ARG);
return mi_base_malloc(heap, total MI_SOURCE_XARG);
}
// Expand in place or fail
mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
mi_decl_restrict void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
if (p == NULL) return NULL;
size_t size = mi_usable_size(p);
if (newsize > size) return NULL;
return p; // it fits
}
mi_decl_allocator void* _mi_heap_source_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_PARAM) {
if (p == NULL) return _mi_heap_source_malloc_zero(heap,newsize,zero MI_SOURCE_ARG);
mi_decl_restrict void* _mi_base_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero MI_SOURCE_XPARAM) {
if (p == NULL) return _mi_base_malloc_zero(heap,newsize,zero MI_SOURCE_XARG);
size_t size = mi_usable_size(p);
if (newsize <= size && newsize >= (size / 2)) {
return p; // reallocation still fits and not more than 50% waste
}
void* newp = mi_heap_source_malloc(heap,newsize MI_SOURCE_ARG);
void* newp = mi_base_malloc(heap, newsize MI_SOURCE_XARG);
if (mi_likely(newp != NULL)) {
if (zero && newsize > size) {
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
@ -561,36 +562,36 @@ mi_decl_allocator void* _mi_heap_source_realloc_zero(mi_heap_t* heap, void* p, s
return newp;
}
MI_ALLOC_API2(void*, realloc, mi_heap_t*,heap, void*,p, size_t,newsize)
MI_ALLOC_API2(void*, realloc, mi_heap_t*, heap, void*, p, size_t, newsize)
{
return _mi_heap_source_realloc_zero(heap, p, newsize, false MI_SOURCE_ARG);
return _mi_base_realloc_zero(heap, p, newsize, false MI_SOURCE_XARG);
}
MI_ALLOC_API3(void*, reallocn, mi_heap_t*,heap, void*,p, size_t,count, size_t,size)
MI_ALLOC_API3(void*, reallocn, mi_heap_t*, heap, void*, p, size_t, count, size_t, size)
{
size_t total;
if (mi_count_size_overflow(count, size, &total)) return NULL;
return mi_heap_source_realloc(heap, p, total MI_SOURCE_ARG);
return mi_base_realloc(heap, p, total MI_SOURCE_XARG);
}
// Reallocate but free `p` on errors
MI_ALLOC_API2(void*, reallocf, mi_heap_t*, heap, void*, p, size_t, newsize)
{
void* newp = mi_heap_source_realloc(heap, p, newsize MI_SOURCE_ARG);
void* newp = mi_base_realloc(heap, p, newsize MI_SOURCE_XARG);
if (newp==NULL && p!=NULL) mi_free(p);
return newp;
}
MI_ALLOC_API2(void*, rezalloc, mi_heap_t*, heap, void*, p, size_t, newsize)
{
return _mi_heap_source_realloc_zero(heap, p, newsize, true MI_SOURCE_ARG);
return _mi_base_realloc_zero(heap, p, newsize, true MI_SOURCE_XARG);
}
MI_ALLOC_API3(void*, recalloc, mi_heap_t*, heap, void*, p, size_t, count, size_t, size)
{
size_t total;
if (mi_count_size_overflow(count, size, &total)) return NULL;
return _mi_heap_source_realloc_zero(heap, p, total, true MI_SOURCE_ARG);
return _mi_base_realloc_zero(heap, p, total, true MI_SOURCE_XARG);
}
@ -604,7 +605,7 @@ MI_ALLOC_API1(char*, strdup, mi_heap_t*,heap, const char*,s)
{
if (s == NULL) return NULL;
size_t n = strlen(s);
char* t = (char*)mi_heap_source_malloc(heap, n+1 MI_SOURCE_ARG);
char* t = (char*)mi_base_malloc(heap, n+1 MI_SOURCE_XARG);
if (t != NULL) memcpy(t, s, n + 1);
return t;
}
@ -616,7 +617,7 @@ MI_ALLOC_API2(char*, strndup, mi_heap_t*, heap, const char*, s, size_t, n)
if (s == NULL) return NULL;
size_t m = strlen(s);
if (n > m) n = m;
char* t = (char*)mi_heap_source_malloc(heap, n+1 MI_SOURCE_ARG);
char* t = (char*)mi_base_malloc(heap, n+1 MI_SOURCE_XARG);
if (t == NULL) return NULL;
memcpy(t, s, n);
t[n] = 0;
@ -646,7 +647,7 @@ MI_ALLOC_API2(char*, realpath, mi_heap_t*, heap, const char*, fname, char*, reso
return resolved_name;
}
else {
return mi_heap_source_strndup(heap, buf, PATH_MAX MI_SOURCE_ARG);
return mi_base_strndup(heap, buf, PATH_MAX MI_SOURCE_XARG);
}
}
#else
@ -672,7 +673,7 @@ MI_ALLOC_API2(char*, realpath, mi_heap_t*, heap, const char*, fname, char*, reso
char* buf = (char*)mi_malloc(n+1);
if (buf==NULL) return NULL;
char* rname = realpath(fname,buf);
char* result = mi_heap_source_strndup(heap, rname, n MI_SOURCE_ARG); // ok if `rname==NULL`
char* result = mi_base_strndup(heap, rname, n MI_SOURCE_XARG); // ok if `rname==NULL`
mi_free(buf);
return result;
}
@ -733,90 +734,72 @@ static bool mi_try_new_handler(bool nothrow) {
}
#endif
static mi_decl_noinline void* mi_source_try_new(size_t size, bool nothrow MI_SOURCE_PARAM) {
static mi_decl_noinline void* mi_base_try_new(size_t size, bool nothrow MI_SOURCE_XPARAM) {
void* p = NULL;
while(p == NULL && mi_try_new_handler(nothrow)) {
p = mi_source_malloc(size MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc, size);
}
return p;
}
inline mi_decl_allocator void* mi_source_new(size_t size MI_SOURCE_PARAM) {
void* p = mi_source_malloc(size MI_SOURCE_ARG);
if (mi_unlikely(p == NULL)) return mi_source_try_new(size, false MI_SOURCE_ARG);
MI_NEW_API1(void*, new, size_t, size)
{
void* p = MI_SOURCE_ARG(mi_malloc, size);
if (mi_unlikely(p == NULL)) return mi_base_try_new(size, false MI_SOURCE_XARG);
return p;
}
mi_decl_allocator void* mi_new(size_t size) {
return mi_source_new(size MI_SOURCE_RET());
}
mi_decl_allocator void* mi_source_new_nothrow(size_t size MI_SOURCE_PARAM) {
void* p = mi_source_malloc(size MI_SOURCE_ARG);
if (mi_unlikely(p == NULL)) return mi_source_try_new(size, true MI_SOURCE_ARG);
MI_NEW_API1(void*, new_nothrow, size_t, size)
{
void* p = MI_SOURCE_ARG(mi_malloc, size);
if (mi_unlikely(p == NULL)) return mi_base_try_new(size, true MI_SOURCE_XARG);
return p;
}
mi_decl_allocator void* mi_new_nothrow(size_t size) {
return mi_source_new_nothrow(size MI_SOURCE_RET());
}
mi_decl_allocator void* mi_source_new_aligned(size_t size, size_t alignment MI_SOURCE_PARAM) {
MI_NEW_API2(void*, new_aligned, size_t, size, size_t, alignment)
{
void* p;
do {
p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment);
}
while(p == NULL && mi_try_new_handler(false));
return p;
}
mi_decl_allocator void* mi_new_aligned(size_t size, size_t alignment) {
return mi_source_new_aligned(size, alignment MI_SOURCE_RET());
}
mi_decl_allocator void* mi_source_new_aligned_nothrow(size_t size, size_t alignment MI_SOURCE_PARAM) {
MI_NEW_API2(void*, new_aligned_nothrow, size_t, size, size_t, alignment)
{
void* p;
do {
p = mi_source_malloc_aligned(size, alignment MI_SOURCE_ARG);
p = MI_SOURCE_ARG(mi_malloc_aligned, size, alignment);
}
while(p == NULL && mi_try_new_handler(true));
return p;
}
mi_decl_allocator void* mi_new_aligned_nothrow(size_t size, size_t alignment) {
return mi_source_new_aligned_nothrow(size, alignment MI_SOURCE_RET());
}
mi_decl_allocator void* mi_source_new_n(size_t count, size_t size MI_SOURCE_PARAM) {
MI_NEW_API2(void*, new_n, size_t, count, size_t, size)
{
size_t total;
if (mi_unlikely(mi_count_size_overflow(count, size, &total))) {
mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
return NULL;
}
else {
return mi_source_new(total MI_SOURCE_ARG);
return MI_SOURCE_ARG(mi_new, total);
}
}
mi_decl_allocator void* mi_new_n(size_t count, size_t size) {
return mi_source_new_n(count, size MI_SOURCE_RET());
}
mi_decl_allocator void* mi_source_new_realloc(void* p, size_t newsize MI_SOURCE_PARAM) {
MI_NEW_API2(void*, new_realloc, void*, p, size_t, newsize)
{
void* q;
do {
q = mi_source_realloc(p, newsize MI_SOURCE_ARG);
q = MI_SOURCE_ARG(mi_realloc, p, newsize);
} while (q == NULL && mi_try_new_handler(false));
return q;
}
mi_decl_allocator void* mi_new_realloc(void* p, size_t newsize) {
return mi_source_new_realloc(p, newsize MI_SOURCE_RET());
}
mi_decl_allocator void* mi_source_new_reallocn(void* p, size_t newcount, size_t size MI_SOURCE_PARAM) {
MI_NEW_API3(void*, new_reallocn, void*, p, size_t, newcount, size_t, size)
{
size_t total;
if (mi_unlikely(mi_count_size_overflow(newcount, size, &total))) {
mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc
@ -824,11 +807,7 @@ mi_decl_allocator void* mi_source_new_reallocn(void* p, size_t newcount, size_t
}
void* q;
do {
q = mi_source_realloc(p, total MI_SOURCE_ARG);
q = MI_SOURCE_ARG(mi_realloc, p, total);
} while (q == NULL && mi_try_new_handler(false));
return q;
}
mi_decl_allocator void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
return mi_source_new_reallocn(p, newcount, size MI_SOURCE_RET());
}

4
src/init.c
View file

@ -33,9 +33,9 @@ const mi_page_t _mi_page_empty = {
#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
#if defined(MI_PADDING) && (MI_INTPTR_SIZE >= 8)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#elif defined(MI_PADDING)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#elif defined(MI_PADDING)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#else
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
#endif

8
src/options.c
View file

@ -386,6 +386,7 @@ void _mi_error_message(int err, const char* fmt, ...) {
// lsb=1: bit 63-19: relative file name char* (to `mi_fname_base`), bit 18-1: line number
// lsb=0: bit 63-01: return address
// -----------------------------------------------------------------------------------------------
#ifndef NDEBUG
static const char* mi_debug_fname_base = "mimalloc_fname_base";
#define MI_FNAME_SHIFT 16
@ -393,14 +394,14 @@ static const char* mi_debug_fname_base = "mimalloc_fname_base";
#define MI_LINE_MASK ((1L << MI_LINE_SHIFT) - 1)
mi_source_t mi_source_ret(void* return_address) {
mi_source_t source = { ((intptr_t)return_address << 1) };
mi_source_t source = { ((long long)return_address << 1) };
return source;
}
mi_source_t mi_source_loc(const char* fname, int lineno ) {
ptrdiff_t delta = fname - mi_debug_fname_base;
long long delta = fname - mi_debug_fname_base;
mi_assert_internal(((delta << MI_FNAME_SHIFT) >> MI_FNAME_SHIFT) == delta);
mi_source_t source = { ((((intptr_t)delta) << MI_FNAME_SHIFT) | ((lineno << 1) & MI_LINE_MASK) | 1) };
mi_source_t source = { ((((long long)delta) << MI_FNAME_SHIFT) | ((lineno << 1) & MI_LINE_MASK) | 1) };
return source;
}
@ -419,6 +420,7 @@ void* mi_source_unpack(mi_source_t source, const char** fname, int* lineno) {
return ((void*)(source.src >> 1));
}
}
#endif
// -----------------------------------------------------------------------------------------------
// Error message for a specific heap block

4
src/page.c
View file

@ -774,7 +774,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.
void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_PARAM) mi_attr_noexcept
void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept
{
mi_assert_internal(heap != NULL);
@ -815,5 +815,5 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_PARAM) mi_attr_n
mi_assert_internal(mi_page_block_size(page) >= size);
// and try again, this time succeeding! (i.e. this should never recurse)
return _mi_page_malloc(heap, page, size MI_SOURCE_ARG);
return _mi_page_malloc(heap, page, size MI_SOURCE_XARG);
}

1
src/static.c
View file

@ -6,6 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file
-----------------------------------------------------------------------------*/
#define _DEFAULT_SOURCE
#define MI_NO_SOURCE_DEBUG
#include "mimalloc.h"
#include "mimalloc-internal.h"