kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-07-06 11:34:38 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge with dev

Browse source
This commit is contained in:
daan 2019-07-08 15:42:42 -07:00
commit 493dfc4b82
22 changed files with 321 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

131
src/alloc-override.c
View file

@ -25,15 +25,19 @@ terms of the MIT license. A copy of the license can be found in the file
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
// use aliasing to alias the exported function to one of our `mi_` functions
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default")))
#define MI_FORWARD1(fun,x) MI_FORWARD(fun)
#define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)
#define MI_FORWARD0(fun,x) MI_FORWARD(fun)
#define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default")))
#define MI_FORWARD1(fun,x) MI_FORWARD(fun)
#define MI_FORWARD2(fun,x,y) MI_FORWARD(fun)
#define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun)
#define MI_FORWARD0(fun,x) MI_FORWARD(fun)
#define MI_FORWARD02(fun,x,y) MI_FORWARD(fun)
#else
// use forwarding by calling our `mi_` function
#define MI_FORWARD1(fun,x) { return fun(x); }
#define MI_FORWARD2(fun,x,y) { return fun(x,y); }
#define MI_FORWARD0(fun,x) { fun(x); }
#define MI_FORWARD1(fun,x) { return fun(x); }
#define MI_FORWARD2(fun,x,y) { return fun(x,y); }
#define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); }
#define MI_FORWARD0(fun,x) { fun(x); }
#define MI_FORWARD02(fun,x,y) { fun(x,y); }
#endif
#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE)
@ -59,9 +63,7 @@ terms of the MIT license. A copy of the license can be found in the file
void* malloc(size_t size) mi_attr_noexcept MI_FORWARD1(mi_malloc, size);
void* calloc(size_t size, size_t n) mi_attr_noexcept MI_FORWARD2(mi_calloc, size, n);
void* realloc(void* p, size_t newsize) mi_attr_noexcept MI_FORWARD2(mi_realloc, p, newsize);
void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p);
//char* strdup(const char* s) MI_FORWARD1(mi_strdup, s);
//char* strndup(const char* s, size_t n) MI_FORWARD2(mi_strndup, s, n);
void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p);
#endif
#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
@ -81,17 +83,42 @@ terms of the MIT license. A copy of the license can be found in the file
#include <new>
void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p);
void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p);
void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_malloc,n);
void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_malloc,n);
#if (__cplusplus >= 201703L)
void* operator new( std::size_t n, std::align_val_t align) noexcept(false) MI_FORWARD2(mi_malloc_aligned,n,align);
void* operator new[]( std::size_t n, std::align_val_t align) noexcept(false) MI_FORWARD2(mi_malloc_aligned,n,align);
void* operator new(std::size_t n) noexcept(false) { return mi_new(n); }
void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); }
void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n);
void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n);
#if (__cplusplus >= 201402L)
void operator delete (void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz);
void operator delete[](void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz);
#endif
#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast<size_t>(al)); }
void operator delete (void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast<size_t>(al)); };
void operator delete[](void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast<size_t>(al)); };
void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); }
void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); }
void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast<size_t>(al)); }
void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast<size_t>(al)); }
#endif
#else
// ------------------------------------------------------
// With a C compiler we override the new/delete operators
// by defining the mangled C++ names of the operators (as
// With a C compiler we cannot override the new/delete operators
// as the standard requires calling into `get_new_handler` and/or
// throwing C++ exceptions (and we cannot do that from C). So, we
// hope the standard new uses `malloc` internally which will be
// redirected anyways.
// ------------------------------------------------------
#if 0
// ------------------------------------------------------
// Override by defining the mangled C++ names of the operators (as
// used by GCC and CLang).
// See <https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling>
// ------------------------------------------------------
@ -110,6 +137,7 @@ terms of the MIT license. A copy of the license can be found in the file
#else
#error "define overloads for new/delete for this platform (just for performance, can be skipped)"
#endif
#endif
#endif // __cplusplus
@ -121,57 +149,18 @@ extern "C" {
// Posix & Unix functions definitions
// ------------------------------------------------------
#include <errno.h>
#ifndef EINVAL
#define EINVAL 22
#endif
#ifndef ENOMEM
#define ENOMEM 12
#endif
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize);
size_t malloc_size(void* p) MI_FORWARD1(mi_usable_size,p);
size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p);
void cfree(void* p) MI_FORWARD0(mi_free, p);
int posix_memalign(void** p, size_t alignment, size_t size) {
// TODO: the spec says we should return EINVAL also if alignment is not a power of 2.
// The spec also dictates we should not modify `*p` on an error. (issue#27)
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
if (alignment % sizeof(void*) != 0) return EINVAL; // no `p==NULL` check as it is declared as non-null
if ((alignment & (~alignment + 1)) != alignment) return EINVAL; // not a power of 2
void* q = mi_malloc_aligned(size, alignment);
if (q==NULL && size != 0) return ENOMEM;
*p = q;
return 0;
}
void* memalign(size_t alignment, size_t size) {
return mi_malloc_aligned(size, alignment);
}
void* valloc(size_t size) {
return mi_malloc_aligned(size, _mi_os_page_size());
}
void* pvalloc(size_t size) {
size_t psize = _mi_os_page_size();
if (size >= SIZE_MAX - psize) return NULL; // overflow
size_t asize = ((size + psize - 1) / psize) * psize;
return mi_malloc_aligned(asize, psize);
}
void* aligned_alloc(size_t alignment, size_t size) {
return mi_malloc_aligned(size, alignment);
}
void* reallocarray( void* p, size_t count, size_t size ) { // BSD
void* newp = mi_reallocn(p,count,size);
if (newp==NULL) errno = ENOMEM;
return newp;
}
// no forwarding here due to aliasing/name mangling issues
void* valloc(size_t size) { return mi_valloc(size); }
void* pvalloc(size_t size) { return mi_pvalloc(size); }
void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); }
void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }
void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }
#if defined(__GLIBC__) && defined(__linux__)
// forward __libc interface (needed for glibc-based Linux distributions)
@ -181,18 +170,10 @@ void* reallocarray( void* p, size_t count, size_t size ) { // BSD
void __libc_free(void* p) MI_FORWARD0(mi_free,p);
void __libc_cfree(void* p) MI_FORWARD0(mi_free,p);
void* __libc_memalign(size_t alignment, size_t size) {
return memalign(alignment,size);
}
void* __libc_valloc(size_t size) {
return valloc(size);
}
void* __libc_pvalloc(size_t size) {
return pvalloc(size);
}
int __posix_memalign(void** p, size_t alignment, size_t size) {
return posix_memalign(p,alignment,size);
}
void* __libc_valloc(size_t size) { return mi_valloc(size); }
void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); }
void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); }
int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); }
#endif
#ifdef __cplusplus

82
src/alloc-posix.c Normal file
View file

@ -0,0 +1,82 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018,2019, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
// ------------------------------------------------------------------------
// mi prefixed publi definitions of various Posix, Unix, and C++ functions
// for convenience and used when overriding these functions.
// ------------------------------------------------------------------------
#include "mimalloc.h"
#include "mimalloc-internal.h"
// ------------------------------------------------------
// Posix & Unix functions definitions
// ------------------------------------------------------
#include <errno.h>
#ifndef EINVAL
#define EINVAL 22
#endif
#ifndef ENOMEM
#define ENOMEM 12
#endif
size_t mi_malloc_size(const void* p) mi_attr_noexcept {
return mi_usable_size(p);
}
size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept {
return mi_usable_size(p);
}
void mi_cfree(void* p) mi_attr_noexcept {
mi_free(p);
}
int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
// Note: The spec dictates we should not modify `*p` on an error. (issue#27)
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
if (p == NULL) return EINVAL;
if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment
if ((alignment & (alignment - 1)) != 0) return EINVAL; // not a power of 2
void* q = mi_malloc_aligned(size, alignment);
if (q==NULL && size != 0) return ENOMEM;
*p = q;
return 0;
}
int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
return mi_posix_memalign(p, alignment, size);
}
void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
return mi_malloc_aligned(size, alignment);
}
void* mi_valloc(size_t size) mi_attr_noexcept {
return mi_malloc_aligned(size, _mi_os_page_size());
}
void* mi_pvalloc(size_t size) mi_attr_noexcept {
size_t psize = _mi_os_page_size();
if (size >= SIZE_MAX - psize) return NULL; // overflow
size_t asize = ((size + psize - 1) / psize) * psize;
return mi_malloc_aligned(asize, psize);
}
void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
return mi_malloc_aligned(size, alignment);
}
void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD
void* newp = mi_reallocn(p,count,size);
if (newp==NULL) errno = ENOMEM;
return newp;
}

74
src/alloc.c
View file

@ -106,12 +106,12 @@ void* mi_zalloc(size_t size) mi_attr_noexcept {
// multi-threaded free
static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block)
{
mi_thread_free_t tfree;
mi_thread_free_t tfreex;
mi_thread_free_t tfree = {0};
mi_thread_free_t tfreex = {0};
bool use_delayed;
do {
tfreex = tfree = page->thread_free;
tfreex.value = tfree.value = page->thread_free.value;
use_delayed = (tfree.delayed == MI_USE_DELAYED_FREE);
if (mi_unlikely(use_delayed)) {
// unlikely: this only happens on the first concurrent free in a page that is in the full list
@ -143,7 +143,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
// and reset the MI_DELAYED_FREEING flag
do {
tfreex = tfree = page->thread_free;
tfreex.value = tfree.value = page->thread_free.value;
tfreex.delayed = MI_NO_DELAYED_FREE;
} while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value));
}
@ -177,7 +177,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block
// Adjust a block that was allocated aligned, to the actual start of the block in the page.
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, void* p) {
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) {
mi_assert_internal(page!=NULL && p!=NULL);
size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL);
size_t adjust = (diff % page->block_size);
@ -256,7 +256,7 @@ void _mi_free_delayed_block(mi_block_t* block) {
}
// Bytes available in a block
size_t mi_usable_size(void* p) mi_attr_noexcept {
size_t mi_usable_size(const void* p) mi_attr_noexcept {
if (p==NULL) return 0;
const mi_segment_t* segment = _mi_ptr_segment(p);
const mi_page_t* page = _mi_segment_page_of(segment,p);
@ -282,8 +282,11 @@ size_t mi_usable_size(void* p) mi_attr_noexcept {
#ifdef __cplusplus
void* _mi_externs[] = {
(void*)&_mi_page_malloc,
(void*)&mi_malloc_small,
(void*)&mi_malloc,
(void*)&mi_malloc_small,
(void*)&mi_heap_malloc,
(void*)&mi_heap_zalloc,
(void*)&mi_heap_malloc_small
};
@ -294,6 +297,24 @@ void* _mi_externs[] = {
// Allocation extensions
// ------------------------------------------------------
void mi_free_size(void* p, size_t size) mi_attr_noexcept {
UNUSED_RELEASE(size);
mi_assert(size <= mi_usable_size(p));
mi_free(p);
}
void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept {
UNUSED_RELEASE(alignment);
mi_assert(((uintptr_t)p % alignment) == 0);
mi_free_size(p,size);
}
void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
UNUSED_RELEASE(alignment);
mi_assert(((uintptr_t)p % alignment) == 0);
mi_free(p);
}
extern inline void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(count,size,&total)) return NULL;
@ -444,3 +465,42 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name)
char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept {
return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name);
}
#ifdef __cplusplus
#include <new>
static mi_decl_noinline void* mi_new_try(std::size_t n) noexcept(false) {
void* p;
do {
std::new_handler h = std::get_new_handler();
if (h==NULL) throw std::bad_alloc();
h();
// and try again
p = mi_malloc(n);
} while (p==NULL);
return p;
}
// spit out `new_try` for better assembly code
void* mi_new(std::size_t n) noexcept(false) {
void* p = mi_malloc(n);
if (mi_likely(p != NULL)) return p;
else return mi_new_try(n);
}
#if (__cplusplus > 201402L || defined(__cpp_aligned_new))
// for aligned allocation its fine as it is not inlined anyways
void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) {
void* p;
while ((p = mi_malloc_aligned(n,static_cast<size_t>(alignment))) == NULL) {
std::new_handler h = std::get_new_handler();
if (h==NULL) throw std::bad_alloc();
h();
// and try again
};
return p;
}
#endif
#endif

6
src/heap.c
View file

@ -84,7 +84,7 @@ typedef enum mi_collect_e {
static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
UNUSED(arg2);
UNUSED(heap);
mi_collect_t collect = (mi_collect_t)arg_collect;
mi_collect_t collect = *((mi_collect_t*)arg_collect);
_mi_page_free_collect(page);
if (mi_page_all_free(page)) {
// no more used blocks, free the page. TODO: should we retire here and be less aggressive?
@ -131,7 +131,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
_mi_heap_delayed_free(heap);
// collect all pages owned by this thread
mi_heap_visit_pages(heap, &mi_heap_page_collect, (void*)(collect), NULL);
mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL);
mi_assert_internal( collect != ABANDON || heap->thread_delayed_free == NULL );
// collect segment caches
@ -480,7 +480,7 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
// Visit all heap pages as areas
static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) {
if (visitor == NULL) return false;
return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, visitor, arg);
return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, (void*)(visitor), arg); // note: function pointer to void* :-{
}
// Just to pass arguments

2
src/init.c
View file

@ -426,7 +426,7 @@ static void mi_process_done(void) {
// C++: use static initialization to detect process start
static bool _mi_process_init(void) {
mi_process_init();
return (mi_main_thread_id != 0);
return (_mi_heap_main.thread_id != 0);
}
static bool mi_initialized = _mi_process_init();

2
src/page-queue.c
View file

@ -56,7 +56,7 @@ static inline uint8_t mi_bsr32(uint32_t x);
#include <intrin.h>
static inline uint8_t mi_bsr32(uint32_t x) {
uint32_t idx;
_BitScanReverse(&idx, x);
_BitScanReverse((DWORD*)&idx, x);
return idx;
}
#elif defined(__GNUC__) || defined(__clang__)

8
src/page.c
View file

@ -114,7 +114,7 @@ void _mi_page_use_delayed_free(mi_page_t* page, bool enable) {
mi_thread_free_t tfreex;
do {
tfreex = tfree = page->thread_free;
tfreex.value = tfree.value = page->thread_free.value;
tfreex.delayed = (enable ? MI_USE_DELAYED_FREE : MI_NO_DELAYED_FREE);
if (mi_unlikely(tfree.delayed == MI_DELAYED_FREEING)) {
mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
@ -137,10 +137,10 @@ void _mi_page_use_delayed_free(mi_page_t* page, bool enable) {
static void mi_page_thread_free_collect(mi_page_t* page)
{
mi_block_t* head;
mi_thread_free_t tfree;
mi_thread_free_t tfreex;
mi_thread_free_t tfree = {0};
mi_thread_free_t tfreex = {0};
do {
tfreex = tfree = page->thread_free;
tfreex.value = tfree.value = page->thread_free.value;
head = (mi_block_t*)((uintptr_t)tfree.head << MI_TF_PTR_SHIFT);
tfreex.head = 0;
} while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex.value, tfree.value));

1
src/static.c
View file

@ -21,5 +21,6 @@ terms of the MIT license. A copy of the license can be found in the file
#include "heap.c"
#include "alloc.c"
#include "alloc-aligned.c"
#include "alloc-posix.c"
#include "init.c"
#include "options.c"