mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-07-07 03:48:42 +03:00
merge from dev
This commit is contained in:
daan
commit
ad47cab97c
142 changed files with 3326 additions and 1819 deletions
|
|
@ -14,31 +14,14 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
// Aligned Allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept {
|
||||
// note: we don't require `size > offset`, we just guarantee that
|
||||
// the address at offset is aligned regardless of the allocated size.
|
||||
mi_assert(alignment > 0);
|
||||
if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
|
||||
if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
|
||||
// Fallback primitive aligned allocation -- split out for better codegen
|
||||
static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept
|
||||
{
|
||||
mi_assert_internal(size <= PTRDIFF_MAX);
|
||||
mi_assert_internal(alignment!=0 && _mi_is_power_of_two(alignment) && alignment <= MI_ALIGNMENT_MAX);
|
||||
|
||||
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
|
||||
|
||||
// try if there is a small block available with just the right alignment
|
||||
const size_t padsize = size + MI_PADDING_SIZE;
|
||||
if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) {
|
||||
mi_page_t* page = _mi_heap_get_free_small_page(heap,padsize);
|
||||
const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0;
|
||||
if (mi_likely(page->free != NULL && is_aligned))
|
||||
{
|
||||
#if MI_STAT>1
|
||||
mi_heap_stat_increase( heap, malloc, size);
|
||||
#endif
|
||||
void* p = _mi_page_malloc(heap,page,padsize); // TODO: inline _mi_page_malloc
|
||||
mi_assert_internal(p != NULL);
|
||||
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
|
||||
if (zero) _mi_block_zero_init(page,p,size);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
// use regular allocation if it is guaranteed to fit the alignment constraints
|
||||
if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) {
|
||||
|
|
@ -46,7 +29,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
|
|||
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
// otherwise over-allocate
|
||||
void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero);
|
||||
if (p == NULL) return NULL;
|
||||
|
|
@ -55,21 +38,90 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
|
|||
uintptr_t adjust = alignment - (((uintptr_t)p + offset) & align_mask);
|
||||
mi_assert_internal(adjust <= alignment);
|
||||
void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust));
|
||||
if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true);
|
||||
if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true);
|
||||
mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
|
||||
mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) );
|
||||
mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p));
|
||||
return aligned_p;
|
||||
}
|
||||
|
||||
// Primitive aligned allocation
|
||||
static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept
|
||||
{
|
||||
// note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size.
|
||||
mi_assert(alignment > 0);
|
||||
if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) { // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
|
||||
#if MI_DEBUG > 0
|
||||
_mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment);
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
if (mi_unlikely(alignment > MI_ALIGNMENT_MAX)) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers)
|
||||
#if MI_DEBUG > 0
|
||||
_mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)\n", MI_ALIGNMENT_MAX, size, alignment);
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
|
||||
#if MI_DEBUG > 0
|
||||
_mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
|
||||
const size_t padsize = size + MI_PADDING_SIZE; // note: cannot overflow due to earlier size > PTRDIFF_MAX check
|
||||
|
||||
// try first if there happens to be a small block available with just the right alignment
|
||||
if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) {
|
||||
mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize);
|
||||
const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0;
|
||||
if (mi_likely(page->free != NULL && is_aligned))
|
||||
{
|
||||
#if MI_STAT>1
|
||||
mi_heap_stat_increase(heap, malloc, size);
|
||||
#endif
|
||||
void* p = _mi_page_malloc(heap, page, padsize); // TODO: inline _mi_page_malloc
|
||||
mi_assert_internal(p != NULL);
|
||||
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
|
||||
if (zero) { _mi_block_zero_init(page, p, size); }
|
||||
return p;
|
||||
}
|
||||
}
|
||||
// fallback
|
||||
return mi_heap_malloc_zero_aligned_at_fallback(heap, size, alignment, offset, zero);
|
||||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Optimized mi_heap_malloc_aligned / mi_malloc_aligned
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false);
|
||||
}
|
||||
|
||||
mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
|
||||
return mi_heap_malloc_aligned_at(heap, size, alignment, 0);
|
||||
#if !MI_PADDING
|
||||
// without padding, any small sized allocation is naturally aligned (see also `_mi_segment_page_start`)
|
||||
if (!_mi_is_power_of_two(alignment)) return NULL;
|
||||
if (mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX))
|
||||
#else
|
||||
// with padding, we can only guarantee this for fixed alignments
|
||||
if (mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2)))
|
||||
&& size <= MI_SMALL_SIZE_MAX))
|
||||
#endif
|
||||
{
|
||||
// fast path for common alignment and size
|
||||
return mi_heap_malloc_small(heap, size);
|
||||
}
|
||||
else {
|
||||
return mi_heap_malloc_aligned_at(heap, size, alignment, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Aligned Allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
|
||||
return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true);
|
||||
}
|
||||
|
|
@ -113,6 +165,10 @@ mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t align
|
|||
}
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Aligned re-allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
|
||||
mi_assert(alignment > 0);
|
||||
if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -192,14 +192,16 @@ static malloc_introspection_t mi_introspect = {
|
|||
.log = &intro_log,
|
||||
.force_lock = &intro_force_lock,
|
||||
.force_unlock = &intro_force_unlock,
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && \
|
||||
MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
|
||||
.statistics = &intro_statistics,
|
||||
.zone_locked = &intro_zone_locked,
|
||||
#endif
|
||||
};
|
||||
|
||||
static malloc_zone_t mi_malloc_zone = {
|
||||
// note: even with designators, the order is important for C++ compilation
|
||||
//.reserved1 = NULL,
|
||||
//.reserved2 = NULL,
|
||||
.size = &zone_size,
|
||||
.malloc = &zone_malloc,
|
||||
.calloc = &zone_calloc,
|
||||
|
|
@ -211,19 +213,21 @@ static malloc_zone_t mi_malloc_zone = {
|
|||
.batch_malloc = &zone_batch_malloc,
|
||||
.batch_free = &zone_batch_free,
|
||||
.introspect = &mi_introspect,
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
|
||||
#if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
|
||||
.version = 10,
|
||||
#else
|
||||
.version = 9,
|
||||
#endif
|
||||
// switch to version 9+ on OSX 10.6 to support memalign.
|
||||
.memalign = &zone_memalign,
|
||||
.free_definite_size = &zone_free_definite_size,
|
||||
.pressure_relief = &zone_pressure_relief,
|
||||
#if defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7
|
||||
#if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
|
||||
.claimed_address = &zone_claimed_address,
|
||||
.version = 10
|
||||
#else
|
||||
.version = 9
|
||||
#endif
|
||||
#else
|
||||
.version = 4
|
||||
.version = 4,
|
||||
#endif
|
||||
};
|
||||
|
||||
|
|
@ -416,8 +420,7 @@ __attribute__((constructor)) // seems not supported by g++-11 on the M1
|
|||
static void _mi_macos_override_malloc() {
|
||||
malloc_zone_t* purgeable_zone = NULL;
|
||||
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && \
|
||||
MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
|
||||
#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
|
||||
// force the purgeable zone to exist to avoid strange bugs
|
||||
if (malloc_default_purgeable_zone) {
|
||||
purgeable_zone = malloc_default_purgeable_zone();
|
||||
|
|
|
|||
|
|
@ -231,7 +231,6 @@ extern "C" {
|
|||
size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); }
|
||||
int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); }
|
||||
|
||||
|
||||
// `aligned_alloc` is only available when __USE_ISOC11 is defined.
|
||||
// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot
|
||||
// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9.
|
||||
|
|
@ -246,6 +245,7 @@ extern "C" {
|
|||
void cfree(void* p) { mi_free(p); }
|
||||
void* pvalloc(size_t size) { return mi_pvalloc(size); }
|
||||
void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); }
|
||||
int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); }
|
||||
void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }
|
||||
void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); }
|
||||
|
||||
|
|
|
|||
|
|
@ -56,9 +56,9 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept
|
|||
// Note: The spec dictates we should not modify `*p` on an error. (issue#27)
|
||||
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
|
||||
if (p == NULL) return EINVAL;
|
||||
if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment
|
||||
if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
|
||||
void* q = (mi_malloc_satisfies_alignment(alignment, size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment));
|
||||
if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment
|
||||
if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
|
||||
void* q = mi_malloc_aligned(size, alignment);
|
||||
if (q==NULL && size != 0) return ENOMEM;
|
||||
mi_assert_internal(((uintptr_t)q % alignment) == 0);
|
||||
*p = q;
|
||||
|
|
@ -66,7 +66,7 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept
|
|||
}
|
||||
|
||||
mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
|
||||
void* p = (mi_malloc_satisfies_alignment(alignment,size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment));
|
||||
void* p = mi_malloc_aligned(size, alignment);
|
||||
mi_assert_internal(((uintptr_t)p % alignment) == 0);
|
||||
return p;
|
||||
}
|
||||
|
|
@ -83,22 +83,40 @@ mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept {
|
|||
}
|
||||
|
||||
mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
|
||||
if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL;
|
||||
if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
|
||||
void* p = (mi_malloc_satisfies_alignment(alignment, size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment));
|
||||
if (mi_unlikely((size&(alignment-1)) != 0)) { // C11 requires alignment>0 && integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
|
||||
#if MI_DEBUG > 0
|
||||
_mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment);
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
// C11 also requires alignment to be a power-of-two which is checked in mi_malloc_aligned
|
||||
void* p = mi_malloc_aligned(size, alignment);
|
||||
mi_assert_internal(((uintptr_t)p % alignment) == 0);
|
||||
return p;
|
||||
}
|
||||
|
||||
void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD
|
||||
void* newp = mi_reallocn(p,count,size);
|
||||
if (newp==NULL) errno = ENOMEM;
|
||||
if (newp==NULL) { errno = ENOMEM; }
|
||||
return newp;
|
||||
}
|
||||
|
||||
int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD
|
||||
mi_assert(p != NULL);
|
||||
if (p == NULL) {
|
||||
errno = EINVAL;
|
||||
return EINVAL;
|
||||
}
|
||||
void** op = (void**)p;
|
||||
void* newp = mi_reallocarray(*op, count, size);
|
||||
if (mi_unlikely(newp == NULL)) return errno;
|
||||
*op = newp;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
|
||||
void* res = mi_expand(p, newsize);
|
||||
if (res == NULL) errno = ENOMEM;
|
||||
if (res == NULL) { errno = ENOMEM; }
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
|
|||
56
src/init.c
56
src/init.c
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2022, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -268,12 +268,6 @@ static bool _mi_heap_done(mi_heap_t* heap) {
|
|||
|
||||
static void _mi_thread_done(mi_heap_t* default_heap);
|
||||
|
||||
#ifdef __wasi__
|
||||
// no pthreads in the WebAssembly Standard Interface
|
||||
#elif !defined(_WIN32)
|
||||
#define MI_USE_PTHREADS
|
||||
#endif
|
||||
|
||||
#if defined(_WIN32) && defined(MI_SHARED_LIB)
|
||||
// nothing to do as it is done in DllMain
|
||||
#elif defined(_WIN32) && !defined(MI_SHARED_LIB)
|
||||
|
|
@ -293,7 +287,6 @@ static void _mi_thread_done(mi_heap_t* default_heap);
|
|||
#elif defined(MI_USE_PTHREADS)
|
||||
// use pthread local storage keys to detect thread ending
|
||||
// (and used with MI_TLS_PTHREADS for the default heap)
|
||||
#include <pthread.h>
|
||||
pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
|
||||
static void mi_pthread_done(void* value) {
|
||||
if (value!=NULL) _mi_thread_done((mi_heap_t*)value);
|
||||
|
|
@ -493,6 +486,14 @@ void mi_process_init(void) mi_attr_noexcept {
|
|||
#endif
|
||||
_mi_verbose_message("secure level: %d\n", MI_SECURE);
|
||||
mi_thread_init();
|
||||
|
||||
#if defined(_WIN32) && !defined(MI_SHARED_LIB)
|
||||
// When building as a static lib the FLS cleanup happens to early for the main thread.
|
||||
// To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
|
||||
// will not call _mi_thread_done on the (still executing) main thread. See issue #508.
|
||||
FlsSetValue(mi_fls_key, NULL);
|
||||
#endif
|
||||
|
||||
mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL)
|
||||
|
||||
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
|
||||
|
|
@ -522,8 +523,7 @@ static void mi_process_done(void) {
|
|||
process_done = true;
|
||||
|
||||
#if defined(_WIN32) && !defined(MI_SHARED_LIB)
|
||||
FlsSetValue(mi_fls_key, NULL); // don't call main-thread callback
|
||||
FlsFree(mi_fls_key); // call thread-done on all threads to prevent dangling callback pointer if statically linked with a DLL; Issue #208
|
||||
FlsFree(mi_fls_key); // call thread-done on all threads (except the main thread) to prevent dangling callback pointer if statically linked with a DLL; Issue #208
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB)
|
||||
|
|
@ -557,6 +557,24 @@ static void mi_process_done(void) {
|
|||
return TRUE;
|
||||
}
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
// MSVC: use data section magic for static libraries
|
||||
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
|
||||
static int _mi_process_init(void) {
|
||||
mi_process_load();
|
||||
return 0;
|
||||
}
|
||||
typedef int(*_mi_crt_callback_t)(void);
|
||||
#if defined(_M_X64) || defined(_M_ARM64)
|
||||
__pragma(comment(linker, "/include:" "_mi_msvc_initu"))
|
||||
#pragma section(".CRT$XIU", long, read)
|
||||
#else
|
||||
__pragma(comment(linker, "/include:" "__mi_msvc_initu"))
|
||||
#endif
|
||||
#pragma data_seg(".CRT$XIU")
|
||||
extern "C" _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init };
|
||||
#pragma data_seg()
|
||||
|
||||
#elif defined(__cplusplus)
|
||||
// C++: use static initialization to detect process start
|
||||
static bool _mi_process_init(void) {
|
||||
|
|
@ -571,24 +589,6 @@ static void mi_process_done(void) {
|
|||
mi_process_load();
|
||||
}
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
// MSVC: use data section magic for static libraries
|
||||
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
|
||||
static int _mi_process_init(void) {
|
||||
mi_process_load();
|
||||
return 0;
|
||||
}
|
||||
typedef int(*_crt_cb)(void);
|
||||
#if defined(_M_X64) || defined(_M_ARM64)
|
||||
__pragma(comment(linker, "/include:" "_mi_msvc_initu"))
|
||||
#pragma section(".CRT$XIU", long, read)
|
||||
#else
|
||||
__pragma(comment(linker, "/include:" "__mi_msvc_initu"))
|
||||
#endif
|
||||
#pragma data_seg(".CRT$XIU")
|
||||
_crt_cb _mi_msvc_initu[] = { &_mi_process_init };
|
||||
#pragma data_seg()
|
||||
|
||||
#else
|
||||
#pragma message("define a way to call mi_process_load on your platform")
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -257,22 +257,37 @@ static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop
|
|||
|
||||
// When overriding malloc, we may recurse into mi_vfprintf if an allocation
|
||||
// inside the C runtime causes another message.
|
||||
// In some cases (like on macOS) the loader already allocates which
|
||||
// calls into mimalloc; if we then access thread locals (like `recurse`)
|
||||
// this may crash as the access may call _tlv_bootstrap that tries to
|
||||
// (recursively) invoke malloc again to allocate space for the thread local
|
||||
// variables on demand. This is why we use a _mi_preloading test on such
|
||||
// platforms. However, C code generator may move the initial thread local address
|
||||
// load before the `if` and we therefore split it out in a separate funcion.
|
||||
static mi_decl_thread bool recurse = false;
|
||||
|
||||
static mi_decl_noinline bool mi_recurse_enter_prim(void) {
|
||||
if (recurse) return false;
|
||||
recurse = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
static mi_decl_noinline void mi_recurse_exit_prim(void) {
|
||||
recurse = false;
|
||||
}
|
||||
|
||||
static bool mi_recurse_enter(void) {
|
||||
#if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
|
||||
if (_mi_preloading()) return true;
|
||||
#endif
|
||||
if (recurse) return false;
|
||||
recurse = true;
|
||||
return true;
|
||||
return mi_recurse_enter_prim();
|
||||
}
|
||||
|
||||
static void mi_recurse_exit(void) {
|
||||
#if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
|
||||
if (_mi_preloading()) return;
|
||||
#endif
|
||||
recurse = false;
|
||||
mi_recurse_exit_prim();
|
||||
}
|
||||
|
||||
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) {
|
||||
|
|
@ -471,14 +486,25 @@ void _mi_error_message(int err, const char* fmt, ...) {
|
|||
// --------------------------------------------------------
|
||||
|
||||
static void mi_strlcpy(char* dest, const char* src, size_t dest_size) {
|
||||
dest[0] = 0;
|
||||
strncpy(dest, src, dest_size - 1);
|
||||
dest[dest_size - 1] = 0;
|
||||
if (dest==NULL || src==NULL || dest_size == 0) return;
|
||||
// copy until end of src, or when dest is (almost) full
|
||||
while (*src != 0 && dest_size > 1) {
|
||||
*dest++ = *src++;
|
||||
dest_size--;
|
||||
}
|
||||
// always zero terminate
|
||||
*dest = 0;
|
||||
}
|
||||
|
||||
static void mi_strlcat(char* dest, const char* src, size_t dest_size) {
|
||||
strncat(dest, src, dest_size - 1);
|
||||
dest[dest_size - 1] = 0;
|
||||
if (dest==NULL || src==NULL || dest_size == 0) return;
|
||||
// find end of string in the dest buffer
|
||||
while (*dest != 0 && dest_size > 1) {
|
||||
dest++;
|
||||
dest_size--;
|
||||
}
|
||||
// and catenate
|
||||
mi_strlcpy(dest, src, dest_size);
|
||||
}
|
||||
|
||||
#ifdef MI_NO_GETENV
|
||||
|
|
@ -596,15 +622,26 @@ static void mi_option_init(mi_option_desc_t* desc) {
|
|||
else if (*end == 'M') { value *= MI_KiB; end++; }
|
||||
else if (*end == 'G') { value *= MI_MiB; end++; }
|
||||
else { value = (value + MI_KiB - 1) / MI_KiB; }
|
||||
if (*end == 'B') { end++; }
|
||||
if (end[0] == 'I' && end[1] == 'B') { end += 2; }
|
||||
else if (*end == 'B') { end++; }
|
||||
}
|
||||
if (*end == 0) {
|
||||
desc->value = value;
|
||||
desc->init = INITIALIZED;
|
||||
}
|
||||
else {
|
||||
_mi_warning_message("environment option mimalloc_%s has an invalid value: %s\n", desc->name, buf);
|
||||
// set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.
|
||||
desc->init = DEFAULTED;
|
||||
if (desc->option == mi_option_verbose && desc->value == 0) {
|
||||
// if the 'mimalloc_verbose' env var has a bogus value we'd never know
|
||||
// (since the value defaults to 'off') so in that case briefly enable verbose
|
||||
desc->value = 1;
|
||||
_mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name );
|
||||
desc->value = 0;
|
||||
}
|
||||
else {
|
||||
_mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name );
|
||||
}
|
||||
}
|
||||
}
|
||||
mi_assert_internal(desc->init != UNINIT);
|
||||
|
|
|
|||
229
src/os.c
229
src/os.c
|
|
@ -88,6 +88,7 @@ static void* mi_align_down_ptr(void* p, size_t alignment) {
|
|||
return (void*)_mi_align_down((uintptr_t)p, alignment);
|
||||
}
|
||||
|
||||
|
||||
// page size (initialized properly in `os_init`)
|
||||
static size_t os_page_size = 4096;
|
||||
|
||||
|
|
@ -231,7 +232,7 @@ void _mi_os_init(void)
|
|||
#elif defined(__wasi__)
|
||||
void _mi_os_init() {
|
||||
os_overcommit = false;
|
||||
os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB
|
||||
os_page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
|
||||
os_alloc_granularity = 16;
|
||||
}
|
||||
|
||||
|
|
@ -241,7 +242,7 @@ static void os_detect_overcommit(void) {
|
|||
#if defined(__linux__)
|
||||
int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
if (fd < 0) return;
|
||||
char buf[128];
|
||||
char buf[32];
|
||||
ssize_t nread = read(fd, &buf, sizeof(buf));
|
||||
close(fd);
|
||||
// <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
|
||||
|
|
@ -273,9 +274,20 @@ void _mi_os_init() {
|
|||
#endif
|
||||
|
||||
|
||||
#if defined(MADV_NORMAL)
|
||||
static int mi_madvise(void* addr, size_t length, int advice) {
|
||||
#if defined(__sun)
|
||||
return madvise((caddr_t)addr, length, advice); // Solaris needs cast (issue #520)
|
||||
#else
|
||||
return madvise(addr, length, advice);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).
|
||||
----------------------------------------------------------- */
|
||||
free memory
|
||||
-------------------------------------------------------------- */
|
||||
|
||||
static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats_t* stats)
|
||||
{
|
||||
|
|
@ -283,7 +295,7 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats
|
|||
bool err = false;
|
||||
#if defined(_WIN32)
|
||||
err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
|
||||
#elif defined(MI_USE_SBRK)
|
||||
#elif defined(MI_USE_SBRK) || defined(__wasi__)
|
||||
err = 0; // sbrk heap cannot be shrunk
|
||||
#else
|
||||
err = (munmap(addr, size) == -1);
|
||||
|
|
@ -303,6 +315,10 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats
|
|||
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size);
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Raw allocation on Windows (VirtualAlloc)
|
||||
-------------------------------------------------------------- */
|
||||
|
||||
#ifdef _WIN32
|
||||
static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
#if (MI_INTPTR_SIZE >= 8)
|
||||
|
|
@ -326,7 +342,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
|
|||
#endif
|
||||
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
|
||||
// on modern Windows try use VirtualAlloc2 for aligned allocation
|
||||
if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
|
||||
if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
|
||||
MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
|
||||
reqs.Alignment = try_alignment;
|
||||
MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
|
||||
|
|
@ -375,52 +391,93 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
|
|||
return p;
|
||||
}
|
||||
|
||||
#elif defined(MI_USE_SBRK)
|
||||
#define MI_SBRK_FAIL ((void*)(-1))
|
||||
static void* mi_sbrk_heap_grow(size_t size, size_t try_alignment) {
|
||||
void* pbase0 = sbrk(0);
|
||||
if (pbase0 == MI_SBRK_FAIL) {
|
||||
_mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size);
|
||||
/* -----------------------------------------------------------
|
||||
Raw allocation using `sbrk` or `wasm_memory_grow`
|
||||
-------------------------------------------------------------- */
|
||||
|
||||
#elif defined(MI_USE_SBRK) || defined(__wasi__)
|
||||
#if defined(MI_USE_SBRK)
|
||||
static void* mi_memory_grow( size_t size ) {
|
||||
void* p = sbrk(size);
|
||||
if (p == (void*)(-1)) return NULL;
|
||||
#if !defined(__wasi__) // on wasi this is always zero initialized already (?)
|
||||
memset(p,0,size);
|
||||
#endif
|
||||
return p;
|
||||
}
|
||||
#elif defined(__wasi__)
|
||||
static void* mi_memory_grow( size_t size ) {
|
||||
size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size()))
|
||||
: __builtin_wasm_memory_size(0));
|
||||
if (base == SIZE_MAX) return NULL;
|
||||
return (void*)(base * _mi_os_page_size());
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
#endif
|
||||
|
||||
static void* mi_heap_grow(size_t size, size_t try_alignment) {
|
||||
void* p = NULL;
|
||||
if (try_alignment <= 1) {
|
||||
// `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now)
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_lock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
p = mi_memory_grow(size);
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_unlock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
void* base = NULL;
|
||||
size_t alloc_size = 0;
|
||||
// to allocate aligned use a lock to try to avoid thread interaction
|
||||
// between getting the current size and actual allocation
|
||||
// (also, `sbrk` is not thread safe in general)
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_lock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
{
|
||||
void* current = mi_memory_grow(0); // get current size
|
||||
if (current != NULL) {
|
||||
void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space
|
||||
alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size());
|
||||
base = mi_memory_grow(alloc_size);
|
||||
}
|
||||
}
|
||||
#if defined(MI_USE_PTHREADS)
|
||||
pthread_mutex_unlock(&mi_heap_grow_mutex);
|
||||
#endif
|
||||
if (base != NULL) {
|
||||
p = mi_align_up_ptr(base, try_alignment);
|
||||
if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) {
|
||||
// another thread used wasm_memory_grow/sbrk in-between and we do not have enough
|
||||
// space after alignment. Give up (and waste the space as we cannot shrink :-( )
|
||||
// (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align)
|
||||
p = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (p == NULL) {
|
||||
_mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment);
|
||||
errno = ENOMEM;
|
||||
return NULL;
|
||||
}
|
||||
uintptr_t base = (uintptr_t)pbase0;
|
||||
uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment);
|
||||
size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size());
|
||||
mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0);
|
||||
if (alloc_size < size) return NULL;
|
||||
void* pbase1 = sbrk(alloc_size);
|
||||
if (pbase1 == MI_SBRK_FAIL) {
|
||||
_mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes, %zu requested)\n", size, alloc_size);
|
||||
errno = ENOMEM;
|
||||
return NULL;
|
||||
}
|
||||
mi_assert(pbase0 == pbase1);
|
||||
return (void*)aligned_base;
|
||||
mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 );
|
||||
return p;
|
||||
}
|
||||
|
||||
#elif defined(__wasi__)
|
||||
// currently unused as we use sbrk() on wasm
|
||||
static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
|
||||
uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size();
|
||||
uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment);
|
||||
size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size());
|
||||
mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0);
|
||||
if (alloc_size < size) return NULL;
|
||||
if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) {
|
||||
_mi_warning_message("unable to allocate wasm_memory_grow() OS memory (%zu bytes, %zu requested)\n", size, alloc_size);
|
||||
errno = ENOMEM;
|
||||
return NULL;
|
||||
}
|
||||
return (void*)aligned_base;
|
||||
}
|
||||
|
||||
#else
|
||||
/* -----------------------------------------------------------
|
||||
Raw allocation on Unix's (mmap)
|
||||
-------------------------------------------------------------- */
|
||||
#else
|
||||
#define MI_OS_USE_MMAP
|
||||
static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
|
||||
MI_UNUSED(try_alignment);
|
||||
#if defined(MAP_ALIGNED) // BSD
|
||||
if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
size_t n = mi_bsr(try_alignment);
|
||||
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
|
||||
flags |= MAP_ALIGNED(n);
|
||||
|
|
@ -430,8 +487,8 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr
|
|||
}
|
||||
}
|
||||
#elif defined(MAP_ALIGN) // Solaris
|
||||
if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0);
|
||||
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
|
||||
void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
}
|
||||
|
|
@ -543,7 +600,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
|
|||
// However, some systems only allow THP if called with explicit `madvise`, so
|
||||
// when large OS pages are enabled for mimalloc, we call `madvise` anyways.
|
||||
if (allow_large && use_large_os_page(size, try_alignment)) {
|
||||
if (madvise(p, size, MADV_HUGEPAGE) == 0) {
|
||||
if (mi_madvise(p, size, MADV_HUGEPAGE) == 0) {
|
||||
*is_large = true; // possibly
|
||||
};
|
||||
}
|
||||
|
|
@ -552,7 +609,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
|
|||
struct memcntl_mha cmd = {0};
|
||||
cmd.mha_pagesize = large_os_page_size;
|
||||
cmd.mha_cmd = MHA_MAPSIZE_VA;
|
||||
if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
|
||||
if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
|
||||
*is_large = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -584,7 +641,7 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base;
|
|||
|
||||
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size)
|
||||
{
|
||||
if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL;
|
||||
if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;
|
||||
size = _mi_align_up(size, MI_SEGMENT_SIZE);
|
||||
if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
|
||||
#if (MI_SECURE>0)
|
||||
|
|
@ -614,13 +671,16 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
|
|||
}
|
||||
#endif
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Primitive allocation from the OS.
|
||||
-------------------------------------------------------------- */
|
||||
|
||||
// Primitive allocation from the OS.
|
||||
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
|
||||
static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) {
|
||||
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
|
||||
if (size == 0) return NULL;
|
||||
if (!commit) allow_large = false;
|
||||
if (try_alignment == 0) try_alignment = 1; // avoid 0 to ensure there will be no divide by zero when aligning
|
||||
|
||||
void* p = NULL;
|
||||
/*
|
||||
|
|
@ -637,14 +697,10 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo
|
|||
int flags = MEM_RESERVE;
|
||||
if (commit) flags |= MEM_COMMIT;
|
||||
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
|
||||
#elif defined(MI_USE_SBRK)
|
||||
#elif defined(MI_USE_SBRK) || defined(__wasi__)
|
||||
MI_UNUSED(allow_large);
|
||||
*is_large = false;
|
||||
p = mi_sbrk_heap_grow(size, try_alignment);
|
||||
#elif defined(__wasi__)
|
||||
MI_UNUSED(allow_large);
|
||||
*is_large = false;
|
||||
p = mi_wasm_heap_grow(size, try_alignment);
|
||||
p = mi_heap_grow(size, try_alignment);
|
||||
#else
|
||||
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
|
||||
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
|
||||
|
|
@ -712,9 +768,9 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
|
|||
}
|
||||
#else
|
||||
// overallocate...
|
||||
p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats);
|
||||
p = mi_os_mem_alloc(over_size, 1, commit, false, is_large, stats);
|
||||
if (p == NULL) return NULL;
|
||||
// and selectively unmap parts around the over-allocated area.
|
||||
// and selectively unmap parts around the over-allocated area. (noop on sbrk)
|
||||
void* aligned_p = mi_align_up_ptr(p, alignment);
|
||||
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
|
||||
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
|
||||
|
|
@ -722,7 +778,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
|
|||
mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
|
||||
if (pre_size > 0) mi_os_mem_free(p, pre_size, commit, stats);
|
||||
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats);
|
||||
// we can return the aligned pointer on `mmap` systems
|
||||
// we can return the aligned pointer on `mmap` (and sbrk) systems
|
||||
p = aligned_p;
|
||||
#endif
|
||||
}
|
||||
|
|
@ -833,8 +889,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
|
|||
|
||||
#if defined(_WIN32)
|
||||
if (commit) {
|
||||
// if the memory was already committed, the call succeeds but it is not zero'd
|
||||
// *is_zero = true;
|
||||
// *is_zero = true; // note: if the memory was already committed, the call succeeds but the memory is not zero'd
|
||||
void* p = VirtualAlloc(start, csize, MEM_COMMIT, PAGE_READWRITE);
|
||||
err = (p == start ? 0 : GetLastError());
|
||||
}
|
||||
|
|
@ -844,23 +899,40 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
|
|||
}
|
||||
#elif defined(__wasi__)
|
||||
// WebAssembly guests can't control memory protection
|
||||
#elif defined(MAP_FIXED)
|
||||
if (!commit) {
|
||||
// use mmap with MAP_FIXED to discard the existing memory (and reduce commit charge)
|
||||
void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
|
||||
if (p != start) { err = errno; }
|
||||
}
|
||||
else {
|
||||
// for commit, just change the protection
|
||||
#elif 0 && defined(MAP_FIXED) && !defined(__APPLE__)
|
||||
// Linux: disabled for now as mmap fixed seems much more expensive than MADV_DONTNEED (and splits VMA's?)
|
||||
if (commit) {
|
||||
// commit: just change the protection
|
||||
err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) { err = errno; }
|
||||
//#if defined(MADV_FREE_REUSE)
|
||||
// while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; }
|
||||
//#endif
|
||||
}
|
||||
else {
|
||||
// decommit: use mmap with MAP_FIXED to discard the existing memory (and reduce rss)
|
||||
const int fd = mi_unix_mmap_fd();
|
||||
void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
|
||||
if (p != start) { err = errno; }
|
||||
}
|
||||
#else
|
||||
err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
|
||||
if (err != 0) { err = errno; }
|
||||
// Linux, macOSX and others.
|
||||
if (commit) {
|
||||
// commit: ensure we can access the area
|
||||
err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
|
||||
if (err != 0) { err = errno; }
|
||||
}
|
||||
else {
|
||||
#if defined(MADV_DONTNEED) && MI_DEBUG == 0 && MI_SECURE == 0
|
||||
// decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
|
||||
// (on the other hand, MADV_FREE would be good enough.. it is just not reflected in the stats :-( )
|
||||
err = madvise(start, csize, MADV_DONTNEED);
|
||||
#else
|
||||
// decommit: just disable access (also used in debug and secure mode to trap on illegal access)
|
||||
err = mprotect(start, csize, PROT_NONE);
|
||||
if (err != 0) { err = errno; }
|
||||
#endif
|
||||
//#if defined(MADV_FREE_REUSE)
|
||||
// while ((err = mi_madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; }
|
||||
//#endif
|
||||
}
|
||||
#endif
|
||||
if (err != 0) {
|
||||
_mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err);
|
||||
|
|
@ -921,16 +993,16 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
|
|||
static _Atomic(size_t) advice = ATOMIC_VAR_INIT(MADV_FREE);
|
||||
int oadvice = (int)mi_atomic_load_relaxed(&advice);
|
||||
int err;
|
||||
while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; };
|
||||
while ((err = mi_madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; };
|
||||
if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) {
|
||||
// if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
|
||||
mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);
|
||||
err = madvise(start, csize, MADV_DONTNEED);
|
||||
err = mi_madvise(start, csize, MADV_DONTNEED);
|
||||
}
|
||||
#elif defined(__wasi__)
|
||||
int err = 0;
|
||||
#else
|
||||
int err = madvise(start, csize, MADV_DONTNEED);
|
||||
int err = mi_madvise(start, csize, MADV_DONTNEED);
|
||||
#endif
|
||||
if (err != 0) {
|
||||
_mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, errno);
|
||||
|
|
@ -1209,8 +1281,8 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
|
|||
}
|
||||
}
|
||||
mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size);
|
||||
if (pages_reserved != NULL) *pages_reserved = page;
|
||||
if (psize != NULL) *psize = page * MI_HUGE_OS_PAGE_SIZE;
|
||||
if (pages_reserved != NULL) { *pages_reserved = page; }
|
||||
if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; }
|
||||
return (page == 0 ? NULL : start);
|
||||
}
|
||||
|
||||
|
|
@ -1222,6 +1294,7 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
|
|||
while (size >= MI_HUGE_OS_PAGE_SIZE) {
|
||||
_mi_os_free(base, MI_HUGE_OS_PAGE_SIZE, stats);
|
||||
size -= MI_HUGE_OS_PAGE_SIZE;
|
||||
base += MI_HUGE_OS_PAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -621,7 +621,11 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
|
|||
page->keys[0] = _mi_heap_random_next(heap);
|
||||
page->keys[1] = _mi_heap_random_next(heap);
|
||||
#endif
|
||||
#if MI_DEBUG > 0
|
||||
page->is_zero = false; // ensure in debug mode we initialize with MI_DEBUG_UNINIT, see issue #501
|
||||
#else
|
||||
page->is_zero = page->is_zero_init;
|
||||
#endif
|
||||
|
||||
mi_assert_internal(page->capacity == 0);
|
||||
mi_assert_internal(page->free == NULL);
|
||||
|
|
|
|||
22
src/random.c
22
src/random.c
|
|
@ -160,7 +160,8 @@ uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {
|
|||
/* ----------------------------------------------------------------------------
|
||||
To initialize a fresh random context we rely on the OS:
|
||||
- Windows : BCryptGenRandom (or RtlGenRandom)
|
||||
- osX,bsd,wasi: arc4random_buf
|
||||
- macOS : CCRandomGenerateBytes, arc4random_buf
|
||||
- bsd,wasi : arc4random_buf
|
||||
- Linux : getrandom,/dev/urandom
|
||||
If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
|
@ -191,7 +192,24 @@ static bool os_random_buf(void* buf, size_t buf_len) {
|
|||
}
|
||||
#endif
|
||||
|
||||
#elif defined(ANDROID) || defined(XP_DARWIN) || defined(__APPLE__) || defined(__DragonFly__) || \
|
||||
#elif defined(__APPLE__)
|
||||
#include <AvailabilityMacros.h>
|
||||
#if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10
|
||||
#include <CommonCrypto/CommonRandom.h>
|
||||
#endif
|
||||
static bool os_random_buf(void* buf, size_t buf_len) {
|
||||
#if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15
|
||||
// We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf
|
||||
// may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html>
|
||||
return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
|
||||
#else
|
||||
// fall back on older macOS
|
||||
arc4random_buf(buf, buf_len);
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
#elif defined(__ANDROID__) || defined(__DragonFly__) || \
|
||||
defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \
|
||||
defined(__sun) // todo: what to use with __wasi__?
|
||||
#include <stdlib.h>
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ typedef struct mem_region_s {
|
|||
mi_bitmap_field_t commit; // track if committed per block
|
||||
mi_bitmap_field_t reset; // track if reset per block
|
||||
_Atomic(size_t) arena_memid; // if allocated from a (huge page) arena
|
||||
size_t padding; // round to 8 fields
|
||||
_Atomic(size_t) padding; // round to 8 fields (needs to be atomic for msvc, see issue #508)
|
||||
} mem_region_t;
|
||||
|
||||
// The region map
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue