mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-07-06 11:34:38 +03:00
Merge branch 'dev' into netbsd_reallocarr
This commit is contained in:
Daan
GitHub
commit
1b09877bc4
37 changed files with 1377 additions and 758 deletions
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018,2020 Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2021 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -25,7 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_memory_order(name) std::memory_order_##name
|
||||
#elif defined(_MSC_VER)
|
||||
// Use MSVC C wrapper for C11 atomics
|
||||
#define _Atomic(tp) tp
|
||||
#define _Atomic(tp) tp
|
||||
#define ATOMIC_VAR_INIT(x) x
|
||||
#define mi_atomic(name) mi_atomic_##name
|
||||
#define mi_memory_order(name) mi_memory_order_##name
|
||||
|
|
@ -118,7 +118,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
|
|||
|
||||
// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics.
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <Windows.h>
|
||||
#include <windows.h>
|
||||
#include <intrin.h>
|
||||
#ifdef _WIN64
|
||||
typedef LONG64 msc_intptr_t;
|
||||
|
|
@ -173,7 +173,7 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt
|
|||
}
|
||||
static inline void mi_atomic_thread_fence(mi_memory_order mo) {
|
||||
(void)(mo);
|
||||
_Atomic(uintptr_t)x = 0;
|
||||
_Atomic(uintptr_t) x = 0;
|
||||
mi_atomic_exchange_explicit(&x, 1, mo);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
|
||||
|
|
@ -295,7 +295,7 @@ static inline void mi_atomic_yield(void) {
|
|||
}
|
||||
#elif defined(__aarch64__)
|
||||
static inline void mi_atomic_yield(void) {
|
||||
asm volatile("wfe");
|
||||
__asm__ volatile("wfe");
|
||||
}
|
||||
#elif (defined(__arm__) && __ARM_ARCH__ >= 7)
|
||||
static inline void mi_atomic_yield(void) {
|
||||
|
|
@ -307,7 +307,7 @@ static inline void mi_atomic_yield(void) {
|
|||
}
|
||||
#elif defined(__armel__) || defined(__ARMEL__)
|
||||
static inline void mi_atomic_yield(void) {
|
||||
asm volatile ("nop" ::: "memory");
|
||||
__asm__ volatile ("nop" ::: "memory");
|
||||
}
|
||||
#endif
|
||||
#elif defined(__sun)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_decl_noinline __declspec(noinline)
|
||||
#define mi_decl_thread __declspec(thread)
|
||||
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
|
||||
#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
|
||||
#define mi_decl_noinline __attribute__((noinline))
|
||||
#define mi_decl_thread __thread
|
||||
#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
|
||||
|
|
@ -32,6 +32,16 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_decl_cache_align
|
||||
#endif
|
||||
|
||||
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
|
||||
#define __wasi__
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus)
|
||||
#define mi_decl_externc extern "C"
|
||||
#else
|
||||
#define mi_decl_externc
|
||||
#endif
|
||||
|
||||
// "options.c"
|
||||
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
|
||||
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
|
||||
|
|
@ -46,14 +56,15 @@ void _mi_random_init(mi_random_ctx_t* ctx);
|
|||
void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
|
||||
uintptr_t _mi_random_next(mi_random_ctx_t* ctx);
|
||||
uintptr_t _mi_heap_random_next(mi_heap_t* heap);
|
||||
uintptr_t _os_random_weak(uintptr_t extra_seed);
|
||||
uintptr_t _mi_os_random_weak(uintptr_t extra_seed);
|
||||
static inline uintptr_t _mi_random_shuffle(uintptr_t x);
|
||||
|
||||
// init.c
|
||||
extern mi_decl_cache_align mi_stats_t _mi_stats_main;
|
||||
extern mi_decl_cache_align const mi_page_t _mi_page_empty;
|
||||
bool _mi_is_main_thread(void);
|
||||
bool _mi_preloading(); // true while the C runtime is not ready
|
||||
size_t _mi_current_thread_count(void);
|
||||
bool _mi_preloading(void); // true while the C runtime is not ready
|
||||
|
||||
// os.c
|
||||
size_t _mi_os_page_size(void);
|
||||
|
|
@ -61,6 +72,7 @@ void _mi_os_init(void); // called fro
|
|||
void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data
|
||||
void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data
|
||||
size_t _mi_os_good_alloc_size(size_t size);
|
||||
bool _mi_os_has_overcommit(void);
|
||||
|
||||
// memory.c
|
||||
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld);
|
||||
|
|
@ -90,7 +102,7 @@ void _mi_abandoned_await_readers(void);
|
|||
// "page.c"
|
||||
void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc;
|
||||
|
||||
void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks
|
||||
void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks
|
||||
void _mi_page_unfull(mi_page_t* page);
|
||||
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page
|
||||
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
|
||||
|
|
@ -176,11 +188,11 @@ bool _mi_page_is_valid(mi_page_t* page);
|
|||
/* -----------------------------------------------------------
|
||||
Inlined definitions
|
||||
----------------------------------------------------------- */
|
||||
#define UNUSED(x) (void)(x)
|
||||
#define MI_UNUSED(x) (void)(x)
|
||||
#if (MI_DEBUG>0)
|
||||
#define UNUSED_RELEASE(x)
|
||||
#define MI_UNUSED_RELEASE(x)
|
||||
#else
|
||||
#define UNUSED_RELEASE(x) UNUSED(x)
|
||||
#define MI_UNUSED_RELEASE(x) MI_UNUSED(x)
|
||||
#endif
|
||||
|
||||
#define MI_INIT4(x) x(),x(),x(),x()
|
||||
|
|
@ -236,18 +248,18 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size)
|
|||
}
|
||||
|
||||
// Overflow detecting multiply
|
||||
#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5
|
||||
#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5))
|
||||
#include <limits.h> // UINT_MAX, ULONG_MAX
|
||||
#if defined(_CLOCK_T) // for Illumos
|
||||
#undef _CLOCK_T
|
||||
#endif
|
||||
static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
|
||||
#if (SIZE_MAX == UINT_MAX)
|
||||
return __builtin_umul_overflow(count, size, total);
|
||||
#elif (SIZE_MAX == ULONG_MAX)
|
||||
return __builtin_umull_overflow(count, size, total);
|
||||
#if (SIZE_MAX == ULONG_MAX)
|
||||
return __builtin_umull_overflow(count, size, (unsigned long *)total);
|
||||
#elif (SIZE_MAX == UINT_MAX)
|
||||
return __builtin_umul_overflow(count, size, (unsigned int *)total);
|
||||
#else
|
||||
return __builtin_umulll_overflow(count, size, total);
|
||||
return __builtin_umulll_overflow(count, size, (unsigned long long *)total);
|
||||
#endif
|
||||
}
|
||||
#else /* __builtin_umul_overflow is unavailable */
|
||||
|
|
@ -293,8 +305,9 @@ extern bool _mi_process_is_initialized;
|
|||
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
|
||||
|
||||
#if defined(MI_MALLOC_OVERRIDE)
|
||||
#if defined(__MACH__) // OSX
|
||||
#if defined(__APPLE__) // macOS
|
||||
#define MI_TLS_SLOT 89 // seems unused?
|
||||
// #define MI_TLS_RECURSE_GUARD 1
|
||||
// other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89)
|
||||
// see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
|
||||
#elif defined(__OpenBSD__)
|
||||
|
|
@ -324,14 +337,20 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) {
|
|||
#elif defined(MI_TLS_PTHREAD)
|
||||
#include <pthread.h>
|
||||
extern pthread_key_t _mi_heap_default_key;
|
||||
#else
|
||||
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
|
||||
#endif
|
||||
|
||||
// Default heap to allocate from (if not using TLS- or pthread slots).
|
||||
// Do not use this directly but use through `mi_heap_get_default()` (or the unchecked `mi_get_default_heap`).
|
||||
// This thread local variable is only used when neither MI_TLS_SLOT, MI_TLS_PTHREAD, or MI_TLS_PTHREAD_SLOT_OFS are defined.
|
||||
// However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356).
|
||||
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
|
||||
|
||||
|
||||
static inline mi_heap_t* mi_get_default_heap(void) {
|
||||
#if defined(MI_TLS_SLOT)
|
||||
mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT);
|
||||
return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
|
||||
if (mi_unlikely(heap == NULL)) { heap = (mi_heap_t*)&_mi_heap_empty; } //_mi_heap_empty_get(); }
|
||||
return heap;
|
||||
#elif defined(MI_TLS_PTHREAD_SLOT_OFS)
|
||||
mi_heap_t* heap = *mi_tls_pthread_heap_slot();
|
||||
return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
|
||||
|
|
@ -339,7 +358,7 @@ static inline mi_heap_t* mi_get_default_heap(void) {
|
|||
mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key));
|
||||
return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
|
||||
#else
|
||||
#if defined(MI_TLS_RECURSE_GUARD)
|
||||
#if defined(MI_TLS_RECURSE_GUARD)
|
||||
if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get();
|
||||
#endif
|
||||
return _mi_heap_default;
|
||||
|
|
@ -395,11 +414,11 @@ static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) {
|
|||
}
|
||||
|
||||
// used internally
|
||||
static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) {
|
||||
static inline size_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) {
|
||||
// if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages
|
||||
ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment;
|
||||
mi_assert_internal(diff >= 0 && (size_t)diff < MI_SEGMENT_SIZE);
|
||||
uintptr_t idx = (uintptr_t)diff >> segment->page_shift;
|
||||
size_t idx = (size_t)diff >> segment->page_shift;
|
||||
mi_assert_internal(idx < segment->capacity);
|
||||
mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0);
|
||||
return idx;
|
||||
|
|
@ -407,7 +426,7 @@ static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, con
|
|||
|
||||
// Get the page containing the pointer
|
||||
static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) {
|
||||
uintptr_t idx = _mi_segment_page_idx_of(segment, p);
|
||||
size_t idx = _mi_segment_page_idx_of(segment, p);
|
||||
return &((mi_segment_t*)segment)->pages[idx];
|
||||
}
|
||||
|
||||
|
|
@ -423,7 +442,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) {
|
|||
return _mi_segment_page_of(_mi_ptr_segment(p), p);
|
||||
}
|
||||
|
||||
// Get the block size of a page (special cased for huge objects)
|
||||
// Get the block size of a page (special case for huge objects)
|
||||
static inline size_t mi_page_block_size(const mi_page_t* page) {
|
||||
const size_t bsize = page->xblock_size;
|
||||
mi_assert_internal(bsize > 0);
|
||||
|
|
@ -565,8 +584,8 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) {
|
|||
mi_segment_t* segmentp = _mi_ptr_segment(p);
|
||||
mi_segment_t* segmentq = _mi_ptr_segment(q);
|
||||
if (segmentp != segmentq) return false;
|
||||
uintptr_t idxp = _mi_segment_page_idx_of(segmentp, p);
|
||||
uintptr_t idxq = _mi_segment_page_idx_of(segmentq, q);
|
||||
size_t idxp = _mi_segment_page_idx_of(segmentp, p);
|
||||
size_t idxq = _mi_segment_page_idx_of(segmentq, q);
|
||||
return (idxp == idxq);
|
||||
}
|
||||
|
||||
|
|
@ -593,7 +612,7 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl
|
|||
#ifdef MI_ENCODE_FREELIST
|
||||
return (mi_block_t*)mi_ptr_decode(null, block->next, keys);
|
||||
#else
|
||||
UNUSED(keys); UNUSED(null);
|
||||
MI_UNUSED(keys); MI_UNUSED(null);
|
||||
return (mi_block_t*)block->next;
|
||||
#endif
|
||||
}
|
||||
|
|
@ -602,7 +621,7 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const
|
|||
#ifdef MI_ENCODE_FREELIST
|
||||
block->next = mi_ptr_encode(null, next, keys);
|
||||
#else
|
||||
UNUSED(keys); UNUSED(null);
|
||||
MI_UNUSED(keys); MI_UNUSED(null);
|
||||
block->next = (mi_encoded_t)next;
|
||||
#endif
|
||||
}
|
||||
|
|
@ -618,7 +637,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t*
|
|||
}
|
||||
return next;
|
||||
#else
|
||||
UNUSED(page);
|
||||
MI_UNUSED(page);
|
||||
return mi_block_nextx(page,block,NULL);
|
||||
#endif
|
||||
}
|
||||
|
|
@ -627,7 +646,7 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c
|
|||
#ifdef MI_ENCODE_FREELIST
|
||||
mi_block_set_nextx(page,block,next, page->keys);
|
||||
#else
|
||||
UNUSED(page);
|
||||
MI_UNUSED(page);
|
||||
mi_block_set_nextx(page,block,next,NULL);
|
||||
#endif
|
||||
}
|
||||
|
|
@ -663,13 +682,14 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
|
|||
int _mi_os_numa_node_get(mi_os_tld_t* tld);
|
||||
size_t _mi_os_numa_node_count_get(void);
|
||||
|
||||
extern size_t _mi_numa_node_count;
|
||||
extern _Atomic(size_t) _mi_numa_node_count;
|
||||
static inline int _mi_os_numa_node(mi_os_tld_t* tld) {
|
||||
if (mi_likely(_mi_numa_node_count == 1)) return 0;
|
||||
if (mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1)) return 0;
|
||||
else return _mi_os_numa_node_get(tld);
|
||||
}
|
||||
static inline size_t _mi_os_numa_node_count(void) {
|
||||
if (mi_likely(_mi_numa_node_count>0)) return _mi_numa_node_count;
|
||||
const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count);
|
||||
if (mi_likely(count>0)) return count;
|
||||
else return _mi_os_numa_node_count_get();
|
||||
}
|
||||
|
||||
|
|
@ -681,7 +701,7 @@ static inline size_t _mi_os_numa_node_count(void) {
|
|||
#if defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
|
||||
static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
|
||||
// Windows: works on Intel and ARM in both 32- and 64-bit
|
||||
return (uintptr_t)NtCurrentTeb();
|
||||
}
|
||||
|
|
@ -695,23 +715,24 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept {
|
|||
const size_t ofs = (slot*sizeof(void*));
|
||||
#if defined(__i386__)
|
||||
__asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // 32-bit always uses GS
|
||||
#elif defined(__MACH__) && defined(__x86_64__)
|
||||
#elif defined(__APPLE__) && defined(__x86_64__)
|
||||
__asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
|
||||
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
|
||||
__asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
void** tcb; UNUSED(ofs);
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
|
||||
res = tcb[slot];
|
||||
#elif defined(__aarch64__)
|
||||
void** tcb; UNUSED(ofs);
|
||||
#if defined(__APPLE__) // issue #343
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
#if defined(__APPLE__) // M1, issue #343
|
||||
__asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb));
|
||||
#else
|
||||
tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits
|
||||
#else
|
||||
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
|
||||
#endif
|
||||
#endif
|
||||
res = tcb[slot];
|
||||
#endif
|
||||
return res;
|
||||
|
|
@ -722,34 +743,40 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
|
|||
const size_t ofs = (slot*sizeof(void*));
|
||||
#if defined(__i386__)
|
||||
__asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
|
||||
#elif defined(__MACH__) && defined(__x86_64__)
|
||||
#elif defined(__APPLE__) && defined(__x86_64__)
|
||||
__asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS
|
||||
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
|
||||
__asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
|
||||
__asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
|
||||
#elif defined(__x86_64__)
|
||||
__asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
|
||||
__asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
|
||||
#elif defined(__arm__)
|
||||
void** tcb; UNUSED(ofs);
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
|
||||
tcb[slot] = value;
|
||||
#elif defined(__aarch64__)
|
||||
void** tcb; UNUSED(ofs);
|
||||
#if defined(__APPLE__) // issue #343
|
||||
void** tcb; MI_UNUSED(ofs);
|
||||
#if defined(__APPLE__) // M1, issue #343
|
||||
__asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb));
|
||||
#else
|
||||
tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits
|
||||
#else
|
||||
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
|
||||
#endif
|
||||
#endif
|
||||
tcb[slot] = value;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
|
||||
// in all our targets, slot 0 is the pointer to the thread control block
|
||||
static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
|
||||
#if defined(__arm__) || (defined(__BIONIC__) && defined(__aarch64__))
|
||||
// issue #384, #495: on arm32 and arm32/arm64 Android, slot 1 is the thread ID (pointer to pthread internal struct)
|
||||
return (uintptr_t)mi_tls_slot(1);
|
||||
#else
|
||||
// in all our other targets, slot 0 is the pointer to the thread control block
|
||||
return (uintptr_t)mi_tls_slot(0);
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
// otherwise use standard C
|
||||
static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
|
||||
static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
|
||||
return (uintptr_t)&_mi_heap_default;
|
||||
}
|
||||
#endif
|
||||
|
|
@ -891,7 +918,7 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
|
|||
// This is used for example in `mi_realloc`.
|
||||
// -------------------------------------------------------------------------------
|
||||
|
||||
#if (__GNUC__ >= 4) || defined(__clang__)
|
||||
#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
|
||||
// On GCC/CLang we provide a hint that the pointers are word aligned.
|
||||
#include <string.h>
|
||||
static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018,2019 Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018,2019 Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2020 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -10,14 +10,14 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
|
||||
#include <stddef.h> // ptrdiff_t
|
||||
#include <stdint.h> // uintptr_t, uint16_t, etc
|
||||
#include <mimalloc-atomic.h> // _Atomic
|
||||
#include "mimalloc-atomic.h" // _Atomic
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(disable:4214) // bitfield is not int
|
||||
#endif
|
||||
|
||||
// Minimal alignment necessary. On most platforms 16 bytes are needed
|
||||
// due to SSE registers for example. This must be at least `MI_INTPTR_SIZE`
|
||||
// due to SSE registers for example. This must be at least `sizeof(void*)`
|
||||
#ifndef MI_MAX_ALIGN_SIZE
|
||||
#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
|
||||
#endif
|
||||
|
|
@ -67,6 +67,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define MI_ENCODE_FREELIST 1
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Platform specific values
|
||||
// ------------------------------------------------------
|
||||
|
|
@ -83,20 +84,43 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
// or otherwise one might define an intptr_t type that is larger than a pointer...
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if INTPTR_MAX == 9223372036854775807LL
|
||||
#if INTPTR_MAX > INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)
|
||||
#elif INTPTR_MAX == INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (3)
|
||||
#elif INTPTR_MAX == 2147483647LL
|
||||
#elif INTPTR_MAX == INT32_MAX
|
||||
# define MI_INTPTR_SHIFT (2)
|
||||
#else
|
||||
#error platform must be 32 or 64 bits
|
||||
#error platform pointers must be 32, 64, or 128 bits
|
||||
#endif
|
||||
|
||||
#if SIZE_MAX == UINT64_MAX
|
||||
# define MI_SIZE_SHIFT (3)
|
||||
typedef int64_t mi_ssize_t;
|
||||
#elif SIZE_MAX == UINT32_MAX
|
||||
# define MI_SIZE_SHIFT (2)
|
||||
typedef int32_t mi_ssize_t;
|
||||
#else
|
||||
#error platform objects must be 32 or 64 bits
|
||||
#endif
|
||||
|
||||
#if (SIZE_MAX/2) > LONG_MAX
|
||||
# define MI_ZU(x) x##ULL
|
||||
# define MI_ZI(x) x##LL
|
||||
#else
|
||||
# define MI_ZU(x) x##UL
|
||||
# define MI_ZI(x) x##L
|
||||
#endif
|
||||
|
||||
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
|
||||
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
|
||||
|
||||
#define KiB ((size_t)1024)
|
||||
#define MiB (KiB*KiB)
|
||||
#define GiB (MiB*KiB)
|
||||
#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT)
|
||||
#define MI_SIZE_BITS (MI_SIZE_SIZE*8)
|
||||
|
||||
#define MI_KiB (MI_ZU(1024))
|
||||
#define MI_MiB (MI_KiB*MI_KiB)
|
||||
#define MI_GiB (MI_MiB*MI_KiB)
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
|
|
@ -105,18 +129,18 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
|
||||
// Main tuning parameters for segment and page sizes
|
||||
// Sizes for 64-bit, divide by two for 32-bit
|
||||
#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb
|
||||
#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb
|
||||
#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb
|
||||
#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb
|
||||
#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB
|
||||
#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB
|
||||
#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB
|
||||
#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB
|
||||
|
||||
// Derived constants
|
||||
#define MI_SEGMENT_SIZE (1UL<<MI_SEGMENT_SHIFT)
|
||||
#define MI_SEGMENT_MASK ((uintptr_t)MI_SEGMENT_SIZE - 1)
|
||||
#define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT)
|
||||
#define MI_SEGMENT_MASK (MI_SEGMENT_SIZE - 1)
|
||||
|
||||
#define MI_SMALL_PAGE_SIZE (1UL<<MI_SMALL_PAGE_SHIFT)
|
||||
#define MI_MEDIUM_PAGE_SIZE (1UL<<MI_MEDIUM_PAGE_SHIFT)
|
||||
#define MI_LARGE_PAGE_SIZE (1UL<<MI_LARGE_PAGE_SHIFT)
|
||||
#define MI_SMALL_PAGE_SIZE (MI_ZU(1)<<MI_SMALL_PAGE_SHIFT)
|
||||
#define MI_MEDIUM_PAGE_SIZE (MI_ZU(1)<<MI_MEDIUM_PAGE_SHIFT)
|
||||
#define MI_LARGE_PAGE_SIZE (MI_ZU(1)<<MI_LARGE_PAGE_SHIFT)
|
||||
|
||||
#define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE)
|
||||
#define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE)
|
||||
|
|
@ -124,9 +148,9 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
|
||||
// The max object size are checked to not waste more than 12.5% internally over the page sizes.
|
||||
// (Except for large pages since huge objects are allocated in 4MiB chunks)
|
||||
#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 16kb
|
||||
#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb
|
||||
#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2mb
|
||||
#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 16KiB
|
||||
#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB
|
||||
#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2MiB
|
||||
#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
|
||||
#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)
|
||||
|
||||
|
|
@ -140,9 +164,17 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
// Used as a special value to encode block sizes in 32 bits.
|
||||
#define MI_HUGE_BLOCK_SIZE ((uint32_t)MI_HUGE_OBJ_SIZE_MAX)
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Mimalloc pages contain allocated blocks
|
||||
// ------------------------------------------------------
|
||||
|
||||
// The free lists use encoded next fields
|
||||
// (Only actually encodes when MI_ENCODED_FREELIST is defined.)
|
||||
typedef uintptr_t mi_encoded_t;
|
||||
typedef uintptr_t mi_encoded_t;
|
||||
|
||||
// thread id's
|
||||
typedef size_t mi_threadid_t;
|
||||
|
||||
// free lists contain blocks
|
||||
typedef struct mi_block_s {
|
||||
|
|
@ -249,13 +281,13 @@ typedef struct mi_page_s {
|
|||
|
||||
|
||||
typedef enum mi_page_kind_e {
|
||||
MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment
|
||||
MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment
|
||||
MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment
|
||||
MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages inside a segment
|
||||
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
|
||||
MI_PAGE_HUGE // huge blocks (>512kb) are put into a single page in a segment of the exact size (but still 2mb aligned)
|
||||
MI_PAGE_HUGE // huge blocks (>512KiB) are put into a single page in a segment of the exact size (but still 2MiB aligned)
|
||||
} mi_page_kind_t;
|
||||
|
||||
// Segments are large allocated memory blocks (2mb on 64 bit) from
|
||||
// Segments are large allocated memory blocks (2MiB on 64 bit) from
|
||||
// the OS. Inside segments we allocated fixed size _pages_ that
|
||||
// contain blocks.
|
||||
typedef struct mi_segment_s {
|
||||
|
|
@ -270,7 +302,7 @@ typedef struct mi_segment_s {
|
|||
struct mi_segment_s* prev;
|
||||
|
||||
size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
|
||||
size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long)
|
||||
size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim if it is too long)
|
||||
|
||||
size_t used; // count of pages in use (`used <= capacity`)
|
||||
size_t capacity; // count of available pages (`#free + used`)
|
||||
|
|
@ -279,8 +311,8 @@ typedef struct mi_segment_s {
|
|||
uintptr_t cookie; // verify addresses in secure mode: `_mi_ptr_cookie(segment) == segment->cookie`
|
||||
|
||||
// layout like this to optimize access in `mi_free`
|
||||
size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`).
|
||||
_Atomic(uintptr_t) thread_id; // unique id of the thread owning this segment
|
||||
size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`).
|
||||
_Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment
|
||||
mi_page_kind_t page_kind; // kind of pages: small, large, or huge
|
||||
mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages
|
||||
} mi_segment_t;
|
||||
|
|
@ -319,7 +351,7 @@ typedef struct mi_random_cxt_s {
|
|||
} mi_random_ctx_t;
|
||||
|
||||
|
||||
// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows
|
||||
// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
|
||||
#if (MI_PADDING)
|
||||
typedef struct mi_padding_s {
|
||||
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
|
||||
|
|
@ -341,7 +373,7 @@ struct mi_heap_s {
|
|||
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
|
||||
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
|
||||
_Atomic(mi_block_t*) thread_delayed_free;
|
||||
uintptr_t thread_id; // thread this heap belongs too
|
||||
mi_threadid_t thread_id; // thread this heap belongs too
|
||||
uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
|
||||
uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list
|
||||
mi_random_ctx_t random; // random number context used for secure allocation
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#ifndef MIMALLOC_H
|
||||
#define MIMALLOC_H
|
||||
|
||||
#define MI_MALLOC_VERSION 170 // major + 2 digits minor
|
||||
#define MI_MALLOC_VERSION 174 // major + 2 digits minor
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Compiler specific attributes
|
||||
|
|
@ -26,7 +26,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
|
||||
#if defined(__cplusplus) && (__cplusplus >= 201703)
|
||||
#define mi_decl_nodiscard [[nodiscard]]
|
||||
#elif (__GNUC__ >= 4) || defined(__clang__) // includes clang, icc, and clang-cl
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl
|
||||
#define mi_decl_nodiscard __attribute__((warn_unused_result))
|
||||
#elif (_MSC_VER >= 1700)
|
||||
#define mi_decl_nodiscard _Check_return_
|
||||
|
|
@ -58,8 +58,12 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#define mi_attr_alloc_align(p)
|
||||
#elif defined(__GNUC__) // includes clang and icc
|
||||
#if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT)
|
||||
#define mi_decl_export __attribute__((visibility("default")))
|
||||
#else
|
||||
#define mi_decl_export
|
||||
#endif
|
||||
#define mi_cdecl // leads to warnings... __attribute__((cdecl))
|
||||
#define mi_decl_export __attribute__((visibility("default")))
|
||||
#define mi_decl_restrict
|
||||
#define mi_attr_malloc __attribute__((malloc))
|
||||
#if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
|
||||
|
|
@ -267,7 +271,6 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size
|
|||
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
|
||||
mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
|
||||
|
||||
|
||||
// deprecated
|
||||
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept;
|
||||
|
||||
|
|
@ -306,6 +309,7 @@ typedef enum mi_option_e {
|
|||
mi_option_reset_decommits,
|
||||
mi_option_large_os_pages, // implies eager commit
|
||||
mi_option_reserve_huge_os_pages,
|
||||
mi_option_reserve_huge_os_pages_at,
|
||||
mi_option_reserve_os_memory,
|
||||
mi_option_segment_cache,
|
||||
mi_option_page_reset,
|
||||
|
|
@ -342,6 +346,7 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value);
|
|||
mi_decl_export void mi_cfree(void* p) mi_attr_noexcept;
|
||||
mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept;
|
||||
mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept;
|
||||
|
|
@ -384,6 +389,7 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
|
|||
// ---------------------------------------------------------------------------------------------
|
||||
#ifdef __cplusplus
|
||||
|
||||
#include <cstddef> // std::size_t
|
||||
#include <cstdint> // PTRDIFF_MAX
|
||||
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
|
||||
#include <type_traits> // std::true_type
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue