mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-07-06 19:38:41 +03:00
Merge branch 'dev' into dev
This commit is contained in:
Daan
GitHub
commit
f33aa58d88
79 changed files with 3509 additions and 3833 deletions
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023 Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -72,6 +72,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed))
|
||||
#define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_exchange_relaxed(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(relaxed))
|
||||
#define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release))
|
||||
#define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel))
|
||||
#define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed))
|
||||
|
|
@ -110,6 +111,7 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
|
|||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,(tp*)x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x)
|
||||
#else
|
||||
|
|
@ -118,6 +120,7 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
|
|||
#define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des)
|
||||
#define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des)
|
||||
#define mi_atomic_exchange_ptr_relaxed(tp,p,x) mi_atomic_exchange_relaxed(p,x)
|
||||
#define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x)
|
||||
#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x)
|
||||
#endif
|
||||
|
|
@ -402,19 +405,46 @@ static inline void mi_atomic_yield(void) {
|
|||
|
||||
|
||||
// ----------------------------------------------------------------------
|
||||
// Locks are only used for abandoned segment visiting in `arena.c`
|
||||
// Locks
|
||||
// These do not have to be recursive and should be light-weight
|
||||
// in-process only locks. Only used for reserving arena's and to
|
||||
// maintain the abandoned list.
|
||||
// ----------------------------------------------------------------------
|
||||
#if _MSC_VER
|
||||
#pragma warning(disable:26110) // unlock with holding lock
|
||||
#endif
|
||||
|
||||
#define mi_lock(lock) for(bool _go = (mi_lock_acquire(lock),true); _go; (mi_lock_release(lock), _go=false) )
|
||||
|
||||
#if defined(_WIN32)
|
||||
|
||||
#if 1
|
||||
#define mi_lock_t SRWLOCK // slim reader-writer lock
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return TryAcquireSRWLockExclusive(lock);
|
||||
}
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
AcquireSRWLockExclusive(lock);
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
ReleaseSRWLockExclusive(lock);
|
||||
}
|
||||
static inline void mi_lock_init(mi_lock_t* lock) {
|
||||
InitializeSRWLock(lock);
|
||||
}
|
||||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
|
||||
#else
|
||||
#define mi_lock_t CRITICAL_SECTION
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return TryEnterCriticalSection(lock);
|
||||
}
|
||||
static inline bool mi_lock_acquire(mi_lock_t* lock) {
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
EnterCriticalSection(lock);
|
||||
return true;
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
LeaveCriticalSection(lock);
|
||||
|
|
@ -426,16 +456,22 @@ static inline void mi_lock_done(mi_lock_t* lock) {
|
|||
DeleteCriticalSection(lock);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#elif defined(MI_USE_PTHREADS)
|
||||
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
|
||||
#define mi_lock_t pthread_mutex_t
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return (pthread_mutex_trylock(lock) == 0);
|
||||
}
|
||||
static inline bool mi_lock_acquire(mi_lock_t* lock) {
|
||||
return (pthread_mutex_lock(lock) == 0);
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
const int err = pthread_mutex_lock(lock);
|
||||
if (err != 0) {
|
||||
_mi_error_message(err, "internal error: lock cannot be acquired\n");
|
||||
}
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
pthread_mutex_unlock(lock);
|
||||
|
|
@ -447,18 +483,16 @@ static inline void mi_lock_done(mi_lock_t* lock) {
|
|||
pthread_mutex_destroy(lock);
|
||||
}
|
||||
|
||||
/*
|
||||
#elif defined(__cplusplus)
|
||||
|
||||
#include <mutex>
|
||||
#define mi_lock_t std::mutex
|
||||
|
||||
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
||||
return lock->lock_try_acquire();
|
||||
return lock->try_lock();
|
||||
}
|
||||
static inline bool mi_lock_acquire(mi_lock_t* lock) {
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
lock->lock();
|
||||
return true;
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
lock->unlock();
|
||||
|
|
@ -469,7 +503,6 @@ static inline void mi_lock_init(mi_lock_t* lock) {
|
|||
static inline void mi_lock_done(mi_lock_t* lock) {
|
||||
(void)(lock);
|
||||
}
|
||||
*/
|
||||
|
||||
#else
|
||||
|
||||
|
|
@ -482,12 +515,11 @@ static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
|
|||
uintptr_t expected = 0;
|
||||
return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1);
|
||||
}
|
||||
static inline bool mi_lock_acquire(mi_lock_t* lock) {
|
||||
static inline void mi_lock_acquire(mi_lock_t* lock) {
|
||||
for (int i = 0; i < 1000; i++) { // for at most 1000 tries?
|
||||
if (mi_lock_try_acquire(lock)) return true;
|
||||
if (mi_lock_try_acquire(lock)) return;
|
||||
mi_atomic_yield();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
static inline void mi_lock_release(mi_lock_t* lock) {
|
||||
mi_atomic_store_release(lock, (uintptr_t)0);
|
||||
|
|
@ -502,6 +534,4 @@ static inline void mi_lock_done(mi_lock_t* lock) {
|
|||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
#endif // __MIMALLOC_ATOMIC_H
|
||||
|
|
|
|||
|
|
@ -31,16 +31,19 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_decl_thread __declspec(thread)
|
||||
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
|
||||
#define mi_decl_weak
|
||||
#define mi_decl_hidden
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
|
||||
#define mi_decl_noinline __attribute__((noinline))
|
||||
#define mi_decl_thread __thread
|
||||
#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
|
||||
#define mi_decl_weak __attribute__((weak))
|
||||
#define mi_decl_hidden __attribute__((visibility("hidden")))
|
||||
#else
|
||||
#define mi_decl_noinline
|
||||
#define mi_decl_thread __thread // hope for the best :-)
|
||||
#define mi_decl_cache_align
|
||||
#define mi_decl_weak
|
||||
#define mi_decl_hidden
|
||||
#endif
|
||||
|
||||
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
|
||||
|
|
@ -53,82 +56,100 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_decl_externc
|
||||
#endif
|
||||
|
||||
// "libc.c"
|
||||
#include <stdarg.h>
|
||||
void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
|
||||
void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
|
||||
char _mi_toupper(char c);
|
||||
int _mi_strnicmp(const char* s, const char* t, size_t n);
|
||||
void _mi_strlcpy(char* dest, const char* src, size_t dest_size);
|
||||
void _mi_strlcat(char* dest, const char* src, size_t dest_size);
|
||||
size_t _mi_strlen(const char* s);
|
||||
size_t _mi_strnlen(const char* s, size_t max_len);
|
||||
bool _mi_getenv(const char* name, char* result, size_t result_size);
|
||||
|
||||
// "options.c"
|
||||
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
|
||||
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
|
||||
void _mi_warning_message(const char* fmt, ...);
|
||||
void _mi_verbose_message(const char* fmt, ...);
|
||||
void _mi_trace_message(const char* fmt, ...);
|
||||
void _mi_options_init(void);
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
|
||||
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
|
||||
void _mi_warning_message(const char* fmt, ...);
|
||||
void _mi_verbose_message(const char* fmt, ...);
|
||||
void _mi_trace_message(const char* fmt, ...);
|
||||
void _mi_options_init(void);
|
||||
long _mi_option_get_fast(mi_option_t option);
|
||||
void _mi_error_message(int err, const char* fmt, ...);
|
||||
|
||||
// random.c
|
||||
void _mi_random_init(mi_random_ctx_t* ctx);
|
||||
void _mi_random_init_weak(mi_random_ctx_t* ctx);
|
||||
void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);
|
||||
void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
|
||||
uintptr_t _mi_random_next(mi_random_ctx_t* ctx);
|
||||
uintptr_t _mi_heap_random_next(mi_heap_t* heap);
|
||||
uintptr_t _mi_os_random_weak(uintptr_t extra_seed);
|
||||
void _mi_random_init(mi_random_ctx_t* ctx);
|
||||
void _mi_random_init_weak(mi_random_ctx_t* ctx);
|
||||
void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx);
|
||||
void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
|
||||
uintptr_t _mi_random_next(mi_random_ctx_t* ctx);
|
||||
uintptr_t _mi_heap_random_next(mi_heap_t* heap);
|
||||
uintptr_t _mi_os_random_weak(uintptr_t extra_seed);
|
||||
static inline uintptr_t _mi_random_shuffle(uintptr_t x);
|
||||
|
||||
// init.c
|
||||
extern mi_decl_cache_align mi_stats_t _mi_stats_main;
|
||||
extern mi_decl_cache_align const mi_page_t _mi_page_empty;
|
||||
bool _mi_is_main_thread(void);
|
||||
size_t _mi_current_thread_count(void);
|
||||
bool _mi_preloading(void); // true while the C runtime is not initialized yet
|
||||
void _mi_thread_done(mi_heap_t* heap);
|
||||
void _mi_thread_data_collect(void);
|
||||
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
|
||||
extern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty;
|
||||
void _mi_process_load(void);
|
||||
void mi_cdecl _mi_process_done(void);
|
||||
bool _mi_is_redirected(void);
|
||||
bool _mi_allocator_init(const char** message);
|
||||
void _mi_allocator_done(void);
|
||||
bool _mi_is_main_thread(void);
|
||||
size_t _mi_current_thread_count(void);
|
||||
bool _mi_preloading(void); // true while the C runtime is not initialized yet
|
||||
void _mi_thread_done(mi_heap_t* heap);
|
||||
void _mi_thread_data_collect(void);
|
||||
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
|
||||
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
|
||||
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
|
||||
mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
|
||||
void _mi_heap_guarded_init(mi_heap_t* heap);
|
||||
|
||||
// os.c
|
||||
void _mi_os_init(void); // called from process init
|
||||
void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats);
|
||||
void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats);
|
||||
void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats);
|
||||
void _mi_os_init(void); // called from process init
|
||||
void* _mi_os_alloc(size_t size, mi_memid_t* memid);
|
||||
void _mi_os_free(void* p, size_t size, mi_memid_t memid);
|
||||
void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid);
|
||||
|
||||
size_t _mi_os_page_size(void);
|
||||
size_t _mi_os_good_alloc_size(size_t size);
|
||||
bool _mi_os_has_overcommit(void);
|
||||
bool _mi_os_has_virtual_reserve(void);
|
||||
size_t _mi_os_page_size(void);
|
||||
size_t _mi_os_good_alloc_size(size_t size);
|
||||
bool _mi_os_has_overcommit(void);
|
||||
bool _mi_os_has_virtual_reserve(void);
|
||||
|
||||
bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats);
|
||||
bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
|
||||
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
|
||||
bool _mi_os_protect(void* addr, size_t size);
|
||||
bool _mi_os_unprotect(void* addr, size_t size);
|
||||
bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats);
|
||||
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats);
|
||||
bool _mi_os_reset(void* addr, size_t size);
|
||||
bool _mi_os_commit(void* p, size_t size, bool* is_zero);
|
||||
bool _mi_os_decommit(void* addr, size_t size);
|
||||
bool _mi_os_protect(void* addr, size_t size);
|
||||
bool _mi_os_unprotect(void* addr, size_t size);
|
||||
bool _mi_os_purge(void* p, size_t size);
|
||||
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size);
|
||||
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats);
|
||||
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats);
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
|
||||
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);
|
||||
|
||||
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
|
||||
bool _mi_os_use_large_page(size_t size, size_t alignment);
|
||||
size_t _mi_os_large_page_size(void);
|
||||
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
|
||||
bool _mi_os_use_large_page(size_t size, size_t alignment);
|
||||
size_t _mi_os_large_page_size(void);
|
||||
|
||||
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
|
||||
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
|
||||
|
||||
// arena.c
|
||||
mi_arena_id_t _mi_arena_id_none(void);
|
||||
void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats);
|
||||
void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld);
|
||||
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld);
|
||||
bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id);
|
||||
bool _mi_arena_contains(const void* p);
|
||||
void _mi_arenas_collect(bool force_purge, mi_stats_t* stats);
|
||||
void _mi_arena_unsafe_destroy_all(mi_stats_t* stats);
|
||||
void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid);
|
||||
void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid);
|
||||
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid);
|
||||
bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id);
|
||||
bool _mi_arena_contains(const void* p);
|
||||
void _mi_arenas_collect(bool force_purge);
|
||||
void _mi_arena_unsafe_destroy_all(void);
|
||||
|
||||
bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment);
|
||||
void _mi_arena_segment_mark_abandoned(mi_segment_t* segment);
|
||||
bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment);
|
||||
void _mi_arena_segment_mark_abandoned(mi_segment_t* segment);
|
||||
|
||||
void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid);
|
||||
void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size);
|
||||
void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid);
|
||||
void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size);
|
||||
|
||||
typedef struct mi_arena_field_cursor_s { // abstract struct
|
||||
size_t os_list_count; // max entries to visit in the OS abandoned list
|
||||
|
|
@ -144,61 +165,64 @@ mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* pr
|
|||
void _mi_arena_field_cursor_done(mi_arena_field_cursor_t* current);
|
||||
|
||||
// "segment-map.c"
|
||||
void _mi_segment_map_allocated_at(const mi_segment_t* segment);
|
||||
void _mi_segment_map_freed_at(const mi_segment_t* segment);
|
||||
void _mi_segment_map_allocated_at(const mi_segment_t* segment);
|
||||
void _mi_segment_map_freed_at(const mi_segment_t* segment);
|
||||
void _mi_segment_map_unsafe_destroy(void);
|
||||
|
||||
// "segment.c"
|
||||
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
|
||||
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
|
||||
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
|
||||
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size);
|
||||
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld);
|
||||
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
|
||||
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
|
||||
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size);
|
||||
|
||||
#if MI_HUGE_PAGE_ABANDON
|
||||
void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
|
||||
void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
|
||||
#else
|
||||
void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
|
||||
void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
|
||||
#endif
|
||||
|
||||
void _mi_segments_collect(bool force, mi_segments_tld_t* tld);
|
||||
void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
|
||||
bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment);
|
||||
bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
void _mi_segments_collect(bool force, mi_segments_tld_t* tld);
|
||||
void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
|
||||
bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment);
|
||||
bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
// "page.c"
|
||||
void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc;
|
||||
void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc;
|
||||
|
||||
void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks
|
||||
void _mi_page_unfull(mi_page_t* page);
|
||||
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page
|
||||
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
|
||||
void _mi_heap_delayed_free_all(mi_heap_t* heap);
|
||||
bool _mi_heap_delayed_free_partial(mi_heap_t* heap);
|
||||
void _mi_heap_collect_retired(mi_heap_t* heap, bool force);
|
||||
void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks
|
||||
void _mi_page_unfull(mi_page_t* page);
|
||||
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page
|
||||
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
|
||||
void _mi_page_force_abandon(mi_page_t* page);
|
||||
|
||||
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
|
||||
bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
|
||||
size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);
|
||||
void _mi_deferred_free(mi_heap_t* heap, bool force);
|
||||
void _mi_heap_delayed_free_all(mi_heap_t* heap);
|
||||
bool _mi_heap_delayed_free_partial(mi_heap_t* heap);
|
||||
void _mi_heap_collect_retired(mi_heap_t* heap, bool force);
|
||||
|
||||
void _mi_page_free_collect(mi_page_t* page,bool force);
|
||||
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments
|
||||
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
|
||||
bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
|
||||
size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);
|
||||
void _mi_deferred_free(mi_heap_t* heap, bool force);
|
||||
|
||||
size_t _mi_bin_size(uint8_t bin); // for stats
|
||||
uint8_t _mi_bin(size_t size); // for stats
|
||||
void _mi_page_free_collect(mi_page_t* page,bool force);
|
||||
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments
|
||||
|
||||
size_t _mi_bin_size(uint8_t bin); // for stats
|
||||
uint8_t _mi_bin(size_t size); // for stats
|
||||
|
||||
// "heap.c"
|
||||
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag);
|
||||
void _mi_heap_destroy_pages(mi_heap_t* heap);
|
||||
void _mi_heap_collect_abandon(mi_heap_t* heap);
|
||||
void _mi_heap_set_default_direct(mi_heap_t* heap);
|
||||
bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid);
|
||||
void _mi_heap_unsafe_destroy_all(void);
|
||||
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag);
|
||||
void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
|
||||
bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
|
||||
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag);
|
||||
void _mi_heap_destroy_pages(mi_heap_t* heap);
|
||||
void _mi_heap_collect_abandon(mi_heap_t* heap);
|
||||
void _mi_heap_set_default_direct(mi_heap_t* heap);
|
||||
bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid);
|
||||
void _mi_heap_unsafe_destroy_all(mi_heap_t* heap);
|
||||
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag);
|
||||
void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
|
||||
bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
// "stats.c"
|
||||
void _mi_stats_done(mi_stats_t* stats);
|
||||
void _mi_stats_done(mi_stats_t* stats);
|
||||
mi_msecs_t _mi_clock_now(void);
|
||||
mi_msecs_t _mi_clock_end(mi_msecs_t start);
|
||||
mi_msecs_t _mi_clock_start(void);
|
||||
|
|
@ -215,18 +239,6 @@ bool _mi_free_delayed_block(mi_block_t* block);
|
|||
void _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration
|
||||
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);
|
||||
|
||||
// "libc.c"
|
||||
#include <stdarg.h>
|
||||
void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
|
||||
void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
|
||||
char _mi_toupper(char c);
|
||||
int _mi_strnicmp(const char* s, const char* t, size_t n);
|
||||
void _mi_strlcpy(char* dest, const char* src, size_t dest_size);
|
||||
void _mi_strlcat(char* dest, const char* src, size_t dest_size);
|
||||
size_t _mi_strlen(const char* s);
|
||||
size_t _mi_strnlen(const char* s, size_t max_len);
|
||||
bool _mi_getenv(const char* name, char* result, size_t result_size);
|
||||
|
||||
#if MI_DEBUG>1
|
||||
bool _mi_page_is_valid(mi_page_t* page);
|
||||
#endif
|
||||
|
|
@ -322,6 +334,7 @@ static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
// Align a pointer upwards
|
||||
static inline void* mi_align_up_ptr(void* p, size_t alignment) {
|
||||
return (void*)_mi_align_up((uintptr_t)p, alignment);
|
||||
|
|
@ -402,7 +415,7 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot
|
|||
Heap functions
|
||||
------------------------------------------------------------------------------------------- */
|
||||
|
||||
extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap
|
||||
extern mi_decl_hidden const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap
|
||||
|
||||
static inline bool mi_heap_is_backing(const mi_heap_t* heap) {
|
||||
return (heap->tld->heap_backing == heap);
|
||||
|
|
@ -410,11 +423,11 @@ static inline bool mi_heap_is_backing(const mi_heap_t* heap) {
|
|||
|
||||
static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
|
||||
mi_assert_internal(heap != NULL);
|
||||
return (heap != &_mi_heap_empty);
|
||||
return (heap != NULL && heap != &_mi_heap_empty);
|
||||
}
|
||||
|
||||
static inline uintptr_t _mi_ptr_cookie(const void* p) {
|
||||
extern mi_heap_t _mi_heap_main;
|
||||
extern mi_decl_hidden mi_heap_t _mi_heap_main;
|
||||
mi_assert_internal(_mi_heap_main.cookie != 0);
|
||||
return ((uintptr_t)p ^ _mi_heap_main.cookie);
|
||||
}
|
||||
|
|
@ -562,7 +575,7 @@ static inline bool mi_page_immediate_available(const mi_page_t* page) {
|
|||
}
|
||||
|
||||
// is more than 7/8th of a page in use?
|
||||
static inline bool mi_page_mostly_used(const mi_page_t* page) {
|
||||
static inline bool mi_page_is_mostly_used(const mi_page_t* page) {
|
||||
if (page==NULL) return true;
|
||||
uint16_t frac = page->reserved / 8U;
|
||||
return (page->reserved - page->used <= frac);
|
||||
|
|
@ -593,6 +606,39 @@ static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) {
|
|||
page->flags.x.has_aligned = has_aligned;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------------
|
||||
Guarded objects
|
||||
------------------------------------------------------------------- */
|
||||
#if MI_GUARDED
|
||||
static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {
|
||||
const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;
|
||||
return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED);
|
||||
}
|
||||
|
||||
static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) {
|
||||
// this code is written to result in fast assembly as it is on the hot path for allocation
|
||||
const size_t count = heap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time..
|
||||
if mi_likely(count != 0) {
|
||||
// no sample
|
||||
heap->guarded_sample_count = count;
|
||||
return false;
|
||||
}
|
||||
else if (size >= heap->guarded_size_min && size <= heap->guarded_size_max) {
|
||||
// use guarded allocation
|
||||
heap->guarded_sample_count = heap->guarded_sample_rate; // reset
|
||||
return (heap->guarded_sample_rate != 0);
|
||||
}
|
||||
else {
|
||||
// failed size criteria, rewind count (but don't write to an empty heap)
|
||||
if (heap->guarded_sample_rate != 0) { heap->guarded_sample_count = 1; }
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
/* -------------------------------------------------------------------
|
||||
Encoding/Decoding the free list next pointers
|
||||
|
|
@ -651,6 +697,16 @@ static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const
|
|||
return mi_rotl(x ^ keys[1], keys[0]) + keys[0];
|
||||
}
|
||||
|
||||
static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) {
|
||||
const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys));
|
||||
// make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951)
|
||||
#ifdef MI_BIG_ENDIAN
|
||||
return (x & 0x00FFFFFF);
|
||||
#else
|
||||
return (x & 0xFFFFFF00);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
|
||||
mi_track_mem_defined(block,sizeof(mi_block_t));
|
||||
mi_block_t* next;
|
||||
|
|
@ -731,7 +787,7 @@ static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool
|
|||
|
||||
static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
|
||||
if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros
|
||||
#if (MI_INTPTR_SIZE==8)
|
||||
#if (MI_INTPTR_SIZE>=8)
|
||||
// by Sebastiano Vigna, see: <http://xoshiro.di.unimi.it/splitmix64.c>
|
||||
x ^= x >> 30;
|
||||
x *= 0xbf58476d1ce4e5b9UL;
|
||||
|
|
@ -753,13 +809,13 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
|
|||
// Optimize numa node access for the common case (= one node)
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
int _mi_os_numa_node_get(mi_os_tld_t* tld);
|
||||
int _mi_os_numa_node_get(void);
|
||||
size_t _mi_os_numa_node_count_get(void);
|
||||
|
||||
extern _Atomic(size_t) _mi_numa_node_count;
|
||||
static inline int _mi_os_numa_node(mi_os_tld_t* tld) {
|
||||
extern mi_decl_hidden _Atomic(size_t) _mi_numa_node_count;
|
||||
static inline int _mi_os_numa_node(void) {
|
||||
if mi_likely(mi_atomic_load_relaxed(&_mi_numa_node_count) == 1) { return 0; }
|
||||
else return _mi_os_numa_node_get(tld);
|
||||
else return _mi_os_numa_node_get();
|
||||
}
|
||||
static inline size_t _mi_os_numa_node_count(void) {
|
||||
const size_t count = mi_atomic_load_relaxed(&_mi_numa_node_count);
|
||||
|
|
@ -821,16 +877,18 @@ static inline size_t mi_ctz(uintptr_t x) {
|
|||
}
|
||||
|
||||
#else
|
||||
static inline size_t mi_ctz32(uint32_t x) {
|
||||
|
||||
static inline size_t mi_ctz_generic32(uint32_t x) {
|
||||
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
|
||||
static const unsigned char debruijn[32] = {
|
||||
static const uint8_t debruijn[32] = {
|
||||
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
|
||||
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
|
||||
};
|
||||
if (x==0) return 32;
|
||||
return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27];
|
||||
return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27];
|
||||
}
|
||||
static inline size_t mi_clz32(uint32_t x) {
|
||||
|
||||
static inline size_t mi_clz_generic32(uint32_t x) {
|
||||
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
|
||||
static const uint8_t debruijn[32] = {
|
||||
31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
|
||||
|
|
@ -842,28 +900,37 @@ static inline size_t mi_clz32(uint32_t x) {
|
|||
x |= x >> 4;
|
||||
x |= x >> 8;
|
||||
x |= x >> 16;
|
||||
return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27];
|
||||
return debruijn[(uint32_t)(x * (uint32_t)(0x07C4ACDDU)) >> 27];
|
||||
}
|
||||
|
||||
static inline size_t mi_clz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
#if (MI_INTPTR_BITS <= 32)
|
||||
return mi_clz32((uint32_t)x);
|
||||
#else
|
||||
size_t count = mi_clz32((uint32_t)(x >> 32));
|
||||
if (count < 32) return count;
|
||||
return (32 + mi_clz32((uint32_t)x));
|
||||
#endif
|
||||
static inline size_t mi_ctz(size_t x) {
|
||||
if (x==0) return MI_SIZE_BITS;
|
||||
#if (MI_SIZE_BITS <= 32)
|
||||
return mi_ctz_generic32((uint32_t)x);
|
||||
#else
|
||||
const uint32_t lo = (uint32_t)x;
|
||||
if (lo != 0) {
|
||||
return mi_ctz_generic32(lo);
|
||||
}
|
||||
else {
|
||||
return (32 + mi_ctz_generic32((uint32_t)(x>>32)));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
static inline size_t mi_ctz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
#if (MI_INTPTR_BITS <= 32)
|
||||
return mi_ctz32((uint32_t)x);
|
||||
#else
|
||||
size_t count = mi_ctz32((uint32_t)x);
|
||||
if (count < 32) return count;
|
||||
return (32 + mi_ctz32((uint32_t)(x>>32)));
|
||||
#endif
|
||||
|
||||
static inline size_t mi_clz(size_t x) {
|
||||
if (x==0) return MI_SIZE_BITS;
|
||||
#if (MI_SIZE_BITS <= 32)
|
||||
return mi_clz_generic32((uint32_t)x);
|
||||
#else
|
||||
const uint32_t hi = (uint32_t)(x>>32);
|
||||
if (hi != 0) {
|
||||
return mi_clz_generic32(hi);
|
||||
}
|
||||
else {
|
||||
return 32 + mi_clz_generic32((uint32_t)x);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -885,8 +952,9 @@ static inline size_t mi_bsr(uintptr_t x) {
|
|||
#if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))
|
||||
#include <intrin.h>
|
||||
extern bool _mi_cpu_has_fsrm;
|
||||
extern bool _mi_cpu_has_erms;
|
||||
static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
|
||||
if (_mi_cpu_has_fsrm) {
|
||||
if ((_mi_cpu_has_fsrm && n <= 128) || (_mi_cpu_has_erms && n > 128)) {
|
||||
__movsb((unsigned char*)dst, (const unsigned char*)src, n);
|
||||
}
|
||||
else {
|
||||
|
|
@ -894,7 +962,7 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
|
|||
}
|
||||
}
|
||||
static inline void _mi_memzero(void* dst, size_t n) {
|
||||
if (_mi_cpu_has_fsrm) {
|
||||
if ((_mi_cpu_has_fsrm && n <= 128) || (_mi_cpu_has_erms && n > 128)) {
|
||||
__stosb((unsigned char*)dst, 0, n);
|
||||
}
|
||||
else {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
|
||||
Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
|
|
@ -25,6 +25,8 @@ typedef struct mi_os_mem_config_s {
|
|||
size_t page_size; // default to 4KiB
|
||||
size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)
|
||||
size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)
|
||||
size_t physical_memory; // physical memory size
|
||||
size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization)
|
||||
bool has_overcommit; // can we reserve more memory than can be actually committed?
|
||||
bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
|
||||
bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
|
||||
|
|
@ -41,9 +43,10 @@ int _mi_prim_free(void* addr, size_t size );
|
|||
// If `commit` is false, the virtual memory range only needs to be reserved (with no access)
|
||||
// which will later be committed explicitly using `_mi_prim_commit`.
|
||||
// `is_zero` is set to true if the memory was zero initialized (as on most OS's)
|
||||
// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored.
|
||||
// pre: !commit => !allow_large
|
||||
// try_alignment >= _mi_os_page_size() and a power of 2
|
||||
int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
|
||||
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr);
|
||||
|
||||
// Commit memory. Returns error code or 0 on success.
|
||||
// For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
|
||||
|
|
@ -116,14 +119,13 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap);
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Thread id: `_mi_prim_thread_id()`
|
||||
//
|
||||
// Getting the thread id should be performant as it is called in the
|
||||
// fast path of `_mi_free` and we specialize for various platforms as
|
||||
// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
|
||||
// We only require _mi_prim_thread_id() to return a unique id
|
||||
// for each thread (unequal to zero).
|
||||
// Access to TLS (thread local storage) slots.
|
||||
// We need fast access to both a unique thread id (in `free.c:mi_free`) and
|
||||
// to a thread-local heap pointer (in `alloc.c:mi_malloc`).
|
||||
// To achieve this we use specialized code for various platforms.
|
||||
//-------------------------------------------------------------------
|
||||
|
||||
// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot.
|
||||
|
|
@ -135,14 +137,14 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap);
|
|||
// but unfortunately we can not detect support reliably (see issue #883)
|
||||
// We also use it on Apple OS as we use a TLS slot for the default heap there.
|
||||
#if defined(__GNUC__) && ( \
|
||||
(defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
|
||||
(defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
|
||||
|| (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \
|
||||
|| (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
|
||||
|| (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \
|
||||
|| (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
|
||||
|| (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
|
||||
)
|
||||
|
||||
#define MI_HAS_TLS_SLOT
|
||||
#define MI_HAS_TLS_SLOT 1
|
||||
|
||||
static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
|
||||
void* res;
|
||||
|
|
@ -203,8 +205,52 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce
|
|||
#endif
|
||||
}
|
||||
|
||||
#elif _WIN32 && MI_WIN_USE_FIXED_TLS && !defined(MI_WIN_USE_FLS)
|
||||
|
||||
// On windows we can store the thread-local heap at a fixed TLS slot to avoid
|
||||
// thread-local initialization checks in the fast path. This uses a fixed location
|
||||
// in the TCB though (last user-reserved slot by default) which may clash with other applications.
|
||||
|
||||
#define MI_HAS_TLS_SLOT 2 // 2 = we can reliable initialize the slot (saving a test on each malloc)
|
||||
|
||||
#if MI_WIN_USE_FIXED_TLS > 1
|
||||
#define MI_TLS_SLOT (MI_WIN_USE_FIXED_TLS)
|
||||
#elif MI_SIZE_SIZE == 4
|
||||
#define MI_TLS_SLOT (0x710) // Last user-reserved slot <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block>
|
||||
// #define MI_TLS_SLOT (0xF0C) // Last TlsSlot (might clash with other app reserved slot)
|
||||
#else
|
||||
#define MI_TLS_SLOT (0x888) // Last user-reserved slot <https://en.wikipedia.org/wiki/Win32_Thread_Information_Block>
|
||||
// #define MI_TLS_SLOT (0x1678) // Last TlsSlot (might clash with other app reserved slot)
|
||||
#endif
|
||||
|
||||
static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept {
|
||||
#if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC)
|
||||
return (void*)__readgsqword((unsigned long)slot); // direct load at offset from gs
|
||||
#elif _M_IX86 && !defined(_M_ARM64EC)
|
||||
return (void*)__readfsdword((unsigned long)slot); // direct load at offset from fs
|
||||
#else
|
||||
return ((void**)NtCurrentTeb())[slot / sizeof(void*)];
|
||||
#endif
|
||||
}
|
||||
static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
|
||||
((void**)NtCurrentTeb())[slot / sizeof(void*)] = value;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Get a fast unique thread id.
|
||||
//
|
||||
// Getting the thread id should be performant as it is called in the
|
||||
// fast path of `_mi_free` and we specialize for various platforms as
|
||||
// inlined definitions. Regular code should call `init.c:_mi_thread_id()`.
|
||||
// We only require _mi_prim_thread_id() to return a unique id
|
||||
// for each thread (unequal to zero).
|
||||
//-------------------------------------------------------------------
|
||||
|
||||
|
||||
// Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id
|
||||
// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
|
||||
// Nevertheless, it seems needed on older graviton platforms (see issue #851).
|
||||
|
|
@ -248,7 +294,7 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
|||
return (uintptr_t)__builtin_thread_pointer();
|
||||
}
|
||||
|
||||
#elif defined(MI_HAS_TLS_SLOT)
|
||||
#elif MI_HAS_TLS_SLOT
|
||||
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
#if defined(__BIONIC__)
|
||||
|
|
@ -275,7 +321,8 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
|||
|
||||
|
||||
/* ----------------------------------------------------------------------------------------
|
||||
The thread local default heap: `_mi_prim_get_default_heap()`
|
||||
Get the thread local default heap: `_mi_prim_get_default_heap()`
|
||||
|
||||
This is inlined here as it is on the fast path for allocation functions.
|
||||
|
||||
On most platforms (Windows, Linux, FreeBSD, NetBSD, etc), this just returns a
|
||||
|
|
@ -312,19 +359,21 @@ static inline mi_heap_t* mi_prim_get_default_heap(void);
|
|||
#endif
|
||||
|
||||
|
||||
#if defined(MI_TLS_SLOT)
|
||||
#if MI_TLS_SLOT
|
||||
# if !defined(MI_HAS_TLS_SLOT)
|
||||
# error "trying to use a TLS slot for the default heap, but the mi_prim_tls_slot primitives are not defined"
|
||||
# endif
|
||||
|
||||
static inline mi_heap_t* mi_prim_get_default_heap(void) {
|
||||
mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT);
|
||||
#if MI_HAS_TLS_SLOT == 1 // check if the TLS slot is initialized
|
||||
if mi_unlikely(heap == NULL) {
|
||||
#ifdef __GNUC__
|
||||
__asm(""); // prevent conditional load of the address of _mi_heap_empty
|
||||
#endif
|
||||
heap = (mi_heap_t*)&_mi_heap_empty;
|
||||
}
|
||||
#endif
|
||||
return heap;
|
||||
}
|
||||
|
||||
|
|
@ -366,7 +415,4 @@ static inline mi_heap_t* mi_prim_get_default_heap(void) {
|
|||
#endif // mi_prim_get_default_heap()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#endif // MIMALLOC_PRIM_H
|
||||
|
|
|
|||
|
|
@ -72,6 +72,13 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#endif
|
||||
#endif
|
||||
|
||||
// Use guard pages behind objects of a certain size (set by the MIMALLOC_DEBUG_GUARDED_MIN/MAX options)
|
||||
// Padding should be disabled when using guard pages
|
||||
// #define MI_GUARDED 1
|
||||
#if defined(MI_GUARDED)
|
||||
#define MI_PADDING 0
|
||||
#endif
|
||||
|
||||
// Reserve extra padding at the end of each block to be more resilient against heap block overflows.
|
||||
// The padding can detect buffer overflow on free.
|
||||
#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW))
|
||||
|
|
@ -225,6 +232,13 @@ typedef struct mi_block_s {
|
|||
mi_encoded_t next;
|
||||
} mi_block_t;
|
||||
|
||||
#if MI_GUARDED
|
||||
// we always align guarded pointers in a block at an offset
|
||||
// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones
|
||||
#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0))
|
||||
#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED)
|
||||
#endif
|
||||
|
||||
|
||||
// The delayed flags are used for efficient multi-threaded free-ing
|
||||
typedef enum mi_delayed_e {
|
||||
|
|
@ -248,7 +262,7 @@ typedef union mi_page_flags_s {
|
|||
#else
|
||||
// under thread sanitizer, use a byte for each flag to suppress warning, issue #130
|
||||
typedef union mi_page_flags_s {
|
||||
uint16_t full_aligned;
|
||||
uint32_t full_aligned;
|
||||
struct {
|
||||
uint8_t in_full;
|
||||
uint8_t has_aligned;
|
||||
|
|
@ -363,7 +377,7 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
|
|||
|
||||
typedef struct mi_memid_os_info {
|
||||
void* base; // actual base address of the block (used for offset aligned allocations)
|
||||
size_t alignment; // alignment at allocation
|
||||
size_t size; // full allocation size
|
||||
} mi_memid_os_info_t;
|
||||
|
||||
typedef struct mi_memid_arena_info {
|
||||
|
|
@ -402,7 +416,8 @@ typedef struct mi_segment_s {
|
|||
// segment fields
|
||||
struct mi_segment_s* next; // must be the first (non-constant) segment field -- see `segment.c:segment_init`
|
||||
struct mi_segment_s* prev;
|
||||
bool was_reclaimed; // true if it was reclaimed (used to limit on-free reclamation)
|
||||
bool was_reclaimed; // true if it was reclaimed (used to limit reclaim-on-free reclamation)
|
||||
bool dont_free; // can be temporarily true to ensure the segment is not freed
|
||||
|
||||
size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
|
||||
size_t abandoned_visits; // count how often this segment is visited for reclaiming (to force reclaim if it is too long)
|
||||
|
|
@ -488,6 +503,13 @@ struct mi_heap_s {
|
|||
mi_heap_t* next; // list of heaps per thread
|
||||
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
|
||||
uint8_t tag; // custom tag, can be used for separating heaps based on the object types
|
||||
#if MI_GUARDED
|
||||
size_t guarded_size_min; // minimal size for guarded objects
|
||||
size_t guarded_size_max; // maximal size for guarded objects
|
||||
size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages)
|
||||
size_t guarded_sample_seed; // starting sample count
|
||||
size_t guarded_sample_count; // current sample count (counting down to 0)
|
||||
#endif
|
||||
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
|
||||
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
|
||||
};
|
||||
|
|
@ -580,24 +602,34 @@ typedef struct mi_stats_s {
|
|||
mi_stat_counter_t arena_count;
|
||||
mi_stat_counter_t arena_crossover_count;
|
||||
mi_stat_counter_t arena_rollback_count;
|
||||
mi_stat_counter_t guarded_alloc_count;
|
||||
#if MI_STAT>1
|
||||
mi_stat_count_t normal_bins[MI_BIN_HUGE+1];
|
||||
#endif
|
||||
} mi_stats_t;
|
||||
|
||||
|
||||
// add to stat keeping track of the peak
|
||||
void _mi_stat_increase(mi_stat_count_t* stat, size_t amount);
|
||||
void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
|
||||
// adjust stat in special cases to compensate for double counting
|
||||
void _mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount);
|
||||
void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount);
|
||||
// counters can just be increased
|
||||
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
|
||||
|
||||
#if (MI_STAT)
|
||||
#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount)
|
||||
#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount)
|
||||
#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount)
|
||||
#define mi_stat_adjust_increase(stat,amount) _mi_stat_adjust_increase( &(stat), amount)
|
||||
#define mi_stat_adjust_decrease(stat,amount) _mi_stat_adjust_decrease( &(stat), amount)
|
||||
#else
|
||||
#define mi_stat_increase(stat,amount) (void)0
|
||||
#define mi_stat_decrease(stat,amount) (void)0
|
||||
#define mi_stat_counter_increase(stat,amount) (void)0
|
||||
#define mi_stat_increase(stat,amount) ((void)0)
|
||||
#define mi_stat_decrease(stat,amount) ((void)0)
|
||||
#define mi_stat_counter_increase(stat,amount) ((void)0)
|
||||
#define mi_stat_adjuct_increase(stat,amount) ((void)0)
|
||||
#define mi_stat_adjust_decrease(stat,amount) ((void)0)
|
||||
#endif
|
||||
|
||||
#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount)
|
||||
|
|
@ -633,12 +665,6 @@ typedef struct mi_segment_queue_s {
|
|||
mi_segment_t* last;
|
||||
} mi_segment_queue_t;
|
||||
|
||||
// OS thread local data
|
||||
typedef struct mi_os_tld_s {
|
||||
size_t region_idx; // start point for next allocation
|
||||
mi_stats_t* stats; // points to tld stats
|
||||
} mi_os_tld_t;
|
||||
|
||||
// Segments thread local data
|
||||
typedef struct mi_segments_tld_s {
|
||||
mi_segment_queue_t small_free; // queue of segments with free small pages
|
||||
|
|
@ -651,7 +677,6 @@ typedef struct mi_segments_tld_s {
|
|||
size_t reclaim_count;// number of reclaimed (abandoned) segments
|
||||
mi_subproc_t* subproc; // sub-process this thread belongs to.
|
||||
mi_stats_t* stats; // points to tld stats
|
||||
mi_os_tld_t* os; // points to os tld
|
||||
} mi_segments_tld_t;
|
||||
|
||||
// Thread local data
|
||||
|
|
@ -661,7 +686,6 @@ struct mi_tld_s {
|
|||
mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted)
|
||||
mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates)
|
||||
mi_segments_tld_t segments; // segment tld
|
||||
mi_os_tld_t os; // os tld
|
||||
mi_stats_t stats; // statistics
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue