kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-10 01:09:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge from dev3

Browse source
This commit is contained in:
daanx 2024-12-21 15:52:15 -08:00
commit 5de5550c63
18 changed files with 835 additions and 653 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
ide/vs2022/mimalloc-test.vcxproj
View file

@ -272,14 +272,14 @@
<SubSystem>Console</SubSystem> <SubSystem>Console</SubSystem>
</Link> </Link>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\test\main-override-static.c" />
</ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="mimalloc.vcxproj"> <ProjectReference Include="mimalloc.vcxproj">
<Project>{abb5eae7-b3e6-432e-b636-333449892ea6}</Project> <Project>{abb5eae7-b3e6-432e-b636-333449892ea6}</Project>
</ProjectReference> </ProjectReference>
</ItemGroup> </ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\test\main-override-static.c" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>

10
include/mimalloc.h
View file

@ -279,7 +279,7 @@ mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_commit
mi_decl_export void mi_debug_show_arenas(bool show_pages, bool show_inuse, bool show_committed) mi_attr_noexcept; mi_decl_export void mi_debug_show_arenas(bool show_pages, bool show_inuse, bool show_committed) mi_attr_noexcept;
// Experimental: heaps associated with specific memory arena's // Experimental: heaps associated with specific memory arena's
typedef int mi_arena_id_t; typedef void* mi_arena_id_t;
mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept;
@ -326,7 +326,13 @@ mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min,
//mi_decl_export void mi_os_decommit(void* p, size_t size); //mi_decl_export void mi_os_decommit(void* p, size_t size);
mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size); mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* size);
mi_decl_export bool mi_arena_reload(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, mi_arena_id_t* arena_id); mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_arena_id_t* arena_id);
mi_decl_export bool mi_heap_reload(mi_heap_t* heap, mi_arena_id_t arena);
mi_decl_export void mi_heap_unload(mi_heap_t* heap);
// Is a pointer contained in the given arena area?
mi_decl_export bool mi_arena_contains(mi_arena_id_t arena_id, const void* p);
// ------------------------------------------------------ // ------------------------------------------------------
// Convenience // Convenience

58
include/mimalloc/atomic.h
View file

@ -1,5 +1,5 @@
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
Copyright (c) 2018-2023 Microsoft Research, Daan Leijen Copyright (c) 2018-2024 Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution. "LICENSE" at the root of this distribution.
@ -407,19 +407,45 @@ static inline void mi_atomic_yield(void) {
// ---------------------------------------------------------------------- // ----------------------------------------------------------------------
// Locks are only used for abandoned segment visiting in `arena.c` // Locks
// These should be light-weight in-process only locks.
// Only used for reserving arena's and to maintain the abandoned list.
// ---------------------------------------------------------------------- // ----------------------------------------------------------------------
#if _MSC_VER
#pragma warning(disable:26110) // unlock with holding lock
#endif
#define mi_lock(lock) for(bool _go = (mi_lock_acquire(lock),true); _go; (mi_lock_release(lock), _go=false) )
#if defined(_WIN32) #if defined(_WIN32)
#if 1
#define mi_lock_t SRWLOCK // slim reader-writer lock
static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return TryAcquireSRWLockExclusive(lock);
}
static inline void mi_lock_acquire(mi_lock_t* lock) {
AcquireSRWLockExclusive(lock);
}
static inline void mi_lock_release(mi_lock_t* lock) {
ReleaseSRWLockExclusive(lock);
}
static inline void mi_lock_init(mi_lock_t* lock) {
InitializeSRWLock(lock);
}
static inline void mi_lock_done(mi_lock_t* lock) {
(void)(lock);
}
#else
#define mi_lock_t CRITICAL_SECTION #define mi_lock_t CRITICAL_SECTION
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return TryEnterCriticalSection(lock); return TryEnterCriticalSection(lock);
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
EnterCriticalSection(lock); EnterCriticalSection(lock);
return true;
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
LeaveCriticalSection(lock); LeaveCriticalSection(lock);
@ -431,16 +457,22 @@ static inline void mi_lock_done(mi_lock_t* lock) {
DeleteCriticalSection(lock); DeleteCriticalSection(lock);
} }
#endif
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
void _mi_error_message(int err, const char* fmt, ...);
#define mi_lock_t pthread_mutex_t #define mi_lock_t pthread_mutex_t
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return (pthread_mutex_trylock(lock) == 0); return (pthread_mutex_trylock(lock) == 0);
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
return (pthread_mutex_lock(lock) == 0); const int err = pthread_mutex_lock(lock);
if (err != 0) {
_mi_error_message(err, "internal error: lock cannot be acquired\n");
}
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
pthread_mutex_unlock(lock); pthread_mutex_unlock(lock);
@ -452,18 +484,16 @@ static inline void mi_lock_done(mi_lock_t* lock) {
pthread_mutex_destroy(lock); pthread_mutex_destroy(lock);
} }
/*
#elif defined(__cplusplus) #elif defined(__cplusplus)
#include <mutex> #include <mutex>
#define mi_lock_t std::mutex #define mi_lock_t std::mutex
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return lock->lock_try_acquire(); return lock->try_lock();
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
lock->lock(); lock->lock();
return true;
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
lock->unlock(); lock->unlock();
@ -474,7 +504,6 @@ static inline void mi_lock_init(mi_lock_t* lock) {
static inline void mi_lock_done(mi_lock_t* lock) { static inline void mi_lock_done(mi_lock_t* lock) {
(void)(lock); (void)(lock);
} }
*/
#else #else
@ -487,12 +516,11 @@ static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
uintptr_t expected = 0; uintptr_t expected = 0;
return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1); return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1);
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
for (int i = 0; i < 1000; i++) { // for at most 1000 tries? for (int i = 0; i < 1000; i++) { // for at most 1000 tries?
if (mi_lock_try_acquire(lock)) return true; if (mi_lock_try_acquire(lock)) return;
mi_atomic_yield(); mi_atomic_yield();
} }
return true;
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
mi_atomic_store_release(lock, (uintptr_t)0); mi_atomic_store_release(lock, (uintptr_t)0);
@ -507,6 +535,4 @@ static inline void mi_lock_done(mi_lock_t* lock) {
#endif #endif
#endif // MI_ATOMIC_H #endif // MI_ATOMIC_H

28
include/mimalloc/internal.h
View file

@ -90,7 +90,6 @@ uintptr_t _mi_os_random_weak(uintptr_t extra_seed);
static inline uintptr_t _mi_random_shuffle(uintptr_t x); static inline uintptr_t _mi_random_shuffle(uintptr_t x);
// init.c // init.c
extern mi_decl_cache_align mi_stats_t _mi_stats_main;
extern mi_decl_cache_align const mi_page_t _mi_page_empty; extern mi_decl_cache_align const mi_page_t _mi_page_empty;
void _mi_process_load(void); void _mi_process_load(void);
void mi_cdecl _mi_process_done(void); void mi_cdecl _mi_process_done(void);
@ -101,8 +100,10 @@ bool _mi_is_main_thread(void);
size_t _mi_current_thread_count(void); size_t _mi_current_thread_count(void);
bool _mi_preloading(void); // true while the C runtime is not initialized yet bool _mi_preloading(void); // true while the C runtime is not initialized yet
void _mi_thread_done(mi_heap_t* heap); void _mi_thread_done(mi_heap_t* heap);
mi_tld_t* _mi_tld(void); // current tld: `_mi_tld() == _mi_heap_get_default()->tld`
mi_tld_t* _mi_tld(void); // current tld: `_mi_tld() == _mi_heap_get_default()->tld`
mi_subproc_t* _mi_subproc(void);
mi_subproc_t* _mi_subproc_main(void);
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
size_t _mi_thread_seq_id(void) mi_attr_noexcept; size_t _mi_thread_seq_id(void) mi_attr_noexcept;
@ -142,10 +143,12 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t m
// arena.c // arena.c
mi_arena_id_t _mi_arena_id_none(void); mi_arena_id_t _mi_arena_id_none(void);
void _mi_arena_init(void); mi_arena_t* _mi_arena_from_id(mi_arena_id_t id);
void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, size_t tseq, mi_memid_t* memid);
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, size_t tseq, mi_memid_t* memid); void* _mi_arena_alloc(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid);
bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); void* _mi_arena_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid);
void _mi_arena_free(void* p, size_t size, mi_memid_t memid);
bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena);
bool _mi_arena_contains(const void* p); bool _mi_arena_contains(const void* p);
void _mi_arenas_collect(bool force_purge); void _mi_arenas_collect(bool force_purge);
void _mi_arena_unsafe_destroy_all(void); void _mi_arena_unsafe_destroy_all(void);
@ -201,6 +204,7 @@ void _mi_heap_page_reclaim(mi_heap_t* heap, mi_page_t* page);
// "stats.c" // "stats.c"
void _mi_stats_done(mi_stats_t* stats); void _mi_stats_done(mi_stats_t* stats);
void _mi_stats_merge_from(mi_stats_t* to, mi_stats_t* from);
mi_msecs_t _mi_clock_now(void); mi_msecs_t _mi_clock_now(void);
mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_end(mi_msecs_t start);
mi_msecs_t _mi_clock_start(void); mi_msecs_t _mi_clock_start(void);
@ -418,11 +422,11 @@ static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
return (heap != &_mi_heap_empty); return (heap != &_mi_heap_empty);
} }
static inline uintptr_t _mi_ptr_cookie(const void* p) { //static inline uintptr_t _mi_ptr_cookie(const void* p) {
extern mi_heap_t _mi_heap_main; // extern mi_heap_t _mi_heap_main;
mi_assert_internal(_mi_heap_main.cookie != 0); // mi_assert_internal(_mi_heap_main.cookie != 0);
return ((uintptr_t)p ^ _mi_heap_main.cookie); // return ((uintptr_t)p ^ _mi_heap_main.cookie);
} //}
/* ----------------------------------------------------------- /* -----------------------------------------------------------
@ -524,7 +528,7 @@ static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {
if (heap != NULL) { if (heap != NULL) {
page->heap = heap; page->heap = heap;
page->heap_tag = heap->tag; page->heap_tag = heap->tag;
mi_atomic_store_release(&page->xthread_id, heap->thread_id); mi_atomic_store_release(&page->xthread_id, heap->tld->thread_id);
} }
else { else {
page->heap = NULL; page->heap = NULL;

127
include/mimalloc/types.h
View file

@ -243,9 +243,6 @@ typedef size_t mi_page_flags_t;
// atomically in `free.c:mi_free_block_mt`. // atomically in `free.c:mi_free_block_mt`.
typedef uintptr_t mi_thread_free_t; typedef uintptr_t mi_thread_free_t;
// Sub processes are used to keep memory separate between them (e.g. multiple interpreters in CPython)
typedef struct mi_subproc_s mi_subproc_t;
// A heap can serve only specific objects signified by its heap tag (e.g. various object types in CPython) // A heap can serve only specific objects signified by its heap tag (e.g. various object types in CPython)
typedef uint8_t mi_heaptag_t; typedef uint8_t mi_heaptag_t;
@ -296,10 +293,9 @@ typedef struct mi_page_s {
uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) or padding canary uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) or padding canary
#endif #endif
mi_heap_t* heap; // heap this threads belong to. mi_heap_t* heap; // the heap owning this page (or NULL for abandoned pages)
struct mi_page_s* next; // next page owned by the heap with the same `block_size` struct mi_page_s* next; // next page owned by the heap with the same `block_size`
struct mi_page_s* prev; // previous page owned by the heap with the same `block_size` struct mi_page_s* prev; // previous page owned by the heap with the same `block_size`
mi_subproc_t* subproc; // sub-process of this heap
mi_memid_t memid; // provenance of the page memory mi_memid_t memid; // provenance of the page memory
} mi_page_t; } mi_page_t;
@ -380,7 +376,7 @@ typedef struct mi_random_cxt_s {
// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows // In debug mode there is a padding structure at the end of the blocks to check for buffer overflows
#if (MI_PADDING) #if MI_PADDING
typedef struct mi_padding_s { typedef struct mi_padding_s {
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
@ -397,19 +393,14 @@ typedef struct mi_padding_s {
// A heap owns a set of pages. // A heap owns a set of pages.
struct mi_heap_s { struct mi_heap_s {
mi_tld_t* tld; mi_tld_t* tld; // thread-local data
// _Atomic(mi_block_t*) thread_delayed_free; mi_arena_t* exclusive_arena; // if the heap should only allocate from a specific arena (or NULL)
mi_threadid_t thread_id; // thread this heap belongs too
mi_arena_id_t arena_id; // arena id if the heap belongs to a specific arena (or 0)
uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list
mi_random_ctx_t random; // random number context used for secure allocation mi_random_ctx_t random; // random number context used for secure allocation
size_t page_count; // total number of pages in the `pages` queues. size_t page_count; // total number of pages in the `pages` queues.
size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)
size_t page_retired_max; // largest retired index into the `pages` array. size_t page_retired_max; // largest retired index into the `pages` array.
mi_heap_t* next; // list of heaps per thread mi_heap_t* next; // list of heaps per thread
mi_memid_t memid; // provenance of the heap struct itseft (meta or os)
long generic_count;
long full_page_retain; // how many full pages can be retained per queue (before abondoning them) long full_page_retain; // how many full pages can be retained per queue (before abondoning them)
bool allow_page_reclaim; // `true` if this heap should not reclaim abandoned pages bool allow_page_reclaim; // `true` if this heap should not reclaim abandoned pages
bool allow_page_abandon; // `true` if this heap can abandon pages to reduce memory footprint bool allow_page_abandon; // `true` if this heap can abandon pages to reduce memory footprint
@ -422,7 +413,8 @@ struct mi_heap_s {
size_t guarded_sample_count; // current sample count (counting down to 0) size_t guarded_sample_count; // current sample count (counting down to 0)
#endif #endif
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin")
mi_memid_t memid; // provenance of the heap struct itself (meta or os)
}; };
@ -480,53 +472,89 @@ typedef struct mi_stats_s {
mi_stat_counter_t arena_count; mi_stat_counter_t arena_count;
mi_stat_counter_t guarded_alloc_count; mi_stat_counter_t guarded_alloc_count;
#if MI_STAT>1 #if MI_STAT>1
mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; mi_stat_count_t normal_bins[MI_BIN_COUNT];
#endif #endif
} mi_stats_t; } mi_stats_t;
// add to stat keeping track of the peak // add to stat keeping track of the peak
void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); void __mi_stat_increase(mi_stat_count_t* stat, size_t amount);
void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount);
void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount);
// adjust stat in special cases to compensate for double counting // adjust stat in special cases to compensate for double counting
void _mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount, bool on_alloc); void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount, bool on_alloc);
void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount, bool on_free); void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount, bool on_free);
void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount, bool on_alloc);
void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount, bool on_free);
// counters can just be increased // counters can just be increased
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
#if (MI_STAT) #if (MI_STAT)
#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) #define mi_debug_stat_increase(stat,amount) __mi_stat_increase( &(stat), amount)
#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) #define mi_debug_stat_decrease(stat,amount) __mi_stat_decrease( &(stat), amount)
#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) #define mi_debug_stat_counter_increase(stat,amount) __mi_stat_counter_increase( &(stat), amount)
#define mi_stat_adjust_increase(stat,amnt,b) _mi_stat_adjust_increase( &(stat), amnt, b) #define mi_debug_stat_increase_mt(stat,amount) __mi_stat_increase_mt( &(stat), amount)
#define mi_stat_adjust_decrease(stat,amnt,b) _mi_stat_adjust_decrease( &(stat), amnt, b) #define mi_debug_stat_decrease_mt(stat,amount) __mi_stat_decrease_mt( &(stat), amount)
#define mi_debug_stat_counter_increase_mt(stat,amount) __mi_stat_counter_increase_mt( &(stat), amount)
#define mi_debug_stat_adjust_increase_mt(stat,amnt,b) __mi_stat_adjust_increase_mt( &(stat), amnt, b)
#define mi_debug_stat_adjust_decrease_mt(stat,amnt,b) __mi_stat_adjust_decrease_mt( &(stat), amnt, b)
#else #else
#define mi_stat_increase(stat,amount) ((void)0) #define mi_debug_stat_increase(stat,amount) ((void)0)
#define mi_stat_decrease(stat,amount) ((void)0) #define mi_debug_stat_decrease(stat,amount) ((void)0)
#define mi_stat_counter_increase(stat,amount) ((void)0) #define mi_debug_stat_counter_increase(stat,amount) ((void)0)
#define mi_stat_adjuct_increase(stat,amnt,b) ((void)0) #define mi_debug_stat_increase_mt(stat,amount) ((void)0)
#define mi_stat_adjust_decrease(stat,amnt,b) ((void)0) #define mi_debug_stat_decrease_mt(stat,amount) ((void)0)
#define mi_debug_stat_counter_increase_mt(stat,amount) ((void)0)
#define mi_debug_stat_adjust_increase(stat,amnt,b) ((void)0)
#define mi_debug_stat_adjust_decrease(stat,amnt,b) ((void)0)
#endif #endif
#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) #define mi_subproc_stat_counter_increase(subproc,stat,amount) __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount)
#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) #define mi_subproc_stat_increase(subproc,stat,amount) __mi_stat_increase_mt( &(subproc)->stats.stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) #define mi_subproc_stat_decrease(subproc,stat,amount) __mi_stat_decrease_mt( &(subproc)->stats.stat, amount)
#define mi_subproc_stat_adjust_increase(subproc,stat,amnt,b) __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amnt, b)
#define mi_subproc_stat_adjust_decrease(subproc,stat,amnt,b) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amnt, b)
#define mi_os_stat_counter_increase(stat,amount) mi_subproc_stat_counter_increase(_mi_subproc(),stat,amount)
#define mi_os_stat_increase(stat,amount) mi_subproc_stat_increase(_mi_subproc(),stat,amount)
#define mi_os_stat_decrease(stat,amount) mi_subproc_stat_decrease(_mi_subproc(),stat,amount)
#define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase( &(heap)->tld->stats.stat, amount)
#define mi_heap_stat_increase(heap,stat,amount) __mi_stat_increase( &(heap)->tld->stats.stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) __mi_stat_decrease( &(heap)->tld->stats.stat, amount)
#define mi_debug_heap_stat_counter_increase(heap,stat,amount) mi_debug_stat_counter_increase( (heap)->tld->stats.stat, amount)
#define mi_debug_heap_stat_increase(heap,stat,amount) mi_debug_stat_increase( (heap)->tld->stats.stat, amount)
#define mi_debug_heap_stat_decrease(heap,stat,amount) mi_debug_stat_decrease( (heap)->tld->stats.stat, amount)
// ------------------------------------------------------ // ------------------------------------------------------
// Sub processes do not reclaim or visit segments // Sub processes use separate arena's and no heaps/pages/blocks
// from other sub processes // are shared between sub processes.
// The subprocess structure contains essentially all static variables (except per subprocess :-))
//
// Each thread should belong to one sub-process only
// ------------------------------------------------------ // ------------------------------------------------------
struct mi_subproc_s { #define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)
_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // count of abandoned pages for this sub-process // 160 arenas is enough for ~2 TiB memory
_Atomic(size_t) abandoned_os_list_count; // count of abandoned pages in the os-list
mi_lock_t abandoned_os_lock; // lock for the abandoned os pages list (outside of arena's) (this lock protect list operations) typedef struct mi_subproc_s {
mi_lock_t abandoned_os_visit_lock; // ensure only one thread per subproc visits the abandoned os list _Atomic(size_t) arena_count; // current count of arena's
mi_page_t* abandoned_os_list; // doubly-linked list of abandoned pages outside of arena's (in OS allocated memory) _Atomic(mi_arena_t*) arenas[MI_MAX_ARENAS]; // arena's of this sub-process
mi_page_t* abandoned_os_list_tail; // the tail-end of the list mi_lock_t arena_reserve_lock; // lock to ensure arena's get reserved one at a time
mi_memid_t memid; // provenance of this memory block _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged
};
_Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages for this sub-process
mi_page_t* os_abandoned_pages; // list of pages that OS allocated and not in an arena (only used if `mi_option_visit_abandoned` is on)
mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations)
mi_memid_t memid; // provenance of this memory block (meta or OS)
mi_stats_t stats; // sub-process statistics (tld stats are merged in on thread termination)
} mi_subproc_t;
// ------------------------------------------------------ // ------------------------------------------------------
// Thread Local data // Thread Local data
@ -535,20 +563,21 @@ struct mi_subproc_s {
// Milliseconds as in `int64_t` to avoid overflows // Milliseconds as in `int64_t` to avoid overflows
typedef int64_t mi_msecs_t; typedef int64_t mi_msecs_t;
// Thread local data // Thread local data
struct mi_tld_s { struct mi_tld_s {
unsigned long long heartbeat; // monotonic heartbeat count mi_threadid_t thread_id; // thread id of this thread
size_t thread_seq; // thread sequence id (linear count of created threads)
mi_subproc_t* subproc; // sub-process this thread belongs to.
mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted)
mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates)
mi_subproc_t* subproc; // sub-process this thread belongs to. unsigned long long heartbeat; // monotonic heartbeat count
size_t tseq; // thread sequence id
mi_memid_t memid; // provenance of the tld memory itself (meta or OS)
bool recurse; // true if deferred was called; used to prevent infinite recursion. bool recurse; // true if deferred was called; used to prevent infinite recursion.
bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks) bool is_in_threadpool; // true if this thread is part of a threadpool (and can run arbitrary tasks)
mi_stats_t stats; // statistics mi_stats_t stats; // statistics
mi_memid_t memid; // provenance of the tld memory itself (meta or OS)
}; };
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Error codes passed to `_mi_fatal_error` Error codes passed to `_mi_fatal_error`
All are recoverable but EFAULT is a serious error and aborts by default in secure mode. All are recoverable but EFAULT is a serious error and aborts by default in secure mode.

4
src/alloc-aligned.c
View file

@ -193,9 +193,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
if mi_likely(is_aligned) if mi_likely(is_aligned)
{ {
#if MI_STAT>1 mi_debug_heap_stat_increase(heap, malloc, size);
mi_heap_stat_increase(heap, malloc, size);
#endif
void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen
mi_assert_internal(p != NULL); mi_assert_internal(p != NULL);
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);

4
src/alloc.c
View file

@ -134,7 +134,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap,
mi_assert(size <= MI_SMALL_SIZE_MAX); mi_assert(size <= MI_SMALL_SIZE_MAX);
#if MI_DEBUG #if MI_DEBUG
const uintptr_t tid = _mi_thread_id(); const uintptr_t tid = _mi_thread_id();
mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local mi_assert(heap->tld->thread_id == 0 || heap->tld->thread_id == tid); // heaps are thread local
#endif #endif
#if (MI_PADDING || MI_GUARDED) #if (MI_PADDING || MI_GUARDED)
if (size == 0) { size = sizeof(void*); } if (size == 0) { size = sizeof(void*); }
@ -188,7 +188,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z
else { else {
// regular allocation // regular allocation
mi_assert(heap!=NULL); mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local mi_assert(heap->tld->thread_id == 0 || heap->tld->thread_id == _mi_thread_id()); // heaps are thread local
void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic
mi_track_malloc(p,size,zero); mi_track_malloc(p,size,zero);

10
src/arena-meta.c
View file

@ -64,10 +64,11 @@ static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) {
// allocate a fresh meta page and add it to the global list. // allocate a fresh meta page and add it to the global list.
static mi_meta_page_t* mi_meta_page_zalloc(void) { static mi_meta_page_t* mi_meta_page_zalloc(void) {
// allocate a fresh arena slice // allocate a fresh arena slice
// note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again..
mi_memid_t memid; mi_memid_t memid;
mi_meta_page_t* mpage = (mi_meta_page_t*)_mi_arena_alloc_aligned(MI_ARENA_SLICE_SIZE, MI_ARENA_SLICE_ALIGN, 0, mi_meta_page_t* mpage = (mi_meta_page_t*)_mi_arena_alloc_aligned(_mi_subproc(), MI_ARENA_SLICE_SIZE, MI_ARENA_SLICE_ALIGN, 0,
true /* commit*/, true /* allow large */, true /* commit*/, true /* allow large */,
_mi_arena_id_none(), 0 /* tseq */, &memid ); NULL /* req arena */, 0 /* thread_seq */, &memid);
if (mpage == NULL) return NULL; if (mpage == NULL) return NULL;
mi_assert_internal(_mi_is_aligned(mpage,MI_META_PAGE_ALIGN)); mi_assert_internal(_mi_is_aligned(mpage,MI_META_PAGE_ALIGN));
if (!memid.initially_zero) { if (!memid.initially_zero) {
@ -147,11 +148,8 @@ mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) {
_mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE); _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE);
mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count); mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count);
} }
else if (mi_memid_is_os(memid)) {
_mi_os_free(p, size, memid);
}
else { else {
mi_assert_internal(mi_memid_needs_no_free(memid)); _mi_arena_free(p,size,memid);
} }
} }

467
src/arena.c
View file

@ -35,7 +35,7 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
// A memory arena descriptor // A memory arena descriptor
typedef struct mi_arena_s { typedef struct mi_arena_s {
mi_memid_t memid; // memid of the memory area mi_memid_t memid; // memid of the memory area
mi_arena_id_t id; // arena id (> 0 where `arena == arenas[arena->id - 1]`) mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'in' this->subproc->arenas`)
size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`) size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
size_t info_slices; // initial slices reserved for the arena bitmaps size_t info_slices; // initial slices reserved for the arena bitmaps
@ -43,7 +43,6 @@ typedef struct mi_arena_s {
bool is_exclusive; // only allow allocations if specifically for this arena bool is_exclusive; // only allow allocations if specifically for this arena
bool is_large; // memory area consists of large- or huge OS pages (always committed) bool is_large; // memory area consists of large- or huge OS pages (always committed)
_Atomic(mi_msecs_t) purge_expire; // expiration time when slices can be purged from `slices_purge`. _Atomic(mi_msecs_t) purge_expire; // expiration time when slices can be purged from `slices_purge`.
_Atomic(mi_msecs_t) purge_expire_extend; // the purge expiration may be extended by a bit
mi_bbitmap_t* slices_free; // is the slice free? (a binned bitmap with size classes) mi_bbitmap_t* slices_free; // is the slice free? (a binned bitmap with size classes)
mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible) mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible)
@ -57,64 +56,40 @@ typedef struct mi_arena_s {
} mi_arena_t; } mi_arena_t;
#define MI_MAX_ARENAS (160) // Limited for now (and takes up .bss).. but arena's scale up exponentially (see `mi_arena_reserve`)
// 160 arenas is enough for ~2 TiB memory
// The available arenas
static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS];
static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0
static mi_lock_t mi_arena_reserve_lock;
void _mi_arena_init(void) {
mi_lock_init(&mi_arena_reserve_lock);
}
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Arena id's Arena id's
id = arena_index + 1
----------------------------------------------------------- */ ----------------------------------------------------------- */
size_t mi_arena_id_index(mi_arena_id_t id) {
return (size_t)(id <= 0 ? MI_MAX_ARENAS : id - 1);
}
static mi_arena_id_t mi_arena_id_create(size_t arena_index) {
mi_assert_internal(arena_index < MI_MAX_ARENAS);
return (int)arena_index + 1;
}
mi_arena_id_t _mi_arena_id_none(void) { mi_arena_id_t _mi_arena_id_none(void) {
return 0; return NULL;
} }
static bool mi_arena_id_is_suitable(mi_arena_id_t arena_id, bool arena_is_exclusive, mi_arena_id_t req_arena_id) { mi_arena_t* _mi_arena_from_id(mi_arena_id_t id) {
return ((!arena_is_exclusive && req_arena_id == _mi_arena_id_none()) || return (mi_arena_t*)id;
(arena_id == req_arena_id));
} }
bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id) {
static bool mi_arena_id_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena) {
return ((arena == req_arena) || // they match,
(req_arena == NULL && !arena->is_exclusive)); // or the arena is not exclusive, and we didn't request a specific one
}
bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_arena) {
if (memid.memkind == MI_MEM_ARENA) { if (memid.memkind == MI_MEM_ARENA) {
const mi_arena_t* arena = memid.mem.arena.arena; return mi_arena_id_is_suitable(memid.mem.arena.arena, request_arena);
return mi_arena_id_is_suitable(arena->id, arena->is_exclusive, request_arena_id);
} }
else { else {
return mi_arena_id_is_suitable(_mi_arena_id_none(), false, request_arena_id); return mi_arena_id_is_suitable(NULL, request_arena);
} }
} }
size_t mi_arena_get_count(void) { size_t mi_arenas_get_count(mi_subproc_t* subproc) {
return mi_atomic_load_relaxed(&mi_arena_count); return mi_atomic_load_relaxed(&subproc->arena_count);
} }
mi_arena_t* mi_arena_from_index(size_t idx) { mi_arena_t* mi_arena_from_index(mi_subproc_t* subproc, size_t idx) {
mi_assert_internal(idx < mi_arena_get_count()); mi_assert_internal(idx < mi_arenas_get_count(subproc));
return mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[idx]); return mi_atomic_load_ptr_relaxed(mi_arena_t, &subproc->arenas[idx]);
}
mi_arena_t* mi_arena_from_id(mi_arena_id_t id) {
return mi_arena_from_index(mi_arena_id_index(id));
} }
static size_t mi_arena_info_slices(mi_arena_t* arena) { static size_t mi_arena_info_slices(mi_arena_t* arena) {
@ -152,9 +127,7 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) {
// Arena area // Arena area
void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) { void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) {
if (size != NULL) *size = 0; if (size != NULL) *size = 0;
const size_t arena_index = mi_arena_id_index(arena_id); mi_arena_t* arena = _mi_arena_from_id(arena_id);
if (arena_index >= MI_MAX_ARENAS) return NULL;
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_index]);
if (arena == NULL) return NULL; if (arena == NULL) return NULL;
if (size != NULL) { *size = mi_size_of_slices(arena->slice_count); } if (size != NULL) { *size = mi_size_of_slices(arena->slice_count); }
return mi_arena_start(arena); return mi_arena_start(arena);
@ -240,14 +213,14 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count); mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
// adjust the stats so we don't double count the commits // adjust the stats so we don't double count the commits
if (already_committed_count > 0) { if (already_committed_count > 0) {
_mi_stat_adjust_decrease(&_mi_stats_main.committed, mi_size_of_slices(already_committed_count), true /* on alloc */); mi_subproc_stat_adjust_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count), true /* on alloc */);
} }
// now actually commit // now actually commit
bool commit_zero = false; bool commit_zero = false;
if (!_mi_os_commit(p, mi_size_of_slices(slice_count), &commit_zero)) { if (!_mi_os_commit(p, mi_size_of_slices(slice_count), &commit_zero)) {
// failed to commit (todo: give warning?) // failed to commit (todo: give warning?)
if (already_committed_count > 0) { if (already_committed_count > 0) {
_mi_stat_increase(&_mi_stats_main.committed, mi_size_of_slices(already_committed_count)); mi_subproc_stat_increase(arena->subproc, committed, mi_size_of_slices(already_committed_count));
} }
memid->initially_committed = false; memid->initially_committed = false;
} }
@ -269,7 +242,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
// if the OS has overcommit, and this is the first time we access these pages, then // if the OS has overcommit, and this is the first time we access these pages, then
// count the commit now (as at arena reserve we didn't count those commits as these are on-demand) // count the commit now (as at arena reserve we didn't count those commits as these are on-demand)
if (_mi_os_has_overcommit() && touched_slices > 0) { if (_mi_os_has_overcommit() && touched_slices > 0) {
_mi_stat_increase(&_mi_stats_main.committed, mi_size_of_slices(touched_slices)); mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices));
} }
} }
// tool support // tool support
@ -293,13 +266,12 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
} }
// try to reserve a fresh arena space static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t req_arena_id, mi_arena_id_t* arena_id)
{
// if (_mi_preloading()) return false; // use OS only while pre loading
if (req_arena_id != _mi_arena_id_none()) return false;
const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count); // try to reserve a fresh arena space
static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_large, mi_arena_id_t* arena_id)
{
const size_t arena_count = mi_arenas_get_count(subproc);
if (arena_count > (MI_MAX_ARENAS - 4)) return false; if (arena_count > (MI_MAX_ARENAS - 4)) return false;
// calc reserve // calc reserve
@ -341,18 +313,18 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re
// on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice
// is actually allocated for the first time it will be counted. // is actually allocated for the first time it will be counted.
const bool adjust = (overcommit && arena_commit); const bool adjust = (overcommit && arena_commit);
if (adjust) { _mi_stat_adjust_decrease(&_mi_stats_main.committed, arena_reserve, true /* on alloc */); } if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve, true /* on alloc */); }
// and try to reserve the arena // and try to reserve the arena
int err = mi_reserve_os_memory_ex(arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
if (err != 0) { if (err != 0) {
if (adjust) { _mi_stat_adjust_increase(&_mi_stats_main.committed, arena_reserve, true); } // roll back if (adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve, true); } // roll back
// failed, try a smaller size? // failed, try a smaller size?
const size_t small_arena_reserve = (MI_SIZE_BITS == 32 ? 128*MI_MiB : 1*MI_GiB); const size_t small_arena_reserve = (MI_SIZE_BITS == 32 ? 128*MI_MiB : 1*MI_GiB);
if (adjust) { _mi_stat_adjust_decrease(&_mi_stats_main.committed, arena_reserve, true); } if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve, true); }
if (arena_reserve > small_arena_reserve) { if (arena_reserve > small_arena_reserve) {
// try again // try again
err = mi_reserve_os_memory_ex(small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); err = mi_reserve_os_memory_ex(small_arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
if (err != 0 && adjust) { _mi_stat_adjust_increase(&_mi_stats_main.committed, arena_reserve, true); } // roll back if (err != 0 && adjust) { mi_subproc_stat_adjust_increase( subproc, committed, arena_reserve, true); } // roll back
} }
} }
return (err==0); return (err==0);
@ -365,32 +337,27 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re
Arena iteration Arena iteration
----------------------------------------------------------- */ ----------------------------------------------------------- */
static inline bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_id_t req_arena_id, int numa_node, bool allow_large) { static inline bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_t* req_arena, int numa_node, bool allow_large) {
if (!allow_large && arena->is_large) return false; if (!allow_large && arena->is_large) return false;
if (!mi_arena_id_is_suitable(arena->id, arena->is_exclusive, req_arena_id)) return false; if (!mi_arena_id_is_suitable(arena, req_arena)) return false;
if (req_arena_id == _mi_arena_id_none()) { // if not specific, check numa affinity if (req_arena == NULL) { // if not specific, check numa affinity
const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node); const bool numa_suitable = (numa_node < 0 || arena->numa_node < 0 || arena->numa_node == numa_node);
if (!numa_suitable) return false; if (!numa_suitable) return false;
} }
return true; return true;
} }
#define mi_forall_arenas(subproc, req_arena, tseq, name_arena) { \
#define mi_forall_arenas(req_arena_id, tseq, name_arena) \ const size_t _arena_count = mi_arenas_get_count(subproc); \
{ \ const size_t _arena_cycle = (_arena_count == 0 ? 0 : _arena_count - 1); /* first search the arenas below the last one */ \
const size_t _arena_count = mi_arena_get_count(); \
if (_arena_count > 0) { \
const size_t _arena_cycle = _arena_count - 1; /* first search the arenas below the last one */ \
size_t _start; \
if (req_arena_id == _mi_arena_id_none()) { \
/* always start searching in the arena's below the max */ \ /* always start searching in the arena's below the max */ \
_start = (_arena_cycle <= 1 ? 0 : (tseq % _arena_cycle)); \ size_t _start = (_arena_cycle <= 1 ? 0 : (tseq % _arena_cycle)); \
for (size_t _i = 0; _i < _arena_count; _i++) { \
mi_arena_t* name_arena; \
if (req_arena != NULL) { \
name_arena = req_arena; /* if there is a specific req_arena, only search that one */\
} \ } \
else { \ else { \
_start = mi_arena_id_index(req_arena_id); \
mi_assert_internal(_start < _arena_count); \
} \
for (size_t _i = 0; _i < _arena_count; _i++) { \
size_t _idx; \ size_t _idx; \
if (_i < _arena_cycle) { \ if (_i < _arena_cycle) { \
_idx = _i + _start; \ _idx = _i + _start; \
@ -399,19 +366,20 @@ static inline bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_id_t req_are
else { \ else { \
_idx = _i; /* remaining arena's */ \ _idx = _i; /* remaining arena's */ \
} \ } \
mi_arena_t* const name_arena = mi_arena_from_index(_idx); \ name_arena = mi_arena_from_index(subproc,_idx); \
} \
if (name_arena != NULL) \ if (name_arena != NULL) \
{ {
#define mi_forall_arenas_end() \ #define mi_forall_arenas_end() \
} \ } \
if (req_arena_id != _mi_arena_id_none()) break; \ if (req_arena != NULL) break; \
} \ } \
}} }
#define mi_forall_suitable_arenas(req_arena_id, tseq, allow_large, name_arena) \ #define mi_forall_suitable_arenas(subproc, req_arena, tseq, allow_large, name_arena) \
mi_forall_arenas(req_arena_id,tseq,name_arena) { \ mi_forall_arenas(subproc, req_arena,tseq,name_arena) { \
if (mi_arena_is_suitable(name_arena, req_arena_id, -1 /* todo: numa node */, allow_large)) { \ if (mi_arena_is_suitable(name_arena, req_arena, -1 /* todo: numa node */, allow_large)) { \
#define mi_forall_suitable_arenas_end() \ #define mi_forall_suitable_arenas_end() \
}} \ }} \
@ -422,17 +390,16 @@ static inline bool mi_arena_is_suitable(mi_arena_t* arena, mi_arena_id_t req_are
----------------------------------------------------------- */ ----------------------------------------------------------- */
// allocate slices from the arenas // allocate slices from the arenas
static mi_decl_noinline void* mi_arena_try_find_free( static mi_decl_noinline void* mi_arenas_try_find_free(
size_t slice_count, size_t alignment, mi_subproc_t* subproc, size_t slice_count, size_t alignment,
bool commit, bool allow_large, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid)
mi_arena_id_t req_arena_id, size_t tseq, mi_memid_t* memid)
{ {
mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_OBJ_SIZE)); mi_assert_internal(slice_count <= mi_slice_count_of_size(MI_ARENA_MAX_OBJ_SIZE));
mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
if (alignment > MI_ARENA_SLICE_ALIGN) return NULL; if (alignment > MI_ARENA_SLICE_ALIGN) return NULL;
// search arena's // search arena's
mi_forall_suitable_arenas(req_arena_id, tseq, allow_large, arena) mi_forall_suitable_arenas(subproc, req_arena, tseq, allow_large, arena)
{ {
void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid); void* p = mi_arena_try_alloc_at(arena, slice_count, commit, tseq, memid);
if (p != NULL) return p; if (p != NULL) return p;
@ -442,42 +409,42 @@ static mi_decl_noinline void* mi_arena_try_find_free(
} }
// Allocate slices from the arena's -- potentially allocating a fresh arena // Allocate slices from the arena's -- potentially allocating a fresh arena
static mi_decl_noinline void* mi_arena_try_alloc( static mi_decl_noinline void* mi_arenas_try_alloc(
mi_subproc_t* subproc,
size_t slice_count, size_t alignment, size_t slice_count, size_t alignment,
bool commit, bool allow_large, bool commit, bool allow_large,
mi_arena_id_t req_arena_id, size_t tseq, mi_memid_t* memid) mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid)
{ {
mi_assert(slice_count <= MI_ARENA_MAX_OBJ_SLICES); mi_assert(slice_count <= MI_ARENA_MAX_OBJ_SLICES);
mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
void* p; void* p;
again:
// try to find free slices in the arena's // try to find free slices in the arena's
p = mi_arena_try_find_free(slice_count, alignment, commit, allow_large, req_arena_id, tseq, memid); p = mi_arenas_try_find_free(subproc, slice_count, alignment, commit, allow_large, req_arena, tseq, memid);
if (p != NULL) return p; if (p != NULL) return p;
// did we need a specific arena? // did we need a specific arena?
if (req_arena_id != _mi_arena_id_none()) return NULL; if (req_arena != NULL) return NULL;
// don't create arena's while preloading (todo: or should we?) // don't create arena's while preloading (todo: or should we?)
if (_mi_preloading()) return NULL; if (_mi_preloading()) return NULL;
// otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?) // otherwise, try to reserve a new arena -- but one thread at a time.. (todo: allow 2 or 4 to reduce contention?)
if (mi_lock_try_acquire(&mi_arena_reserve_lock)) { const size_t arena_count = mi_arenas_get_count(subproc);
mi_lock(&subproc->arena_reserve_lock) {
if (arena_count == mi_arenas_get_count(subproc)) {
// we are the first to enter the lock, reserve a fresh arena
mi_arena_id_t arena_id = 0; mi_arena_id_t arena_id = 0;
bool ok = mi_arena_reserve(mi_size_of_slices(slice_count), allow_large, req_arena_id, &arena_id); mi_arena_reserve(subproc, mi_size_of_slices(slice_count), allow_large, &arena_id);
mi_lock_release(&mi_arena_reserve_lock);
if (ok) {
// and try allocate in there
mi_assert_internal(req_arena_id == _mi_arena_id_none());
p = mi_arena_try_find_free(slice_count, alignment, commit, allow_large, req_arena_id, tseq, memid);
if (p != NULL) return p;
}
} }
else { else {
// if we are racing with another thread wait until the new arena is reserved (todo: a better yield?) // another thread already reserved a new arena
mi_atomic_yield();
goto again;
} }
}
// try once more to allocate in the new arena
mi_assert_internal(req_arena == NULL);
p = mi_arenas_try_find_free(subproc, slice_count, alignment, commit, allow_large, req_arena, tseq, memid);
if (p != NULL) return p;
return NULL; return NULL;
} }
@ -504,10 +471,10 @@ static void* mi_arena_os_alloc_aligned(
// Allocate large sized memory // Allocate large sized memory
void* _mi_arena_alloc_aligned( void* _mi_arena_alloc_aligned( mi_subproc_t* subproc,
size_t size, size_t alignment, size_t align_offset, size_t size, size_t alignment, size_t align_offset,
bool commit, bool allow_large, bool commit, bool allow_large,
mi_arena_id_t req_arena_id, size_t tseq, mi_memid_t* memid) mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid)
{ {
mi_assert_internal(memid != NULL); mi_assert_internal(memid != NULL);
mi_assert_internal(size > 0); mi_assert_internal(size > 0);
@ -517,23 +484,22 @@ void* _mi_arena_alloc_aligned(
// try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data) // try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data)
if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // is arena allocation allowed? if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // is arena allocation allowed?
req_arena_id == _mi_arena_id_none() && // not a specific arena?
size >= MI_ARENA_MIN_OBJ_SIZE && size <= MI_ARENA_MAX_OBJ_SIZE && // and not too small/large size >= MI_ARENA_MIN_OBJ_SIZE && size <= MI_ARENA_MAX_OBJ_SIZE && // and not too small/large
alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0) // and good alignment alignment <= MI_ARENA_SLICE_ALIGN && align_offset == 0) // and good alignment
{ {
const size_t slice_count = mi_slice_count_of_size(size); const size_t slice_count = mi_slice_count_of_size(size);
void* p = mi_arena_try_alloc(slice_count, alignment, commit, allow_large, req_arena_id, tseq, memid); void* p = mi_arenas_try_alloc(subproc,slice_count, alignment, commit, allow_large, req_arena, tseq, memid);
if (p != NULL) return p; if (p != NULL) return p;
} }
// fall back to the OS // fall back to the OS
void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena_id, memid); void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena, memid);
return p; return p;
} }
void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, size_t tseq, mi_memid_t* memid) void* _mi_arena_alloc(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid)
{ {
return _mi_arena_alloc_aligned(size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena_id, tseq, memid); return _mi_arena_alloc_aligned(subproc, size, MI_ARENA_SLICE_SIZE, 0, commit, allow_large, req_arena, tseq, memid);
} }
@ -542,7 +508,7 @@ void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t
Arena page allocation Arena page allocation
----------------------------------------------------------- */ ----------------------------------------------------------- */
static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, mi_subproc_t* subproc, mi_heaptag_t heap_tag, bool* keep_abandoned) { static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena, mi_heaptag_t heap_tag, bool* keep_abandoned) {
// found an abandoned page of the right size // found an abandoned page of the right size
mi_page_t* const page = (mi_page_t*)mi_arena_slice_start(arena, slice_index); mi_page_t* const page = (mi_page_t*)mi_arena_slice_start(arena, slice_index);
// can we claim ownership? // can we claim ownership?
@ -554,9 +520,9 @@ static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena,
*keep_abandoned = true; *keep_abandoned = true;
return false; return false;
} }
if (subproc != page->subproc || heap_tag != page->heap_tag) { if (heap_tag != page->heap_tag) {
// wrong sub-process or heap_tag.. we need to unown again // wrong heap_tag.. we need to unown again
// note: this normally never happens unless subprocesses/heaptags are actually used. // note: this normally never happens unless heaptags are actually used.
// (an unown might free the page, and depending on that we can keep it in the abandoned map or not) // (an unown might free the page, and depending on that we can keep it in the abandoned map or not)
// note: a minor wrinkle: the page will still be mapped but the abandoned map entry is (temporarily) clear at this point. // note: a minor wrinkle: the page will still be mapped but the abandoned map entry is (temporarily) clear at this point.
// so we cannot check in `mi_arena_free` for this invariant to hold. // so we cannot check in `mi_arena_free` for this invariant to hold.
@ -564,31 +530,31 @@ static bool mi_arena_try_claim_abandoned(size_t slice_index, mi_arena_t* arena,
*keep_abandoned = !freed; *keep_abandoned = !freed;
return false; return false;
} }
// yes, we can reclaim it, keep the abandaned map entry clear // yes, we can reclaim it, keep the abandoned map entry clear
*keep_abandoned = false; *keep_abandoned = false;
return true; return true;
} }
static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t block_size, mi_arena_id_t req_arena_id, mi_heaptag_t heaptag, mi_tld_t* tld) static mi_page_t* mi_arena_page_try_find_abandoned(mi_subproc_t* subproc, size_t slice_count, size_t block_size, mi_arena_t* req_arena, mi_heaptag_t heaptag, size_t tseq)
{ {
MI_UNUSED(slice_count); MI_UNUSED(slice_count);
const size_t bin = _mi_bin(block_size); const size_t bin = _mi_bin(block_size);
mi_assert_internal(bin < MI_BIN_COUNT); mi_assert_internal(bin < MI_BIN_COUNT);
// any abandoned in our size class? // any abandoned in our size class?
mi_subproc_t* const subproc = tld->subproc;
mi_assert_internal(subproc != NULL); mi_assert_internal(subproc != NULL);
if (mi_atomic_load_relaxed(&subproc->abandoned_count[bin]) == 0) return NULL; if (mi_atomic_load_relaxed(&subproc->abandoned_count[bin]) == 0) {
return NULL;
}
// search arena's // search arena's
const bool allow_large = true; const bool allow_large = true;
size_t tseq = tld->tseq; mi_forall_suitable_arenas(subproc, req_arena, tseq, allow_large, arena)
mi_forall_suitable_arenas(req_arena_id, tseq, allow_large, arena)
{ {
size_t slice_index; size_t slice_index;
mi_bitmap_t* const bitmap = arena->pages_abandoned[bin]; mi_bitmap_t* const bitmap = arena->pages_abandoned[bin];
if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena, subproc, heaptag)) { if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena, heaptag)) {
// found an abandoned page of the right size // found an abandoned page of the right size
// and claimed ownership. // and claimed ownership.
mi_page_t* page = (mi_page_t*)mi_arena_slice_start(arena, slice_index); mi_page_t* page = (mi_page_t*)mi_arena_slice_start(arena, slice_index);
@ -596,8 +562,8 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mi_arena_has_page(arena,page)); mi_assert_internal(mi_arena_has_page(arena,page));
mi_atomic_decrement_relaxed(&subproc->abandoned_count[bin]); mi_atomic_decrement_relaxed(&subproc->abandoned_count[bin]);
_mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1); mi_subproc_stat_decrease( arena->subproc, pages_abandoned, 1);
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_alloc, 1); mi_subproc_stat_counter_increase(arena->subproc, pages_reclaim_on_alloc, 1);
_mi_page_free_collect(page, false); // update `used` count _mi_page_free_collect(page, false); // update `used` count
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
@ -615,8 +581,8 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
return NULL; return NULL;
} }
static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_size, size_t block_alignment, static mi_page_t* mi_arena_page_alloc_fresh(mi_subproc_t* subproc, size_t slice_count, size_t block_size, size_t block_alignment,
mi_arena_id_t req_arena_id, mi_tld_t* tld) mi_arena_t* req_arena, size_t tseq)
{ {
const bool allow_large = true; const bool allow_large = true;
const bool commit = true; const bool commit = true;
@ -630,7 +596,7 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_siz
!os_align && // not large alignment !os_align && // not large alignment
slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large
{ {
page = (mi_page_t*)mi_arena_try_alloc(slice_count, page_alignment, commit, allow_large, req_arena_id, tld->tseq, &memid); page = (mi_page_t*)mi_arenas_try_alloc(subproc, slice_count, page_alignment, commit, allow_large, req_arena, tseq, &memid);
if (page != NULL) { if (page != NULL) {
mi_assert_internal(mi_bitmap_is_clearN(memid.mem.arena.arena->pages, memid.mem.arena.slice_index, memid.mem.arena.slice_count)); mi_assert_internal(mi_bitmap_is_clearN(memid.mem.arena.arena->pages, memid.mem.arena.slice_index, memid.mem.arena.slice_count));
mi_bitmap_set(memid.mem.arena.arena->pages, memid.mem.arena.slice_index); mi_bitmap_set(memid.mem.arena.arena->pages, memid.mem.arena.slice_index);
@ -642,10 +608,10 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_siz
if (os_align) { if (os_align) {
// note: slice_count already includes the page // note: slice_count already includes the page
mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment)); mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));
page = (mi_page_t*)mi_arena_os_alloc_aligned(mi_size_of_slices(slice_count), block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena_id, &memid); page = (mi_page_t*)mi_arena_os_alloc_aligned(mi_size_of_slices(slice_count), block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, &memid);
} }
else { else {
page = (mi_page_t*)mi_arena_os_alloc_aligned(mi_size_of_slices(slice_count), page_alignment, 0 /* align offset */, commit, allow_large, req_arena_id, &memid); page = (mi_page_t*)mi_arena_os_alloc_aligned(mi_size_of_slices(slice_count), page_alignment, 0 /* align offset */, commit, allow_large, req_arena, &memid);
} }
} }
@ -706,11 +672,13 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_siz
else { else {
page->block_size_shift = 0; page->block_size_shift = 0;
} }
// and own it
mi_page_try_claim_ownership(page);
// register in the page map
_mi_page_map_register(page); _mi_page_map_register(page);
mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_page_try_claim_ownership(page);
mi_assert_internal(mi_page_block_size(page) == block_size); mi_assert_internal(mi_page_block_size(page) == block_size);
mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_owned(page));
@ -718,17 +686,17 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_siz
} }
static mi_page_t* mi_arena_page_allocN(mi_heap_t* heap, size_t slice_count, size_t block_size) { static mi_page_t* mi_arena_page_allocN(mi_heap_t* heap, size_t slice_count, size_t block_size) {
const mi_arena_id_t req_arena_id = heap->arena_id; mi_arena_t* req_arena = heap->exclusive_arena;
mi_tld_t* const tld = heap->tld; mi_tld_t* const tld = heap->tld;
// 1. look for an abandoned page // 1. look for an abandoned page
mi_page_t* page = mi_arena_page_try_find_abandoned(slice_count, block_size, req_arena_id, heap->tag, tld); mi_page_t* page = mi_arena_page_try_find_abandoned(tld->subproc, slice_count, block_size, req_arena, heap->tag, tld->thread_seq);
if (page != NULL) { if (page != NULL) {
return page; // return as abandoned return page; // return as abandoned
} }
// 2. find a free block, potentially allocating a new arena // 2. find a free block, potentially allocating a new arena
page = mi_arena_page_alloc_fresh(slice_count, block_size, 1, req_arena_id, tld); page = mi_arena_page_alloc_fresh(tld->subproc, slice_count, block_size, 1, req_arena, tld->thread_seq);
if (page != NULL) { if (page != NULL) {
mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);
_mi_page_init(heap, page); _mi_page_init(heap, page);
@ -740,13 +708,13 @@ static mi_page_t* mi_arena_page_allocN(mi_heap_t* heap, size_t slice_count, size
static mi_page_t* mi_singleton_page_alloc(mi_heap_t* heap, size_t block_size, size_t block_alignment) { static mi_page_t* mi_singleton_page_alloc(mi_heap_t* heap, size_t block_size, size_t block_alignment) {
const mi_arena_id_t req_arena_id = heap->arena_id; mi_arena_t* req_arena = heap->exclusive_arena;
mi_tld_t* const tld = heap->tld; mi_tld_t* const tld = heap->tld;
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size());
const size_t slice_count = mi_slice_count_of_size(info_size + block_size); const size_t slice_count = mi_slice_count_of_size(info_size + block_size);
mi_page_t* page = mi_arena_page_alloc_fresh(slice_count, block_size, block_alignment, req_arena_id, tld); mi_page_t* page = mi_arena_page_alloc_fresh(tld->subproc, slice_count, block_size, block_alignment, req_arena, tld->thread_seq);
if (page == NULL) return NULL; if (page == NULL) return NULL;
mi_assert(page != NULL); mi_assert(page != NULL);
@ -785,14 +753,13 @@ mi_page_t* _mi_arena_page_alloc(mi_heap_t* heap, size_t block_size, size_t block
return page; return page;
} }
static void mi_arena_free(void* p, size_t size, mi_memid_t memid);
void _mi_arena_page_free(mi_page_t* page) { void _mi_arena_page_free(mi_page_t* page) {
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_all_free(page)); mi_assert_internal(mi_page_all_free(page));
mi_assert_internal(page->next==NULL); mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(page->next==NULL && page->prev==NULL);
#if MI_DEBUG>1 #if MI_DEBUG>1
if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
@ -811,11 +778,12 @@ void _mi_arena_page_free(mi_page_t* page) {
} }
#endif #endif
// unregister page
_mi_page_map_unregister(page); _mi_page_map_unregister(page);
if (page->memid.memkind == MI_MEM_ARENA) { if (page->memid.memkind == MI_MEM_ARENA) {
mi_bitmap_clear(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index); mi_bitmap_clear(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index);
} }
mi_arena_free(page, mi_memid_size(page->memid), page->memid); _mi_arena_free(page, mi_memid_size(page->memid), page->memid);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
@ -828,9 +796,8 @@ void _mi_arena_page_abandon(mi_page_t* page) {
mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_all_free(page)); mi_assert_internal(!mi_page_all_free(page));
mi_assert_internal(page->next==NULL); mi_assert_internal(page->next==NULL && page->prev == NULL);
mi_subproc_t* subproc = page->subproc;
if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
// make available for allocations // make available for allocations
size_t bin = _mi_bin(mi_page_block_size(page)); size_t bin = _mi_bin(mi_page_block_size(page));
@ -845,13 +812,25 @@ void _mi_arena_page_abandon(mi_page_t* page) {
mi_page_set_abandoned_mapped(page); mi_page_set_abandoned_mapped(page);
const bool wasclear = mi_bitmap_set(arena->pages_abandoned[bin], slice_index); const bool wasclear = mi_bitmap_set(arena->pages_abandoned[bin], slice_index);
MI_UNUSED(wasclear); mi_assert_internal(wasclear); MI_UNUSED(wasclear); mi_assert_internal(wasclear);
mi_atomic_increment_relaxed(&subproc->abandoned_count[bin]); mi_atomic_increment_relaxed(&arena->subproc->abandoned_count[bin]);
mi_subproc_stat_increase(arena->subproc, pages_abandoned, 1);
} }
else { else {
// page is full (or a singleton), page is OS/externally allocated // page is full (or a singleton), or the page is OS/externally allocated
// leave as is; it will be reclaimed when an object is free'd in the page // leave as is; it will be reclaimed when an object is free'd in the page
mi_subproc_t* subproc = _mi_subproc();
// but for non-arena pages, add to the subproc list so these can be visited
if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
mi_lock(&subproc->os_abandoned_pages_lock) {
// push in front
page->prev = NULL;
page->next = subproc->os_abandoned_pages;
if (page->next != NULL) { page->next->prev = page; }
subproc->os_abandoned_pages = page;
}
}
mi_subproc_stat_increase(_mi_subproc(), pages_abandoned, 1);
} }
_mi_stat_increase(&_mi_stats_main.pages_abandoned, 1);
_mi_page_unown(page); _mi_page_unown(page);
} }
@ -868,8 +847,9 @@ bool _mi_arena_page_try_reabandon_to_mapped(mi_page_t* page) {
return false; return false;
} }
else { else {
_mi_stat_counter_increase(&_mi_stats_main.pages_reabandon_full, 1); mi_subproc_t* subproc = _mi_subproc();
_mi_stat_adjust_decrease(&_mi_stats_main.pages_abandoned, 1, true /* on alloc */); // adjust as we are not abandoning fresh mi_subproc_stat_counter_increase( subproc, pages_reabandon_full, 1);
mi_subproc_stat_adjust_decrease( subproc, pages_abandoned, 1, true /* on alloc */); // adjust as we are not abandoning fresh
_mi_arena_page_abandon(page); _mi_arena_page_abandon(page);
return true; return true;
} }
@ -896,14 +876,24 @@ void _mi_arena_page_unabandon(mi_page_t* page) {
// this busy waits until a concurrent reader (from alloc_abandoned) is done // this busy waits until a concurrent reader (from alloc_abandoned) is done
mi_bitmap_clear_once_set(arena->pages_abandoned[bin], slice_index); mi_bitmap_clear_once_set(arena->pages_abandoned[bin], slice_index);
mi_page_clear_abandoned_mapped(page); mi_page_clear_abandoned_mapped(page);
mi_atomic_decrement_relaxed(&page->subproc->abandoned_count[bin]); mi_atomic_decrement_relaxed(&arena->subproc->abandoned_count[bin]);
mi_subproc_stat_decrease(arena->subproc, pages_abandoned, 1);
} }
else { else {
// page is full (or a singleton), page is OS/nly allocated // page is full (or a singleton), page is OS allocated
// nothing to do mi_subproc_t* subproc = _mi_subproc();
// TODO: maintain count of these as well? mi_subproc_stat_decrease(_mi_subproc(), pages_abandoned, 1);
// if not an arena page, remove from the subproc os pages list
if (page->memid.memkind != MI_MEM_ARENA && mi_option_is_enabled(mi_option_visit_abandoned)) {
mi_lock(&subproc->os_abandoned_pages_lock) {
if (page->prev != NULL) { page->prev->next = page->next; }
if (page->next != NULL) { page->next->prev = page->prev; }
if (subproc->os_abandoned_pages == page) { subproc->os_abandoned_pages = page->next; }
page->next = NULL;
page->prev = NULL;
}
}
} }
_mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1);
} }
void _mi_arena_reclaim_all_abandoned(mi_heap_t* heap) { void _mi_arena_reclaim_all_abandoned(mi_heap_t* heap) {
@ -919,7 +909,7 @@ void _mi_arena_reclaim_all_abandoned(mi_heap_t* heap) {
static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices); static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_t slices);
static void mi_arenas_try_purge(bool force, bool visit_all); static void mi_arenas_try_purge(bool force, bool visit_all);
static void mi_arena_free(void* p, size_t size, mi_memid_t memid) { void _mi_arena_free(void* p, size_t size, mi_memid_t memid) {
if (p==NULL) return; if (p==NULL) return;
if (size==0) return; if (size==0) return;
@ -981,12 +971,21 @@ void _mi_arenas_collect(bool force_purge) {
mi_arenas_try_purge(force_purge, force_purge /* visit all? */); mi_arenas_try_purge(force_purge, force_purge /* visit all? */);
} }
// Is a pointer contained in the given arena area?
bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) {
mi_arena_t* arena = _mi_arena_from_id(arena_id);
return (mi_arena_start(arena) <= (const uint8_t*)p &&
mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p);
}
// Is a pointer inside any of our arenas? // Is a pointer inside any of our arenas?
bool _mi_arena_contains(const void* p) { bool _mi_arena_contains(const void* p) {
const size_t max_arena = mi_arena_get_count(); mi_subproc_t* subproc = _mi_subproc();
const size_t max_arena = mi_arenas_get_count(subproc);
for (size_t i = 0; i < max_arena; i++) { for (size_t i = 0; i < max_arena; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
if (arena != NULL && mi_arena_start(arena) <= (const uint8_t*)p && mi_arena_start(arena) + mi_size_of_slices(arena->slice_count) >(const uint8_t*)p) { if (arena != NULL && mi_arena_contains(arena,p)) {
return true; return true;
} }
} }
@ -1001,14 +1000,14 @@ bool _mi_arena_contains(const void* p) {
// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` // destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
// for dynamic libraries that are unloaded and need to release all their allocated memory. // for dynamic libraries that are unloaded and need to release all their allocated memory.
static void mi_arenas_unsafe_destroy(void) { static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {
const size_t max_arena = mi_arena_get_count(); const size_t max_arena = mi_arenas_get_count(subproc);
size_t new_max_arena = 0; size_t new_max_arena = 0;
for (size_t i = 0; i < max_arena; i++) { for (size_t i = 0; i < max_arena; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
if (arena != NULL) { if (arena != NULL) {
// mi_lock_done(&arena->abandoned_visit_lock); // mi_lock_done(&arena->abandoned_visit_lock);
mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL); mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
if (mi_memkind_is_os(arena->memid.memkind)) { if (mi_memkind_is_os(arena->memid.memkind)) {
_mi_os_free(mi_arena_start(arena), mi_arena_size(arena), arena->memid); _mi_os_free(mi_arena_start(arena), mi_arena_size(arena), arena->memid);
} }
@ -1017,14 +1016,14 @@ static void mi_arenas_unsafe_destroy(void) {
// try to lower the max arena. // try to lower the max arena.
size_t expected = max_arena; size_t expected = max_arena;
mi_atomic_cas_strong_acq_rel(&mi_arena_count, &expected, new_max_arena); mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, new_max_arena);
} }
// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` // destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
// for dynamic libraries that are unloaded and need to release all their allocated memory. // for dynamic libraries that are unloaded and need to release all their allocated memory.
void _mi_arena_unsafe_destroy_all(void) { void _mi_arena_unsafe_destroy_all(void) {
mi_arenas_unsafe_destroy(); mi_arenas_unsafe_destroy(_mi_subproc());
_mi_arenas_collect(true /* force purge */); // purge non-owned arenas _mi_arenas_collect(true /* force purge */); // purge non-owned arenas
} }
@ -1033,40 +1032,36 @@ void _mi_arena_unsafe_destroy_all(void) {
Add an arena. Add an arena.
----------------------------------------------------------- */ ----------------------------------------------------------- */
static bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id, mi_stats_t* stats) { static bool mi_arena_add(mi_subproc_t* subproc, mi_arena_t* arena, mi_arena_id_t* arena_id) {
mi_assert_internal(arena != NULL); mi_assert_internal(arena != NULL);
mi_assert_internal(arena->slice_count > 0); mi_assert_internal(arena->slice_count > 0);
if (arena_id != NULL) { *arena_id = -1; } if (arena_id != NULL) { *arena_id = NULL; }
// first try to find a NULL entry // first try to find a NULL entry
const size_t count = mi_arena_get_count(); const size_t count = mi_arenas_get_count(subproc);
size_t i; size_t i;
for (i = 0; i < count; i++) { for (i = 0; i < count; i++) {
if (mi_arena_from_index(i) == NULL) { if (mi_arena_from_index(subproc,i) == NULL) {
arena->id = mi_arena_id_create(i);
mi_arena_t* expected = NULL; mi_arena_t* expected = NULL;
if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &mi_arenas[i], &expected, arena)) { if (mi_atomic_cas_ptr_strong_release(mi_arena_t, &subproc->arenas[i], &expected, arena)) {
// success // success
if (arena_id != NULL) { *arena_id = arena->id; } if (arena_id != NULL) { *arena_id = arena; }
return true; return true;
} }
else {
arena->id = _mi_arena_id_none();
}
} }
} }
// otherwise increase the max // otherwise increase the max
i = mi_atomic_increment_acq_rel(&mi_arena_count); i = mi_atomic_increment_acq_rel(&subproc->arena_count);
if (i >= MI_MAX_ARENAS) { if (i >= MI_MAX_ARENAS) {
mi_atomic_decrement_acq_rel(&mi_arena_count); mi_atomic_decrement_acq_rel(&subproc->arena_count);
arena->subproc = NULL;
return false; return false;
} }
_mi_stat_counter_increase(&stats->arena_count,1); mi_subproc_stat_counter_increase(arena->subproc, arena_count, 1);
arena->id = mi_arena_id_create(i); mi_atomic_store_ptr_release(mi_arena_t,&subproc->arenas[i], arena);
mi_atomic_store_ptr_release(mi_arena_t,&mi_arenas[i], arena); if (arena_id != NULL) { *arena_id = arena; }
if (arena_id != NULL) { *arena_id = arena->id; }
return true; return true;
} }
@ -1099,7 +1094,7 @@ static mi_bbitmap_t* mi_arena_bbitmap_init(size_t slice_count, uint8_t** base) {
} }
static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept
{ {
mi_assert(!is_large || (memid.initially_committed && memid.is_pinned)); mi_assert(!is_large || (memid.initially_committed && memid.is_pinned));
mi_assert(_mi_is_aligned(start,MI_ARENA_SLICE_SIZE)); mi_assert(_mi_is_aligned(start,MI_ARENA_SLICE_SIZE));
@ -1138,7 +1133,7 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int
} }
// init // init
arena->id = _mi_arena_id_none(); arena->subproc = subproc;
arena->memid = memid; arena->memid = memid;
arena->is_exclusive = exclusive; arena->is_exclusive = exclusive;
arena->slice_count = slice_count; arena->slice_count = slice_count;
@ -1146,7 +1141,6 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int
arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1)
arena->is_large = is_large; arena->is_large = is_large;
arena->purge_expire = 0; arena->purge_expire = 0;
arena->purge_expire_extend = 0;
// mi_lock_init(&arena->abandoned_visit_lock); // mi_lock_init(&arena->abandoned_visit_lock);
// init bitmaps // init bitmaps
@ -1176,7 +1170,7 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int
mi_bitmap_setN(arena->slices_dirty, 0, info_slices, NULL); mi_bitmap_setN(arena->slices_dirty, 0, info_slices, NULL);
} }
return mi_arena_add(arena, arena_id, &_mi_stats_main); return mi_arena_add(subproc, arena, arena_id);
} }
@ -1187,18 +1181,18 @@ bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is
memid.initially_committed = is_committed; memid.initially_committed = is_committed;
memid.initially_zero = is_zero; memid.initially_zero = is_zero;
memid.is_pinned = is_large; memid.is_pinned = is_large;
return mi_manage_os_memory_ex2(start, size, is_large, numa_node, exclusive, memid, arena_id); return mi_manage_os_memory_ex2(_mi_subproc(), start, size, is_large, numa_node, exclusive, memid, arena_id);
} }
// Reserve a range of regular OS memory // Reserve a range of regular OS memory
int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) {
if (arena_id != NULL) *arena_id = _mi_arena_id_none(); if (arena_id != NULL) *arena_id = _mi_arena_id_none();
size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice
mi_memid_t memid; mi_memid_t memid;
void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid); void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid);
if (start == NULL) return ENOMEM; if (start == NULL) return ENOMEM;
const bool is_large = memid.is_pinned; // todo: use separate is_large field? const bool is_large = memid.is_pinned; // todo: use separate is_large field?
if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) { if (!mi_manage_os_memory_ex2(subproc, start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) {
_mi_os_free_ex(start, size, commit, memid); _mi_os_free_ex(start, size, commit, memid);
_mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
return ENOMEM; return ENOMEM;
@ -1209,6 +1203,11 @@ int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exc
return 0; return 0;
} }
// Reserve a range of regular OS memory
int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
return mi_reserve_os_memory_ex2(_mi_subproc(), size, commit, allow_large, exclusive, arena_id);
}
// Manage a range of regular OS memory // Manage a range of regular OS memory
bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept {
return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL); return mi_manage_os_memory_ex(start, size, is_committed, is_large, is_zero, numa_node, false /* exclusive? */, NULL);
@ -1333,16 +1332,18 @@ static size_t mi_debug_show_bbitmap(const char* header, size_t slice_count, mi_b
void mi_debug_show_arenas(bool show_pages, bool show_inuse, bool show_committed) mi_attr_noexcept { void mi_debug_show_arenas(bool show_pages, bool show_inuse, bool show_committed) mi_attr_noexcept {
size_t max_arenas = mi_arena_get_count(); mi_subproc_t* subproc = _mi_subproc();
size_t max_arenas = mi_arenas_get_count(subproc);
size_t free_total = 0; size_t free_total = 0;
size_t slice_total = 0; size_t slice_total = 0;
//size_t abandoned_total = 0; //size_t abandoned_total = 0;
size_t page_total = 0; size_t page_total = 0;
for (size_t i = 0; i < max_arenas; i++) { for (size_t i = 0; i < max_arenas; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]);
if (arena == NULL) break; if (arena == NULL) break;
mi_assert(arena->subproc == subproc);
slice_total += arena->slice_count; slice_total += arena->slice_count;
_mi_output_message("arena %zu at %p: %zu slices (%zu MiB)%s\n", i, arena, arena->slice_count, mi_size_of_slices(arena->slice_count)/MI_MiB, (arena->memid.is_pinned ? ", pinned" : "")); _mi_output_message("arena %zu at %p: %zu slices (%zu MiB)%s, subproc: %p\n", i, arena, arena->slice_count, mi_size_of_slices(arena->slice_count)/MI_MiB, (arena->memid.is_pinned ? ", pinned" : ""), arena->subproc);
if (show_inuse) { if (show_inuse) {
free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL); free_total += mi_debug_show_bbitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL);
} }
@ -1368,7 +1369,7 @@ void mi_debug_show_arenas(bool show_pages, bool show_inuse, bool show_committed)
----------------------------------------------------------- */ ----------------------------------------------------------- */
// reserve at a specific numa node // reserve at a specific numa node
int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept { int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept {
if (arena_id != NULL) *arena_id = -1; if (arena_id != NULL) *arena_id = NULL;
if (pages==0) return 0; if (pages==0) return 0;
if (numa_node < -1) numa_node = -1; if (numa_node < -1) numa_node = -1;
if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count(); if (numa_node >= 0) numa_node = numa_node % _mi_os_numa_node_count();
@ -1382,7 +1383,7 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m
} }
_mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);
if (!mi_manage_os_memory_ex2(p, hsize, true, numa_node, exclusive, memid, arena_id)) { if (!mi_manage_os_memory_ex2(_mi_subproc(), p, hsize, true, numa_node, exclusive, memid, arena_id)) {
_mi_os_free(p, hsize, memid); _mi_os_free(p, hsize, memid);
return ENOMEM; return ENOMEM;
} }
@ -1455,7 +1456,7 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c
// update committed bitmap // update committed bitmap
if (needs_recommit) { if (needs_recommit) {
_mi_stat_adjust_decrease(&_mi_stats_main.committed, mi_size_of_slices(slice_count - already_committed), false /* on freed */); mi_subproc_stat_adjust_decrease( arena->subproc, committed, mi_size_of_slices(slice_count - already_committed), false /* on freed */);
mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
} }
return needs_recommit; return needs_recommit;
@ -1475,14 +1476,15 @@ static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_
} }
else { else {
// schedule purge // schedule purge
const mi_msecs_t expire = _mi_clock_now() + delay;
mi_msecs_t expire0 = 0; mi_msecs_t expire0 = 0;
if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, _mi_clock_now() + delay)) { if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {
// expiration was not yet set // expiration was not yet set
mi_atomic_storei64_release(&arena->purge_expire_extend, 0); // maybe set the global arenas expire as well (if it wasn't set already)
mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire);
} }
else if (mi_atomic_loadi64_acquire(&arena->purge_expire_extend) < 10*delay) { // limit max extension time else {
// already an expiration was set // already an expiration was set
mi_atomic_addi64_acq_rel(&arena->purge_expire_extend, (mi_msecs_t)(delay/10)); // add smallish extra delay
} }
mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL); mi_bitmap_setN(arena->slices_purge, slice_index, slice_count, NULL);
} }
@ -1536,19 +1538,14 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
{ {
// check pre-conditions // check pre-conditions
if (arena->memid.is_pinned) return false; if (arena->memid.is_pinned) return false;
mi_msecs_t expire_base = mi_atomic_loadi64_relaxed(&arena->purge_expire);
mi_msecs_t expire_extend = mi_atomic_loadi64_relaxed(&arena->purge_expire_extend);
const mi_msecs_t expire = expire_base + expire_extend;
if (expire == 0) return false;
// expired yet? // expired yet?
if (!force && expire > now) return false; mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
if (!force && (expire == 0 || expire > now)) return false;
// reset expire (if not already set concurrently) // reset expire (if not already set concurrently)
if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire_base, (mi_msecs_t)0)) { mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0);
mi_atomic_storei64_release(&arena->purge_expire_extend, (mi_msecs_t)0); // and also reset the extend mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1);
}
_mi_stat_counter_increase(&_mi_stats_main.arena_purges, 1);
// go through all purge info's (with max MI_BFIELD_BITS ranges at a time) // go through all purge info's (with max MI_BFIELD_BITS ranges at a time)
// this also clears those ranges atomically (so any newly freed blocks will get purged next // this also clears those ranges atomically (so any newly freed blocks will get purged next
@ -1560,30 +1557,46 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
} }
static void mi_arenas_try_purge(bool force, bool visit_all) { static void mi_arenas_try_purge(bool force, bool visit_all)
{
if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled
const size_t max_arena = mi_arena_get_count(); // check if any arena needs purging?
mi_tld_t* tld = _mi_tld();
mi_subproc_t* subproc = tld->subproc;
const mi_msecs_t now = _mi_clock_now();
mi_msecs_t arenas_expire = mi_atomic_load_acquire(&subproc->purge_expire);
if (!force && (arenas_expire == 0 || arenas_expire < now)) return;
const size_t max_arena = mi_arenas_get_count(subproc);
if (max_arena == 0) return; if (max_arena == 0) return;
// allow only one thread to purge at a time // allow only one thread to purge at a time
static mi_atomic_guard_t purge_guard; static mi_atomic_guard_t purge_guard;
mi_atomic_guard(&purge_guard) mi_atomic_guard(&purge_guard)
{ {
const mi_msecs_t now = _mi_clock_now(); // increase global expire: at most one purge per delay cycle
const size_t arena_start = _mi_tld()->tseq % max_arena; mi_atomic_store_release(&subproc->purge_expire, now + mi_arena_purge_delay());
size_t max_purge_count = (visit_all ? max_arena : 1); const size_t arena_start = tld->thread_seq % max_arena;
size_t max_purge_count = (visit_all ? max_arena : 2);
bool all_visited = true;
for (size_t _i = 0; _i < max_arena; _i++) { for (size_t _i = 0; _i < max_arena; _i++) {
size_t i = _i + arena_start; size_t i = _i + arena_start;
if (i >= max_arena) { i -= max_arena; } if (i >= max_arena) { i -= max_arena; }
mi_arena_t* arena = mi_arena_from_index(i); mi_arena_t* arena = mi_arena_from_index(subproc,i);
if (arena != NULL) { if (arena != NULL) {
if (mi_arena_try_purge(arena, now, force)) { if (mi_arena_try_purge(arena, now, force)) {
if (max_purge_count <= 1) break; if (max_purge_count <= 1) {
all_visited = false;
break;
}
max_purge_count--; max_purge_count--;
} }
} }
} }
if (all_visited) {
mi_atomic_store_release(&subproc->purge_expire, (mi_msecs_t)0);
}
} }
} }
@ -1612,13 +1625,7 @@ static bool mi_arena_pages_reregister(mi_arena_t* arena) {
} }
mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) { mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t* accessed_size, size_t* full_size) {
const size_t count = mi_arena_get_count(); mi_arena_t* arena = _mi_arena_from_id(arena_id);
const size_t arena_idx = mi_arena_id_index(arena_id);
if (count <= arena_idx) {
_mi_warning_message("arena id is invalid (%zu)\n", arena_id);
return false;
}
mi_arena_t* arena = mi_arena_from_id(arena_id);
if (arena==NULL) { if (arena==NULL) {
return false; return false;
} }
@ -1649,15 +1656,22 @@ mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t*
_mi_page_map_unregister_range(arena, asize); _mi_page_map_unregister_range(arena, asize);
// set the entry to NULL // set the entry to NULL
mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[arena_idx], NULL); mi_subproc_t* subproc = arena->subproc;
if (arena_idx + 1 == count) { // try adjust the count? const size_t count = mi_arenas_get_count(subproc);
for(size_t i = 0; i < count; i++) {
if (mi_arena_from_index(subproc, i) == arena) {
mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL);
if (i + 1 == count) { // try adjust the count?
size_t expected = count; size_t expected = count;
mi_atomic_cas_strong_acq_rel(&mi_arena_count, &expected, count-1); mi_atomic_cas_strong_acq_rel(&subproc->arena_count, &expected, count-1);
}
break;
}
} }
return true; return true;
} }
mi_decl_export bool mi_arena_reload(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, mi_arena_id_t* arena_id) { mi_decl_export bool mi_arena_reload(void* start, size_t size, mi_arena_id_t* arena_id) {
// assume the memory area is already containing the arena // assume the memory area is already containing the arena
if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
if (start == NULL || size == 0) return false; if (start == NULL || size == 0) return false;
@ -1679,13 +1693,10 @@ mi_decl_export bool mi_arena_reload(void* start, size_t size, bool is_committed,
_mi_warning_message("the reloaded arena is not exclusive\n"); _mi_warning_message("the reloaded arena is not exclusive\n");
return false; return false;
} }
arena->memid.is_pinned = is_large;
arena->memid.initially_committed = is_committed;
arena->memid.initially_zero = is_zero;
arena->is_exclusive = true; arena->is_exclusive = true;
arena->is_large = is_large; arena->subproc = _mi_subproc();
arena->id = _mi_arena_id_none(); if (!mi_arena_add(arena->subproc, arena, arena_id)) {
if (!mi_arena_add(arena, arena_id, &_mi_stats_main)) {
return false; return false;
} }
mi_arena_pages_reregister(arena); mi_arena_pages_reregister(arena);

11
src/bitmap.c
View file

@ -114,7 +114,9 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_
do { do {
if mi_unlikely((old&mask) == 0) { if mi_unlikely((old&mask) == 0) {
old = mi_atomic_load_acquire(b); old = mi_atomic_load_acquire(b);
if ((old&mask)==0) { _mi_stat_counter_increase(&_mi_stats_main.pages_unabandon_busy_wait, 1); } if ((old&mask)==0) {
mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);
}
while ((old&mask)==0) { // busy wait while ((old&mask)==0) { // busy wait
mi_atomic_yield(); mi_atomic_yield();
old = mi_atomic_load_acquire(b); old = mi_atomic_load_acquire(b);
@ -1151,7 +1153,6 @@ static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, si
typedef struct mi_claim_fun_data_s { typedef struct mi_claim_fun_data_s {
mi_arena_t* arena; mi_arena_t* arena;
mi_subproc_t* subproc;
mi_heaptag_t heap_tag; mi_heaptag_t heap_tag;
} mi_claim_fun_data_t; } mi_claim_fun_data_t;
@ -1165,7 +1166,7 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk
const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx; const size_t slice_index = (chunk_idx * MI_BCHUNK_BITS) + cidx;
mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap)); mi_assert_internal(slice_index < mi_bitmap_max_bits(bitmap));
bool keep_set = true; bool keep_set = true;
if ((*claim_fun)(slice_index, claim_data->arena, claim_data->subproc, claim_data->heap_tag, &keep_set)) { if ((*claim_fun)(slice_index, claim_data->arena, claim_data->heap_tag, &keep_set)) {
// success! // success!
mi_assert_internal(!keep_set); mi_assert_internal(!keep_set);
*pidx = slice_index; *pidx = slice_index;
@ -1190,9 +1191,9 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk
// Find a set bit in the bitmap and try to atomically clear it and claim it. // Find a set bit in the bitmap and try to atomically clear it and claim it.
// (Used to find pages in the pages_abandoned bitmaps.) // (Used to find pages in the pages_abandoned bitmaps.)
mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
mi_claim_fun_t* claim, mi_arena_t* arena, mi_subproc_t* subproc, mi_heaptag_t heap_tag) mi_claim_fun_t* claim, mi_arena_t* arena, mi_heaptag_t heap_tag)
{ {
mi_claim_fun_data_t claim_data = { arena, subproc, heap_tag }; mi_claim_fun_data_t claim_data = { arena, heap_tag };
return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data); return mi_bitmap_find(bitmap, tseq, 1, pidx, &mi_bitmap_try_find_and_claim_visit, (void*)claim, &claim_data);
} }

4
src/bitmap.h
View file

@ -177,13 +177,13 @@ static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) {
// Called once a bit is cleared to see if the memory slice can be claimed. // Called once a bit is cleared to see if the memory slice can be claimed.
typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, mi_subproc_t* subproc, mi_heaptag_t heap_tag, bool* keep_set); typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, mi_heaptag_t heap_tag, bool* keep_set);
// Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true. // Find a set bits in the bitmap, atomically clear it, and check if `claim` returns true.
// If not claimed, continue on (potentially setting the bit again depending on `keep_set`). // If not claimed, continue on (potentially setting the bit again depending on `keep_set`).
// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`. // Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
mi_claim_fun_t* claim, mi_arena_t* arena, mi_subproc_t* subproc, mi_heaptag_t heap_tag ); mi_claim_fun_t* claim, mi_arena_t* arena, mi_heaptag_t heap_tag );
// Atomically clear a bit but only if it is set. Will block otherwise until the bit is set. // Atomically clear a bit but only if it is set. Will block otherwise until the bit is set.

6
src/free.c
View file

@ -234,15 +234,15 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page) {
mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag); mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag);
if ((tagheap != NULL) && // don't reclaim across heap object types if ((tagheap != NULL) && // don't reclaim across heap object types
(tagheap->allow_page_reclaim) && // we are allowed to reclaim abandoned pages (tagheap->allow_page_reclaim) && // we are allowed to reclaim abandoned pages
(page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? ) // (page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? )
(_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?) (_mi_arena_memid_is_suitable(page->memid, tagheap->exclusive_arena)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?)
) )
{ {
if (mi_page_queue(tagheap, page->block_size)->first != NULL) { // don't reclaim for an block_size we don't use if (mi_page_queue(tagheap, page->block_size)->first != NULL) { // don't reclaim for an block_size we don't use
// first remove it from the abandoned pages in the arena -- this waits for any readers to finish // first remove it from the abandoned pages in the arena -- this waits for any readers to finish
_mi_arena_page_unabandon(page); _mi_arena_page_unabandon(page);
_mi_heap_page_reclaim(tagheap, page); _mi_heap_page_reclaim(tagheap, page);
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1); mi_heap_stat_counter_increase(tagheap, pages_reclaim_on_free, 1);
return; return;
} }
} }

96
src/heap.c
View file

@ -184,7 +184,7 @@ mi_heap_t* mi_heap_get_backing(void) {
mi_assert_internal(heap!=NULL); mi_assert_internal(heap!=NULL);
mi_heap_t* bheap = heap->tld->heap_backing; mi_heap_t* bheap = heap->tld->heap_backing;
mi_assert_internal(bheap!=NULL); mi_assert_internal(bheap!=NULL);
mi_assert_internal(bheap->thread_id == _mi_thread_id()); mi_assert_internal(bheap->tld->thread_id == _mi_thread_id());
return bheap; return bheap;
} }
@ -196,8 +196,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_arena_id_t arena_id, bool noreclaim, uint
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->memid = memid; heap->memid = memid;
heap->tld = tld; // avoid reading the thread-local tld during initialization heap->tld = tld; // avoid reading the thread-local tld during initialization
heap->thread_id = _mi_thread_id(); heap->exclusive_arena = _mi_arena_from_id(arena_id);
heap->arena_id = arena_id;
heap->allow_page_reclaim = !noreclaim; heap->allow_page_reclaim = !noreclaim;
heap->allow_page_abandon = (!noreclaim && mi_option_get(mi_option_full_page_retain) >= 0); heap->allow_page_abandon = (!noreclaim && mi_option_get(mi_option_full_page_retain) >= 0);
heap->full_page_retain = mi_option_get_clamp(mi_option_full_page_retain, -1, 32); heap->full_page_retain = mi_option_get_clamp(mi_option_full_page_retain, -1, 32);
@ -220,8 +219,8 @@ void _mi_heap_init(mi_heap_t* heap, mi_arena_id_t arena_id, bool noreclaim, uint
_mi_random_split(&heap->tld->heap_backing->random, &heap->random); _mi_random_split(&heap->tld->heap_backing->random, &heap->random);
} }
heap->cookie = _mi_heap_random_next(heap) | 1; heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap); //heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap); //heap->keys[1] = _mi_heap_random_next(heap);*/
_mi_heap_guarded_init(heap); _mi_heap_guarded_init(heap);
// push on the thread local heaps list // push on the thread local heaps list
@ -234,7 +233,15 @@ mi_heap_t* _mi_heap_create(int heap_tag, bool allow_destroy, mi_arena_id_t arena
mi_assert(heap_tag >= 0 && heap_tag < 256); mi_assert(heap_tag >= 0 && heap_tag < 256);
// allocate and initialize a heap // allocate and initialize a heap
mi_memid_t memid; mi_memid_t memid;
mi_heap_t* heap = (mi_heap_t*)_mi_meta_zalloc(sizeof(mi_heap_t), &memid); mi_heap_t* heap;
if (arena_id == _mi_arena_id_none()) {
heap = (mi_heap_t*)_mi_meta_zalloc(sizeof(mi_heap_t), &memid);
}
else {
// heaps associated wita a specific arena are allocated in that arena
// note: takes up at least one slice which is quite wasteful...
heap = (mi_heap_t*)_mi_arena_alloc(_mi_subproc(), _mi_align_up(sizeof(mi_heap_t),MI_ARENA_MIN_OBJ_SIZE), true, true, _mi_arena_from_id(arena_id), tld->thread_seq, &memid);
}
if (heap==NULL) { if (heap==NULL) {
_mi_error_message(ENOMEM, "unable to allocate heap meta-data\n"); _mi_error_message(ENOMEM, "unable to allocate heap meta-data\n");
return NULL; return NULL;
@ -260,7 +267,7 @@ mi_decl_nodiscard mi_heap_t* mi_heap_new(void) {
} }
bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid) { bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid) {
return _mi_arena_memid_is_suitable(memid, heap->arena_id); return _mi_arena_memid_is_suitable(memid, heap->exclusive_arena);
} }
uintptr_t _mi_heap_random_next(mi_heap_t* heap) { uintptr_t _mi_heap_random_next(mi_heap_t* heap) {
@ -279,7 +286,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) {
} }
// called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources. // called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources.
static void mi_heap_free(mi_heap_t* heap) { static void mi_heap_free(mi_heap_t* heap, bool do_free_mem) {
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap)); mi_assert_internal(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap==NULL || !mi_heap_is_initialized(heap)) return;
@ -306,8 +313,10 @@ static void mi_heap_free(mi_heap_t* heap) {
mi_assert_internal(heap->tld->heaps != NULL); mi_assert_internal(heap->tld->heaps != NULL);
// and free the used memory // and free the used memory
if (do_free_mem) {
_mi_meta_free(heap, sizeof(*heap), heap->memid); _mi_meta_free(heap, sizeof(*heap), heap->memid);
} }
}
// return a heap on the same thread as `heap` specialized for the specified tag (if it exists) // return a heap on the same thread as `heap` specialized for the specified tag (if it exists)
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) {
@ -402,7 +411,7 @@ void mi_heap_destroy(mi_heap_t* heap) {
#endif #endif
// free all pages // free all pages
_mi_heap_destroy_pages(heap); _mi_heap_destroy_pages(heap);
mi_heap_free(heap); mi_heap_free(heap,true);
} }
#endif #endif
} }
@ -461,20 +470,11 @@ void mi_heap_delete(mi_heap_t* heap)
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap==NULL || !mi_heap_is_initialized(heap)) return;
/*
mi_heap_t* bheap = heap->tld->heap_backing;
if (bheap != heap && mi_heaps_are_compatible(bheap,heap)) {
// transfer still used pages to the backing heap
mi_heap_absorb(bheap, heap);
}
else
*/
{
// abandon all pages // abandon all pages
_mi_heap_collect_abandon(heap); _mi_heap_collect_abandon(heap);
}
mi_assert_internal(heap->page_count==0); mi_assert_internal(heap->page_count==0);
mi_heap_free(heap); mi_heap_free(heap,true);
} }
mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { mi_heap_t* mi_heap_set_default(mi_heap_t* heap) {
@ -488,7 +488,63 @@ mi_heap_t* mi_heap_set_default(mi_heap_t* heap) {
} }
/* -----------------------------------------------------------
Load/unload heaps
----------------------------------------------------------- */
void mi_heap_unload(mi_heap_t* heap) {
mi_assert(mi_heap_is_initialized(heap));
mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (heap->exclusive_arena == NULL) {
_mi_warning_message("cannot unload heaps that are not associated with an exclusive arena\n");
return;
}
// abandon all pages so all thread'id in the pages are cleared
_mi_heap_collect_abandon(heap);
mi_assert_internal(heap->page_count==0);
// remove from heap list
mi_heap_free(heap, false /* but don't actually free the memory */);
// disassociate from the current thread-local and static state
heap->tld = NULL;
return;
}
bool mi_heap_reload(mi_heap_t* heap, mi_arena_id_t arena_id) {
mi_assert(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return false;
if (heap->exclusive_arena == NULL) {
_mi_warning_message("cannot reload heaps that were not associated with an exclusive arena\n");
return false;
}
if (heap->tld != NULL) {
_mi_warning_message("cannot reload heaps that were not unloaded first\n");
return false;
}
mi_arena_t* arena = _mi_arena_from_id(arena_id);
if (heap->exclusive_arena != arena) {
_mi_warning_message("trying to reload a heap at a different arena address: %p vs %p\n", heap->exclusive_arena, arena);
return false;
}
mi_assert_internal(heap->page_count==0);
// re-associate from the current thread-local and static state
heap->tld = _mi_tld();
// reinit direct pages (as we may be in a different process)
mi_assert_internal(heap->page_count == 0);
for (int i = 0; i < MI_PAGES_DIRECT; i++) {
heap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty;
}
// push on the thread local heaps list
heap->next = heap->tld->heaps;
heap->tld->heaps = heap;
return true;
}
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Analysis Analysis

265
src/init.c
View file

@ -11,7 +11,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include <string.h> // memcpy, memset #include <string.h> // memcpy, memset
#include <stdlib.h> // atexit #include <stdlib.h> // atexit
#define MI_MEMID_STATIC {{{NULL,0}}, MI_MEM_STATIC, true /* pinned */, true /* committed */, false /* zero */ } #define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ }
#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC)
// Empty page used to initialize the small free pages array // Empty page used to initialize the small free pages array
const mi_page_t _mi_page_empty = { const mi_page_t _mi_page_empty = {
@ -30,11 +31,10 @@ const mi_page_t _mi_page_empty = {
0, // heap tag 0, // heap tag
false, // is_zero false, // is_zero
#if (MI_PADDING || MI_ENCODE_FREELIST) #if (MI_PADDING || MI_ENCODE_FREELIST)
{ 0, 0 }, { 0, 0 }, // keys
#endif #endif
NULL, // xheap NULL, // xheap
NULL, NULL, // next, prev NULL, NULL, // next, prev
NULL, // subproc
MI_MEMID_STATIC // memid MI_MEMID_STATIC // memid
}; };
@ -96,19 +96,30 @@ const mi_page_t _mi_page_empty = {
// may lead to allocation itself on some platforms) // may lead to allocation itself on some platforms)
// -------------------------------------------------------- // --------------------------------------------------------
mi_decl_cache_align const mi_heap_t _mi_heap_empty = { static mi_decl_cache_align mi_subproc_t subproc_main;
NULL,
// MI_ATOMIC_VAR_INIT(NULL), // thread delayed free static mi_decl_cache_align mi_tld_t tld_empty = {
0, // thread_id 0, // thread_id
0, // arena_id 0, // thread_seq
&subproc_main, // subproc
NULL, // heap_backing
NULL, // heaps list
0, // heartbeat
false, // recurse
false, // is_in_threadpool
{ MI_STATS_NULL }, // stats
MI_MEMID_STATIC // memid
};
mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
&tld_empty, // tld
NULL, // exclusive_arena
0, // cookie 0, // cookie
{ 0, 0 }, // keys //{ 0, 0 }, // keys
{ {0}, {0}, 0, true }, // random { {0}, {0}, 0, true }, // random
0, // page count 0, // page count
MI_BIN_FULL, 0, // page retired min/max MI_BIN_FULL, 0, // page retired min/max
NULL, // next NULL, // next
MI_MEMID_STATIC, // memid
0,
0, // full page retain 0, // full page retain
false, // can reclaim false, // can reclaim
true, // can eager abandon true, // can eager abandon
@ -117,47 +128,34 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) 0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`)
#endif #endif
MI_SMALL_PAGES_EMPTY, MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY MI_PAGE_QUEUES_EMPTY,
MI_MEMID_STATIC
}; };
extern mi_heap_t heap_main;
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
return _mi_prim_thread_id();
}
// the thread-local default heap for allocation
mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
extern mi_heap_t _mi_heap_main;
static mi_decl_cache_align mi_subproc_t mi_subproc_default;
static mi_decl_cache_align mi_tld_t tld_main = { static mi_decl_cache_align mi_tld_t tld_main = {
0, 0, // thread_id
&_mi_heap_main, // heap_backing 0, // thread_seq
&_mi_heap_main, // heaps list &subproc_main, // subproc
&mi_subproc_default, // subproc &heap_main, // heap_backing
0, // tseq &heap_main, // heaps list
MI_MEMID_STATIC, // memid 0, // heartbeat
false, // recurse false, // recurse
false, // is_in_threadpool false, // is_in_threadpool
{ MI_STATS_NULL } // stats { MI_STATS_NULL }, // stats
MI_MEMID_STATIC // memid
}; };
mi_decl_cache_align mi_heap_t _mi_heap_main = { mi_decl_cache_align mi_heap_t heap_main = {
&tld_main, &tld_main, // thread local data
// MI_ATOMIC_VAR_INIT(NULL), // thread delayed free list NULL, // exclusive arena
0, // thread id
0, // initial cookie 0, // initial cookie
0, // arena id //{ 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!)
{ 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!)
{ {0x846ca68b}, {0}, 0, true }, // random { {0x846ca68b}, {0}, 0, true }, // random
0, // page count 0, // page count
MI_BIN_FULL, 0, // page retired min/max MI_BIN_FULL, 0, // page retired min/max
NULL, // next heap NULL, // next heap
MI_MEMID_STATIC, // memid
0,
2, // full page retain 2, // full page retain
true, // allow page reclaim true, // allow page reclaim
true, // allow page abandon true, // allow page abandon
@ -166,9 +164,19 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
#endif #endif
MI_SMALL_PAGES_EMPTY, MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY MI_PAGE_QUEUES_EMPTY,
MI_MEMID_STATIC
}; };
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
return _mi_prim_thread_id();
}
// the thread-local default heap for allocation
mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
mi_stats_t _mi_stats_main = { MI_STATS_NULL }; mi_stats_t _mi_stats_main = { MI_STATS_NULL };
@ -212,30 +220,46 @@ void _mi_heap_guarded_init(mi_heap_t* heap) {
} }
#endif #endif
// Initialize main subproc
static void mi_subproc_main_init(void) {
if (subproc_main.memid.memkind != MI_MEM_STATIC) {
subproc_main.memid = _mi_memid_create(MI_MEM_STATIC);
mi_lock_init(&subproc_main.os_abandoned_pages_lock);
mi_lock_init(&subproc_main.arena_reserve_lock);
}
}
// Initialize main tld
static void mi_tld_main_init(void) {
if (tld_main.thread_id == 0) {
tld_main.thread_id = _mi_prim_thread_id();
}
}
// Initialization of the (statically allocated) main heap, and the main tld and subproc.
static void mi_heap_main_init(void) { static void mi_heap_main_init(void) {
if (_mi_heap_main.cookie == 0) { if (heap_main.cookie == 0) {
_mi_heap_main.thread_id = _mi_thread_id(); mi_subproc_main_init();
_mi_heap_main.cookie = 1; mi_tld_main_init();
// heap
heap_main.cookie = 1;
#if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB) #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB)
_mi_random_init_weak(&_mi_heap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking _mi_random_init_weak(&heap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking
#else #else
_mi_random_init(&_mi_heap_main.random); _mi_random_init(&heap_main.random);
#endif #endif
_mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main); heap_main.cookie = _mi_heap_random_next(&heap_main);
_mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); //heap_main.keys[0] = _mi_heap_random_next(&heap_main);
_mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); //heap_main.keys[1] = _mi_heap_random_next(&heap_main);
mi_lock_init(&mi_subproc_default.abandoned_os_lock); _mi_heap_guarded_init(&heap_main);
mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock); heap_main.allow_page_abandon = (mi_option_get(mi_option_full_page_retain) >= 0);
_mi_heap_guarded_init(&_mi_heap_main); heap_main.full_page_retain = mi_option_get_clamp(mi_option_full_page_retain, -1, 32);
_mi_heap_main.allow_page_abandon = (mi_option_get(mi_option_full_page_retain) >= 0);
_mi_heap_main.full_page_retain = mi_option_get_clamp(mi_option_full_page_retain, -1, 32);
} }
} }
mi_heap_t* _mi_heap_main_get(void) { mi_heap_t* heap_main_get(void) {
mi_heap_main_init(); mi_heap_main_init();
return &_mi_heap_main; return &heap_main;
} }
@ -243,14 +267,21 @@ mi_heap_t* _mi_heap_main_get(void) {
Thread local data Thread local data
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Thread sequence number // Count current and total created threads
static _Atomic(size_t) mi_tcount; static _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1);
static _Atomic(size_t) thread_total_count;
size_t _mi_current_thread_count(void) {
return mi_atomic_load_relaxed(&thread_count);
}
// The mimalloc thread local data // The mimalloc thread local data
mi_decl_thread mi_tld_t* mi_tld; mi_decl_thread mi_tld_t* thread_tld = &tld_empty;
// Allocate fresh tld // Allocate fresh tld
static mi_tld_t* mi_tld_alloc(void) { static mi_tld_t* mi_tld_alloc(void) {
mi_atomic_increment_relaxed(&thread_count);
if (_mi_is_main_thread()) { if (_mi_is_main_thread()) {
return &tld_main; return &tld_main;
} }
@ -267,8 +298,9 @@ static mi_tld_t* mi_tld_alloc(void) {
tld->memid = memid; tld->memid = memid;
tld->heap_backing = NULL; tld->heap_backing = NULL;
tld->heaps = NULL; tld->heaps = NULL;
tld->subproc = &mi_subproc_default; tld->subproc = &subproc_main;
tld->tseq = mi_atomic_add_acq_rel(&mi_tcount, 1); tld->thread_id = _mi_prim_thread_id();
tld->thread_seq = mi_atomic_add_acq_rel(&thread_total_count, 1);
tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool(); tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool();
return tld; return tld;
} }
@ -278,27 +310,49 @@ static mi_tld_t* mi_tld_alloc(void) {
mi_decl_noinline static void mi_tld_free(void) { mi_decl_noinline static void mi_tld_free(void) {
mi_tld_t* tld = _mi_tld(); mi_tld_t* tld = _mi_tld();
mi_tld = MI_TLD_INVALID; if (tld != NULL && tld != MI_TLD_INVALID) {
_mi_stats_done(&tld->stats);
_mi_meta_free(tld, sizeof(mi_tld_t), tld->memid); _mi_meta_free(tld, sizeof(mi_tld_t), tld->memid);
} }
tld = MI_TLD_INVALID;
mi_atomic_decrement_relaxed(&thread_count);
}
mi_decl_noinline mi_tld_t* _mi_tld(void) { mi_decl_noinline mi_tld_t* _mi_tld(void) {
if (mi_tld == MI_TLD_INVALID) { mi_tld_t* tld = thread_tld;
_mi_error_message(EFAULT, "internal error: tld accessed after the thread terminated\n"); if (tld == MI_TLD_INVALID) {
mi_tld = NULL; _mi_error_message(EFAULT, "internal error: tld is accessed after the thread terminated\n");
thread_tld = &tld_empty;
} }
if (mi_tld==NULL) { if (tld==&tld_empty) {
mi_tld = mi_tld_alloc(); thread_tld = tld = mi_tld_alloc();
} }
return mi_tld; return tld;
} }
mi_subproc_t* _mi_subproc(void) {
// should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()`
// todo: this will still fail on OS systems where the first access to a thread-local causes allocation.
// on such systems we can check for this with the _mi_prim_get_default_heap as those are protected (by being
// stored in a TLS slot for example)
mi_heap_t* heap = mi_prim_get_default_heap();
if (heap == NULL || heap == &_mi_heap_empty) {
return _mi_subproc_main();
}
else {
return thread_tld->subproc; // don't call `_mi_tld()`
}
}
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Sub process Sub process
----------------------------------------------------------- */ ----------------------------------------------------------- */
mi_subproc_t* _mi_subproc_main(void) {
return &subproc_main;
}
mi_subproc_id_t mi_subproc_main(void) { mi_subproc_id_t mi_subproc_main(void) {
return NULL; return NULL;
} }
@ -307,42 +361,44 @@ mi_subproc_id_t mi_subproc_new(void) {
mi_memid_t memid; mi_memid_t memid;
mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid); mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(sizeof(mi_subproc_t),&memid);
if (subproc == NULL) return NULL; if (subproc == NULL) return NULL;
subproc->abandoned_os_list = NULL;
subproc->memid = memid; subproc->memid = memid;
mi_lock_init(&subproc->abandoned_os_lock); mi_lock_init(&subproc->os_abandoned_pages_lock);
mi_lock_init(&subproc->abandoned_os_visit_lock); mi_lock_init(&subproc->arena_reserve_lock);
return subproc; return subproc;
} }
mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) {
return (subproc_id == NULL ? &mi_subproc_default : (mi_subproc_t*)subproc_id); return (subproc_id == NULL ? &subproc_main : (mi_subproc_t*)subproc_id);
} }
void mi_subproc_delete(mi_subproc_id_t subproc_id) { void mi_subproc_delete(mi_subproc_id_t subproc_id) {
if (subproc_id == NULL) return; if (subproc_id == NULL) return;
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
// check if there are no abandoned segments still.. // check if there are os pages still..
bool safe_to_delete = false; bool safe_to_delete = false;
if (mi_lock_acquire(&subproc->abandoned_os_lock)) { mi_lock(&subproc->os_abandoned_pages_lock) {
if (subproc->abandoned_os_list == NULL) { if (subproc->os_abandoned_pages == NULL) {
safe_to_delete = true; safe_to_delete = true;
} }
mi_lock_release(&subproc->abandoned_os_lock);
} }
if (!safe_to_delete) return; if (!safe_to_delete) return;
// merge stats back into the main subproc?
_mi_stats_merge_from(&_mi_subproc_main()->stats, &subproc->stats);
// safe to release // safe to release
// todo: should we refcount subprocesses? // todo: should we refcount subprocesses?
mi_lock_done(&subproc->abandoned_os_lock); mi_lock_done(&subproc->os_abandoned_pages_lock);
mi_lock_done(&subproc->abandoned_os_visit_lock); mi_lock_done(&subproc->arena_reserve_lock);
_mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid); _mi_meta_free(subproc, sizeof(mi_subproc_t), subproc->memid);
} }
void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {
mi_heap_t* heap = mi_heap_get_default(); mi_tld_t* tld = _mi_tld();
if (heap == NULL) return; if (tld == NULL) return;
mi_assert(heap->tld->subproc == &mi_subproc_default); mi_assert(tld->subproc == &subproc_main);
if (heap->tld->subproc != &mi_subproc_default) return; if (tld->subproc != &subproc_main) return;
heap->tld->subproc = _mi_subproc_from_id(subproc_id); tld->subproc = _mi_subproc_from_id(subproc_id);
} }
@ -354,10 +410,10 @@ void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) {
static bool _mi_thread_heap_init(void) { static bool _mi_thread_heap_init(void) {
if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true;
if (_mi_is_main_thread()) { if (_mi_is_main_thread()) {
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization // mi_assert_internal(heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization
// the main heap is statically allocated // the main heap is statically allocated
mi_heap_main_init(); mi_heap_main_init();
_mi_heap_set_default_direct(&_mi_heap_main); _mi_heap_set_default_direct(&heap_main);
//mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap()); //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap());
} }
else { else {
@ -374,7 +430,7 @@ static bool _mi_thread_heap_init(void) {
_mi_heap_set_default_direct(heap); _mi_heap_set_default_direct(heap);
// now that the heap is set for this thread, we can set the thread-local tld. // now that the heap is set for this thread, we can set the thread-local tld.
mi_tld = tld; thread_tld = tld;
} }
return false; return false;
} }
@ -385,7 +441,7 @@ static bool _mi_thread_heap_done(mi_heap_t* heap) {
if (!mi_heap_is_initialized(heap)) return true; if (!mi_heap_is_initialized(heap)) return true;
// reset default heap // reset default heap
_mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); _mi_heap_set_default_direct(_mi_is_main_thread() ? &heap_main : (mi_heap_t*)&_mi_heap_empty);
// switch to backing heap // switch to backing heap
heap = heap->tld->heap_backing; heap = heap->tld->heap_backing;
@ -405,22 +461,19 @@ static bool _mi_thread_heap_done(mi_heap_t* heap) {
mi_assert_internal(mi_heap_is_backing(heap)); mi_assert_internal(mi_heap_is_backing(heap));
// collect if not the main thread // collect if not the main thread
if (heap != &_mi_heap_main) { if (heap != &heap_main) {
_mi_heap_collect_abandon(heap); _mi_heap_collect_abandon(heap);
} }
// merge stats
_mi_stats_done(&heap->tld->stats);
// free heap meta data // free heap meta data
_mi_meta_free(heap, sizeof(mi_heap_t), heap->memid); _mi_meta_free(heap, sizeof(mi_heap_t), heap->memid);
if (heap == &_mi_heap_main) { if (heap == &heap_main) {
#if 0 #if 0
// never free the main thread even in debug mode; if a dll is linked statically with mimalloc, // never free the main thread even in debug mode; if a dll is linked statically with mimalloc,
// there may still be delete/free calls after the mi_fls_done is called. Issue #207 // there may still be delete/free calls after the mi_fls_done is called. Issue #207
_mi_heap_destroy_pages(heap); _mi_heap_destroy_pages(heap);
mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main); mi_assert_internal(heap->tld->heap_backing == &heap_main);
#endif #endif
} }
@ -451,19 +504,14 @@ static void mi_process_setup_auto_thread_done(void) {
if (tls_initialized) return; if (tls_initialized) return;
tls_initialized = true; tls_initialized = true;
_mi_prim_thread_init_auto_done(); _mi_prim_thread_init_auto_done();
_mi_heap_set_default_direct(&_mi_heap_main); _mi_heap_set_default_direct(&heap_main);
} }
bool _mi_is_main_thread(void) { bool _mi_is_main_thread(void) {
return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); return (tld_main.thread_id==0 || tld_main.thread_id == _mi_thread_id());
} }
static _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1);
size_t _mi_current_thread_count(void) {
return mi_atomic_load_relaxed(&thread_count);
}
// This is called from the `mi_malloc_generic` // This is called from the `mi_malloc_generic`
void mi_thread_init(void) mi_attr_noexcept void mi_thread_init(void) mi_attr_noexcept
@ -476,8 +524,7 @@ void mi_thread_init(void) mi_attr_noexcept
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
if (_mi_thread_heap_init()) return; // returns true if already initialized if (_mi_thread_heap_init()) return; // returns true if already initialized
_mi_stat_increase(&_mi_stats_main.threads, 1); mi_subproc_stat_increase(_mi_subproc_main(), threads, 1);
mi_atomic_increment_relaxed(&thread_count);
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
} }
@ -499,11 +546,10 @@ void _mi_thread_done(mi_heap_t* heap)
} }
// adjust stats // adjust stats
mi_atomic_decrement_relaxed(&thread_count); mi_subproc_stat_decrease(_mi_subproc_main(), threads, 1);
_mi_stat_decrease(&_mi_stats_main.threads, 1);
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps...
if (heap->thread_id != _mi_thread_id()) return; if (heap->tld->thread_id != _mi_prim_thread_id()) return;
// abandon the thread local heap // abandon the thread local heap
_mi_thread_heap_done(heap); // returns true if already ran _mi_thread_heap_done(heap); // returns true if already ran
@ -562,7 +608,7 @@ void _mi_process_load(void) {
} }
// reseed random // reseed random
_mi_random_reinit_if_weak(&_mi_heap_main.random); _mi_random_reinit_if_weak(&heap_main.random);
} }
#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) #if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))
@ -589,7 +635,7 @@ void mi_process_init(void) mi_attr_noexcept {
// ensure we are called once // ensure we are called once
static mi_atomic_once_t process_init; static mi_atomic_once_t process_init;
#if _MSC_VER < 1920 #if _MSC_VER < 1920
mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main mi_heap_main_init(); // vs2017 can dynamically re-initialize heap_main
#endif #endif
if (!mi_atomic_once(&process_init)) return; if (!mi_atomic_once(&process_init)) return;
_mi_process_is_initialized = true; _mi_process_is_initialized = true;
@ -597,10 +643,11 @@ void mi_process_init(void) mi_attr_noexcept {
mi_process_setup_auto_thread_done(); mi_process_setup_auto_thread_done();
mi_detect_cpu_features(); mi_detect_cpu_features();
mi_subproc_main_init();
mi_tld_main_init();
mi_heap_main_init();
_mi_os_init(); _mi_os_init();
_mi_page_map_init(); _mi_page_map_init();
_mi_arena_init();
mi_heap_main_init();
#if MI_DEBUG #if MI_DEBUG
_mi_verbose_message("debug level : %d\n", MI_DEBUG); _mi_verbose_message("debug level : %d\n", MI_DEBUG);
#endif #endif
@ -611,7 +658,7 @@ void mi_process_init(void) mi_attr_noexcept {
#endif #endif
mi_thread_init(); mi_thread_init();
#if defined(_WIN32) #if defined(_WIN32) && defined(MI_WIN_USE_FLS)
// On windows, when building as a static lib the FLS cleanup happens to early for the main thread. // On windows, when building as a static lib the FLS cleanup happens to early for the main thread.
// To avoid this, set the FLS value for the main thread to NULL so the fls cleanup // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
// will not call _mi_thread_done on the (still executing) main thread. See issue #508. // will not call _mi_thread_done on the (still executing) main thread. See issue #508.
@ -672,7 +719,7 @@ void mi_cdecl _mi_process_done(void) {
mi_stats_print(NULL); mi_stats_print(NULL);
} }
_mi_allocator_done(); _mi_allocator_done();
_mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id);
os_preloading = true; // don't call the C runtime anymore os_preloading = true; // don't call the C runtime anymore
} }

28
src/os.c
View file

@ -114,9 +114,9 @@ static void mi_os_prim_free(void* addr, size_t size, bool still_committed) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
} }
if (still_committed) { if (still_committed) {
_mi_stat_decrease(&os_stats->committed, size); mi_os_stat_decrease(committed, size);
} }
_mi_stat_decrease(&os_stats->reserved, size); mi_os_stat_decrease(reserved, size);
} }
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) { void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) {
@ -171,11 +171,11 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large); _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
} }
_mi_stat_counter_increase(&os_stats->mmap_calls, 1); mi_os_stat_counter_increase(mmap_calls, 1);
if (p != NULL) { if (p != NULL) {
_mi_stat_increase(&os_stats->reserved, size); mi_os_stat_increase(reserved, size);
if (commit) { if (commit) {
_mi_stat_increase(&os_stats->committed, size); mi_os_stat_increase(committed, size);
// seems needed for asan (or `mimalloc-test-api` fails) // seems needed for asan (or `mimalloc-test-api` fails)
#ifdef MI_TRACK_ASAN #ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); } if (*is_zero) { mi_track_mem_defined(p,size); }
@ -379,8 +379,8 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
bool _mi_os_commit(void* addr, size_t size, bool* is_zero) { bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
if (is_zero != NULL) { *is_zero = false; } if (is_zero != NULL) { *is_zero = false; }
_mi_stat_increase(&os_stats->committed, size); // use size for precise commit vs. decommit mi_os_stat_increase(committed, size); // use size for precise commit vs. decommit
_mi_stat_counter_increase(&os_stats->commit_calls, 1); mi_os_stat_counter_increase(commit_calls, 1);
// page align range // page align range
size_t csize; size_t csize;
@ -408,7 +408,7 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) { static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) {
mi_assert_internal(needs_recommit!=NULL); mi_assert_internal(needs_recommit!=NULL);
_mi_stat_decrease(&os_stats->committed, size); mi_os_stat_decrease(committed, size);
// page align // page align
size_t csize; size_t csize;
@ -440,8 +440,8 @@ bool _mi_os_reset(void* addr, size_t size) {
size_t csize; size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize); void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
_mi_stat_increase(&os_stats->reset, csize); mi_os_stat_increase(reset, csize);
_mi_stat_counter_increase(&os_stats->reset_calls, 1); mi_os_stat_counter_increase(reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
memset(start, 0, csize); // pretend it is eagerly reset memset(start, 0, csize); // pretend it is eagerly reset
@ -460,8 +460,8 @@ bool _mi_os_reset(void* addr, size_t size) {
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset) bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset)
{ {
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
_mi_stat_counter_increase(&os_stats->purge_calls, 1); mi_os_stat_counter_increase(purge_calls, 1);
_mi_stat_increase(&os_stats->purged, size); mi_os_stat_increase(purged, size);
if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?
!_mi_preloading()) // don't decommit during preloading (unsafe) !_mi_preloading()) // don't decommit during preloading (unsafe)
@ -595,8 +595,8 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// success, record it // success, record it
page++; // increase before timeout check (see issue #711) page++; // increase before timeout check (see issue #711)
_mi_stat_increase(&os_stats->committed, MI_HUGE_OS_PAGE_SIZE); mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
_mi_stat_increase(&os_stats->reserved, MI_HUGE_OS_PAGE_SIZE); mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
// check for timeout // check for timeout
if (max_msecs > 0) { if (max_msecs > 0) {

15
src/page.c
View file

@ -387,9 +387,9 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue?
if (pq->last==page && pq->first==page) { // the only page in the queue? if (pq->last==page && pq->first==page) { // the only page in the queue?
mi_stat_counter_increase(_mi_stats_main.page_no_retire,1);
page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);
mi_heap_t* heap = mi_page_heap(page); mi_heap_t* heap = mi_page_heap(page);
mi_debug_heap_stat_counter_increase(heap, page_no_retire, 1);
page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4);
mi_assert_internal(pq >= heap->pages); mi_assert_internal(pq >= heap->pages);
const size_t index = pq - heap->pages; const size_t index = pq - heap->pages;
mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE); mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);
@ -554,7 +554,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page) {
size_t page_size; size_t page_size;
//uint8_t* page_start = //uint8_t* page_start =
mi_page_area(page, &page_size); mi_page_area(page, &page_size);
mi_heap_stat_counter_increase(heap, pages_extended, 1); mi_debug_heap_stat_counter_increase(heap, pages_extended, 1);
// calculate the extend count // calculate the extend count
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
@ -583,7 +583,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page) {
} }
// enable the new free list // enable the new free list
page->capacity += (uint16_t)extend; page->capacity += (uint16_t)extend;
mi_heap_stat_increase(heap, page_committed, extend * bsize); mi_debug_heap_stat_increase(heap, page_committed, extend * bsize);
mi_assert_expensive(mi_page_is_valid_init(page)); mi_assert_expensive(mi_page_is_valid_init(page));
} }
@ -591,7 +591,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page) {
void _mi_page_init(mi_heap_t* heap, mi_page_t* page) { void _mi_page_init(mi_heap_t* heap, mi_page_t* page) {
mi_assert(page != NULL); mi_assert(page != NULL);
mi_page_set_heap(page, heap); mi_page_set_heap(page, heap);
page->subproc = heap->tld->subproc;
size_t page_size; size_t page_size;
uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start); uint8_t* page_start = mi_page_area(page, &page_size); MI_UNUSED(page_start);
mi_track_mem_noaccess(page_start,page_size); mi_track_mem_noaccess(page_start,page_size);
@ -682,7 +682,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, m
_mi_page_free(page_candidate, pq); _mi_page_free(page_candidate, pq);
page_candidate = page; page_candidate = page;
} }
else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // prefer to reuse fuller pages (in the hope the less used page gets freed)
else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page) && !mi_page_is_expandable(page)) {
page_candidate = page; page_candidate = page;
} }
// if we find a non-expandable candidate, or searched for N pages, return with the best candidate // if we find a non-expandable candidate, or searched for N pages, return with the best candidate
@ -708,7 +709,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, m
page = next; page = next;
} // for each page } // for each page
mi_heap_stat_counter_increase(heap, searches, count); mi_debug_heap_stat_counter_increase(heap, searches, count);
// set the page to the best candidate // set the page to the best candidate
if (page_candidate != NULL) { if (page_candidate != NULL) {

99
src/stats.c
View file

@ -19,16 +19,9 @@ terms of the MIT license. A copy of the license can be found in the file
Statistics operations Statistics operations
----------------------------------------------------------- */ ----------------------------------------------------------- */
static bool mi_is_in_main(void* stat) { static void mi_stat_update_mt(mi_stat_count_t* stat, int64_t amount) {
return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main
&& (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));
}
static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
if (amount == 0) return; if (amount == 0) return;
if mi_unlikely(mi_is_in_main(stat)) // add atomically
{
// add atomically (for abandoned pages)
int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount); int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);
mi_atomic_maxi64_relaxed(&stat->peak, current + amount); mi_atomic_maxi64_relaxed(&stat->peak, current + amount);
if (amount > 0) { if (amount > 0) {
@ -38,7 +31,9 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
mi_atomic_addi64_relaxed(&stat->freed, -amount); mi_atomic_addi64_relaxed(&stat->freed, -amount);
} }
} }
else {
static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
if (amount == 0) return;
// add thread local // add thread local
stat->current += amount; stat->current += amount;
if (stat->current > stat->peak) stat->peak = stat->current; if (stat->current > stat->peak) stat->peak = stat->current;
@ -49,21 +44,20 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
stat->freed += -amount; stat->freed += -amount;
} }
} }
}
// Adjust stats to compensate; for example before committing a range, // Adjust stats to compensate; for example before committing a range,
// first adjust downwards with parts that were already committed so // first adjust downwards with parts that were already committed so
// we avoid double counting. // we avoid double counting.
static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount, bool on_alloc) { static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount, bool on_alloc) {
if (amount == 0) return; if (amount == 0) return;
if mi_unlikely(mi_is_in_main(stat))
{
// adjust atomically // adjust atomically
mi_atomic_addi64_relaxed(&stat->current, amount); mi_atomic_addi64_relaxed(&stat->current, amount);
mi_atomic_addi64_relaxed((on_alloc ? &stat->allocated : &stat->freed), amount); mi_atomic_addi64_relaxed((on_alloc ? &stat->allocated : &stat->freed), amount);
} }
else {
// don't affect the peak static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount, bool on_alloc) {
if (amount == 0) return;
stat->current += amount; stat->current += amount;
if (on_alloc) { if (on_alloc) {
stat->allocated += amount; stat->allocated += amount;
@ -72,35 +66,46 @@ static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount, bool on_alloc)
stat->freed += amount; stat->freed += amount;
} }
} }
}
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) {
if (mi_is_in_main(stat)) {
mi_atomic_addi64_relaxed(&stat->count, 1); mi_atomic_addi64_relaxed(&stat->count, 1);
mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount);
} }
else {
void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
stat->count++; stat->count++;
stat->total += amount; stat->total += amount;
} }
}
void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) { void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {
mi_stat_update_mt(stat, (int64_t)amount);
}
void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
mi_stat_update(stat, (int64_t)amount); mi_stat_update(stat, (int64_t)amount);
} }
void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) {
mi_stat_update_mt(stat, -((int64_t)amount));
}
void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
mi_stat_update(stat, -((int64_t)amount)); mi_stat_update(stat, -((int64_t)amount));
} }
void _mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount, bool on_alloc) { void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount, bool on_alloc) {
mi_stat_adjust_mt(stat, (int64_t)amount, on_alloc);
}
void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount, bool on_alloc) {
mi_stat_adjust(stat, (int64_t)amount, on_alloc); mi_stat_adjust(stat, (int64_t)amount, on_alloc);
} }
void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount, bool on_alloc) { void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount, bool on_alloc) {
mi_stat_adjust_mt(stat, -((int64_t)amount), on_alloc);
}
void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount, bool on_alloc) {
mi_stat_adjust(stat, -((int64_t)amount), on_alloc); mi_stat_adjust(stat, -((int64_t)amount), on_alloc);
} }
// must be thread safe as it is called from stats_merge // must be thread safe as it is called from stats_merge
static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) { static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) {
if (stat==src) return; if (stat==src) return;
@ -401,36 +406,37 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
static mi_msecs_t mi_process_start; // = 0 static mi_msecs_t mi_process_start; // = 0
static mi_stats_t* mi_stats_get_default(void) { // return thread local stats
mi_heap_t* heap = mi_heap_get_default(); static mi_stats_t* mi_get_tld_stats(void) {
return &heap->tld->stats; return &_mi_tld()->stats;
}
static void mi_stats_merge_from(mi_stats_t* stats) {
if (stats != &_mi_stats_main) {
mi_stats_add(&_mi_stats_main, stats);
memset(stats, 0, sizeof(mi_stats_t));
}
} }
void mi_stats_reset(void) mi_attr_noexcept { void mi_stats_reset(void) mi_attr_noexcept {
mi_stats_t* stats = mi_stats_get_default(); mi_stats_t* stats = mi_get_tld_stats();
if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); } mi_subproc_t* subproc = _mi_subproc();
memset(&_mi_stats_main, 0, sizeof(mi_stats_t)); if (stats != &subproc->stats) { _mi_memzero(stats, sizeof(mi_stats_t)); }
_mi_memzero(&subproc->stats, sizeof(mi_stats_t));
if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); }; if (mi_process_start == 0) { mi_process_start = _mi_clock_start(); };
} }
void mi_stats_merge(void) mi_attr_noexcept { void _mi_stats_merge_from(mi_stats_t* to, mi_stats_t* from) {
mi_stats_merge_from( mi_stats_get_default() ); if (to != from) {
mi_stats_add(to, from);
_mi_memzero(from, sizeof(mi_stats_t));
}
} }
void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done` void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done`
mi_stats_merge_from(stats); _mi_stats_merge_from(&_mi_subproc()->stats, stats);
}
void mi_stats_merge(void) mi_attr_noexcept {
_mi_stats_done( mi_get_tld_stats() );
} }
void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
mi_stats_merge_from(mi_stats_get_default()); mi_stats_merge();
_mi_stats_print(&_mi_stats_main, out, arg); _mi_stats_print(&_mi_subproc()->stats, out, arg);
} }
void mi_stats_print(void* out) mi_attr_noexcept { void mi_stats_print(void* out) mi_attr_noexcept {
@ -439,7 +445,7 @@ void mi_stats_print(void* out) mi_attr_noexcept {
} }
void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept { void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept {
_mi_stats_print(mi_stats_get_default(), out, arg); _mi_stats_print(mi_get_tld_stats(), out, arg);
} }
@ -473,11 +479,12 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start) {
mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept
{ {
mi_subproc_t* subproc = _mi_subproc();
mi_process_info_t pinfo; mi_process_info_t pinfo;
_mi_memzero_var(pinfo); _mi_memzero_var(pinfo);
pinfo.elapsed = _mi_clock_end(mi_process_start); pinfo.elapsed = _mi_clock_end(mi_process_start);
pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.current)); pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)));
pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&_mi_stats_main.committed.peak)); pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)));
pinfo.current_rss = pinfo.current_commit; pinfo.current_rss = pinfo.current_commit;
pinfo.peak_rss = pinfo.peak_commit; pinfo.peak_rss = pinfo.peak_commit;
pinfo.utime = 0; pinfo.utime = 0;

16
test/test-stress.c
View file

@ -44,20 +44,18 @@ static int ITER = 10;
static int THREADS = 4; static int THREADS = 4;
static int SCALE = 10; static int SCALE = 10;
static int ITER = 20; static int ITER = 20;
#define ALLOW_LARGE false
#elif 0 #elif 0
static int THREADS = 32; static int THREADS = 32;
static int SCALE = 50; static int SCALE = 50;
static int ITER = 50; static int ITER = 50;
#define ALLOW_LARGE false #elif 1
#elif 0 static int THREADS = 32;
static int THREADS = 64; static int SCALE = 25;
static int SCALE = 400; static int ITER = 50;
static int ITER = 10;
#define ALLOW_LARGE true #define ALLOW_LARGE true
#else #else
static int THREADS = 32; // more repeatable if THREADS <= #processors static int THREADS = 32; // more repeatable if THREADS <= #processors
static int SCALE = 25; // scaling factor static int SCALE = 50; // scaling factor
static int ITER = 50; // N full iterations destructing and re-creating all threads static int ITER = 50; // N full iterations destructing and re-creating all threads
#endif #endif
@ -66,7 +64,7 @@ static int ITER = 50; // N full iterations destructing and re-creating a
#define STRESS // undefine for leak test #define STRESS // undefine for leak test
#ifndef ALLOW_LARGE #ifndef ALLOW_LARGE
#define ALLOW_LARGE true #define ALLOW_LARGE false
#endif #endif
static bool allow_large_objects = ALLOW_LARGE; // allow very large objects? (set to `true` if SCALE>100) static bool allow_large_objects = ALLOW_LARGE; // allow very large objects? (set to `true` if SCALE>100)
@ -363,7 +361,7 @@ int main(int argc, char** argv) {
#else #else
mi_stats_print(NULL); // so we see rss/commit/elapsed mi_stats_print(NULL); // so we see rss/commit/elapsed
#endif #endif
//mi_stats_print(NULL); mi_stats_print(NULL);
//bench_end_program(); //bench_end_program();
return 0; return 0;
} }