kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

revise free reclaim; ensure unown cannot race with a free

Browse source
This commit is contained in:
daanx 2024-12-03 10:51:13 -08:00
parent 833b091ff9
commit 666c089fc8
10 changed files with 281 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
include/mimalloc/internal.h
View file

@ -143,7 +143,8 @@ void _mi_arena_unsafe_destroy_all(mi_stats_t* stats);
mi_page_t* _mi_arena_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment);
void _mi_arena_page_free(mi_page_t* page);
void _mi_arena_page_abandon(mi_page_t* page);
void _mi_arena_page_unabandon(mi_page_t* page);
void _mi_arena_page_unabandon(mi_page_t* page);
bool _mi_arena_page_try_reabandon_to_mapped(mi_page_t* page);
bool _mi_arena_try_reclaim(mi_heap_t* heap, mi_page_t* page);
void _mi_arena_reclaim_all_abandoned(mi_heap_t* heap);
@ -572,29 +573,6 @@ static inline bool mi_page_is_owned(const mi_page_t* page) {
return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));
}
// Unown a page that is currently owned
static inline void _mi_page_unown(mi_page_t* page) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_thread_id(page)==0);
const uintptr_t old = mi_atomic_and_acq_rel(&page->xthread_free, ~((uintptr_t)1));
mi_assert_internal((old&1)==1); MI_UNUSED(old);
/*
mi_thread_free_t tf_new;
mi_thread_free_t tf_old;
do {
tf_old = mi_atomic_load_relaxed(&page->xthread_free);
mi_assert_internal(mi_tf_is_owned(tf_old));
tf_new = mi_tf_create(mi_tf_block(tf_old), false);
} while (!mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new));
*/
}
// get ownership if it is not yet owned
static inline bool mi_page_try_claim_ownership(mi_page_t* page) {
const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, 1);
return ((old&1)==0);
}
//static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) {
// return mi_tf_make(mi_tf_block(tf),delayed);
@ -638,7 +616,7 @@ static inline bool mi_page_is_full(mi_page_t* page) {
}
// is more than 7/8th of a page in use?
static inline bool mi_page_mostly_used(const mi_page_t* page) {
static inline bool mi_page_is_mostly_used(const mi_page_t* page) {
if (page==NULL) return true;
uint16_t frac = page->reserved / 8U;
return (page->reserved - page->used <= frac);
@ -646,9 +624,22 @@ static inline bool mi_page_mostly_used(const mi_page_t* page) {
static inline bool mi_page_is_abandoned(const mi_page_t* page) {
// note: the xheap field of an abandoned heap is set to the subproc (for fast reclaim-on-free)
return (mi_atomic_load_acquire(&page->xthread_id) == 0);
return (mi_atomic_load_acquire(&page->xthread_id) <= 1);
}
static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) {
return (mi_atomic_load_acquire(&page->xthread_id) == 1);
}
static inline void mi_page_set_abandoned_mapped(mi_page_t* page) {
mi_atomic_or_acq_rel(&page->xthread_id, (uintptr_t)1);
}
static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {
mi_atomic_and_acq_rel(&page->xthread_id, ~(uintptr_t)1);
}
static inline bool mi_page_is_huge(const mi_page_t* page) {
return (page->block_size > MI_LARGE_MAX_OBJ_SIZE || (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.alignment > MI_PAGE_MAX_OVERALLOC_ALIGN));
}
@ -659,6 +650,51 @@ static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size)
}
// Unown a page that is currently owned
static inline void _mi_page_unown_unconditional(mi_page_t* page) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_thread_id(page)==0);
const uintptr_t old = mi_atomic_and_acq_rel(&page->xthread_free, ~((uintptr_t)1));
mi_assert_internal((old&1)==1); MI_UNUSED(old);
/*
mi_thread_free_t tf_new;
mi_thread_free_t tf_old;
do {
tf_old = mi_atomic_load_relaxed(&page->xthread_free);
mi_assert_internal(mi_tf_is_owned(tf_old));
tf_new = mi_tf_create(mi_tf_block(tf_old), false);
} while (!mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new));
*/
}
// get ownership if it is not yet owned
static inline bool mi_page_try_claim_ownership(mi_page_t* page) {
const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, 1);
return ((old&1)==0);
}
static inline void _mi_page_unown(mi_page_t* page) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mi_page_thread_id(page)==0);
mi_thread_free_t tf_new;
mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free);
do {
mi_assert_internal(mi_tf_is_owned(tf_old));
while mi_unlikely(mi_tf_block(tf_old) != NULL) {
_mi_page_free_collect(page, false); // update used
if (mi_page_all_free(page)) { // it may become free just before unowning it
_mi_arena_page_unabandon(page);
_mi_arena_page_free(page);
return;
}
tf_old = mi_atomic_load_relaxed(&page->xthread_free);
}
mi_assert_internal(mi_tf_block(tf_old)==NULL);
tf_new = mi_tf_create(NULL, false);
} while (!mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new));
}
//-----------------------------------------------------------
// Page flags

4
include/mimalloc/types.h
View file

@ -505,6 +505,10 @@ typedef struct mi_stats_s {
mi_stat_count_t giant;
mi_stat_count_t malloc;
mi_stat_counter_t pages_extended;
mi_stat_counter_t pages_reclaim_on_alloc;
mi_stat_counter_t pages_reclaim_on_free;
mi_stat_counter_t pages_reabandon_full;
mi_stat_counter_t pages_unabandon_busy_wait;
mi_stat_counter_t mmap_calls;
mi_stat_counter_t commit_calls;
mi_stat_counter_t reset_calls;

71
src/arena.c
View file

@ -42,7 +42,7 @@ typedef struct mi_arena_s {
bool is_large; // memory area consists of large- or huge OS pages (always committed)
mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited
_Atomic(mi_msecs_t) purge_expire; // expiration time when slices should be decommitted from `slices_decommit`.
mi_bitmap_t slices_free; // is the slice free?
mi_bitmap_t slices_committed; // is the slice committed? (i.e. accessible)
mi_bitmap_t slices_purge; // can the slice be purged? (slice in purge => slice in free)
@ -216,7 +216,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
else {
if (commit_zero) { memid->initially_zero = true; }
}
}
}
}
else {
// no need to commit, but check if already fully committed
@ -355,7 +355,7 @@ static mi_decl_noinline void* mi_arena_try_alloc(
{
mi_assert(slice_count <= MI_ARENA_MAX_OBJ_SLICES);
mi_assert(alignment <= MI_ARENA_SLICE_ALIGN);
// try to find free slices in the arena's
void* p = mi_arena_try_find_free(slice_count, alignment, commit, allow_large, req_arena_id, memid, tld);
if (p != NULL) return p;
@ -457,7 +457,7 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
// try to claim ownership atomically
mi_page_t* page = (mi_page_t*)mi_arena_slice_start(arena, slice_index);
if (!mi_page_try_claim_ownership(page)) {
// a concurrent free already grabbed the page.
// a concurrent free already grabbed the page.
// Restore the abandoned_map to make it available again (unblocking busy waiters)
mi_pairmap_set(pairmap, slice_index);
}
@ -465,6 +465,9 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
// we got ownership, clear the abandoned entry (unblocking busy waiters)
mi_pairmap_clear(pairmap, slice_index);
mi_atomic_decrement_relaxed(&subproc->abandoned_count[bin]);
_mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1);
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_alloc, 1);
_mi_page_free_collect(page, false); // update `used` count
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count));
@ -472,7 +475,7 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count));
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
mi_assert_internal(mi_page_block_size(page) == block_size);
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(mi_page_is_owned(page));
@ -492,11 +495,11 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_siz
const bool commit = true;
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
// try to allocate from free space in arena's
mi_memid_t memid = _mi_memid_none();
mi_page_t* page = NULL;
if (!_mi_option_get_fast(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's?
if (!_mi_option_get_fast(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's?
!os_align && // not large alignment
slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large
{
@ -575,16 +578,16 @@ static mi_page_t* mi_singleton_page_alloc(mi_heap_t* heap, size_t block_size, si
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
const size_t info_size = (os_align ? MI_PAGE_ALIGN : MI_PAGE_INFO_SIZE);
const size_t slice_count = mi_slice_count_of_size(info_size + block_size);
mi_page_t* page = mi_arena_page_alloc_fresh(slice_count, block_size, block_alignment, req_arena_id, tld);
if (page == NULL) return NULL;
mi_assert(page != NULL);
mi_assert(page->reserved == 1);
mi_assert(page->reserved == 1);
mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
return page;
return page;
}
@ -646,17 +649,17 @@ void _mi_arena_page_free(mi_page_t* page) {
Arena abandon
----------------------------------------------------------- */
void _mi_arena_page_abandon(mi_page_t* page) {
static void mi_arena_page_abandon_no_stat(mi_page_t* page) {
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_all_free(page));
mi_assert_internal(page->next==NULL);
mi_subproc_t* subproc = page->subproc;
if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
// make available for allocations
// make available for allocations
size_t bin = _mi_bin(mi_page_block_size(page));
size_t slice_index;
size_t slice_count;
@ -667,6 +670,7 @@ void _mi_arena_page_abandon(mi_page_t* page) {
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count));
// mi_assert_internal(mi_bitmap_is_setN(&arena->slices_dirty, slice_index, slice_count));
mi_page_set_abandoned_mapped(page);
bool were_zero = mi_pairmap_set(&arena->pages_abandoned[bin], slice_index);
MI_UNUSED(were_zero); mi_assert_internal(were_zero);
mi_atomic_increment_relaxed(&subproc->abandoned_count[bin]);
@ -676,34 +680,59 @@ void _mi_arena_page_abandon(mi_page_t* page) {
// leave as is; it will be reclaimed when an object is free'd in the page
}
_mi_page_unown(page);
}
void _mi_arena_page_abandon(mi_page_t* page) {
mi_arena_page_abandon_no_stat(page);
_mi_stat_increase(&_mi_stats_main.pages_abandoned, 1);
}
bool _mi_arena_page_try_reabandon_to_mapped(mi_page_t* page) {
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
mi_assert_internal(!mi_page_is_abandoned_mapped(page));
mi_assert_internal(!mi_page_is_full(page));
mi_assert_internal(!mi_page_all_free(page));
mi_assert_internal(!mi_page_is_singleton(page));
if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) {
return false;
}
else {
_mi_stat_counter_increase(&_mi_stats_main.pages_reabandon_full, 1);
mi_arena_page_abandon_no_stat(page);
return true;
}
}
// called from `mi_free` if trying to unabandon an abandoned page
void _mi_arena_page_unabandon(mi_page_t* page) {
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_is_abandoned(page));
if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
if (mi_page_is_abandoned_mapped(page)) {
mi_assert_internal(page->memid.memkind==MI_MEM_ARENA);
// remove from the abandoned map
size_t bin = _mi_bin(mi_page_block_size(page));
size_t slice_index;
size_t slice_count;
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count));
// this busy waits until a concurrent reader (from alloc_abandoned) is done
mi_pairmap_clear_while_not_busy(&arena->pages_abandoned[bin], slice_index);
mi_page_clear_abandoned_mapped(page);
mi_atomic_decrement_relaxed(&page->subproc->abandoned_count[bin]);
}
else {
// page is full (or a singleton), page is OS/externally allocated
// nothing to do
// nothing to do
// TODO: maintain count of these as well?
}
_mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1);
@ -715,7 +744,7 @@ bool _mi_arena_try_reclaim(mi_heap_t* heap, mi_page_t* page) {
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page);
// if (!mi_page_is_abandoned(page)) return false; // it is not abandoned (anymore)
// note: we can access the page even it is in the meantime reclaimed by another thread since
// we only call this when on free (and thus there is still an object alive in the page)
mi_memid_t memid = page->memid;
@ -967,7 +996,7 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int
arena->is_large = is_large;
arena->purge_expire = 0;
mi_lock_init(&arena->abandoned_visit_lock);
// init bitmaps
mi_bitmap_init(&arena->slices_free,true);
mi_bitmap_init(&arena->slices_committed,true);
@ -1068,7 +1097,7 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_
_mi_memset(buf + k, 'o', MI_BFIELD_BITS);
k += MI_BFIELD_BITS;
}
bit_count += MI_BFIELD_BITS;
bit_count += MI_BFIELD_BITS;
}
_mi_output_message("%s %s\n", prefix, buf);
}

16
src/bitmap.c
View file

@ -80,7 +80,7 @@ static inline bool mi_bfield_atomic_set2(_Atomic(mi_bfield_t)*b, size_t idx, boo
mi_assert_internal(idx < MI_BFIELD_BITS-1);
const size_t mask = (mi_bfield_t)0x03 << idx;
mi_bfield_t old = mi_atomic_load_relaxed(b);
while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)); // try to atomically set the mask bits until success
while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) { }; // try to atomically set the mask bits until success
if (all_already_set!=NULL) { *all_already_set = ((old&mask)==mask); }
return ((old&mask) == 0);
}
@ -90,7 +90,7 @@ static inline bool mi_bfield_atomic_clear2(_Atomic(mi_bfield_t)*b, size_t idx, b
mi_assert_internal(idx < MI_BFIELD_BITS-1);
const size_t mask = (mi_bfield_t)0x03 << idx;
mi_bfield_t old = mi_atomic_load_relaxed(b);
while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)); // try to atomically clear the mask bits until success
while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) { }; // try to atomically clear the mask bits until success
if (all_already_clear!=NULL) { *all_already_clear = ((old&mask) == 0); }
return ((old&mask) == mask);
}
@ -110,7 +110,7 @@ static inline bool mi_bfield_atomic_xset2(mi_bit_t set, _Atomic(mi_bfield_t)*b,
static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) {
mi_assert_internal(mask != 0);
mi_bfield_t old = mi_atomic_load_relaxed(b);
while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)); // try to atomically set the mask bits until success
while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) { }; // try to atomically set the mask bits until success
if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); }
return ((old&mask) == 0);
}
@ -119,7 +119,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t
static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_clear) {
mi_assert_internal(mask != 0);
mi_bfield_t old = mi_atomic_load_relaxed(b);
while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)); // try to atomically clear the mask bits until success
while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) { }; // try to atomically clear the mask bits until success
if (already_clear!=NULL) { *already_clear = mi_bfield_popcount(~(old&mask)); }
return ((old&mask) == mask);
}
@ -1115,16 +1115,18 @@ static inline bool mi_bfield_atomic_clear_while_not_busy(_Atomic(mi_bfield_t)*b,
mi_assert_internal(idx < MI_BFIELD_BITS-1);
const mi_bfield_t mask = ((mi_bfield_t)0x03 << idx);
const mi_bfield_t mask_busy = ((mi_bfield_t)MI_PAIR_BUSY << idx);
mi_bfield_t old;
mi_bfield_t bnew;
mi_bfield_t old = mi_atomic_load_relaxed(b);
do {
old = mi_atomic_load_relaxed(b);
if mi_unlikely((old&mask)==mask_busy) {
old = mi_atomic_load_acquire(b);
if ((old&mask)==mask_busy) {
_mi_stat_counter_increase(&_mi_stats_main.pages_unabandon_busy_wait, 1);
}
while ((old&mask)==mask_busy) { // busy wait
mi_atomic_yield();
old = mi_atomic_load_acquire(b);
}
}
}
bnew = (old & ~mask); // clear
} while (!mi_atomic_cas_weak_acq_rel(b, &old, bnew));

156
src/free.c
View file

@ -128,7 +128,7 @@ void mi_free(void* p) mi_attr_noexcept
{
mi_page_t* const page = mi_checked_ptr_page(p,"mi_free");
if mi_unlikely(page==NULL) return;
const bool is_local = (_mi_prim_thread_id() == mi_page_thread_id(page));
if mi_likely(is_local) { // thread-local free?
if mi_likely(page->flags.full_aligned == 0) { // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned)
@ -156,50 +156,164 @@ void mi_free(void* p) mi_attr_noexcept
static void mi_decl_noinline mi_free_try_reclaim_mt(mi_page_t* page) {
mi_assert_internal(mi_page_is_owned(page));
mi_assert_internal(mi_page_thread_id(page)==0);
#if 1
// we own the page now..
// first remove it from the abandoned pages in the arena -- this waits for any readers to finish
_mi_arena_page_unabandon(page); // this must be before collect
// collect the thread atomic free list
// safe to collect the thread atomic free list
_mi_page_free_collect(page, false); // update `used` count
#if MI_DEBUG > 1
if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
#endif
if (mi_page_all_free(page)) {
// 1. free if the page is free now
if (mi_page_all_free(page))
{
// first remove it from the abandoned pages in the arena (if mapped, this waits for any readers to finish)
_mi_arena_page_unabandon(page);
// we can free the page directly
_mi_arena_page_free(page);
return;
}
else {
// the page has still some blocks in use
// 2. if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations
else if (!mi_page_is_mostly_used(page) && // only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page
!mi_page_is_abandoned_mapped(page) && page->memid.memkind == MI_MEM_ARENA &&
_mi_arena_page_try_reabandon_to_mapped(page))
{
return;
}
// 3. if the page is not too full, we can try to reclaim it for ourselves
else if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 &&
!mi_page_is_mostly_used(page))
{
// the page has still some blocks in use (but not too many)
// reclaim in our heap if compatible, or otherwise abandon again
// todo: optimize this check further?
// note: don't use `mi_heap_get_default()` as we may just have terminated this thread and we should
// not reinitialize the heap for this thread. (can happen due to thread-local destructors for example -- issue #944)
mi_heap_t* const heap = mi_prim_get_default_heap();
if ((_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0) && // only if reclaim on free is allowed
(heap != (mi_heap_t*)&_mi_heap_empty)) // we did not already terminate our thread (can this happen?
if (heap != (mi_heap_t*)&_mi_heap_empty) // we did not already terminate our thread (can this happen?
{
mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag);
if ((tagheap != NULL) && // don't reclaim across heap object types
if ((tagheap != NULL) && // don't reclaim across heap object types
(page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? )
(_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?)
(_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?)
)
{
// make it part of our heap
// first remove it from the abandoned pages in the arena -- this waits for any readers to finish
_mi_arena_page_unabandon(page);
_mi_heap_page_reclaim(tagheap, page);
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1);
return;
}
}
}
}
// not reclaimed or free'd, unown again
_mi_page_unown(page);
#else
if (!mi_page_is_abandoned_mapped(page)) {
// singleton or OS allocated
if (mi_page_is_singleton(page)) {
// free singleton pages
#if MI_DEBUG>1
_mi_page_free_collect(page, false); // update `used` count
mi_assert_internal(mi_page_all_free(page));
#endif
// we can free the page directly
_mi_arena_page_free(page);
return;
}
else {
const bool was_full = mi_page_is_full(page);
_mi_page_free_collect(page,false); // update used
if (mi_page_all_free(page)) {
// no need to unabandon as it is unmapped
_mi_arena_page_free(page);
return;
}
else if (was_full && _mi_arena_page_reabandon_full(page)) {
return;
}
else if (!mi_page_is_mostly_used(page) && _mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0) {
// the page has still some blocks in use (but not too many)
// reclaim in our heap if compatible, or otherwise abandon again
// todo: optimize this check further?
// note: don't use `mi_heap_get_default()` as we may just have terminated this thread and we should
// not reinitialize the heap for this thread. (can happen due to thread-local destructors for example -- issue #944)
mi_heap_t* const heap = mi_prim_get_default_heap();
if (heap != (mi_heap_t*)&_mi_heap_empty) { // we did not already terminate our thread (can this happen?
mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag);
if ((tagheap != NULL) && // don't reclaim across heap object types
(page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? )
(_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?)
)
{
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1);
// make it part of our heap (no need to unabandon as is unmapped)
_mi_heap_page_reclaim(tagheap, page);
return;
}
}
}
}
}
else {
// don't reclaim pages that can be found for fresh page allocations
}
// not reclaimed or free'd, unown again
_mi_page_unown(page);
#endif
}
/*
// we own the page now..
// safe to collect the thread atomic free list
_mi_page_free_collect(page, false); // update `used` count
if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); }
if (mi_page_all_free(page)) {
// first remove it from the abandoned pages in the arena -- this waits for any readers to finish
_mi_arena_page_unabandon(page); // this must be before free'ing
// we can free the page directly
_mi_arena_page_free(page);
return;
}
else if (!mi_page_is_mostly_used(page)) {
// the page has still some blocks in use (but not too many)
// reclaim in our heap if compatible, or otherwise abandon again
// todo: optimize this check further?
// note: don't use `mi_heap_get_default()` as we may just have terminated this thread and we should
// not reinitialize the heap for this thread. (can happen due to thread-local destructors for example -- issue #944)
mi_heap_t* const heap = mi_prim_get_default_heap();
if ((_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0) && // only if reclaim on free is allowed
(heap != (mi_heap_t*)&_mi_heap_empty)) // we did not already terminate our thread (can this happen?
{
mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag);
if ((tagheap != NULL) && // don't reclaim across heap object types
(page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? )
(_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?)
)
{
// first remove it from the abandoned pages in the arena -- this waits for any readers to finish
_mi_arena_page_unabandon(page);
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1);
// make it part of our heap
_mi_heap_page_reclaim(tagheap, page);
return;
}
// we cannot reclaim this page.. abandon it again
_mi_arena_page_abandon(page);
}
}
// Push a block that is owned by another thread (or abandoned) on its page-local thread free list.
// we cannot reclaim this page.. leave it abandoned
// todo: should re-abandon or otherwise a partly used page could never be re-used if the
// objects in it are not freed explicitly.
_mi_page_unown(page);
*/
// Push a block that is owned by another thread (or abandoned) on its page-local thread free list.
static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_block_t* block)
{
// adjust stats (after padding check and potentially recursive `mi_free` above)

2
src/init.c
View file

@ -83,7 +83,7 @@ const mi_page_t _mi_page_empty = {
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 } \
{ 0, 0 }, { 0, 0 }, { 0, 0 } \
MI_STAT_COUNT_END_NULL()
// --------------------------------------------------------

2
src/options.c
View file

@ -143,7 +143,7 @@ static mi_option_desc_t options[_mi_option_last] =
{ MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
{ 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
{ 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) },
{ 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free
{ 0, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free
{ MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
{ 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
#if defined(MI_VISIT_ABANDONED)

2
src/page.c
View file

@ -811,7 +811,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
_mi_page_free(page_candidate, pq);
page_candidate = page;
}
else if (page->used >= page_candidate->used) {
else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) {
page_candidate = page;
}
// if we find a non-expandable candidate, or searched for N pages, return with the best candidate

4
src/stats.c
View file

@ -331,6 +331,10 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, "");
mi_stat_print_ex(&stats->pages, "pages", -1, out, arg, "");
mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg);
mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "-reclaima", out, arg);
mi_stat_counter_print(&stats->pages_reclaim_on_free, "-reclaimf", out, arg);
mi_stat_counter_print(&stats->pages_reabandon_full, "-reabandon", out, arg);
mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "-waits", out, arg);
mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg);
mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg);
mi_stat_counter_print(&stats->arena_count, "arenas", out, arg);

15
test/test-stress.c
View file

@ -43,7 +43,13 @@ static int ITER = 10;
#elif 0
static int THREADS = 4;
static int SCALE = 100;
static int ITER = 10;
#define ALLOW_LARGE false
#elif 1
static int THREADS = 32;
static int SCALE = 50;
static int ITER = 50;
#define ALLOW_LARGE false
#else
static int THREADS = 32; // more repeatable if THREADS <= #processors
static int SCALE = 50; // scaling factor
@ -54,7 +60,12 @@ static int ITER = 50; // N full iterations destructing and re-creating a
#define STRESS // undefine for leak test
static bool allow_large_objects = false; // allow very large objects? (set to `true` if SCALE>100)
#ifndef ALLOW_LARGE
#define ALLOW_LARGE true
#endif
static bool allow_large_objects = ALLOW_LARGE; // allow very large objects? (set to `true` if SCALE>100)
static size_t use_one_size = 0; // use single object size of `N * sizeof(uintptr_t)`?
static bool main_participates = false; // main thread participates as a worker too
@ -332,6 +343,8 @@ int main(int argc, char** argv) {
mi_debug_show_arenas(true,true,false);
#endif
// mi_stats_print(NULL);
#else
mi_stats_print(NULL); // so we see rss/commit/elapsed
#endif
//bench_end_program();
return 0;