From 666c089fc85b67c0773e502856f5b9fb179164cd Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 3 Dec 2024 10:51:13 -0800 Subject: [PATCH] revise free reclaim; ensure unown cannot race with a free --- include/mimalloc/internal.h | 88 ++++++++++++++------ include/mimalloc/types.h | 4 + src/arena.c | 71 +++++++++++----- src/bitmap.c | 16 ++-- src/free.c | 156 +++++++++++++++++++++++++++++++----- src/init.c | 2 +- src/options.c | 2 +- src/page.c | 2 +- src/stats.c | 4 + test/test-stress.c | 15 +++- 10 files changed, 281 insertions(+), 79 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index cee88684..56172bcd 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -143,7 +143,8 @@ void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); mi_page_t* _mi_arena_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment); void _mi_arena_page_free(mi_page_t* page); void _mi_arena_page_abandon(mi_page_t* page); -void _mi_arena_page_unabandon(mi_page_t* page); +void _mi_arena_page_unabandon(mi_page_t* page); +bool _mi_arena_page_try_reabandon_to_mapped(mi_page_t* page); bool _mi_arena_try_reclaim(mi_heap_t* heap, mi_page_t* page); void _mi_arena_reclaim_all_abandoned(mi_heap_t* heap); @@ -572,29 +573,6 @@ static inline bool mi_page_is_owned(const mi_page_t* page) { return mi_tf_is_owned(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free)); } -// Unown a page that is currently owned -static inline void _mi_page_unown(mi_page_t* page) { - mi_assert_internal(mi_page_is_owned(page)); - mi_assert_internal(mi_page_thread_id(page)==0); - const uintptr_t old = mi_atomic_and_acq_rel(&page->xthread_free, ~((uintptr_t)1)); - mi_assert_internal((old&1)==1); MI_UNUSED(old); - /* - mi_thread_free_t tf_new; - mi_thread_free_t tf_old; - do { - tf_old = mi_atomic_load_relaxed(&page->xthread_free); - mi_assert_internal(mi_tf_is_owned(tf_old)); - tf_new = mi_tf_create(mi_tf_block(tf_old), false); - } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new)); - */ -} - -// get ownership if it is not yet owned -static inline bool mi_page_try_claim_ownership(mi_page_t* page) { - const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, 1); - return ((old&1)==0); -} - //static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) { // return mi_tf_make(mi_tf_block(tf),delayed); @@ -638,7 +616,7 @@ static inline bool mi_page_is_full(mi_page_t* page) { } // is more than 7/8th of a page in use? -static inline bool mi_page_mostly_used(const mi_page_t* page) { +static inline bool mi_page_is_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; return (page->reserved - page->used <= frac); @@ -646,9 +624,22 @@ static inline bool mi_page_mostly_used(const mi_page_t* page) { static inline bool mi_page_is_abandoned(const mi_page_t* page) { // note: the xheap field of an abandoned heap is set to the subproc (for fast reclaim-on-free) - return (mi_atomic_load_acquire(&page->xthread_id) == 0); + return (mi_atomic_load_acquire(&page->xthread_id) <= 1); } +static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) { + return (mi_atomic_load_acquire(&page->xthread_id) == 1); +} + +static inline void mi_page_set_abandoned_mapped(mi_page_t* page) { + mi_atomic_or_acq_rel(&page->xthread_id, (uintptr_t)1); +} + +static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) { + mi_atomic_and_acq_rel(&page->xthread_id, ~(uintptr_t)1); +} + + static inline bool mi_page_is_huge(const mi_page_t* page) { return (page->block_size > MI_LARGE_MAX_OBJ_SIZE || (mi_memkind_is_os(page->memid.memkind) && page->memid.mem.os.alignment > MI_PAGE_MAX_OVERALLOC_ALIGN)); } @@ -659,6 +650,51 @@ static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) } +// Unown a page that is currently owned +static inline void _mi_page_unown_unconditional(mi_page_t* page) { + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_thread_id(page)==0); + const uintptr_t old = mi_atomic_and_acq_rel(&page->xthread_free, ~((uintptr_t)1)); + mi_assert_internal((old&1)==1); MI_UNUSED(old); + /* + mi_thread_free_t tf_new; + mi_thread_free_t tf_old; + do { + tf_old = mi_atomic_load_relaxed(&page->xthread_free); + mi_assert_internal(mi_tf_is_owned(tf_old)); + tf_new = mi_tf_create(mi_tf_block(tf_old), false); + } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new)); + */ +} + + +// get ownership if it is not yet owned +static inline bool mi_page_try_claim_ownership(mi_page_t* page) { + const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, 1); + return ((old&1)==0); +} + +static inline void _mi_page_unown(mi_page_t* page) { + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(mi_page_thread_id(page)==0); + mi_thread_free_t tf_new; + mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); + do { + mi_assert_internal(mi_tf_is_owned(tf_old)); + while mi_unlikely(mi_tf_block(tf_old) != NULL) { + _mi_page_free_collect(page, false); // update used + if (mi_page_all_free(page)) { // it may become free just before unowning it + _mi_arena_page_unabandon(page); + _mi_arena_page_free(page); + return; + } + tf_old = mi_atomic_load_relaxed(&page->xthread_free); + } + mi_assert_internal(mi_tf_block(tf_old)==NULL); + tf_new = mi_tf_create(NULL, false); + } while (!mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new)); +} //----------------------------------------------------------- // Page flags diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index dafd25f1..4430cd6c 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -505,6 +505,10 @@ typedef struct mi_stats_s { mi_stat_count_t giant; mi_stat_count_t malloc; mi_stat_counter_t pages_extended; + mi_stat_counter_t pages_reclaim_on_alloc; + mi_stat_counter_t pages_reclaim_on_free; + mi_stat_counter_t pages_reabandon_full; + mi_stat_counter_t pages_unabandon_busy_wait; mi_stat_counter_t mmap_calls; mi_stat_counter_t commit_calls; mi_stat_counter_t reset_calls; diff --git a/src/arena.c b/src/arena.c index 317a7e48..a2343674 100644 --- a/src/arena.c +++ b/src/arena.c @@ -42,7 +42,7 @@ typedef struct mi_arena_s { bool is_large; // memory area consists of large- or huge OS pages (always committed) mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited _Atomic(mi_msecs_t) purge_expire; // expiration time when slices should be decommitted from `slices_decommit`. - + mi_bitmap_t slices_free; // is the slice free? mi_bitmap_t slices_committed; // is the slice committed? (i.e. accessible) mi_bitmap_t slices_purge; // can the slice be purged? (slice in purge => slice in free) @@ -216,7 +216,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( else { if (commit_zero) { memid->initially_zero = true; } } - } + } } else { // no need to commit, but check if already fully committed @@ -355,7 +355,7 @@ static mi_decl_noinline void* mi_arena_try_alloc( { mi_assert(slice_count <= MI_ARENA_MAX_OBJ_SLICES); mi_assert(alignment <= MI_ARENA_SLICE_ALIGN); - + // try to find free slices in the arena's void* p = mi_arena_try_find_free(slice_count, alignment, commit, allow_large, req_arena_id, memid, tld); if (p != NULL) return p; @@ -457,7 +457,7 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl // try to claim ownership atomically mi_page_t* page = (mi_page_t*)mi_arena_slice_start(arena, slice_index); if (!mi_page_try_claim_ownership(page)) { - // a concurrent free already grabbed the page. + // a concurrent free already grabbed the page. // Restore the abandoned_map to make it available again (unblocking busy waiters) mi_pairmap_set(pairmap, slice_index); } @@ -465,6 +465,9 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl // we got ownership, clear the abandoned entry (unblocking busy waiters) mi_pairmap_clear(pairmap, slice_index); mi_atomic_decrement_relaxed(&subproc->abandoned_count[bin]); + _mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1); + _mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_alloc, 1); + _mi_page_free_collect(page, false); // update `used` count mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); @@ -472,7 +475,7 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); - mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_block_size(page) == block_size); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_owned(page)); @@ -492,11 +495,11 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t slice_count, size_t block_siz const bool commit = true; const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); const size_t page_alignment = MI_ARENA_SLICE_ALIGN; - + // try to allocate from free space in arena's mi_memid_t memid = _mi_memid_none(); mi_page_t* page = NULL; - if (!_mi_option_get_fast(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's? + if (!_mi_option_get_fast(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's? !os_align && // not large alignment slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large { @@ -575,16 +578,16 @@ static mi_page_t* mi_singleton_page_alloc(mi_heap_t* heap, size_t block_size, si const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); const size_t info_size = (os_align ? MI_PAGE_ALIGN : MI_PAGE_INFO_SIZE); const size_t slice_count = mi_slice_count_of_size(info_size + block_size); - + mi_page_t* page = mi_arena_page_alloc_fresh(slice_count, block_size, block_alignment, req_arena_id, tld); if (page == NULL) return NULL; - + mi_assert(page != NULL); - mi_assert(page->reserved == 1); + mi_assert(page->reserved == 1); mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - return page; + return page; } @@ -646,17 +649,17 @@ void _mi_arena_page_free(mi_page_t* page) { Arena abandon ----------------------------------------------------------- */ -void _mi_arena_page_abandon(mi_page_t* page) { +static void mi_arena_page_abandon_no_stat(mi_page_t* page) { mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); mi_assert_internal(page->next==NULL); - + mi_subproc_t* subproc = page->subproc; if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { - // make available for allocations + // make available for allocations size_t bin = _mi_bin(mi_page_block_size(page)); size_t slice_index; size_t slice_count; @@ -667,6 +670,7 @@ void _mi_arena_page_abandon(mi_page_t* page) { mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); // mi_assert_internal(mi_bitmap_is_setN(&arena->slices_dirty, slice_index, slice_count)); + mi_page_set_abandoned_mapped(page); bool were_zero = mi_pairmap_set(&arena->pages_abandoned[bin], slice_index); MI_UNUSED(were_zero); mi_assert_internal(were_zero); mi_atomic_increment_relaxed(&subproc->abandoned_count[bin]); @@ -676,34 +680,59 @@ void _mi_arena_page_abandon(mi_page_t* page) { // leave as is; it will be reclaimed when an object is free'd in the page } _mi_page_unown(page); +} + +void _mi_arena_page_abandon(mi_page_t* page) { + mi_arena_page_abandon_no_stat(page); _mi_stat_increase(&_mi_stats_main.pages_abandoned, 1); } +bool _mi_arena_page_try_reabandon_to_mapped(mi_page_t* page) { + mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(page)==page); + mi_assert_internal(mi_page_is_owned(page)); + mi_assert_internal(mi_page_is_abandoned(page)); + mi_assert_internal(!mi_page_is_abandoned_mapped(page)); + mi_assert_internal(!mi_page_is_full(page)); + mi_assert_internal(!mi_page_all_free(page)); + mi_assert_internal(!mi_page_is_singleton(page)); + if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) { + return false; + } + else { + _mi_stat_counter_increase(&_mi_stats_main.pages_reabandon_full, 1); + mi_arena_page_abandon_no_stat(page); + return true; + } +} + // called from `mi_free` if trying to unabandon an abandoned page void _mi_arena_page_unabandon(mi_page_t* page) { mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); - - if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { + + if (mi_page_is_abandoned_mapped(page)) { + mi_assert_internal(page->memid.memkind==MI_MEM_ARENA); // remove from the abandoned map size_t bin = _mi_bin(mi_page_block_size(page)); size_t slice_index; size_t slice_count; mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); - + mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); // this busy waits until a concurrent reader (from alloc_abandoned) is done mi_pairmap_clear_while_not_busy(&arena->pages_abandoned[bin], slice_index); + mi_page_clear_abandoned_mapped(page); mi_atomic_decrement_relaxed(&page->subproc->abandoned_count[bin]); } else { // page is full (or a singleton), page is OS/externally allocated - // nothing to do + // nothing to do // TODO: maintain count of these as well? } _mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1); @@ -715,7 +744,7 @@ bool _mi_arena_try_reclaim(mi_heap_t* heap, mi_page_t* page) { mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); // if (!mi_page_is_abandoned(page)) return false; // it is not abandoned (anymore) - + // note: we can access the page even it is in the meantime reclaimed by another thread since // we only call this when on free (and thus there is still an object alive in the page) mi_memid_t memid = page->memid; @@ -967,7 +996,7 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int arena->is_large = is_large; arena->purge_expire = 0; mi_lock_init(&arena->abandoned_visit_lock); - + // init bitmaps mi_bitmap_init(&arena->slices_free,true); mi_bitmap_init(&arena->slices_committed,true); @@ -1068,7 +1097,7 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_ _mi_memset(buf + k, 'o', MI_BFIELD_BITS); k += MI_BFIELD_BITS; } - bit_count += MI_BFIELD_BITS; + bit_count += MI_BFIELD_BITS; } _mi_output_message("%s %s\n", prefix, buf); } diff --git a/src/bitmap.c b/src/bitmap.c index eb5da086..df25e028 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -80,7 +80,7 @@ static inline bool mi_bfield_atomic_set2(_Atomic(mi_bfield_t)*b, size_t idx, boo mi_assert_internal(idx < MI_BFIELD_BITS-1); const size_t mask = (mi_bfield_t)0x03 << idx; mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)); // try to atomically set the mask bits until success + while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) { }; // try to atomically set the mask bits until success if (all_already_set!=NULL) { *all_already_set = ((old&mask)==mask); } return ((old&mask) == 0); } @@ -90,7 +90,7 @@ static inline bool mi_bfield_atomic_clear2(_Atomic(mi_bfield_t)*b, size_t idx, b mi_assert_internal(idx < MI_BFIELD_BITS-1); const size_t mask = (mi_bfield_t)0x03 << idx; mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)); // try to atomically clear the mask bits until success + while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) { }; // try to atomically clear the mask bits until success if (all_already_clear!=NULL) { *all_already_clear = ((old&mask) == 0); } return ((old&mask) == mask); } @@ -110,7 +110,7 @@ static inline bool mi_bfield_atomic_xset2(mi_bit_t set, _Atomic(mi_bfield_t)*b, static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) { mi_assert_internal(mask != 0); mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)); // try to atomically set the mask bits until success + while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) { }; // try to atomically set the mask bits until success if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); } return ((old&mask) == 0); } @@ -119,7 +119,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_clear) { mi_assert_internal(mask != 0); mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)); // try to atomically clear the mask bits until success + while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) { }; // try to atomically clear the mask bits until success if (already_clear!=NULL) { *already_clear = mi_bfield_popcount(~(old&mask)); } return ((old&mask) == mask); } @@ -1115,16 +1115,18 @@ static inline bool mi_bfield_atomic_clear_while_not_busy(_Atomic(mi_bfield_t)*b, mi_assert_internal(idx < MI_BFIELD_BITS-1); const mi_bfield_t mask = ((mi_bfield_t)0x03 << idx); const mi_bfield_t mask_busy = ((mi_bfield_t)MI_PAIR_BUSY << idx); - mi_bfield_t old; mi_bfield_t bnew; + mi_bfield_t old = mi_atomic_load_relaxed(b); do { - old = mi_atomic_load_relaxed(b); if mi_unlikely((old&mask)==mask_busy) { old = mi_atomic_load_acquire(b); + if ((old&mask)==mask_busy) { + _mi_stat_counter_increase(&_mi_stats_main.pages_unabandon_busy_wait, 1); + } while ((old&mask)==mask_busy) { // busy wait mi_atomic_yield(); old = mi_atomic_load_acquire(b); - } + } } bnew = (old & ~mask); // clear } while (!mi_atomic_cas_weak_acq_rel(b, &old, bnew)); diff --git a/src/free.c b/src/free.c index 4bce6886..6e8514c6 100644 --- a/src/free.c +++ b/src/free.c @@ -128,7 +128,7 @@ void mi_free(void* p) mi_attr_noexcept { mi_page_t* const page = mi_checked_ptr_page(p,"mi_free"); if mi_unlikely(page==NULL) return; - + const bool is_local = (_mi_prim_thread_id() == mi_page_thread_id(page)); if mi_likely(is_local) { // thread-local free? if mi_likely(page->flags.full_aligned == 0) { // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned) @@ -156,50 +156,164 @@ void mi_free(void* p) mi_attr_noexcept static void mi_decl_noinline mi_free_try_reclaim_mt(mi_page_t* page) { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_thread_id(page)==0); - +#if 1 // we own the page now.. - - // first remove it from the abandoned pages in the arena -- this waits for any readers to finish - _mi_arena_page_unabandon(page); // this must be before collect - - // collect the thread atomic free list + // safe to collect the thread atomic free list _mi_page_free_collect(page, false); // update `used` count + #if MI_DEBUG > 1 if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } + #endif - if (mi_page_all_free(page)) { + // 1. free if the page is free now + if (mi_page_all_free(page)) + { + // first remove it from the abandoned pages in the arena (if mapped, this waits for any readers to finish) + _mi_arena_page_unabandon(page); // we can free the page directly _mi_arena_page_free(page); return; } - else { - // the page has still some blocks in use + // 2. if the page is unmapped, try to reabandon so it can possibly be mapped and found for allocations + else if (!mi_page_is_mostly_used(page) && // only reabandon if a full page starts to have enough blocks available to prevent immediate re-abandon of a full page + !mi_page_is_abandoned_mapped(page) && page->memid.memkind == MI_MEM_ARENA && + _mi_arena_page_try_reabandon_to_mapped(page)) + { + return; + } + // 3. if the page is not too full, we can try to reclaim it for ourselves + else if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 && + !mi_page_is_mostly_used(page)) + { + // the page has still some blocks in use (but not too many) // reclaim in our heap if compatible, or otherwise abandon again // todo: optimize this check further? // note: don't use `mi_heap_get_default()` as we may just have terminated this thread and we should // not reinitialize the heap for this thread. (can happen due to thread-local destructors for example -- issue #944) mi_heap_t* const heap = mi_prim_get_default_heap(); - - if ((_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0) && // only if reclaim on free is allowed - (heap != (mi_heap_t*)&_mi_heap_empty)) // we did not already terminate our thread (can this happen? + if (heap != (mi_heap_t*)&_mi_heap_empty) // we did not already terminate our thread (can this happen? { mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag); - if ((tagheap != NULL) && // don't reclaim across heap object types + if ((tagheap != NULL) && // don't reclaim across heap object types (page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? ) - (_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?) + (_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?) ) { - // make it part of our heap + // first remove it from the abandoned pages in the arena -- this waits for any readers to finish + _mi_arena_page_unabandon(page); _mi_heap_page_reclaim(tagheap, page); + _mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1); return; - } + } + } + } + + // not reclaimed or free'd, unown again + _mi_page_unown(page); + +#else + if (!mi_page_is_abandoned_mapped(page)) { + // singleton or OS allocated + if (mi_page_is_singleton(page)) { + // free singleton pages + #if MI_DEBUG>1 + _mi_page_free_collect(page, false); // update `used` count + mi_assert_internal(mi_page_all_free(page)); + #endif + // we can free the page directly + _mi_arena_page_free(page); + return; + } + else { + const bool was_full = mi_page_is_full(page); + _mi_page_free_collect(page,false); // update used + if (mi_page_all_free(page)) { + // no need to unabandon as it is unmapped + _mi_arena_page_free(page); + return; + } + else if (was_full && _mi_arena_page_reabandon_full(page)) { + return; + } + else if (!mi_page_is_mostly_used(page) && _mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0) { + // the page has still some blocks in use (but not too many) + // reclaim in our heap if compatible, or otherwise abandon again + // todo: optimize this check further? + // note: don't use `mi_heap_get_default()` as we may just have terminated this thread and we should + // not reinitialize the heap for this thread. (can happen due to thread-local destructors for example -- issue #944) + mi_heap_t* const heap = mi_prim_get_default_heap(); + if (heap != (mi_heap_t*)&_mi_heap_empty) { // we did not already terminate our thread (can this happen? + mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag); + if ((tagheap != NULL) && // don't reclaim across heap object types + (page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? ) + (_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?) + ) + { + _mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1); + // make it part of our heap (no need to unabandon as is unmapped) + _mi_heap_page_reclaim(tagheap, page); + return; + } + } + } + } + } + else { + // don't reclaim pages that can be found for fresh page allocations + } + + // not reclaimed or free'd, unown again + _mi_page_unown(page); +#endif +} + +/* +// we own the page now.. +// safe to collect the thread atomic free list +_mi_page_free_collect(page, false); // update `used` count +if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } + +if (mi_page_all_free(page)) { + // first remove it from the abandoned pages in the arena -- this waits for any readers to finish + _mi_arena_page_unabandon(page); // this must be before free'ing + // we can free the page directly + _mi_arena_page_free(page); + return; +} +else if (!mi_page_is_mostly_used(page)) { + // the page has still some blocks in use (but not too many) + // reclaim in our heap if compatible, or otherwise abandon again + // todo: optimize this check further? + // note: don't use `mi_heap_get_default()` as we may just have terminated this thread and we should + // not reinitialize the heap for this thread. (can happen due to thread-local destructors for example -- issue #944) + mi_heap_t* const heap = mi_prim_get_default_heap(); + + if ((_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0) && // only if reclaim on free is allowed + (heap != (mi_heap_t*)&_mi_heap_empty)) // we did not already terminate our thread (can this happen? + { + mi_heap_t* const tagheap = _mi_heap_by_tag(heap, page->heap_tag); + if ((tagheap != NULL) && // don't reclaim across heap object types + (page->subproc == tagheap->tld->subproc) && // don't reclaim across sub-processes; todo: make this check faster (integrate with _mi_heap_by_tag ? ) + (_mi_arena_memid_is_suitable(page->memid, tagheap->arena_id)) // don't reclaim across unsuitable arena's; todo: inline arena_is_suitable (?) + ) + { + // first remove it from the abandoned pages in the arena -- this waits for any readers to finish + _mi_arena_page_unabandon(page); + _mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_free, 1); + // make it part of our heap + _mi_heap_page_reclaim(tagheap, page); + return; } - - // we cannot reclaim this page.. abandon it again - _mi_arena_page_abandon(page); } } -// Push a block that is owned by another thread (or abandoned) on its page-local thread free list. +// we cannot reclaim this page.. leave it abandoned +// todo: should re-abandon or otherwise a partly used page could never be re-used if the +// objects in it are not freed explicitly. +_mi_page_unown(page); +*/ + + +// Push a block that is owned by another thread (or abandoned) on its page-local thread free list. static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_block_t* block) { // adjust stats (after padding check and potentially recursive `mi_free` above) diff --git a/src/init.c b/src/init.c index d1670d02..01beb222 100644 --- a/src/init.c +++ b/src/init.c @@ -83,7 +83,7 @@ const mi_page_t _mi_page_empty = { { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ - { 0, 0 } \ + { 0, 0 }, { 0, 0 }, { 0, 0 } \ MI_STAT_COUNT_END_NULL() // -------------------------------------------------------- diff --git a/src/options.c b/src/options.c index 759d096d..1b326cc3 100644 --- a/src/options.c +++ b/src/options.c @@ -143,7 +143,7 @@ static mi_option_desc_t options[_mi_option_last] = { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, - { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free + { 0, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. #if defined(MI_VISIT_ABANDONED) diff --git a/src/page.c b/src/page.c index 4d26dbad..9ea7a979 100644 --- a/src/page.c +++ b/src/page.c @@ -811,7 +811,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p _mi_page_free(page_candidate, pq); page_candidate = page; } - else if (page->used >= page_candidate->used) { + else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { page_candidate = page; } // if we find a non-expandable candidate, or searched for N pages, return with the best candidate diff --git a/src/stats.c b/src/stats.c index 53b18da0..2a793b59 100644 --- a/src/stats.c +++ b/src/stats.c @@ -331,6 +331,10 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, ""); mi_stat_print_ex(&stats->pages, "pages", -1, out, arg, ""); mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out, arg); + mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "-reclaima", out, arg); + mi_stat_counter_print(&stats->pages_reclaim_on_free, "-reclaimf", out, arg); + mi_stat_counter_print(&stats->pages_reabandon_full, "-reabandon", out, arg); + mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "-waits", out, arg); mi_stat_counter_print(&stats->pages_extended, "-extended", out, arg); mi_stat_counter_print(&stats->page_no_retire, "-noretire", out, arg); mi_stat_counter_print(&stats->arena_count, "arenas", out, arg); diff --git a/test/test-stress.c b/test/test-stress.c index ffeb5dea..4c2719aa 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -43,7 +43,13 @@ static int ITER = 10; #elif 0 static int THREADS = 4; static int SCALE = 100; +static int ITER = 10; +#define ALLOW_LARGE false +#elif 1 +static int THREADS = 32; +static int SCALE = 50; static int ITER = 50; +#define ALLOW_LARGE false #else static int THREADS = 32; // more repeatable if THREADS <= #processors static int SCALE = 50; // scaling factor @@ -54,7 +60,12 @@ static int ITER = 50; // N full iterations destructing and re-creating a #define STRESS // undefine for leak test -static bool allow_large_objects = false; // allow very large objects? (set to `true` if SCALE>100) +#ifndef ALLOW_LARGE +#define ALLOW_LARGE true +#endif + +static bool allow_large_objects = ALLOW_LARGE; // allow very large objects? (set to `true` if SCALE>100) + static size_t use_one_size = 0; // use single object size of `N * sizeof(uintptr_t)`? static bool main_participates = false; // main thread participates as a worker too @@ -332,6 +343,8 @@ int main(int argc, char** argv) { mi_debug_show_arenas(true,true,false); #endif // mi_stats_print(NULL); +#else + mi_stats_print(NULL); // so we see rss/commit/elapsed #endif //bench_end_program(); return 0;