From 1b3eb8ef2863b810e99236ec9bf1eb558c67b1f1 Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 29 Feb 2024 19:17:24 -0800 Subject: [PATCH 1/3] quick exit from try_reclaim if no abandoned segments --- src/segment.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/segment.c b/src/segment.c index 3dd37429..08909eb1 100644 --- a/src/segment.c +++ b/src/segment.c @@ -926,6 +926,8 @@ void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, bool* reclaimed, mi_segments_tld_t* tld) { *reclaimed = false; + if (mi_atomic_load_relaxed(&abandoned_count) == 0) return NULL; + mi_segment_t* segment; mi_arena_id_t current_id = 0; size_t current_idx = 0; From 71bcf1c76bd48f6bd3237944e1fd242466228e3e Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 1 Mar 2024 10:31:58 -0800 Subject: [PATCH 2/3] maintain abandoned_count more robustly --- src/arena.c | 21 ++++++++++++++------- src/segment.c | 13 ++----------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/arena.c b/src/arena.c index cd3a7ef2..59db0b8e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -738,6 +738,9 @@ bool _mi_arena_contains(const void* p) { the arena bitmaps. ----------------------------------------------------------- */ +// Maintain these for debug purposes +static mi_decl_cache_align _Atomic(size_t)abandoned_count; + // reclaim a specific abandoned segment; `true` on success. bool _mi_arena_segment_clear_abandoned(mi_memid_t memid ) { @@ -748,11 +751,12 @@ bool _mi_arena_segment_clear_abandoned(mi_memid_t memid ) mi_assert_internal(arena_idx < MI_MAX_ARENAS); mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); mi_assert_internal(arena != NULL); - bool was_abandoned = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); - // mi_assert_internal(was_abandoned); - mi_assert_internal(!was_abandoned || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); + bool was_marked = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); + if (was_marked) { mi_atomic_decrement_relaxed(&abandoned_count); } + // mi_assert_internal(was_marked); + mi_assert_internal(!was_marked || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); - return was_abandoned; + return was_marked; } // mark a specific segment as abandoned @@ -765,15 +769,17 @@ void _mi_arena_segment_mark_abandoned(mi_memid_t memid) mi_assert_internal(arena_idx < MI_MAX_ARENAS); mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); mi_assert_internal(arena != NULL); - const bool was_unset = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); - MI_UNUSED_RELEASE(was_unset); - mi_assert_internal(was_unset); + const bool was_unmarked = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); + if (was_unmarked) { mi_atomic_increment_relaxed(&abandoned_count); } + mi_assert_internal(was_unmarked); mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); } // reclaim abandoned segments mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_id_t* previous_id, size_t* previous_idx ) { + if (mi_atomic_load_relaxed(&abandoned_count) == 0) return false; + const int max_arena = (int)mi_atomic_load_relaxed(&mi_arena_count); int arena_idx = *previous_id; size_t field_idx = mi_bitmap_index_field(*previous_idx); @@ -794,6 +800,7 @@ mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_id_t* previous_id, mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); // try to reclaim it atomically if (_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) { + mi_atomic_decrement_relaxed(&abandoned_count); *previous_idx = bitmap_idx; *previous_id = arena_idx; mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); diff --git a/src/segment.c b/src/segment.c index 08909eb1..36bf2dfc 100644 --- a/src/segment.c +++ b/src/segment.c @@ -753,10 +753,6 @@ by scanning the arena memory (segments outside arena memoryare only reclaimed by a free). ----------------------------------------------------------- */ -// Maintain these for debug purposes -static mi_decl_cache_align _Atomic(size_t)abandoned_count; - - // legacy: Wait until there are no more pending reads on segments that used to be in the abandoned list void _mi_abandoned_await_readers(void) { // nothing needed @@ -782,7 +778,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { mi_segments_track_size(-((long)segment->segment_size), tld); segment->thread_id = 0; segment->abandoned_visits = 0; - _mi_arena_segment_mark_abandoned(segment->memid); mi_atomic_increment_relaxed(&abandoned_count); + _mi_arena_segment_mark_abandoned(segment->memid); } void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { @@ -905,7 +901,6 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { if (mi_atomic_load_relaxed(&segment->thread_id) != 0) return false; // it is not abandoned if (_mi_arena_segment_clear_abandoned(segment->memid)) { // atomically unabandon - mi_atomic_decrement_relaxed(&abandoned_count); mi_segment_t* res = mi_segment_reclaim(segment, heap, 0, NULL, &heap->tld->segments); mi_assert_internal(res == segment); return (res != NULL); @@ -918,7 +913,6 @@ void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { mi_arena_id_t current_id = 0; size_t current_idx = 0; while ((segment = _mi_arena_segment_clear_abandoned_next(¤t_id, ¤t_idx)) != NULL) { - mi_atomic_decrement_relaxed(&abandoned_count); mi_segment_reclaim(segment, heap, 0, NULL, tld); } } @@ -926,15 +920,13 @@ void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, bool* reclaimed, mi_segments_tld_t* tld) { *reclaimed = false; - if (mi_atomic_load_relaxed(&abandoned_count) == 0) return NULL; - + mi_segment_t* segment; mi_arena_id_t current_id = 0; size_t current_idx = 0; long max_tries = mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 1024); // limit the work to bound allocation times while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t_id, ¤t_idx)) != NULL)) { - mi_atomic_decrement_relaxed(&abandoned_count); segment->abandoned_visits++; // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments // and push them into the visited list and use many tries. Perhaps we can skip non-suitable ones in a better way? @@ -962,7 +954,6 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, else { // otherwise, mark it back as abandoned // todo: reset delayed pages in the segment? - mi_atomic_increment_relaxed(&abandoned_count); _mi_arena_segment_mark_abandoned(segment->memid); } } From cf8f73098e3e76968dd45dffd909f1f36e4c26a8 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 1 Mar 2024 10:51:18 -0800 Subject: [PATCH 3/3] start abandoned search randomized --- include/mimalloc/internal.h | 13 ++++++++++--- src/arena.c | 34 ++++++++++++++++++++++------------ src/segment.c | 13 +++++-------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 1143a184..0976fc4e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -124,9 +124,16 @@ bool _mi_arena_contains(const void* p); void _mi_arena_collect(bool force_purge, mi_stats_t* stats); void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); -bool _mi_arena_segment_clear_abandoned(mi_memid_t memid); -void _mi_arena_segment_mark_abandoned(mi_memid_t memid); -mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_id_t* current_id, size_t* current_idx); +bool _mi_arena_segment_clear_abandoned(mi_memid_t memid); +void _mi_arena_segment_mark_abandoned(mi_memid_t memid); + +typedef struct mi_arena_field_cursor_s { // abstract + mi_arena_id_t start; + int count; + size_t bitmap_idx; +} mi_arena_field_cursor_t; +void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current); +mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous); // "segment-map.c" void _mi_segment_map_allocated_at(const mi_segment_t* segment); diff --git a/src/arena.c b/src/arena.c index 59db0b8e..a73c5f52 100644 --- a/src/arena.c +++ b/src/arena.c @@ -775,17 +775,27 @@ void _mi_arena_segment_mark_abandoned(mi_memid_t memid) mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); } -// reclaim abandoned segments -mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_id_t* previous_id, size_t* previous_idx ) -{ - if (mi_atomic_load_relaxed(&abandoned_count) == 0) return false; +// start a cursor at a randomized arena +void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_arena_field_cursor_t* current) { + const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); + current->start = (max_arena == 0 ? 0 : (mi_arena_id_t)( _mi_heap_random_next(heap) % max_arena)); + current->count = 0; + current->bitmap_idx = 0; +} +// reclaim abandoned segments +mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_field_cursor_t* previous ) +{ const int max_arena = (int)mi_atomic_load_relaxed(&mi_arena_count); - int arena_idx = *previous_id; - size_t field_idx = mi_bitmap_index_field(*previous_idx); - size_t bit_idx = mi_bitmap_index_bit_in_field(*previous_idx) + 1; + if (max_arena <= 0 || mi_atomic_load_relaxed(&abandoned_count) == 0) return NULL; + + int count = previous->count; + size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx); + size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx) + 1; // visit arena's (from previous) - for( ; arena_idx < max_arena; arena_idx++, field_idx = 0, bit_idx = 0) { + for (; count < max_arena; count++, field_idx = 0, bit_idx = 0) { + mi_arena_id_t arena_idx = previous->start + count; + if (arena_idx >= max_arena) { arena_idx = arena_idx % max_arena; } // wrap around mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); if (arena != NULL) { // visit the abandoned fields (starting at previous_idx) @@ -801,8 +811,8 @@ mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_id_t* previous_id, // try to reclaim it atomically if (_mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx)) { mi_atomic_decrement_relaxed(&abandoned_count); - *previous_idx = bitmap_idx; - *previous_id = arena_idx; + previous->bitmap_idx = bitmap_idx; + previous->count = count; mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); return (mi_segment_t*)mi_arena_block_start(arena, bitmap_idx); @@ -814,8 +824,8 @@ mi_segment_t* _mi_arena_segment_clear_abandoned_next(mi_arena_id_t* previous_id, } } // no more found - *previous_idx = 0; - *previous_id = 0; + previous->bitmap_idx = 0; + previous->count = 0; return NULL; } diff --git a/src/segment.c b/src/segment.c index 36bf2dfc..f7a43abf 100644 --- a/src/segment.c +++ b/src/segment.c @@ -910,22 +910,19 @@ bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { mi_segment_t* segment; - mi_arena_id_t current_id = 0; - size_t current_idx = 0; - while ((segment = _mi_arena_segment_clear_abandoned_next(¤t_id, ¤t_idx)) != NULL) { + mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, ¤t); + while ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL) { mi_segment_reclaim(segment, heap, 0, NULL, tld); } } static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, bool* reclaimed, mi_segments_tld_t* tld) { - *reclaimed = false; - + *reclaimed = false; mi_segment_t* segment; - mi_arena_id_t current_id = 0; - size_t current_idx = 0; + mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap,¤t); long max_tries = mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 1024); // limit the work to bound allocation times - while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t_id, ¤t_idx)) != NULL)) + while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) { segment->abandoned_visits++; // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments