From 8dd6460d5e8684aba15d9711f8fba0e56e4e4fb8 Mon Sep 17 00:00:00 2001 From: Sergiy Kuryata Date: Sat, 30 Nov 2024 18:44:35 -0800 Subject: [PATCH] Do not discard segments for large object allocations --- src/heap.c | 2 +- src/segment.c | 49 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/heap.c b/src/heap.c index c52721dd..d2c20fb1 100644 --- a/src/heap.c +++ b/src/heap.c @@ -743,7 +743,7 @@ void mi_heap_drop_segment_if_required(mi_heap_t* heap, size_t alloc_block_size) { size_t targetSegmentCount = mi_option_get_size(mi_option_max_segments_per_heap); if ((targetSegmentCount > 0) && - (alloc_block_size <= MI_LARGE_OBJ_SIZE_MAX) && + (alloc_block_size <= MI_MEDIUM_OBJ_SIZE_MAX) && (heap->tld->segments.count >= targetSegmentCount)) { mi_heap_drop_segment(heap, targetSegmentCount); diff --git a/src/segment.c b/src/segment.c index 18e7d3ed..b701a9ae 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1222,6 +1222,7 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s if (slice->slice_count >= slices_needed) { has_page = true; } + free_space_mask |= mi_free_space_mask_from_slicecount(slice->slice_count); } slice = slice + slice->slice_count; } @@ -1344,6 +1345,9 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice if (max_tries <= 0) return NULL; mi_segment_t* segment; + mi_segment_t* best_candidate_segment = NULL; + size_t best_free_space_mask = MI_FREE_SPACE_MASK_ANY; + int candidates_to_check = 5; size_t free_space_mask = mi_free_space_mask_from_blocksize(block_size); mi_arena_field_cursor_t current; _mi_arena_field_cursor_init2(heap, ¤t, free_space_mask); while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(¤t)) != NULL)) @@ -1366,10 +1370,42 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice // found a large enough free span, or a page of the right block_size with free space // we return the result of reclaim (which is usually `segment`) as it might free // the segment due to concurrent frees (in which case `NULL` is returned). - mi_segment_t* segmentToReturn = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); - if (segmentToReturn != NULL) { - mi_segment_increment_reclaimed_stats(); - return segmentToReturn; + mi_segment_t* candidate_segment = segment; + size_t candidates_free_space_mask = segment->free_space_mask & MI_FREE_SPACE_MASK_ANY; // clear the abandoned bit + + // If the requested block size is the largest available in the segment then use this segment + if (free_space_mask > ((~free_space_mask) & candidates_free_space_mask)) { + candidate_segment = mi_segment_reclaim(candidate_segment, heap, block_size, reclaimed, tld); + if (candidate_segment != NULL) { + // Found a better candidate; return the current best candidate to the abandoned list. + if (best_candidate_segment != NULL) { + _mi_arena_segment_mark_abandoned(best_candidate_segment); + } + best_candidate_segment = candidate_segment; + // found a good candidate segment; break the serch loop. + mi_segment_increment_reclaimed_stats(); + return best_candidate_segment; + } + // The current segment was freed. Look for another one. + } else if (candidates_free_space_mask < best_free_space_mask || best_candidate_segment == NULL) { + //candidate_segment = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld); + if (candidate_segment != NULL) { + // Found a better candidate; return the current best candidate to the abandoned list. + if (best_candidate_segment != NULL) { + _mi_arena_segment_mark_abandoned(best_candidate_segment); + } + best_candidate_segment = candidate_segment; + best_free_space_mask = candidates_free_space_mask; + candidate_segment = NULL; + } + } + + if (candidate_segment != NULL) { + _mi_arena_segment_mark_abandoned(candidate_segment); + } + candidates_to_check--; + if (candidates_to_check == 0) { + break; } } else if (segment->abandoned_visits > 3 && is_suitable && mi_option_get_size(mi_option_max_segments_per_heap) == 0) { @@ -1383,6 +1419,11 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice } } + if (best_candidate_segment != NULL) { + mi_segment_increment_reclaimed_stats(); + return mi_segment_reclaim(best_candidate_segment, heap, block_size, reclaimed, tld); + } + mi_segment_increment_reclaim_failed_stats(); return NULL; }