kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-07 07:49:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Try separating pages with different slice counts

Browse source
This commit is contained in:
Sergiy Kuryata 2024-12-04 23:21:43 -08:00
parent 8dd6460d5e
commit a9bd3205be
2 changed files with 122 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/heap.c
View file

@ -653,19 +653,30 @@ bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_vis
return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);
}
mi_segment_t* mi_segments_get_segment_to_drop_by_slice(mi_segments_tld_t* tld);
const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment);
static mi_segment_t* mi_heap_get_segment_to_drop(mi_heap_t* heap) {
mi_page_queue_t* fullPageQueue = &heap->pages[MI_BIN_FULL];
mi_segment_t* segment = NULL;
if (fullPageQueue->first != NULL) {
segment = _mi_ptr_segment(fullPageQueue->first);
int i = 0;
for (mi_page_t* page = fullPageQueue->first->next; page != NULL && i < 3; page = page->next, i++) {
mi_segment_t* temp_segment = _mi_ptr_segment(page);
if (temp_segment->used > segment->used) {
segment = temp_segment;
}
}
}
else {
segment = mi_segments_get_segment_to_drop_by_slice(&heap->tld->segments);
}
return segment;
}
const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment);
// Visit all pages in a segment
static mi_decl_noinline void mi_segment_visit_pages(mi_heap_t* heap, mi_segment_t* segment, heap_page_visitor_fun* fn, void* arg1)
{
@ -743,7 +754,7 @@ void mi_heap_drop_segment_if_required(mi_heap_t* heap, size_t alloc_block_size)
{
size_t targetSegmentCount = mi_option_get_size(mi_option_max_segments_per_heap);
if ((targetSegmentCount > 0) &&
(alloc_block_size <= MI_MEDIUM_OBJ_SIZE_MAX) &&
(alloc_block_size <= MI_LARGE_OBJ_SIZE_MAX) &&
(heap->tld->segments.count >= targetSegmentCount)) {
mi_heap_drop_segment(heap, targetSegmentCount);

150
src/segment.c
View file

@ -204,14 +204,25 @@ static inline size_t mi_slice_index(const mi_slice_t* slice) {
Slice span queues
----------------------------------------------------------- */
static void mi_span_queue_push(mi_span_queue_t* sq, mi_slice_t* slice) {
static void mi_span_queue_push(mi_span_queue_t* sq, mi_slice_t* slice, bool addToHead) {
// todo: or push to the end?
mi_assert_internal(slice->prev == NULL && slice->next==NULL);
slice->prev = NULL; // paranoia
slice->next = NULL;
if (addToHead) {
slice->next = sq->first;
sq->first = slice;
if (slice->next != NULL) slice->next->prev = slice;
else sq->last = slice;
}
else {
// Add to the end of the queue
slice->prev = sq->last;
sq->last = slice;
if (slice->prev != NULL) slice->prev->next = slice;
else sq->first = slice;
}
slice->block_size = 0; // free
}
@ -638,7 +649,7 @@ static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size
}
// and push it on the free page queue (if it was not a huge page)
if (sq != NULL) mi_span_queue_push( sq, slice );
if (sq != NULL) mi_span_queue_push( sq, slice, (slice_index < (segment->slice_entries / 2)) /*add to head?*/);
else slice->block_size = 0; // mark huge page as free anyways
}
@ -770,7 +781,28 @@ static void mi_segment_slice_split(mi_segment_t* segment, mi_slice_t* slice, siz
slice->slice_count = (uint32_t)slice_count;
}
static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) {
mi_segment_t* mi_segments_get_segment_to_drop_by_slice(mi_segments_tld_t* tld) {
mi_segment_t* segment_to_return = NULL;
mi_span_queue_t* sq = mi_span_queue_for(1, tld);
while (sq <= &tld->spans[MI_SEGMENT_BIN_MAX]) {
if (sq->first != NULL) {
segment_to_return = _mi_ptr_segment(sq->first);
int i = 0;
for (mi_slice_t* slice = sq->first->next; slice != NULL && i < 3; slice = slice->next, i++) {
mi_segment_t* segment = _mi_ptr_segment(slice);
if (segment->used > segment_to_return->used) {
segment_to_return = segment;
}
}
break;
}
sq++;
}
return segment_to_return;
}
static mi_page_t* mi_segments_page_find_and_allocate_from_start(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) {
mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX);
// search from best fit up
mi_span_queue_t* sq = mi_span_queue_for(slice_count, tld);
@ -804,6 +836,69 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren
return NULL;
}
static mi_slice_t* mi_segment_slice_split_end(mi_segment_t* segment, mi_slice_t* slice, size_t slice_count, mi_segments_tld_t* tld) {
mi_assert_internal(_mi_ptr_segment(slice) == segment);
mi_assert_internal(slice->slice_count >= slice_count);
mi_assert_internal(slice->block_size > 0); // no more in free queue
if (slice->slice_count <= slice_count) return slice;
mi_assert_internal(segment->kind != MI_SEGMENT_HUGE);
size_t slice_to_return_index = mi_slice_index(slice) + slice->slice_count - slice_count;
size_t current_index = mi_slice_index(slice);
size_t new_count = slice->slice_count - slice_count;
mi_segment_span_free(segment, current_index, new_count, false /* don't purge left-over part */, tld);
mi_slice_t* slice_to_return = &segment->slices[slice_to_return_index];
slice_to_return->slice_count = (uint32_t)slice_count;
return slice_to_return;
}
static mi_page_t* mi_segments_page_find_and_allocate_from_end(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) {
mi_assert_internal(slice_count * MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX);
// search from best fit up
mi_span_queue_t* sq = mi_span_queue_for(slice_count, tld);
if (slice_count == 0) slice_count = 1;
while (sq <= &tld->spans[MI_SEGMENT_BIN_MAX]) {
for (mi_slice_t* slice = sq->last; slice != NULL; slice = slice->prev) {
if (slice->slice_count >= slice_count) {
// found one
mi_segment_t* segment = _mi_ptr_segment(slice);
if (_mi_arena_memid_is_suitable(segment->memid, req_arena_id)) {
// found a suitable page span
mi_span_queue_delete(sq, slice);
if (slice->slice_count > slice_count) {
slice = mi_segment_slice_split_end(segment, slice, slice_count, tld);
//mi_segment_slice_split(segment, slice, slice_count, tld);
}
mi_assert_internal(slice != NULL && slice->slice_count == slice_count && slice->block_size > 0);
mi_page_t* page = mi_segment_span_allocate(segment, mi_slice_index(slice), slice->slice_count, tld);
if (page == NULL) {
// commit failed; return NULL but first restore the slice
mi_segment_span_free_coalesce(slice, tld);
return NULL;
}
return page;
}
}
}
sq++;
}
// could not find a page..
return NULL;
}
static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) {
if (slice_count > 1) {
return mi_segments_page_find_and_allocate_from_end(slice_count, req_arena_id, tld);
}
else {
return mi_segments_page_find_and_allocate_from_start(slice_count, req_arena_id, tld);
}
}
/* -----------------------------------------------------------
Segment allocation
@ -1216,6 +1311,9 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s
// a page has available free blocks of the right size
has_page = true;
}
else if (block_size == 0 && mi_page_has_any_available(page)) {
free_space_mask |= mi_free_space_mask_from_blocksize(page->block_size);
}
}
else {
// empty span
@ -1345,9 +1443,6 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice
if (max_tries <= 0) return NULL;
mi_segment_t* segment;
mi_segment_t* best_candidate_segment = NULL;
size_t best_free_space_mask = MI_FREE_SPACE_MASK_ANY;
int candidates_to_check = 5;
size_t free_space_mask = mi_free_space_mask_from_blocksize(block_size);
mi_arena_field_cursor_t current; _mi_arena_field_cursor_init2(heap, &current, free_space_mask);
while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(&current)) != NULL))
@ -1370,42 +1465,10 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice
// found a large enough free span, or a page of the right block_size with free space
// we return the result of reclaim (which is usually `segment`) as it might free
// the segment due to concurrent frees (in which case `NULL` is returned).
mi_segment_t* candidate_segment = segment;
size_t candidates_free_space_mask = segment->free_space_mask & MI_FREE_SPACE_MASK_ANY; // clear the abandoned bit
// If the requested block size is the largest available in the segment then use this segment
if (free_space_mask > ((~free_space_mask) & candidates_free_space_mask)) {
candidate_segment = mi_segment_reclaim(candidate_segment, heap, block_size, reclaimed, tld);
if (candidate_segment != NULL) {
// Found a better candidate; return the current best candidate to the abandoned list.
if (best_candidate_segment != NULL) {
_mi_arena_segment_mark_abandoned(best_candidate_segment);
}
best_candidate_segment = candidate_segment;
// found a good candidate segment; break the serch loop.
mi_segment_t* segmentToReturn = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld);
if (segmentToReturn != NULL) {
mi_segment_increment_reclaimed_stats();
return best_candidate_segment;
}
// The current segment was freed. Look for another one.
} else if (candidates_free_space_mask < best_free_space_mask || best_candidate_segment == NULL) {
//candidate_segment = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld);
if (candidate_segment != NULL) {
// Found a better candidate; return the current best candidate to the abandoned list.
if (best_candidate_segment != NULL) {
_mi_arena_segment_mark_abandoned(best_candidate_segment);
}
best_candidate_segment = candidate_segment;
best_free_space_mask = candidates_free_space_mask;
candidate_segment = NULL;
}
}
if (candidate_segment != NULL) {
_mi_arena_segment_mark_abandoned(candidate_segment);
}
candidates_to_check--;
if (candidates_to_check == 0) {
break;
return segmentToReturn;
}
}
else if (segment->abandoned_visits > 3 && is_suitable && mi_option_get_size(mi_option_max_segments_per_heap) == 0) {
@ -1419,11 +1482,6 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice
}
}
if (best_candidate_segment != NULL) {
mi_segment_increment_reclaimed_stats();
return mi_segment_reclaim(best_candidate_segment, heap, block_size, reclaimed, tld);
}
mi_segment_increment_reclaim_failed_stats();
return NULL;
}