diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 88142197..8e8b5c9c 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -110,6 +110,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); void _mi_abandoned_await_readers(void); +void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); diff --git a/src/heap.c b/src/heap.c index 416a9a8d..b0cae474 100644 --- a/src/heap.c +++ b/src/heap.c @@ -147,6 +147,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); + // collect abandoned pages + _mi_abandoned_collect(heap, collect >= MI_FORCE, &heap->tld->segments); + // collect segment local caches if (collect >= MI_FORCE) { _mi_segment_thread_collect(&heap->tld->segments); diff --git a/src/segment.c b/src/segment.c index 3001f160..980ca439 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1050,7 +1050,7 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) Abandonment When threads terminate, they can leave segments with -live blocks (reached through other threads). Such segments +live blocks (reachable through other threads). Such segments are "abandoned" and will be reclaimed by other threads to reuse their pages and/or free them eventually @@ -1065,11 +1065,11 @@ or decommitting segments that have a pending read operation. Note: the current implementation is one possible design; another way might be to keep track of abandoned segments -in the regions. This would have the advantage of keeping +in the arenas/segment_cache's. This would have the advantage of keeping all concurrent code in one place and not needing to deal with ABA issues. The drawback is that it is unclear how to scan abandoned segments efficiently in that case as they -would be spread among all other segments in the regions. +would be spread among all other segments in the arenas. ----------------------------------------------------------- */ // Use the bottom 20-bits (on 64-bit) of the aligned segment pointers @@ -1431,7 +1431,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice } else { // otherwise, push on the visited list so it gets not looked at too quickly again - mi_segment_delayed_decommit(segment, true, tld->stats); // decommit if needed + mi_segment_delayed_decommit(segment, true /* force? */, tld->stats); // forced decommit if needed mi_abandoned_visited_push(segment); } } @@ -1439,6 +1439,29 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice } +void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld) +{ + mi_segment_t* segment; + int max_tries = (force ? 16*1024 : 1024); // limit latency + if (force) { + mi_abandoned_visited_revisit(); + } + while ((max_tries-- > 0) && ((segment = mi_abandoned_pop()) != NULL)) { + mi_segment_check_free(segment,0,0,tld); // try to free up pages (due to concurrent frees) + if (segment->used == 0) { + // free the segment (by forced reclaim) to make it available to other threads. + // note: we could in principle optimize this by skipping reclaim and directly + // freeing but that would violate some invariants temporarily) + mi_segment_reclaim(segment, heap, 0, NULL, tld); + } + else { + // otherwise, decommit if needed and push on the visited list + mi_segment_delayed_decommit(segment, force, tld->stats); // forced decommit if needed + mi_abandoned_visited_push(segment); + } + } +} + /* ----------------------------------------------------------- Reclaim or allocate ----------------------------------------------------------- */