diff --git a/include/mimalloc.h b/include/mimalloc.h index 5bca7956..73f82093 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -349,6 +349,7 @@ typedef enum mi_option_e { mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) mi_option_max_segments_per_heap, // max number of segments that heap can own + mi_option_heap_collect_abandoned_interval, // interval (in ms) between collecting abandoned segments when a heap drops exessive segments _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index c52cbcb7..dbc4f403 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -220,12 +220,13 @@ typedef int32_t mi_ssize_t; #define MI_MAX_ALLOC_SIZE PTRDIFF_MAX #endif -#define MI_FREE_SPACE_MASK_BIT_COUNT 31 +#define MI_FREE_SPACE_MASK_BIT_COUNT 63 #define MI_FREE_SPACE_BINS_PER_BIT (MI_BIN_HUGE/MI_FREE_SPACE_MASK_BIT_COUNT) -#define MI_FREE_SPACE_MASK_ALL ((size_t)0xFFFFFFFF) -#define MI_FREE_SPACE_MASK_ABANDONED ((size_t)0x80000000) +#define MI_FREE_SPACE_MASK_ALL ((size_t)0xFFFFFFFFFFFFFFFF) +#define MI_FREE_SPACE_MASK_ABANDONED ((size_t)0x8000000000000000) #define MI_FREE_SPACE_MASK_ANY (MI_FREE_SPACE_MASK_ALL & (~MI_FREE_SPACE_MASK_ABANDONED)) + // ------------------------------------------------------ // Mimalloc pages contain allocated blocks // ------------------------------------------------------ @@ -544,6 +545,7 @@ struct mi_heap_s { size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) size_t page_retired_max; // largest retired index into the `pages` array. mi_heap_t* next; // list of heaps per thread + mi_msecs_t last_abandoned_collect_time; bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom tag, can be used for separating heaps based on the object types mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. diff --git a/src/heap.c b/src/heap.c index e94d7c54..3932bc9e 100644 --- a/src/heap.c +++ b/src/heap.c @@ -230,6 +230,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; + heap->last_abandoned_collect_time = 0; } mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { @@ -702,6 +703,7 @@ static mi_decl_noinline void mi_segment_visit_pages(mi_heap_t* heap, mi_segment_ } void mi_heap_drop_segment(mi_heap_t* heap, size_t targetSegmentCount) { + bool segmentsDropped = false; while (heap->tld->segments.count >= targetSegmentCount) { @@ -711,6 +713,7 @@ void mi_heap_drop_segment(mi_heap_t* heap, size_t targetSegmentCount) { break; } + segmentsDropped = true; // 2. when abandoning, mark all pages to no longer add to delayed_free mi_segment_visit_pages(heap, segmentToAbandon, &mi_heap_page_never_delayed_free, NULL); @@ -723,6 +726,18 @@ void mi_heap_drop_segment(mi_heap_t* heap, size_t targetSegmentCount) { mi_collect_t collect = MI_ABANDON; mi_segment_visit_pages(heap, segmentToAbandon, &mi_heap_page_collect, &collect); } + + if (segmentsDropped) { + mi_msecs_t now_msec = _mi_clock_now(); + long collect_interval = mi_option_get(mi_option_heap_collect_abandoned_interval); + if ((now_msec - heap->last_abandoned_collect_time) >= collect_interval) { + heap->last_abandoned_collect_time = now_msec; + + // collect abandoned segments (in particular, purge expired parts of segments in the abandoned segment list) + // note: forced purge can be quite expensive if many threads are created/destroyed so we do not force on abandonment + _mi_abandoned_collect(heap, false /* force? */, &heap->tld->segments); + } + } } void mi_heap_drop_segment_if_required(mi_heap_t* heap, size_t alloc_block_size) diff --git a/src/init.c b/src/init.c index 6f51ca89..61062a6c 100644 --- a/src/init.c +++ b/src/init.c @@ -123,6 +123,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { 0, // page count MI_BIN_FULL, 0, // page retired min/max NULL, // next + 0, // last_abandoned_collect_time false, // can reclaim 0, // tag MI_SMALL_PAGES_EMPTY, @@ -169,6 +170,7 @@ mi_heap_t _mi_heap_main = { 0, // page count MI_BIN_FULL, 0, // page retired min/max NULL, // next heap + 0, // last_abandoned_collect_time false, // can reclaim 0, // tag MI_SMALL_PAGES_EMPTY, diff --git a/src/options.c b/src/options.c index 099766a6..774ce577 100644 --- a/src/options.c +++ b/src/options.c @@ -94,6 +94,7 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. { 8, UNINIT, MI_OPTION(max_segments_per_heap) }, // max number of segments that heap can own. + { 2000, UNINIT, MI_OPTION(heap_collect_abandoned_interval) }, // max number of segments that heap can own. }; static void mi_option_init(mi_option_desc_t* desc); diff --git a/src/segment.c b/src/segment.c index 231d99ac..b5ecd136 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1101,7 +1101,7 @@ size_t mi_free_space_mask_from_slicecount(uint32_t slice_count) max_size = slice_count * MI_SEGMENT_SLICE_SIZE; } - free_space_mask = mi_free_space_mask_from_blocksize(max_size - 1); + free_space_mask = mi_free_space_mask_from_blocksize(max_size); free_space_mask = free_space_mask | (free_space_mask - 1); // mark all allocations with size < max_size as available return free_space_mask; @@ -1187,6 +1187,7 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s { mi_assert_internal(mi_segment_is_abandoned(segment)); bool has_page = false; + size_t free_space_mask = 0; // for all slices const mi_slice_t* end; @@ -1208,6 +1209,7 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s if (slice->slice_count >= slices_needed) { has_page = true; } + free_space_mask |= mi_free_space_mask_from_slicecount(slice->slice_count); } else if (mi_page_block_size(page) == block_size && mi_page_has_any_available(page)) { // a page has available free blocks of the right size @@ -1222,6 +1224,10 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s } slice = slice + slice->slice_count; } + + if (free_space_mask != 0) { + mi_atomic_or_acq_rel(&segment->free_space_mask, free_space_mask); + } return has_page; }