kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 23:39:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip: purgeable arenas

Browse source
This commit is contained in:
daanx 2023-04-04 11:46:02 -07:00
parent 33d7503fdb
commit 09297ba8cf
5 changed files with 68 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/arena.c
View file

@ -163,7 +163,7 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren
// none of the claimed blocks should be scheduled for a decommit // none of the claimed blocks should be scheduled for a decommit
if (arena->blocks_purge != NULL) { if (arena->blocks_purge != NULL) {
// this is thread safe as a potential purge only decommits parts that are not yet claimed as used (in `in_use`). // this is thread safe as a potential purge only decommits parts that are not yet claimed as used (in `blocks_inuse`).
_mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, needed_bcount, bitmap_index); _mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, needed_bcount, bitmap_index);
} }
@ -176,7 +176,7 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren
*commit = true; *commit = true;
} }
else if (*commit) { else if (*commit) {
// arena not committed as a whole, but commit requested: ensure commit now // commit requested, but the range may not be committed as a whole: ensure it is committed now
bool any_uncommitted; bool any_uncommitted;
_mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted); _mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted);
if (any_uncommitted) { if (any_uncommitted) {
@ -294,7 +294,8 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset
arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_BLOCK_SIZE); arena_reserve = _mi_align_up(arena_reserve, MI_ARENA_BLOCK_SIZE);
if (arena_reserve > 0 && arena_reserve >= size && // eager reserve enabled and large enough? if (arena_reserve > 0 && arena_reserve >= size && // eager reserve enabled and large enough?
req_arena_id == _mi_arena_id_none() && // not exclusive? req_arena_id == _mi_arena_id_none() && // not exclusive?
mi_atomic_load_relaxed(&mi_arena_count) < 3*(MI_MAX_ARENAS/4) ) // not too many arenas already? mi_atomic_load_relaxed(&mi_arena_count) < 3*(MI_MAX_ARENAS/4) && // not too many arenas already?
!_mi_preloading() ) // and not before main runs
{ {
mi_arena_id_t arena_id = 0; mi_arena_id_t arena_id = 0;

4
src/options.c
View file

@ -61,7 +61,7 @@ static mi_option_desc_t options[_mi_option_last] =
// Some of the following options are experimental and not all combinations are valid. Use with care. // Some of the following options are experimental and not all combinations are valid. Use with care.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (8MiB) (but see also `eager_commit_delay`) { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (8MiB) (but see also `eager_commit_delay`)
{ 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, { 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) },
{ 0, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) },
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
{ -1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N { -1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
@ -72,8 +72,6 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(deprecated_segment_reset) }, { 0, UNINIT, MI_OPTION(deprecated_segment_reset) },
#if defined(__NetBSD__) #if defined(__NetBSD__)
{ 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
#elif defined(_WIN32)
{ 4, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)
#else #else
{ 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) { 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)
#endif #endif

5
src/os.c
View file

@ -437,7 +437,10 @@ bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stat
*/ */
// either resets or decommits memory, returns true if the memory was decommitted. // either resets or decommits memory, returns true if the memory was decommitted.
bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats) { bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats)
{
if (!mi_option_is_enabled(mi_option_allow_purge)) return false;
if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?
!_mi_preloading()) // don't decommit during preloading (unsafe) !_mi_preloading()) // don't decommit during preloading (unsafe)
{ {

99
src/segment.c
View file

@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_USE_SEGMENT_CACHE 0 #define MI_USE_SEGMENT_CACHE 0
#define MI_PAGE_HUGE_ALIGN (256*1024) #define MI_PAGE_HUGE_ALIGN (256*1024)
static void mi_segment_delayed_purge(mi_segment_t* segment, bool force, mi_stats_t* stats); static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats);
// ------------------------------------------------------------------- // -------------------------------------------------------------------
@ -391,7 +391,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
} }
// purge delayed decommits now? (no, leave it to the arena) // purge delayed decommits now? (no, leave it to the arena)
// mi_segment_delayed_purge(segment,true,tld->stats); // mi_segment_try_purge(segment,true,tld->stats);
// _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats); // _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
const size_t size = mi_segment_size(segment); const size_t size = mi_segment_size(segment);
@ -476,49 +476,32 @@ static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uin
mi_commit_mask_create(bitidx, bitcount, cm); mi_commit_mask_create(bitidx, bitcount, cm);
} }
static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, size_t size, mi_stats_t* stats) {
mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask)); mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask));
// commit liberal, but decommit conservative // commit liberal
uint8_t* start = NULL; uint8_t* start = NULL;
size_t full_size = 0; size_t full_size = 0;
mi_commit_mask_t mask; mi_commit_mask_t mask;
mi_segment_commit_mask(segment, !commit/*conservative*/, p, size, &start, &full_size, &mask); mi_segment_commit_mask(segment, false /* conservative? */, p, size, &start, &full_size, &mask);
if (mi_commit_mask_is_empty(&mask) || full_size==0) return true; if (mi_commit_mask_is_empty(&mask) || full_size == 0) return true;
if (commit && !mi_commit_mask_all_set(&segment->commit_mask, &mask)) { if (!mi_commit_mask_all_set(&segment->commit_mask, &mask)) {
// committing // committing
bool is_zero = false; bool is_zero = false;
mi_commit_mask_t cmask; mi_commit_mask_t cmask;
mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false; if (!_mi_os_commit(start, full_size, &is_zero, stats)) return false;
mi_commit_mask_set(&segment->commit_mask, &mask); mi_commit_mask_set(&segment->commit_mask, &mask);
} }
else if (!commit && mi_commit_mask_any_set(&segment->commit_mask, &mask)) {
// purging // increase purge expiration when using part of delayed purges -- we assume more allocations are coming soon.
mi_assert_internal((void*)start != (void*)segment); if (mi_commit_mask_any_set(&segment->purge_mask, &mask)) {
if (mi_option_is_enabled(mi_option_allow_purge)) {
if (segment->allow_decommit) {
const bool decommitted = _mi_os_purge(start, full_size, stats); // reset or decommit
if (decommitted) {
mi_commit_mask_t cmask;
mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for double counting
mi_commit_mask_clear(&segment->commit_mask, &mask);
}
}
else if (segment->allow_purge) {
_mi_os_reset(start, full_size, stats);
}
}
}
// increase expiration of reusing part of the delayed decommit
if (commit && mi_commit_mask_any_set(&segment->purge_mask, &mask)) {
segment->purge_expire = _mi_clock_now() + mi_option_get(mi_option_purge_delay); segment->purge_expire = _mi_clock_now() + mi_option_get(mi_option_purge_delay);
} }
// always undo delayed purges
// always clear any delayed purges in our range (as they are either committed now)
mi_commit_mask_clear(&segment->purge_mask, &mask); mi_commit_mask_clear(&segment->purge_mask, &mask);
return true; return true;
} }
@ -528,13 +511,43 @@ static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_
// note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow
if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->purge_mask)) return true; // fully committed if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->purge_mask)) return true; // fully committed
mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); mi_assert_internal(segment->kind != MI_SEGMENT_HUGE);
return mi_segment_commitx(segment,true,p,size,stats); return mi_segment_commit(segment, p, size, stats);
}
static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask));
if (!segment->allow_purge) return true;
// purge conservative
uint8_t* start = NULL;
size_t full_size = 0;
mi_commit_mask_t mask;
mi_segment_commit_mask(segment, true /* conservative? */, p, size, &start, &full_size, &mask);
if (mi_commit_mask_is_empty(&mask) || full_size==0) return true;
if (mi_commit_mask_any_set(&segment->commit_mask, &mask)) {
// purging
mi_assert_internal((void*)start != (void*)segment);
mi_assert_internal(segment->allow_decommit);
const bool decommitted = _mi_os_purge(start, full_size, stats); // reset or decommit
if (decommitted) {
mi_commit_mask_t cmask;
mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for double counting
mi_commit_mask_clear(&segment->commit_mask, &mask);
}
}
// always clear any scheduled purges in our range
mi_commit_mask_clear(&segment->purge_mask, &mask);
return true;
} }
static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
if (!segment->allow_purge) return; if (!segment->allow_purge) return;
if (mi_option_get(mi_option_purge_delay) == 0) { if (mi_option_get(mi_option_purge_delay) == 0) {
mi_segment_commitx(segment, false, p, size, stats); mi_segment_purge(segment, p, size, stats);
} }
else { else {
// register for future purge in the purge mask // register for future purge in the purge mask
@ -551,26 +564,26 @@ static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t
mi_commit_mask_set(&segment->purge_mask, &cmask); mi_commit_mask_set(&segment->purge_mask, &cmask);
mi_msecs_t now = _mi_clock_now(); mi_msecs_t now = _mi_clock_now();
if (segment->purge_expire == 0) { if (segment->purge_expire == 0) {
// no previous decommits, initialize now // no previous purgess, initialize now
segment->purge_expire = now + mi_option_get(mi_option_purge_delay); segment->purge_expire = now + mi_option_get(mi_option_purge_delay);
} }
else if (segment->purge_expire <= now) { else if (segment->purge_expire <= now) {
// previous decommit mask already expired // previous purge mask already expired
if (segment->purge_expire + mi_option_get(mi_option_purge_extend_delay) <= now) { if (segment->purge_expire + mi_option_get(mi_option_purge_extend_delay) <= now) {
mi_segment_delayed_purge(segment, true, stats); mi_segment_try_purge(segment, true, stats);
} }
else { else {
segment->purge_expire = now + mi_option_get(mi_option_purge_extend_delay); // (mi_option_get(mi_option_purge_delay) / 8); // wait a tiny bit longer in case there is a series of free's segment->purge_expire = now + mi_option_get(mi_option_purge_extend_delay); // (mi_option_get(mi_option_purge_delay) / 8); // wait a tiny bit longer in case there is a series of free's
} }
} }
else { else {
// previous decommit mask is not yet expired, increase the expiration by a bit. // previous purge mask is not yet expired, increase the expiration by a bit.
segment->purge_expire += mi_option_get(mi_option_purge_extend_delay); segment->purge_expire += mi_option_get(mi_option_purge_extend_delay);
} }
} }
} }
static void mi_segment_delayed_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) { static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) {
if (!segment->allow_purge || mi_commit_mask_is_empty(&segment->purge_mask)) return; if (!segment->allow_purge || mi_commit_mask_is_empty(&segment->purge_mask)) return;
mi_msecs_t now = _mi_clock_now(); mi_msecs_t now = _mi_clock_now();
if (!force && now < segment->purge_expire) return; if (!force && now < segment->purge_expire) return;
@ -586,7 +599,7 @@ static void mi_segment_delayed_purge(mi_segment_t* segment, bool force, mi_stats
if (count > 0) { if (count > 0) {
uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE); uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE);
size_t size = count * MI_COMMIT_SIZE; size_t size = count * MI_COMMIT_SIZE;
mi_segment_commitx(segment, false, p, size, stats); mi_segment_purge(segment, p, size, stats);
} }
} }
mi_commit_mask_foreach_end() mi_commit_mask_foreach_end()
@ -916,7 +929,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed
segment->allow_decommit = !segment->mem_is_pinned && !segment->mem_is_large; segment->allow_decommit = !segment->mem_is_pinned && !segment->mem_is_large;
segment->allow_purge = mi_option_is_enabled(mi_option_allow_purge) && segment->allow_decommit; segment->allow_purge = segment->allow_decommit && mi_option_is_enabled(mi_option_allow_purge);
if (segment->allow_purge) { if (segment->allow_purge) {
segment->purge_expire = 0; // don't decommit just committed memory // _mi_clock_now() + mi_option_get(mi_option_purge_delay); segment->purge_expire = 0; // don't decommit just committed memory // _mi_clock_now() + mi_option_get(mi_option_purge_delay);
segment->purge_mask = purge_mask; segment->purge_mask = purge_mask;
@ -1278,7 +1291,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
} }
// perform delayed decommits (forcing is much slower on mstress) // perform delayed decommits (forcing is much slower on mstress)
mi_segment_delayed_purge(segment, mi_option_is_enabled(mi_option_abandoned_page_purge) /* force? */, tld->stats); mi_segment_try_purge(segment, mi_option_is_enabled(mi_option_abandoned_page_purge) /* force? */, tld->stats);
// all pages in the segment are abandoned; add it to the abandoned list // all pages in the segment are abandoned; add it to the abandoned list
_mi_stat_increase(&tld->stats->segments_abandoned, 1); _mi_stat_increase(&tld->stats->segments_abandoned, 1);
@ -1467,7 +1480,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slice
} }
else { else {
// otherwise, push on the visited list so it gets not looked at too quickly again // otherwise, push on the visited list so it gets not looked at too quickly again
mi_segment_delayed_purge(segment, true /* force? */, tld->stats); // force purge if needed as we may not visit soon again mi_segment_try_purge(segment, true /* force? */, tld->stats); // force purge if needed as we may not visit soon again
mi_abandoned_visited_push(segment); mi_abandoned_visited_push(segment);
} }
} }
@ -1493,7 +1506,7 @@ void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld)
else { else {
// otherwise, purge if needed and push on the visited list // otherwise, purge if needed and push on the visited list
// note: forced purge can be expensive if many threads are destroyed/created as in mstress. // note: forced purge can be expensive if many threads are destroyed/created as in mstress.
mi_segment_delayed_purge(segment, force, tld->stats); mi_segment_try_purge(segment, force, tld->stats);
mi_abandoned_visited_push(segment); mi_abandoned_visited_push(segment);
} }
} }
@ -1551,7 +1564,7 @@ static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_ki
} }
mi_assert_internal(page != NULL && page->slice_count*MI_SEGMENT_SLICE_SIZE == page_size); mi_assert_internal(page != NULL && page->slice_count*MI_SEGMENT_SLICE_SIZE == page_size);
mi_assert_internal(_mi_ptr_segment(page)->thread_id == _mi_thread_id()); mi_assert_internal(_mi_ptr_segment(page)->thread_id == _mi_thread_id());
mi_segment_delayed_purge(_mi_ptr_segment(page), false, tld->stats); mi_segment_try_purge(_mi_ptr_segment(page), false, tld->stats);
return page; return page;
} }