diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 9b5e87bb..908e6c48 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -166,6 +166,7 @@ typedef struct mi_page_s { uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` bool segment_in_use:1; // `true` if the segment allocated this page bool is_reset:1; // `true` if the page memory was reset + bool is_committed:1; // `true` if the page virtual memory is committed // layout like this to optimize access in `mi_malloc` and `mi_free` mi_page_flags_t flags; diff --git a/src/init.c b/src/init.c index 043c6a55..b289f657 100644 --- a/src/init.c +++ b/src/init.c @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { - 0, false, false, {0}, + 0, false, false, false, {0}, 0, 0, NULL, 0, 0, // free, used, cookie NULL, 0, {0}, diff --git a/src/options.c b/src/options.c index e39cf08b..17997197 100644 --- a/src/options.c +++ b/src/options.c @@ -34,7 +34,7 @@ typedef struct mi_option_desc_s { static mi_option_desc_t options[_mi_option_last] = { { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, - { 0, UNINIT, "eager_commit" }, + { 1, UNINIT, "eager_commit" }, { 0, UNINIT, "eager_region_commit" }, { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, "reset_decommits" }, diff --git a/src/os.c b/src/os.c index 6051a2db..33ba148a 100644 --- a/src/os.c +++ b/src/os.c @@ -391,8 +391,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative. -static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) { +// Commit/Decommit memory. +// Usuelly commit is aligned liberal, while decommit is aligned conservative. +// (but not for the reset version where we want commit to be conservative as well) +static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, mi_stats_t* stats) { // page align in the range, commit liberally, decommit conservative size_t csize; void* start = mi_os_page_align_areax(!commit, addr, size, &csize); @@ -426,13 +428,18 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stat } bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) { - return mi_os_commitx(addr, size, true, stats); + return mi_os_commitx(addr, size, true, false /* conservative? */, stats); } bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) { - return mi_os_commitx(addr, size, false, stats); + return mi_os_commitx(addr, size, false, true /* conservative? */, stats); } +bool _mi_os_commit_unreset(void* addr, size_t size, mi_stats_t* stats) { + return mi_os_commitx(addr, size, true, true /* conservative? */, stats); +} + + // Signal to the OS that the address range is no longer in use // but may be used later again. This will release physical memory // pages and reduce swapping while keeping the memory committed. @@ -446,6 +453,10 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! + #if MI_DEBUG>1 + memset(start, 0, csize); // pretend it is eagerly reset + #endif + #if defined(_WIN32) // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory // (but this is for an access pattern that immediately reuses the memory) @@ -465,7 +476,6 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) DWORD ok = VirtualUnlock(start, csize); if (ok != 0) return false; */ - return true; #else #if defined(MADV_FREE) static int advice = MADV_FREE; @@ -482,8 +492,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno); } //mi_assert(err == 0); - return (err == 0); + if (err != 0) return false; #endif + return true; } // Signal to the OS that the address range is no longer in use @@ -501,7 +512,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_unreset(void* addr, size_t size, mi_stats_t* stats) { if (mi_option_is_enabled(mi_option_reset_decommits)) { - return _mi_os_commit(addr, size, stats); // re-commit it + return _mi_os_commit_unreset(addr, size, stats); // re-commit it (conservatively!) } else { return mi_os_resetx(addr, size, false, stats); diff --git a/src/segment.c b/src/segment.c index 28b626e5..d88f94a5 100644 --- a/src/segment.c +++ b/src/segment.c @@ -202,16 +202,11 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se #define MI_SEGMENT_CACHE_MAX (4) #define MI_SEGMENT_CACHE_FRACTION (8) +// note: returned segment may be partially reset static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t* tld) { if (segment_size != 0 && segment_size != MI_SEGMENT_SIZE) return NULL; mi_segment_t* segment = tld->cache; if (segment == NULL) return NULL; - if (mi_option_is_enabled(mi_option_eager_commit) && - (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) - { - // ensure the memory is available - _mi_mem_unreset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); - } tld->cache_count--; tld->cache = segment->next; segment->next = NULL; @@ -298,9 +293,16 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, protection_still_good = true; // otherwise, the guard pages are still in place } } - if (page_kind != MI_PAGE_SMALL && !mi_option_is_enabled(mi_option_eager_commit) && - (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) { - _mi_mem_commit(segment, segment->segment_size, tld->stats); + if (!mi_option_is_enabled(mi_option_eager_commit)) { + if (page_kind != MI_PAGE_SMALL) { + _mi_mem_commit(segment, segment->segment_size, tld->stats); + } + else { + // ok, commit (and unreset) on demand again + } + } + else if (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)) { + _mi_mem_unreset(segment, segment->segment_size, tld->stats); } } else { @@ -349,7 +351,8 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, segment->cookie = _mi_ptr_cookie(segment); for (uint8_t i = 0; i < segment->capacity; i++) { segment->pages[i].segment_idx = i; - segment->pages[i].is_reset = !commit; + segment->pages[i].is_reset = false; + segment->pages[i].is_committed = commit; } _mi_stat_increase(&tld->stats->page_committed, segment->segment_info_size); //fprintf(stderr,"mimalloc: alloc segment at %p\n", (void*)segment); @@ -421,18 +424,18 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats) for (size_t i = 0; i < segment->capacity; i++) { mi_page_t* page = &segment->pages[i]; if (!page->segment_in_use) { - if (page->is_reset) { + if (page->is_reset || !page->is_committed) { size_t psize; uint8_t* start = _mi_page_start(segment, page, &psize); - page->is_reset = false; - if (mi_option_is_enabled(mi_option_eager_commit)) { - _mi_mem_unreset(start, psize, stats); - } - else { - // note we could allow both lazy commit, and page level reset if we add a `is_commit` flag... - // for now we use commit for both - _mi_mem_commit(start, psize, stats); + mi_assert_internal(!(page->is_reset && !page->is_committed)); + if (!page->is_committed) { + page->is_committed = true; + _mi_mem_commit(start,psize,stats); } + if (page->is_reset) { + page->is_reset = false; + _mi_mem_unreset(start, psize, stats); + } } return page; } @@ -452,6 +455,7 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta UNUSED(stats); mi_assert_internal(page->segment_in_use); mi_assert_internal(mi_page_all_free(page)); + mi_assert_internal(page->is_committed); size_t inuse = page->capacity * page->block_size; _mi_stat_decrease(&stats->page_committed, inuse); _mi_stat_decrease(&stats->pages, 1); @@ -467,10 +471,12 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta // zero the page data uint8_t idx = page->segment_idx; // don't clear the index bool is_reset = page->is_reset; // don't clear the reset flag + bool is_committed = page->is_committed; // don't clear the commit flag memset(page, 0, sizeof(*page)); page->segment_idx = idx; page->segment_in_use = false; page->is_reset = is_reset; + page->is_committed = is_committed; segment->used--; }