kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-04 22:49:32 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

enable non eager-commit flag

Browse source
This commit is contained in:
daan 2019-07-10 07:17:21 -07:00
parent d8c47bfed2
commit 46a9e51f74
5 changed files with 53 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
include/mimalloc-types.h
View file

@ -166,6 +166,7 @@ typedef struct mi_page_s {
uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
bool segment_in_use:1; // `true` if the segment allocated this page bool segment_in_use:1; // `true` if the segment allocated this page
bool is_reset:1; // `true` if the page memory was reset bool is_reset:1; // `true` if the page memory was reset
bool is_committed:1; // `true` if the page virtual memory is committed
// layout like this to optimize access in `mi_malloc` and `mi_free` // layout like this to optimize access in `mi_malloc` and `mi_free`
mi_page_flags_t flags; mi_page_flags_t flags;

2
src/init.c
View file

@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
// Empty page used to initialize the small free pages array // Empty page used to initialize the small free pages array
const mi_page_t _mi_page_empty = { const mi_page_t _mi_page_empty = {
0, false, false, {0}, 0, false, false, false, {0},
0, 0, 0, 0,
NULL, 0, 0, // free, used, cookie NULL, 0, 0, // free, used, cookie
NULL, 0, {0}, NULL, 0, {0},

2
src/options.c
View file

@ -44,7 +44,7 @@ static mi_option_desc_t options[_mi_option_last] = {
#else #else
{ 0, UNINIT, "secure" }, { 0, UNINIT, "secure" },
#endif #endif
{ 0, UNINIT, "show_stats" }, { 1, UNINIT, "show_stats" },
{ MI_DEBUG, UNINIT, "show_errors" }, { MI_DEBUG, UNINIT, "show_errors" },
{ 0, UNINIT, "verbose" } { 0, UNINIT, "verbose" }
}; };

27
src/os.c
View file

@ -382,7 +382,7 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size,
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
if (diff <= 0) return NULL; if (diff <= 0) return NULL;
mi_assert_internal((size_t)diff <= size); mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size));
if (newsize != NULL) *newsize = (size_t)diff; if (newsize != NULL) *newsize = (size_t)diff;
return start; return start;
} }
@ -391,8 +391,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
return mi_os_page_align_areax(true, addr, size, newsize); return mi_os_page_align_areax(true, addr, size, newsize);
} }
// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative. // Commit/Decommit memory.
static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) { // Usuelly commit is aligned liberal, while decommit is aligned conservative.
// (but not for the reset version where we want commit to be conservative as well)
static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, mi_stats_t* stats) {
// page align in the range, commit liberally, decommit conservative // page align in the range, commit liberally, decommit conservative
size_t csize; size_t csize;
void* start = mi_os_page_align_areax(!commit, addr, size, &csize); void* start = mi_os_page_align_areax(!commit, addr, size, &csize);
@ -426,13 +428,18 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stat
} }
bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, stats); return mi_os_commitx(addr, size, true, false /* conservative? */, stats);
} }
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
return mi_os_commitx(addr, size, false, stats); return mi_os_commitx(addr, size, false, true /* conservative? */, stats);
} }
bool _mi_os_commit_unreset(void* addr, size_t size, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, true /* conservative? */, stats);
}
// Signal to the OS that the address range is no longer in use // Signal to the OS that the address range is no longer in use
// but may be used later again. This will release physical memory // but may be used later again. This will release physical memory
// pages and reduce swapping while keeping the memory committed. // pages and reduce swapping while keeping the memory committed.
@ -446,6 +453,10 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
else _mi_stat_decrease(&stats->reset, csize); else _mi_stat_decrease(&stats->reset, csize);
if (!reset) return true; // nothing to do on unreset! if (!reset) return true; // nothing to do on unreset!
#if MI_DEBUG>1
memset(start, 0, csize); // pretend it is eagerly reset
#endif
#if defined(_WIN32) #if defined(_WIN32)
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
// (but this is for an access pattern that immediately reuses the memory) // (but this is for an access pattern that immediately reuses the memory)
@ -465,7 +476,6 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
DWORD ok = VirtualUnlock(start, csize); DWORD ok = VirtualUnlock(start, csize);
if (ok != 0) return false; if (ok != 0) return false;
*/ */
return true;
#else #else
#if defined(MADV_FREE) #if defined(MADV_FREE)
static int advice = MADV_FREE; static int advice = MADV_FREE;
@ -482,8 +492,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
_mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno); _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno);
} }
//mi_assert(err == 0); //mi_assert(err == 0);
return (err == 0); if (err != 0) return false;
#endif #endif
return true;
} }
// Signal to the OS that the address range is no longer in use // Signal to the OS that the address range is no longer in use
@ -501,7 +512,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
bool _mi_os_unreset(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_unreset(void* addr, size_t size, mi_stats_t* stats) {
if (mi_option_is_enabled(mi_option_reset_decommits)) { if (mi_option_is_enabled(mi_option_reset_decommits)) {
return _mi_os_commit(addr, size, stats); // re-commit it return _mi_os_commit_unreset(addr, size, stats); // re-commit it (conservatively!)
} }
else { else {
return mi_os_resetx(addr, size, false, stats); return mi_os_resetx(addr, size, false, stats);

57
src/segment.c
View file

@ -235,7 +235,7 @@ static mi_segment_t* _mi_segment_cache_findx(mi_segments_tld_t* tld, size_t requ
if (mi_option_is_enabled(mi_option_secure)) { if (mi_option_is_enabled(mi_option_secure)) {
_mi_os_unprotect(segment, segment->segment_size); _mi_os_unprotect(segment, segment->segment_size);
} }
if (_mi_os_shrink(segment, segment->segment_size, required, tld->stats)) { if (_mi_os_shrink(segment, segment->segment_size, required, tld->stats)) { // note: double decommit must be (ok on windows)
tld->current_size -= segment->segment_size; tld->current_size -= segment->segment_size;
tld->current_size += required; tld->current_size += required;
segment->segment_size = required; segment->segment_size = required;
@ -253,18 +253,12 @@ static mi_segment_t* _mi_segment_cache_findx(mi_segments_tld_t* tld, size_t requ
return NULL; return NULL;
} }
// note: the returned segment might be reset
static mi_segment_t* mi_segment_cache_find(mi_segments_tld_t* tld, size_t required) { static mi_segment_t* mi_segment_cache_find(mi_segments_tld_t* tld, size_t required) {
mi_segment_t* segment = _mi_segment_cache_findx(tld,required,false); return _mi_segment_cache_findx(tld,required,false);
if (segment != NULL &&
mi_option_is_enabled(mi_option_eager_commit) &&
(mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)))
{
// ensure the memory is available
_mi_os_unreset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats);
}
return segment;
} }
// note: the returned segment might be reset
static mi_segment_t* mi_segment_cache_evict(mi_segments_tld_t* tld) { static mi_segment_t* mi_segment_cache_evict(mi_segments_tld_t* tld) {
// TODO: random eviction instead? // TODO: random eviction instead?
return _mi_segment_cache_findx(tld, 0, true /* from the end */); return _mi_segment_cache_findx(tld, 0, true /* from the end */);
@ -359,9 +353,16 @@ static mi_segment_t* mi_segment_alloc( size_t required, mi_page_kind_t page_kind
protection_still_good = true; // otherwise, the guard pages are still in place protection_still_good = true; // otherwise, the guard pages are still in place
} }
} }
if (page_kind != MI_PAGE_SMALL && !mi_option_is_enabled(mi_option_eager_commit) && if (!mi_option_is_enabled(mi_option_eager_commit)) {
(mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) { if (page_kind != MI_PAGE_SMALL) {
_mi_os_commit(segment, segment->segment_size, tld->stats); _mi_os_commit(segment, segment->segment_size, tld->stats);
}
else {
// ok, commit (and unreset) on demand again
}
}
else if (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)) {
_mi_os_unreset(segment, segment->segment_size, tld->stats);
} }
} }
// and otherwise allocate it from the OS // and otherwise allocate it from the OS
@ -369,11 +370,11 @@ static mi_segment_t* mi_segment_alloc( size_t required, mi_page_kind_t page_kind
segment = (mi_segment_t*)_mi_os_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, os_tld); segment = (mi_segment_t*)_mi_os_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, os_tld);
if (segment == NULL) return NULL; if (segment == NULL) return NULL;
mi_segments_track_size((long)segment_size,tld); mi_segments_track_size((long)segment_size,tld);
if (!commit) {
_mi_os_commit(segment, info_size, tld->stats); // always commit start of the segment
}
} }
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
if (!commit) {
_mi_os_commit(segment,info_size,tld->stats);
}
memset(segment, 0, info_size); memset(segment, 0, info_size);
if (mi_option_is_enabled(mi_option_secure) && !protection_still_good) { if (mi_option_is_enabled(mi_option_secure) && !protection_still_good) {
@ -403,7 +404,8 @@ static mi_segment_t* mi_segment_alloc( size_t required, mi_page_kind_t page_kind
segment->cookie = _mi_ptr_cookie(segment); segment->cookie = _mi_ptr_cookie(segment);
for (uint8_t i = 0; i < segment->capacity; i++) { for (uint8_t i = 0; i < segment->capacity; i++) {
segment->pages[i].segment_idx = i; segment->pages[i].segment_idx = i;
segment->pages[i].is_reset = !commit; segment->pages[i].is_reset = false;
segment->pages[i].is_committed = commit;
} }
_mi_stat_increase(&tld->stats->page_committed, segment->segment_info_size); _mi_stat_increase(&tld->stats->page_committed, segment->segment_info_size);
//fprintf(stderr,"mimalloc: alloc segment at %p\n", (void*)segment); //fprintf(stderr,"mimalloc: alloc segment at %p\n", (void*)segment);
@ -477,18 +479,18 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats)
for (size_t i = 0; i < segment->capacity; i++) { for (size_t i = 0; i < segment->capacity; i++) {
mi_page_t* page = &segment->pages[i]; mi_page_t* page = &segment->pages[i];
if (!page->segment_in_use) { if (!page->segment_in_use) {
if (page->is_reset) { if (page->is_reset || !page->is_committed) {
size_t psize; size_t psize;
uint8_t* start = _mi_page_start(segment, page, &psize); uint8_t* start = _mi_page_start(segment, page, &psize);
page->is_reset = false; mi_assert_internal(!(page->is_reset && !page->is_committed));
if (mi_option_is_enabled(mi_option_eager_commit)) { if (!page->is_committed) {
_mi_os_unreset(start, psize, stats); page->is_committed = true;
} _mi_os_commit(start,psize,stats);
else {
// note we could allow both lazy commit, and page level reset if we add a `is_commit` flag...
// for now we use commit for both
_mi_os_commit(start, psize, stats);
} }
if (page->is_reset) {
page->is_reset = false;
_mi_os_unreset(start, psize, stats);
}
} }
return page; return page;
} }
@ -508,6 +510,7 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta
UNUSED(stats); UNUSED(stats);
mi_assert_internal(page->segment_in_use); mi_assert_internal(page->segment_in_use);
mi_assert_internal(mi_page_all_free(page)); mi_assert_internal(mi_page_all_free(page));
mi_assert_internal(page->is_committed);
size_t inuse = page->capacity * page->block_size; size_t inuse = page->capacity * page->block_size;
_mi_stat_decrease(&stats->page_committed, inuse); _mi_stat_decrease(&stats->page_committed, inuse);
_mi_stat_decrease(&stats->pages, 1); _mi_stat_decrease(&stats->pages, 1);
@ -523,10 +526,12 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta
// zero the page data // zero the page data
uint8_t idx = page->segment_idx; // don't clear the index uint8_t idx = page->segment_idx; // don't clear the index
bool is_reset = page->is_reset; // don't clear the reset flag bool is_reset = page->is_reset; // don't clear the reset flag
bool is_committed = page->is_committed; // don't clear the commit flag
memset(page, 0, sizeof(*page)); memset(page, 0, sizeof(*page));
page->segment_idx = idx; page->segment_idx = idx;
page->segment_in_use = false; page->segment_in_use = false;
page->is_reset = is_reset; page->is_reset = is_reset;
page->is_committed = is_committed;
segment->used--; segment->used--;
} }