diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 503f5ff1..75e34c68 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -44,11 +44,13 @@ void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocat void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data // memory.c -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t* id, mi_os_tld_t* tld); -void* _mi_mem_alloc(size_t size, size_t* id, mi_os_tld_t* tld); +void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld); +void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld); void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats); bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats); +bool _mi_mem_unreset(void* p, size_t size, mi_stats_t* stats); +bool _mi_mem_commit(void* p, size_t size, mi_stats_t* stats); bool _mi_mem_protect(void* addr, size_t size); bool _mi_mem_unprotect(void* addr, size_t size); diff --git a/include/mimalloc.h b/include/mimalloc.h index 479532f9..12383e87 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -221,7 +221,10 @@ typedef enum mi_option_e { mi_option_page_reset, mi_option_cache_reset, mi_option_eager_commit, + mi_option_eager_region_commit, mi_option_large_os_pages, // implies eager commit + mi_option_reset_decommits, + mi_option_reset_discards, mi_option_secure, mi_option_show_stats, mi_option_show_errors, @@ -229,6 +232,7 @@ typedef enum mi_option_e { _mi_option_last } mi_option_t; + mi_decl_export bool mi_option_is_enabled(mi_option_t option); mi_decl_export void mi_option_enable(mi_option_t option, bool enable); mi_decl_export void mi_option_enable_default(mi_option_t option, bool enable); diff --git a/src/alloc.c b/src/alloc.c index 8ae29723..8f8e380e 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -37,7 +37,10 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz block->next = 0; #endif #if (MI_STAT>1) - if(size <= MI_LARGE_SIZE_MAX) mi_heap_stat_increase(heap,normal[_mi_bin(size)], 1); + if(size <= MI_LARGE_SIZE_MAX) { + size_t bin = _mi_bin(size); + mi_heap_stat_increase(heap,normal[bin], 1); + } #endif return block; } diff --git a/src/memory.c b/src/memory.c index 63ffb8a4..92a9c85a 100644 --- a/src/memory.c +++ b/src/memory.c @@ -41,6 +41,7 @@ bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_commit(void* p, size_t size, mi_stats_t* stats); bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats); bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats); void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld); @@ -108,7 +109,7 @@ Commit from a region // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. // (not being able to claim is not considered an error so check for `p != NULL` afterwards). -static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, void** p, size_t* id, mi_os_tld_t* tld) { +static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) { size_t mask = mi_region_block_mask(blocks,bitidx); mi_assert_internal(mask != 0); mi_assert_internal((mask & mi_atomic_read(®ion->map)) == mask); @@ -116,7 +117,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit // ensure the region is reserved void* start = mi_atomic_read_ptr(®ion->start); if (start == NULL) { - start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, mi_option_is_enabled(mi_option_eager_commit), tld); + start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, mi_option_is_enabled(mi_option_eager_region_commit), tld); if (start == NULL) { // failure to allocate from the OS! unclaim the blocks and fail size_t map; @@ -142,7 +143,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit mi_assert_internal(start == mi_atomic_read_ptr(®ion->start)); mi_assert_internal(start != NULL); void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); - if (!mi_option_is_enabled(mi_option_eager_commit)) { + if (commit && !mi_option_is_enabled(mi_option_eager_region_commit)) { _mi_os_commit(blocks_start, mi_good_commit_size(size), tld->stats); // only commit needed size (unless using large OS pages) } @@ -157,7 +158,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. // (not being able to claim is not considered an error so check for `p != NULL` afterwards). -static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, void** p, size_t* id, mi_os_tld_t* tld) { +static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) { mi_assert_internal(p != NULL && id != NULL); mi_assert_internal(blocks < MI_REGION_MAP_BITS); @@ -196,21 +197,21 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc // success, we claimed the blocks atomically // now commit the block memory -- this can still fail - return mi_region_commit_blocks(region, idx, bitidx, blocks, size, p, id, tld); + return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, p, id, tld); } // Try to allocate `blocks` in a `region` at `idx` of a given `size`. Does a quick check before trying to claim. // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // if the blocks were successfully claimed so ensure they are initialized to NULL/0 before the call. // (not being able to claim is not considered an error so check for `p != NULL` afterwards). -static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, void** p, size_t* id, mi_os_tld_t* tld) +static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) { // check if there are available blocks in the region.. mi_assert_internal(idx < MI_REGION_MAX); mem_region_t* region = ®ions[idx]; uintptr_t m = mi_atomic_read(®ion->map); if (m != MI_REGION_MAP_FULL) { // some bits are zero - return mi_region_alloc_blocks(region, idx, blocks, size, p, id, tld); + return mi_region_alloc_blocks(region, idx, blocks, size, commit, p, id, tld); } else { return true; // no error, but no success either @@ -223,7 +224,7 @@ static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, v // Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`. // (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`) -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t* id, mi_os_tld_t* tld) +void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld) { mi_assert_internal(id != NULL && tld != NULL); mi_assert_internal(size > 0); @@ -247,21 +248,21 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t* id, mi_os_tld size_t count = mi_atomic_read(®ions_count); size_t idx = mi_atomic_read(®ion_next_idx); for (size_t visited = 0; visited < count; visited++, idx++) { - if (!mi_region_try_alloc_blocks(idx%count, blocks, size, &p, id, tld)) return NULL; // error + if (!mi_region_try_alloc_blocks(idx%count, blocks, size, commit, &p, id, tld)) return NULL; // error if (p != NULL) break; } if (p == NULL) { // no free range in existing regions -- try to extend beyond the count for (idx = count; idx < MI_REGION_MAX; idx++) { - if (!mi_region_try_alloc_blocks(idx, blocks, size, &p, id, tld)) return NULL; // error + if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error if (p != NULL) break; } } if (p == NULL) { // we could not find a place to allocate, fall back to the os directly - p = _mi_os_alloc_aligned(size, alignment, true, tld); + p = _mi_os_alloc_aligned(size, alignment, commit, tld); } mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0); @@ -270,8 +271,8 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t* id, mi_os_tld // Allocate `size` memory. Return non NULL on success, with a given memory `id`. -void* _mi_mem_alloc(size_t size, size_t* id, mi_os_tld_t* tld) { - return _mi_mem_alloc_aligned(size,0,id,tld); +void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld) { + return _mi_mem_alloc_aligned(size,0,commit,id,tld); } /* ---------------------------------------------------------------------------- @@ -311,11 +312,11 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) { // TODO: implement delayed decommit/reset as these calls are too expensive // if the memory is reused soon. // reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large - if (mi_option_is_enabled(mi_option_eager_commit)) { - // _mi_os_reset(p, size, stats); // 10x slowdown on malloc-large + if (mi_option_is_enabled(mi_option_eager_region_commit)) { + _mi_os_reset(p, size, stats); // 10x slowdown on malloc-large } else { - // _mi_os_decommit(p, size, stats); // 17x slowdown on malloc-large + _mi_os_decommit(p, size, stats); // 17x slowdown on malloc-large } // TODO: should we free empty regions? @@ -336,10 +337,22 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) { Other -----------------------------------------------------------------------------*/ +bool _mi_mem_commit(void* p, size_t size, mi_stats_t* stats) { + return _mi_os_commit(p, size, stats); +} + +bool _mi_mem_decommit(void* p, size_t size, mi_stats_t* stats) { + return _mi_os_decommit(p, size, stats); +} + bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats) { return _mi_os_reset(p, size, stats); } +bool _mi_mem_unreset(void* p, size_t size, mi_stats_t* stats) { + return _mi_os_unreset(p, size, stats); +} + bool _mi_mem_protect(void* p, size_t size) { return _mi_os_protect(p, size); } diff --git a/src/options.c b/src/options.c index 40a08aa8..e39cf08b 100644 --- a/src/options.c +++ b/src/options.c @@ -34,14 +34,17 @@ typedef struct mi_option_desc_s { static mi_option_desc_t options[_mi_option_last] = { { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, - { 1, UNINIT, "eager_commit" }, // on by default as it seems to be faster in general + { 0, UNINIT, "eager_commit" }, + { 0, UNINIT, "eager_region_commit" }, { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { 0, UNINIT, "reset_decommits" }, + { 0, UNINIT, "reset_discards" }, #if MI_SECURE { MI_SECURE, INITIALIZED, "secure" }, // in secure build the environment setting is ignored #else { 0, UNINIT, "secure" }, #endif - { 0, UNINIT, "show_stats" }, + { 1, UNINIT, "show_stats" }, { MI_DEBUG, UNINIT, "show_errors" }, { 0, UNINIT, "verbose" } }; diff --git a/src/os.c b/src/os.c index d5b7cb65..6051a2db 100644 --- a/src/os.c +++ b/src/os.c @@ -382,7 +382,7 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; if (diff <= 0) return NULL; - mi_assert_internal((size_t)diff <= size); + mi_assert_internal((conservative && (size_t)diff <= size) || (!conservative && (size_t)diff >= size)); if (newsize != NULL) *newsize = (size_t)diff; return start; } @@ -391,29 +391,74 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } +// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative. +static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) { + // page align in the range, commit liberally, decommit conservative + size_t csize; + void* start = mi_os_page_align_areax(!commit, addr, size, &csize); + if (csize == 0) return true; + int err = 0; + if (commit) { + _mi_stat_increase(&stats->committed, csize); + _mi_stat_increase(&stats->commit_calls, 1); + } + else { + _mi_stat_decrease(&stats->committed, csize); + } + #if defined(_WIN32) + if (commit) { + void* p = VirtualAlloc(start, csize, MEM_COMMIT, PAGE_READWRITE); + err = (p == start ? 0 : GetLastError()); + } + else { + BOOL ok = VirtualFree(start, csize, MEM_DECOMMIT); + err = (ok ? 0 : GetLastError()); + } + #else + err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); + #endif + if (err != 0) { + _mi_warning_message("commit/decommit error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err); + } + mi_assert_internal(err == 0); + return (err == 0); +} + +bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) { + return mi_os_commitx(addr, size, true, stats); +} + +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) { + return mi_os_commitx(addr, size, false, stats); +} // Signal to the OS that the address range is no longer in use // but may be used later again. This will release physical memory // pages and reduce swapping while keeping the memory committed. // We page align to a conservative area inside the range to reset. -bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { +static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) { // page align conservatively within the range size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return true; - _mi_stat_increase(&stats->reset, csize); + if (reset) _mi_stat_increase(&stats->reset, csize); + else _mi_stat_decrease(&stats->reset, csize); + if (!reset) return true; // nothing to do on unreset! #if defined(_WIN32) // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory // (but this is for an access pattern that immediately reuses the memory) - /* - DWORD ok = DiscardVirtualMemory(start, csize); - return (ok != 0); - */ - void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); - mi_assert(p == start); - if (p != start) return false; + if (mi_option_is_enabled(mi_option_reset_discards)) { + DWORD ok = DiscardVirtualMemory(start, csize); + mi_assert_internal(ok == 0); + if (ok != 0) return false; + } + else { + void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); + mi_assert_internal(p == start); + if (p != start) return false; + } /* // VirtualUnlock removes the memory eagerly from the current working set (which MEM_RESET does lazily on demand) // TODO: put this behind an option? @@ -441,6 +486,29 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { #endif } +// Signal to the OS that the address range is no longer in use +// but may be used later again. This will release physical memory +// pages and reduce swapping while keeping the memory committed. +// We page align to a conservative area inside the range to reset. +bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { + if (mi_option_is_enabled(mi_option_reset_decommits)) { + return _mi_os_decommit(addr,size,stats); + } + else { + return mi_os_resetx(addr, size, true, stats); + } +} + +bool _mi_os_unreset(void* addr, size_t size, mi_stats_t* stats) { + if (mi_option_is_enabled(mi_option_reset_decommits)) { + return _mi_os_commit(addr, size, stats); // re-commit it + } + else { + return mi_os_resetx(addr, size, false, stats); + } +} + + // Protect a region in memory to be not accessible. static bool mi_os_protectx(void* addr, size_t size, bool protect) { // page align conservatively within the range @@ -470,47 +538,7 @@ bool _mi_os_unprotect(void* addr, size_t size) { return mi_os_protectx(addr, size, false); } -// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative. -static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) { - // page align in the range, commit liberally, decommit conservative - size_t csize; - void* start = mi_os_page_align_areax(!commit, addr, size, &csize); - if (csize == 0) return true; - int err = 0; - if (commit) { - _mi_stat_increase(&stats->committed, csize); - _mi_stat_increase(&stats->commit_calls, 1); - } - else { - _mi_stat_decrease(&stats->committed, csize); - } -#if defined(_WIN32) - if (commit) { - void* p = VirtualAlloc(start, csize, MEM_COMMIT, PAGE_READWRITE); - err = (p == start ? 0 : GetLastError()); - } - else { - BOOL ok = VirtualFree(start, csize, MEM_DECOMMIT); - err = (ok ? 0 : GetLastError()); - } -#else - err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); -#endif - if (err != 0) { - _mi_warning_message("commit/decommit error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err); - } - mi_assert_internal(err == 0); - return (err == 0); -} - -bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) { - return mi_os_commitx(addr, size, true, stats); -} - -bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) { - return mi_os_commitx(addr, size, false, stats); -} bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { // page align conservatively within the range diff --git a/src/page.c b/src/page.c index 04464fc5..df02c864 100644 --- a/src/page.c +++ b/src/page.c @@ -456,7 +456,7 @@ static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t e } // enable the new free list page->capacity += (uint16_t)extend; - mi_stat_increase(stats->page_committed, extend * page->block_size); + _mi_stat_increase(&stats->page_committed, extend * page->block_size); } /* ----------------------------------------------------------- @@ -484,13 +484,8 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st if (page->capacity >= page->reserved) return; size_t page_size; - _mi_page_start(_mi_page_segment(page), page, &page_size); - if (page->is_reset) { - page->is_reset = false; - mi_stat_decrease( stats->reset, page_size); - } - - mi_stat_increase( stats->pages_extended, 1); + _mi_page_start(_mi_page_segment(page), page, &page_size); + _mi_stat_increase(&stats->pages_extended, 1); // calculate the extend count size_t extend = page->reserved - page->capacity; @@ -595,7 +590,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p page = next; } // for each page - mi_stat_counter_increase(heap->tld->stats.searches,count); + _mi_stat_counter_increase(&heap->tld->stats.searches,count); if (page == NULL) { page = rpage; diff --git a/src/segment.c b/src/segment.c index f249b02c..28b626e5 100644 --- a/src/segment.c +++ b/src/segment.c @@ -179,8 +179,8 @@ reuse and avoid setting/clearing guard pages in secure mode. ------------------------------------------------------------------------------- */ static void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) { - if (segment_size>=0) mi_stat_increase(tld->stats->segments,1); - else mi_stat_decrease(tld->stats->segments,1); + if (segment_size>=0) _mi_stat_increase(&tld->stats->segments,1); + else _mi_stat_decrease(&tld->stats->segments,1); tld->count += (segment_size >= 0 ? 1 : -1); if (tld->count > tld->peak_count) tld->peak_count = tld->count; tld->current_size += segment_size; @@ -196,6 +196,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se _mi_mem_free(segment, segment_size, segment->memid, tld->stats); } + // The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use, // and no more than 4. #define MI_SEGMENT_CACHE_MAX (4) @@ -205,6 +206,12 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t if (segment_size != 0 && segment_size != MI_SEGMENT_SIZE) return NULL; mi_segment_t* segment = tld->cache; if (segment == NULL) return NULL; + if (mi_option_is_enabled(mi_option_eager_commit) && + (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) + { + // ensure the memory is available + _mi_mem_unreset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); + } tld->cache_count--; tld->cache = segment->next; segment->next = NULL; @@ -230,12 +237,9 @@ static bool mi_segment_cache_full(mi_segments_tld_t* tld) { static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) { mi_assert_internal(!mi_segment_is_in_free_queue(segment, tld)); mi_assert_internal(segment->next == NULL); - if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) { - return false; - } - + if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) return false; mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); - if (mi_option_is_enabled(mi_option_cache_reset) && !mi_option_is_enabled(mi_option_page_reset)) { + if (mi_option_is_enabled(mi_option_cache_reset)) { _mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); } segment->next = tld->cache; @@ -282,6 +286,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_size = (page_kind == MI_PAGE_HUGE ? segment_size : (size_t)1 << page_shift); // Try to get it from our thread local cache first + bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind != MI_PAGE_SMALL); bool protection_still_good = false; mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld); if (segment != NULL) { @@ -293,12 +298,19 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, protection_still_good = true; // otherwise, the guard pages are still in place } } + if (page_kind != MI_PAGE_SMALL && !mi_option_is_enabled(mi_option_eager_commit) && + (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) { + _mi_mem_commit(segment, segment->segment_size, tld->stats); + } } else { // Allocate the segment from the OS size_t memid; - segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &memid, os_tld); + segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &memid, os_tld); if (segment == NULL) return NULL; // failed to allocate + if (!commit) { + _mi_mem_commit(segment, info_size, tld->stats); + } segment->memid = memid; mi_segments_track_size((long)segment_size, tld); } @@ -311,7 +323,8 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, } if (mi_option_is_enabled(mi_option_secure) && !protection_still_good) { - // in secure mode, we set up a protected page in between the segment info and the page data + // in secure mode, we set up a protected page in between the segment info + // and the page data mi_assert_internal( info_size == pre_size - _mi_os_page_size() && info_size % _mi_os_page_size() == 0); _mi_mem_protect( (uint8_t*)segment + info_size, (pre_size - info_size) ); size_t os_page_size = _mi_os_page_size(); @@ -336,8 +349,9 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, segment->cookie = _mi_ptr_cookie(segment); for (uint8_t i = 0; i < segment->capacity; i++) { segment->pages[i].segment_idx = i; + segment->pages[i].is_reset = !commit; } - mi_stat_increase(tld->stats->page_committed, segment->segment_info_size); + _mi_stat_increase(&tld->stats->page_committed, segment->segment_info_size); //fprintf(stderr,"mimalloc: alloc segment at %p\n", (void*)segment); return segment; } @@ -369,10 +383,11 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t mi_assert_expensive(!mi_segment_queue_contains(&tld->small_free, segment)); mi_assert(segment->next == NULL); mi_assert(segment->prev == NULL); - mi_stat_decrease( tld->stats->page_committed, segment->segment_info_size); + _mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size); segment->thread_id = 0; // update reset memory statistics + /* for (uint8_t i = 0; i < segment->capacity; i++) { mi_page_t* page = &segment->pages[i]; if (page->is_reset) { @@ -380,6 +395,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t mi_stat_decrease( tld->stats->reset,mi_page_size(page)); } } + */ if (!force && mi_segment_cache_push(segment, tld)) { // it is put in our cache @@ -399,12 +415,25 @@ static bool mi_segment_has_free(const mi_segment_t* segment) { return (segment->used < segment->capacity); } -static mi_page_t* mi_segment_find_free(mi_segment_t* segment) { +static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats) { mi_assert_internal(mi_segment_has_free(segment)); mi_assert_expensive(mi_segment_is_valid(segment)); for (size_t i = 0; i < segment->capacity; i++) { mi_page_t* page = &segment->pages[i]; if (!page->segment_in_use) { + if (page->is_reset) { + size_t psize; + uint8_t* start = _mi_page_start(segment, page, &psize); + page->is_reset = false; + if (mi_option_is_enabled(mi_option_eager_commit)) { + _mi_mem_unreset(start, psize, stats); + } + else { + // note we could allow both lazy commit, and page level reset if we add a `is_commit` flag... + // for now we use commit for both + _mi_mem_commit(start, psize, stats); + } + } return page; } } @@ -424,17 +453,15 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta mi_assert_internal(page->segment_in_use); mi_assert_internal(mi_page_all_free(page)); size_t inuse = page->capacity * page->block_size; - mi_stat_decrease( stats->page_committed, inuse); - mi_stat_decrease( stats->pages, 1); + _mi_stat_decrease(&stats->page_committed, inuse); + _mi_stat_decrease(&stats->pages, 1); // reset the page memory to reduce memory pressure? if (!page->is_reset && mi_option_is_enabled(mi_option_page_reset)) { size_t psize; uint8_t* start = _mi_page_start(segment, page, &psize); page->is_reset = true; - if (inuse > 0) { - _mi_mem_reset(start, psize, stats); // TODO: just `inuse`? - } + _mi_mem_reset(start, psize, stats); } // zero the page data @@ -503,7 +530,8 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { segment->abandoned_next = (mi_segment_t*)abandoned; } while (!mi_atomic_compare_exchange_ptr((volatile void**)&abandoned, segment, segment->abandoned_next)); mi_atomic_increment(&abandoned_count); - mi_stat_increase( tld->stats->segments_abandoned,1); + _mi_stat_increase(&tld->stats->segments_abandoned,1); + mi_segments_track_size((long)segment->segment_size, tld); } void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { @@ -511,7 +539,7 @@ void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { mi_segment_t* segment = _mi_page_segment(page); mi_assert_expensive(mi_segment_is_valid(segment)); segment->abandoned++; - mi_stat_increase( tld->stats->pages_abandoned, 1); + _mi_stat_increase(&tld->stats->pages_abandoned, 1); mi_assert_internal(segment->abandoned <= segment->used); if (segment->used == segment->abandoned) { // all pages are abandoned, abandon the entire segment @@ -546,7 +574,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen mi_segments_track_size((long)segment->segment_size,tld); mi_assert_internal(segment->next == NULL && segment->prev == NULL); mi_assert_expensive(mi_segment_is_valid(segment)); - mi_stat_decrease(tld->stats->segments_abandoned,1); + _mi_stat_decrease(&tld->stats->segments_abandoned,1); // add its free pages to the the current thread if (segment->page_kind == MI_PAGE_SMALL && mi_segment_has_free(segment)) { mi_segment_enqueue(&tld->small_free, segment); @@ -558,7 +586,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen if (page->segment_in_use) { segment->abandoned--; mi_assert(page->next == NULL); - mi_stat_decrease( tld->stats->pages_abandoned, 1); + _mi_stat_decrease(&tld->stats->pages_abandoned, 1); if (mi_page_all_free(page)) { // if everything free by now, free the page mi_segment_page_clear(segment,page,tld->stats); @@ -589,7 +617,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen // Requires that the page has free pages static mi_page_t* mi_segment_small_page_alloc_in(mi_segment_t* segment, mi_segments_tld_t* tld) { mi_assert_internal(mi_segment_has_free(segment)); - mi_page_t* page = mi_segment_find_free(segment); + mi_page_t* page = mi_segment_find_free(segment, tld->stats); page->segment_in_use = true; segment->used++; mi_assert_internal(segment->used <= segment->capacity); diff --git a/src/stats.c b/src/stats.c index 6b4fc8bb..d4c8dade 100644 --- a/src/stats.c +++ b/src/stats.c @@ -141,7 +141,7 @@ static void mi_printf_amount(int64_t n, int64_t unit, FILE* out, const char* fmt _mi_fprintf(out, (fmt==NULL ? "%11s" : fmt), buf); } -#if MI_STAT>0 + static void mi_print_amount(int64_t n, int64_t unit, FILE* out) { mi_printf_amount(n,unit,out,NULL); } @@ -175,7 +175,8 @@ static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); _mi_fprintf(out,"%10s: %7.1f avg\n", msg, avg); } -#endif + + static void mi_print_header( FILE* out ) { _mi_fprintf(out,"%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count "); @@ -204,33 +205,27 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin #endif -static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim); +static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit); static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_noexcept { if (out == NULL) out = stderr; mi_print_header(out); -#if !defined(MI_STAT) || (MI_STAT==0) - UNUSED(stats); - //_mi_fprintf(out,"(mimalloc built without statistics)\n"); -#else #if MI_STAT>1 mi_stat_count_t normal = { 0,0,0,0 }; mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out); mi_stat_print(&normal, "normal", 1, out); - #endif mi_stat_print(&stats->huge, "huge", 1, out); - #if MI_STAT>1 mi_stat_count_t total = { 0,0,0,0 }; mi_stat_add(&total, &normal, 1); mi_stat_add(&total, &stats->huge, 1); mi_stat_print(&total, "total", 1, out); - #endif _mi_fprintf(out, "malloc requested: "); mi_print_amount(stats->malloc.allocated, 1, out); _mi_fprintf(out, "\n\n"); + #endif mi_stat_print(&stats->reserved, "reserved", 1, out); mi_stat_print(&stats->committed, "committed", 1, out); - mi_stat_print(&stats->reset, "reset", -1, out); + mi_stat_print(&stats->reset, "reset", 1, out); mi_stat_print(&stats->page_committed, "touched", 1, out); mi_stat_print(&stats->segments, "segments", -1, out); mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out); @@ -243,7 +238,6 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n mi_stat_print(&stats->commit_calls, "commits", 0, out); mi_stat_print(&stats->threads, "threads", 0, out); mi_stat_counter_print(&stats->searches, "searches", out); -#endif if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs); @@ -252,9 +246,14 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n size_t peak_rss; size_t page_faults; size_t page_reclaim; - mi_process_info(&user_time, &sys_time, &peak_rss, &page_faults, &page_reclaim); + size_t peak_commit; + mi_process_info(&user_time, &sys_time, &peak_rss, &page_faults, &page_reclaim, &peak_commit); _mi_fprintf(out,"%10s: user: %.3f s, system: %.3f s, faults: %lu, reclaims: %lu, rss: ", "process", user_time, sys_time, (unsigned long)page_faults, (unsigned long)page_reclaim ); mi_printf_amount((int64_t)peak_rss, 1, out, "%s"); + if (peak_commit > 0) { + _mi_fprintf(out,", commit charge: "); + mi_printf_amount((int64_t)peak_commit, 1, out, "%s"); + } _mi_fprintf(out,"\n"); } @@ -362,7 +361,7 @@ static double filetime_secs(const FILETIME* ftime) { double secs = (double)(i.QuadPart) * 1.0e-7; // FILETIME is in 100 nano seconds return secs; } -static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim) { +static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { FILETIME ct; FILETIME ut; FILETIME st; @@ -375,6 +374,7 @@ static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); *peak_rss = (size_t)info.PeakWorkingSetSize; *page_faults = (size_t)info.PageFaultCount; + *peak_commit = (size_t)info.PeakPagefileUsage; *page_reclaim = 0; } @@ -391,7 +391,7 @@ static double timeval_secs(const struct timeval* tv) { return (double)tv->tv_sec + ((double)tv->tv_usec * 1.0e-6); } -static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim) { +static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { struct rusage rusage; getrusage(RUSAGE_SELF, &rusage); #if defined(__APPLE__) && defined(__MACH__) @@ -401,16 +401,18 @@ static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size #endif *page_faults = rusage.ru_majflt; *page_reclaim = rusage.ru_minflt; + *peak_commit = 0; *utime = timeval_secs(&rusage.ru_utime); *stime = timeval_secs(&rusage.ru_stime); } #else #pragma message("define a way to get process info") -static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim) { +static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { *peak_rss = 0; *page_faults = 0; *page_reclaim = 0; + *peak_commit = 0; *utime = 0.0; *stime = 0.0; }