diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj
index 63bc7d1d..87e866bb 100644
--- a/ide/vs2022/mimalloc.vcxproj
+++ b/ide/vs2022/mimalloc.vcxproj
@@ -190,7 +190,7 @@
true
Default
../../include
- MI_DEBUG=3;MI_GUARDED=0;MI_SECURE=4;%(PreprocessorDefinitions);
+ MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions);
CompileAsCpp
false
stdcpp20
diff --git a/include/mimalloc.h b/include/mimalloc.h
index 508e6aec..5f856411 100644
--- a/include/mimalloc.h
+++ b/include/mimalloc.h
@@ -400,6 +400,7 @@ typedef enum mi_option_e {
mi_option_max_page_candidates, // max candidate pages to consider for allocation (=4)
mi_option_max_vabits, // max user space virtual address bits to consider (=48)
mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0)
+ mi_option_page_commit_on_demand, // commit page memory on-demand
_mi_option_last,
// legacy option names
mi_option_large_os_pages = mi_option_allow_large_os_pages,
diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h
index 7c49d590..5b877635 100644
--- a/include/mimalloc/internal.h
+++ b/include/mimalloc/internal.h
@@ -128,7 +128,8 @@ bool _mi_os_decommit(void* addr, size_t size);
bool _mi_os_protect(void* addr, size_t size);
bool _mi_os_unprotect(void* addr, size_t size);
bool _mi_os_purge(void* p, size_t size);
-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset);
+bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size);
+bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);
size_t _mi_os_secure_guard_page_size(void);
bool _mi_os_secure_guard_page_set_at(void* addr, bool is_pinned);
@@ -155,7 +156,7 @@ void* _mi_arenas_alloc(mi_subproc_t* subproc, size_t size, bool commit,
void* _mi_arenas_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, mi_memid_t* memid);
void _mi_arenas_free(void* p, size_t size, mi_memid_t memid);
bool _mi_arenas_contain(const void* p);
-void _mi_arenas_collect(bool force_purge, mi_tld_t* tld);
+void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);
void _mi_arenas_unsafe_destroy_all(mi_tld_t* tld);
mi_page_t* _mi_arenas_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment);
@@ -534,9 +535,12 @@ static inline uint8_t* mi_page_start(const mi_page_t* page) {
return page->page_start;
}
+static inline size_t mi_page_size(const mi_page_t* page) {
+ return mi_page_block_size(page) * page->reserved;
+}
static inline uint8_t* mi_page_area(const mi_page_t* page, size_t* size) {
- if (size) { *size = mi_page_block_size(page) * page->reserved; }
+ if (size) { *size = mi_page_size(page); }
return mi_page_start(page);
}
@@ -564,6 +568,21 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
return mi_page_block_size(page) - MI_PADDING_SIZE;
}
+// This may change if we locate page info outside the page data slices
+static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {
+ return (uint8_t*)page;
+}
+
+// This gives the offset relative to the start slice of a page. This may change if we ever
+// locate page info outside the page-data itself.
+static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {
+ return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;
+}
+
+static inline size_t mi_page_committed(const mi_page_t* page) {
+ return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - (page->page_start - mi_page_slice_start(page)));
+}
+
static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
return page->heap;
}
diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h
index 53c543d0..59ecc99b 100644
--- a/include/mimalloc/types.h
+++ b/include/mimalloc/types.h
@@ -139,6 +139,9 @@ terms of the MIT license. A copy of the license can be found in the file
// We never allocate more than PTRDIFF_MAX (see also )
#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX
+// Minimal commit for a page on-demand commit (should be >= OS page size, and >= MI_ARENA_SLICE_SIZE for correct stats)
+#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE
+
// ------------------------------------------------------
// Arena's are large reserved areas of memory allocated from
// the OS that are managed by mimalloc to efficiently
@@ -290,7 +293,7 @@ typedef struct mi_page_s {
_Atomic(mi_page_flags_t) xflags; // `in_full_queue` and `has_aligned` flags
size_t block_size; // size available in each block (always `>0`)
- uint8_t* page_start; // start of the blocks
+ uint8_t* page_start; // start of the blocks
mi_heaptag_t heap_tag; // tag of the owning heap, used to separate heaps by object type
bool free_is_zero; // `true` if the blocks in the free list are zero initialized
// padding
@@ -301,6 +304,7 @@ typedef struct mi_page_s {
mi_heap_t* heap; // the heap owning this page (or NULL for abandoned pages)
struct mi_page_s* next; // next page owned by the heap with the same `block_size`
struct mi_page_s* prev; // previous page owned by the heap with the same `block_size`
+ size_t slice_committed; // committed size relative to the first arena slice of the page data
mi_memid_t memid; // provenance of the page memory
} mi_page_t;
@@ -322,9 +326,9 @@ typedef struct mi_page_s {
// The max object size are checked to not waste more than 12.5% internally over the page sizes.
// (Except for large pages since huge objects are allocated in 4MiB chunks)
-#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 11 KiB
-#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 128 KiB
-#define MI_LARGE_MAX_OBJ_SIZE ((MI_LARGE_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 1 MiB
+#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 8 KiB
+#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 64 KiB
+#define MI_LARGE_MAX_OBJ_SIZE ((MI_LARGE_PAGE_SIZE-MI_PAGE_INFO_SIZE)/4) // < 512 KiB
#define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)
diff --git a/src/arena.c b/src/arena.c
index bc88acf3..9915cdcf 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -207,12 +207,12 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
size_t already_committed_count = 0;
mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
// adjust the stats so we don't double count the commits
- if (already_committed_count > 0) {
- mi_subproc_stat_adjust_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count), true /* on alloc */);
- }
+ //if (already_committed_count > 0) {
+ // mi_subproc_stat_adjust_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count), true /* on alloc */);
+ //}
// now actually commit
bool commit_zero = false;
- if (!_mi_os_commit(p, mi_size_of_slices(slice_count), &commit_zero)) {
+ if (!_mi_os_commit_ex(p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed_count))) {
// failed to commit (todo: give warning?)
if (already_committed_count > 0) {
mi_subproc_stat_increase(arena->subproc, committed, mi_size_of_slices(already_committed_count));
@@ -562,7 +562,8 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_subproc_t* subproc, size_
_mi_page_free_collect(page, false); // update `used` count
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page);
@@ -578,16 +579,16 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_subproc_t* subproc, size_
// Allocate a fresh page
static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice_count, size_t block_size, size_t block_alignment,
- mi_arena_t* req_arena, size_t tseq)
+ mi_arena_t* req_arena, size_t tseq, bool commit)
{
const bool allow_large = (MI_SECURE < 2); // 2 = guard page at end of each arena page
- const bool commit = true;
const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN);
const size_t page_alignment = MI_ARENA_SLICE_ALIGN;
// try to allocate from free space in arena's
mi_memid_t memid = _mi_memid_none();
mi_page_t* page = NULL;
+ const size_t alloc_size = mi_size_of_slices(slice_count);
if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) && // allowed to allocate from arena's?
!os_align && // not large alignment
slice_count <= MI_ARENA_MAX_OBJ_SLICES) // and not too large
@@ -604,10 +605,10 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice
if (os_align) {
// note: slice_count already includes the page
mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment));
- page = (mi_page_t*)mi_arena_os_alloc_aligned(mi_size_of_slices(slice_count), block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, &memid);
+ page = (mi_page_t*)mi_arena_os_alloc_aligned(alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, &memid);
}
else {
- page = (mi_page_t*)mi_arena_os_alloc_aligned(mi_size_of_slices(slice_count), page_alignment, 0 /* align offset */, commit, allow_large, req_arena, &memid);
+ page = (mi_page_t*)mi_arena_os_alloc_aligned(alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, &memid);
}
}
@@ -617,25 +618,25 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice
// guard page at the end of mimalloc page?
#if MI_SECURE < 2
- const size_t page_noguard_size = mi_size_of_slices(slice_count);
+ const size_t page_noguard_size = alloc_size;
#else
- mi_assert(mi_size_of_slices(slice_count) > _mi_os_secure_guard_page_size());
- const size_t page_noguard_size = mi_size_of_slices(slice_count) - _mi_os_secure_guard_page_size();
+ mi_assert(alloc_size > _mi_os_secure_guard_page_size());
+ const size_t page_noguard_size = alloc_size - _mi_os_secure_guard_page_size();
if (memid.initially_committed) {
_mi_os_secure_guard_page_set_at((uint8_t*)page + page_noguard_size, memid.is_pinned);
}
#endif
// claimed free slices: initialize the page partly
- if (!memid.initially_zero) {
+ if (!memid.initially_zero && memid.initially_committed) {
mi_track_mem_undefined(page, slice_count * MI_ARENA_SLICE_SIZE);
_mi_memzero_aligned(page, sizeof(*page));
}
- else {
+ else if (memid.initially_committed) {
mi_track_mem_defined(page, slice_count * MI_ARENA_SLICE_SIZE);
}
#if MI_DEBUG > 1
- if (memid.initially_zero) {
+ if (memid.initially_zero && memid.initially_committed) {
if (!mi_mem_is_zero(page, page_noguard_size)) {
_mi_error_message(EFAULT, "internal error: page memory was not zero initialized.\n");
memid.initially_zero = false;
@@ -644,6 +645,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice
}
#endif
mi_assert(MI_PAGE_INFO_SIZE >= mi_page_info_size());
+
size_t block_start;
#if MI_GUARDED
// in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size
@@ -668,9 +670,24 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_subproc_t* subproc, size_t slice
}
const size_t reserved = (os_align ? 1 : (page_noguard_size - block_start) / block_size);
mi_assert_internal(reserved > 0 && reserved <= UINT16_MAX);
+
+ // commit first block?
+ size_t commit_size = 0;
+ if (!memid.initially_committed) {
+ commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE);
+ if (commit_size > page_noguard_size) { commit_size = page_noguard_size; }
+ bool is_zero;
+ _mi_os_commit(page, commit_size, &is_zero);
+ if (!memid.initially_zero && !is_zero) {
+ _mi_memzero_aligned(page, commit_size);
+ }
+ }
+
+ // initialize
page->reserved = (uint16_t)reserved;
page->page_start = (uint8_t*)page + block_start;
page->block_size = block_size;
+ page->slice_committed = commit_size;
page->memid = memid;
page->free_is_zero = memid.initially_zero;
if (block_size > 0 && _mi_is_power_of_two(block_size)) {
@@ -704,7 +721,10 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_heap_t* heap, size_t slice_cou
}
// 2. find a free block, potentially allocating a new arena
- page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, 1, req_arena, tld->thread_seq);
+ const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages
+ _mi_os_has_overcommit() || // no need to commit on demand on an OS that already does this for us
+ !mi_option_is_enabled(mi_option_page_commit_on_demand));
+ page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, 1, req_arena, tld->thread_seq, commit);
if (page != NULL) {
mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count);
_mi_page_init(heap, page);
@@ -726,7 +746,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_heap_t* heap, size_t block_s
const size_t slice_count = mi_slice_count_of_size(_mi_align_up(info_size + block_size, _mi_os_secure_guard_page_size()) + _mi_os_secure_guard_page_size());
#endif
- mi_page_t* page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, block_alignment, req_arena, tld->thread_seq);
+ mi_page_t* page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, block_alignment, req_arena, tld->thread_seq, true /* commit singletons always */);
if (page == NULL) return NULL;
mi_assert(page->reserved == 1);
@@ -779,7 +799,7 @@ void _mi_arenas_page_free(mi_page_t* page) {
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_clearN(arena->pages_abandoned[bin], slice_index, 1));
mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1));
// note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
@@ -799,7 +819,21 @@ void _mi_arenas_page_free(mi_page_t* page) {
// unregister page
_mi_page_map_unregister(page);
if (page->memid.memkind == MI_MEM_ARENA) {
- mi_bitmap_clear(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index);
+ mi_arena_t* arena = page->memid.mem.arena.arena;
+ mi_bitmap_clear(arena->pages, page->memid.mem.arena.slice_index);
+ if (page->slice_committed > 0) {
+ // if committed on-demand, set the commit bits to account commit properly
+ mi_assert_internal(mi_memid_size(page->memid) >= page->slice_committed);
+ const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative
+ mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, page->memid.mem.arena.slice_index, total_slices));
+ mi_assert_internal(page->memid.mem.arena.slice_count >= total_slices);
+ if (total_slices > 0) {
+ mi_bitmap_setN(arena->slices_committed, page->memid.mem.arena.slice_index, total_slices, NULL);
+ }
+ }
+ else {
+ mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, page->memid.mem.arena.slice_index, page->memid.mem.arena.slice_count));
+ }
}
_mi_arenas_free(page, mi_memid_size(page->memid), page->memid);
}
@@ -824,7 +858,7 @@ void _mi_arenas_page_abandon(mi_page_t* page) {
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(!mi_page_is_singleton(page));
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
mi_page_set_abandoned_mapped(page);
@@ -889,7 +923,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page) {
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
- mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+ mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
// this busy waits until a concurrent reader (from alloc_abandoned) is done
mi_bitmap_clear_once_set(arena->pages_abandoned[bin], slice_index);
@@ -979,8 +1013,8 @@ void _mi_arenas_free(void* p, size_t size, mi_memid_t memid) {
}
// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired
-void _mi_arenas_collect(bool force_purge, mi_tld_t* tld) {
- mi_arenas_try_purge(force_purge, force_purge /* visit all? */, tld);
+void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) {
+ mi_arenas_try_purge(force_purge, visit_all, tld);
}
@@ -1036,7 +1070,7 @@ static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) {
// for dynamic libraries that are unloaded and need to release all their allocated memory.
void _mi_arenas_unsafe_destroy_all(mi_tld_t* tld) {
mi_arenas_unsafe_destroy(_mi_subproc());
- _mi_arenas_collect(true /* force purge */, tld); // purge non-owned arenas
+ _mi_arenas_collect(true /* force purge */, true /* visit all*/, tld); // purge non-owned arenas
}
@@ -1468,15 +1502,23 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c
void* const p = mi_arena_slice_start(arena, slice_index);
//const bool all_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
size_t already_committed;
- mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed);
+ mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed); // pretend all committed.. (as we lack a clearN call that counts the already set bits..)
const bool all_committed = (already_committed == slice_count);
- const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */);
+ const bool needs_recommit = _mi_os_purge_ex(p, size, all_committed /* allow reset? */, mi_size_of_slices(already_committed));
- // update committed bitmap
if (needs_recommit) {
- mi_subproc_stat_adjust_decrease( arena->subproc, committed, mi_size_of_slices(slice_count - already_committed), false /* on freed */);
+ // no longer committed
mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
+ // we just counted in the purge to decommit all, but the some part was not committed so adjust that here
+ // mi_os_stat_decrease(committed, mi_size_of_slices(slice_count - already_committed));
}
+ else if (!all_committed) {
+ // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed)
+ mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
+ // we adjust the commit count as parts will be re-committed
+ // mi_os_stat_decrease(committed, mi_size_of_slices(already_committed));
+ }
+
return needs_recommit;
}
@@ -1499,6 +1541,7 @@ static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_
if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {
// expiration was not yet set
// maybe set the global arenas expire as well (if it wasn't set already)
+ mi_assert_internal(expire0==0);
mi_atomic_casi64_strong_acq_rel(&arena->subproc->purge_expire, &expire0, expire);
}
else {
@@ -1561,8 +1604,8 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
if (!force && (expire == 0 || expire > now)) return false;
- // reset expire (if not already set concurrently)
- mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0);
+ // reset expire
+ mi_atomic_store_release(&arena->purge_expire, (mi_msecs_t)0);
mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1);
// go through all purge info's (with max MI_BFIELD_BITS ranges at a time)
@@ -1577,33 +1620,36 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
static void mi_arenas_try_purge(bool force, bool visit_all, mi_tld_t* tld)
{
+ // try purge can be called often so try to only run when needed
const long delay = mi_arena_purge_delay();
if (_mi_preloading() || delay <= 0) return; // nothing will be scheduled
// check if any arena needs purging?
mi_subproc_t* subproc = tld->subproc;
const mi_msecs_t now = _mi_clock_now();
- mi_msecs_t arenas_expire = mi_atomic_load_acquire(&subproc->purge_expire);
- if (!force && (arenas_expire == 0 || arenas_expire > now)) return;
+ const mi_msecs_t arenas_expire = mi_atomic_load_acquire(&subproc->purge_expire);
+ if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return;
const size_t max_arena = mi_arenas_get_count(subproc);
if (max_arena == 0) return;
- // allow only one thread to purge at a time
+ // allow only one thread to purge at a time (todo: allow concurrent purging?)
static mi_atomic_guard_t purge_guard;
mi_atomic_guard(&purge_guard)
{
// increase global expire: at most one purge per delay cycle
- mi_atomic_store_release(&subproc->purge_expire, now + delay);
+ if (arenas_expire > now) { mi_atomic_store_release(&subproc->purge_expire, now + (delay/10)); }
const size_t arena_start = tld->thread_seq % max_arena;
- size_t max_purge_count = (visit_all ? max_arena : 2);
+ size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1);
bool all_visited = true;
+ bool any_purged = false;
for (size_t _i = 0; _i < max_arena; _i++) {
size_t i = _i + arena_start;
if (i >= max_arena) { i -= max_arena; }
mi_arena_t* arena = mi_arena_from_index(subproc,i);
if (arena != NULL) {
if (mi_arena_try_purge(arena, now, force)) {
+ any_purged = true;
if (max_purge_count <= 1) {
all_visited = false;
break;
@@ -1612,8 +1658,8 @@ static void mi_arenas_try_purge(bool force, bool visit_all, mi_tld_t* tld)
}
}
}
- if (all_visited) {
- mi_atomic_store_release(&subproc->purge_expire, (mi_msecs_t)0);
+ if (all_visited && !any_purged) {
+ mi_atomic_store_release(&subproc->purge_expire, 0);
}
}
}
diff --git a/src/heap.c b/src/heap.c
index 7c475bc5..85cd5cb0 100644
--- a/src/heap.c
+++ b/src/heap.c
@@ -121,8 +121,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
// collect all pages owned by this thread
mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL);
- // collect arenas (this is program wide so don't force purges on abandonment of threads)
- _mi_arenas_collect(collect == MI_FORCE /* force purge? */, heap->tld);
+ // collect arenas (this is program wide so don't force purges on abandonment of threads)
+ //mi_atomic_storei64_release(&heap->tld->subproc->purge_expire, 1);
+ _mi_arenas_collect(collect == MI_FORCE /* force purge? */, true /* visit all? */, heap->tld);
}
void _mi_heap_collect_abandon(mi_heap_t* heap) {
diff --git a/src/init.c b/src/init.c
index 1fc00404..f821d708 100644
--- a/src/init.c
+++ b/src/init.c
@@ -35,6 +35,7 @@ const mi_page_t _mi_page_empty = {
#endif
NULL, // xheap
NULL, NULL, // next, prev
+ MI_ARENA_SLICE_SIZE, // page_committed
MI_MEMID_STATIC // memid
};
diff --git a/src/options.c b/src/options.c
index 63d8a68f..b613f983 100644
--- a/src/options.c
+++ b/src/options.c
@@ -144,7 +144,7 @@ static mi_option_desc_t options[_mi_option_last] =
#else
{ 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)
#endif
- { 500, UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds
+ { 0, UNINIT, MI_OPTION_LEGACY(purge_delay,reset_delay) }, // purge delay in milli-seconds
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
@@ -175,6 +175,7 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(max_vabits) },
{ MI_DEFAULT_PAGEMAP_COMMIT,
UNINIT, MI_OPTION(pagemap_commit) }, // commit the full pagemap upfront?
+ { 1, UNINIT, MI_OPTION(page_commit_on_demand) },
};
static void mi_option_init(mi_option_desc_t* desc);
diff --git a/src/os.c b/src/os.c
index 399aac6c..79c2bc17 100644
--- a/src/os.c
+++ b/src/os.c
@@ -429,9 +429,9 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
return mi_os_page_align_areax(true, addr, size, newsize);
}
-bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
+bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
if (is_zero != NULL) { *is_zero = false; }
- mi_os_stat_increase(committed, size); // use size for precise commit vs. decommit
+ mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit
mi_os_stat_counter_increase(commit_calls, 1);
// page align range
@@ -458,9 +458,13 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
return true;
}
-static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) {
+bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
+ return _mi_os_commit_ex(addr, size, is_zero, size);
+}
+
+static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stats_size) {
mi_assert_internal(needs_recommit!=NULL);
- mi_os_stat_decrease(committed, size);
+ mi_os_stat_decrease(committed, stats_size);
// page align
size_t csize;
@@ -479,7 +483,7 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) {
bool _mi_os_decommit(void* addr, size_t size) {
bool needs_recommit;
- return mi_os_decommit_ex(addr, size, &needs_recommit);
+ return mi_os_decommit_ex(addr, size, &needs_recommit, size);
}
@@ -509,7 +513,7 @@ bool _mi_os_reset(void* addr, size_t size) {
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
-bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset)
+bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
mi_os_stat_counter_increase(purge_calls, 1);
@@ -519,7 +523,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset)
!_mi_preloading()) // don't decommit during preloading (unsafe)
{
bool needs_recommit = true;
- mi_os_decommit_ex(p, size, &needs_recommit);
+ mi_os_decommit_ex(p, size, &needs_recommit, stats_size);
return needs_recommit;
}
else {
@@ -533,7 +537,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset)
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge(void* p, size_t size) {
- return _mi_os_purge_ex(p, size, true);
+ return _mi_os_purge_ex(p, size, true, size);
}
diff --git a/src/page.c b/src/page.c
index 239d5d6e..aba548e9 100644
--- a/src/page.c
+++ b/src/page.c
@@ -251,8 +251,10 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
}
else {
mi_page_queue_remove(pq, page);
+ mi_tld_t* tld = page->heap->tld;
mi_page_set_heap(page, NULL);
- _mi_arenas_page_abandon(page);
+ _mi_arenas_page_abandon(page);
+ _mi_arenas_collect(false, false, tld); // allow purging
}
}
@@ -263,7 +265,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
mi_assert_internal(pq != NULL);
mi_assert_internal(mi_heap_contains_queue(heap, pq));
mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size);
- #endif
+ #endif
mi_page_t* page = _mi_arenas_page_alloc(heap, block_size, page_alignment);
if (page == NULL) {
// out-of-memory
@@ -359,7 +361,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {
mi_heap_t* heap = page->heap;
mi_page_set_heap(page,NULL);
_mi_arenas_page_free(page);
- _mi_arenas_collect(false, heap->tld); // allow purging
+ _mi_arenas_collect(false, false, heap->tld); // allow purging
}
#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE
@@ -606,6 +608,17 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page) {
mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved);
mi_assert_internal(extend < (1UL<<16));
+ // commit on demand?
+ if (page->slice_committed > 0) {
+ const size_t needed_size = (page->capacity + extend)*bsize;
+ const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE );
+ if (needed_commit > page->slice_committed) {
+ mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0);
+ _mi_os_commit(mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL);
+ page->slice_committed = needed_commit;
+ }
+ }
+
// and append the extend the free list
if (extend < MI_MIN_SLICES || MI_SECURE<3) { //!mi_option_is_enabled(mi_option_secure)) {
mi_page_free_list_extend(page, bsize, extend, &heap->tld->stats );
@@ -635,8 +648,8 @@ void _mi_page_init(mi_heap_t* heap, mi_page_t* page) {
#endif
#if MI_DEBUG>2
if (page->memid.initially_zero) {
- mi_track_mem_defined(page->page_start, page_size);
- mi_assert_expensive(mi_mem_is_zero(page_start, page_size));
+ mi_track_mem_defined(page->page_start, mi_page_committed(page));
+ mi_assert_expensive(mi_mem_is_zero(page_start, mi_page_committed(page)));
}
#endif