From 5a663da9aaca48e90ced03d832f387ad42e976bc Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 24 Dec 2024 20:38:36 -0800 Subject: [PATCH 1/5] fix build warning --- src/arena.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index 083fc35b..bbc0907e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1400,14 +1400,14 @@ void mi_debug_show_arenas(bool show_pages) mi_attr_noexcept { mi_subproc_t* subproc = _mi_subproc(); size_t max_arenas = mi_arenas_get_count(subproc); //size_t free_total = 0; - size_t slice_total = 0; + //size_t slice_total = 0; //size_t abandoned_total = 0; size_t page_total = 0; for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); if (arena == NULL) break; mi_assert(arena->subproc == subproc); - slice_total += arena->slice_count; + // slice_total += arena->slice_count; _mi_output_message("arena %zu at %p: %zu slices (%zu MiB)%s, subproc: %p\n", i, arena, arena->slice_count, mi_size_of_slices(arena->slice_count)/MI_MiB, (arena->memid.is_pinned ? ", pinned" : ""), arena->subproc); //if (show_inuse) { // free_total += mi_debug_show_bitmap("in-use slices", arena->slice_count, arena->slices_free, true, NULL); From ce7eb4db7a746aba77a35fa332d9b01f23430b9b Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 25 Dec 2024 10:49:49 -0800 Subject: [PATCH 2/5] fix page commit-on-demand setting --- src/arena.c | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/arena.c b/src/arena.c index bbc0907e..bd1c3e70 100644 --- a/src/arena.c +++ b/src/arena.c @@ -213,10 +213,6 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( // now actually commit bool commit_zero = false; if (!_mi_os_commit_ex(p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed_count))) { - // failed to commit (todo: give warning?) - if (already_committed_count > 0) { - mi_subproc_stat_increase(arena->subproc, committed, mi_size_of_slices(already_committed_count)); - } memid->initially_committed = false; } else { @@ -308,7 +304,9 @@ static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_ // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice // is actually allocated for the first time it will be counted. const bool adjust = (overcommit && arena_commit); - if (adjust) { mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve, true /* on alloc */); } + if (adjust) { + mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve, true /* on alloc */); + } // and try to reserve the arena int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); if (err != 0) { @@ -562,7 +560,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_subproc_t* subproc, size_ _mi_page_free_collect(page, false); // update `used` count mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_option_is_enabled(mi_option_page_commit_on_demand) || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(page)==page); @@ -722,7 +720,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_heap_t* heap, size_t slice_cou // 2. find a free block, potentially allocating a new arena const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages - (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 1)); + (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); page = mi_arenas_page_alloc_fresh(tld->subproc, slice_count, block_size, 1, req_arena, tld->thread_seq, commit); if (page != NULL) { mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); @@ -798,7 +796,7 @@ void _mi_arenas_page_free(mi_page_t* page) { mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_option_is_enabled(mi_option_page_commit_on_demand) || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->pages_abandoned[bin], slice_index, 1)); mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1)); // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may @@ -857,7 +855,7 @@ void _mi_arenas_page_abandon(mi_page_t* page) { mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); mi_assert_internal(!mi_page_is_singleton(page)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_option_is_enabled(mi_option_page_commit_on_demand) || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); mi_page_set_abandoned_mapped(page); @@ -922,7 +920,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page) { mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(mi_option_is_enabled(mi_option_page_commit_on_demand) || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); // this busy waits until a concurrent reader (from alloc_abandoned) is done mi_bitmap_clear_once_set(arena->pages_abandoned[bin], slice_index); @@ -1161,9 +1159,9 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s mi_arena_t* arena = (mi_arena_t*)start; - // commit & zero if needed + // commit & zero if needed if (!memid.initially_committed) { - // leave a guard OS page decommitted at the end + // leave a guard OS page decommitted at the end _mi_os_commit(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size(), NULL); } else { @@ -1180,7 +1178,7 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s arena->is_exclusive = exclusive; arena->slice_count = slice_count; arena->info_slices = info_slices; - arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) + arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->purge_expire = 0; // mi_lock_init(&arena->abandoned_visit_lock); @@ -1292,7 +1290,7 @@ typedef enum mi_ansi_color_e { } mi_ansi_color_t; static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { - buf[*k] = '\x1b'; + buf[*k] = '\x1b'; buf[*k+1] = '['; buf[*k+2] = (char)(((int)color / 10) + '0'); buf[*k+3] = (char)(((int)color % 10) + '0'); @@ -1342,7 +1340,7 @@ static size_t mi_debug_show_page_bfield(mi_bfield_t field, char* buf, size_t* k, else if (mi_bitmap_is_setN(arena->slices_committed, slice_index + bit, 1)) { c = '_'; color = MI_GRAY; } else { c = '.'; color = MI_GRAY; } } - if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; } + if (bit==MI_BFIELD_BITS-1 && bit_of_page > 1) { c = '>'; } } if (color != prev_color) { mi_debug_color(buf, k, color); @@ -1357,7 +1355,7 @@ static size_t mi_debug_show_page_bfield(mi_bfield_t field, char* buf, size_t* k, #define MI_FIELDS_PER_LINE (4) static size_t mi_debug_show_bitmap(const char* header, size_t slice_count, mi_bitmap_t* bitmap, bool invert, mi_arena_t* arena) { - _mi_output_message("\x1B[37m%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header); + _mi_output_message("\x1B[37m%s (use/commit: \x1B[31m0 - 25%%\x1B[33m - 50%%\x1B[36m - 75%%\x1B[32m - 100%%\x1B[0m)\n", header); size_t bit_count = 0; size_t bit_set_count = 0; for (size_t i = 0; i < mi_bitmap_chunk_count(bitmap) && bit_count < slice_count; i++) { @@ -1506,7 +1504,7 @@ int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserv static long mi_arena_purge_delay(void) { // <0 = no purging allowed, 0=immediate purging, >0=milli-second delay - return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult)); + return (mi_option_get(mi_option_purge_delay) * mi_option_get(mi_option_arena_purge_mult)); } // reset or decommit in an arena and update the commit bitmap @@ -1533,7 +1531,7 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c else if (!all_committed) { // we cannot assume any of these are committed any longer (even with reset since we did setN and may have marked uncommitted slices as committed) mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); - // we adjust the commit count as parts will be re-committed + // we adjust the commit count as parts will be re-committed // mi_os_stat_decrease(committed, mi_size_of_slices(already_committed)); } @@ -1621,7 +1619,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); if (!force && (expire == 0 || expire > now)) return false; - // reset expire + // reset expire mi_atomic_store_release(&arena->purge_expire, (mi_msecs_t)0); mi_subproc_stat_counter_increase(arena->subproc, arena_purges, 1); @@ -1696,8 +1694,8 @@ static bool abandoned_page_visit(mi_page_t* page, mi_abandoned_page_visit_info_t if (page->heap_tag != vinfo->heap_tag) { return true; } // continue mi_heap_area_t area; _mi_heap_area_init(&area, page); - if (!vinfo->visitor(NULL, &area, NULL, area.block_size, vinfo->arg)) { - return false; + if (!vinfo->visitor(NULL, &area, NULL, area.block_size, vinfo->arg)) { + return false; } if (vinfo->visit_blocks) { return _mi_heap_area_visit_blocks(&area, page, vinfo->visitor, vinfo->arg); @@ -1712,7 +1710,7 @@ static bool abandoned_page_visit_at(size_t slice_index, size_t slice_count, mi_a mi_abandoned_page_visit_info_t* vinfo = (mi_abandoned_page_visit_info_t*)arg; mi_page_t* page = (mi_page_t*)mi_arena_slice_start(arena, slice_index); mi_assert_internal(mi_page_is_abandoned_mapped(page)); - return abandoned_page_visit(page, vinfo); + return abandoned_page_visit(page, vinfo); } // Visit all abandoned pages in this subproc. From 15061be4b2fec43ed8bfaa807fd98624366d04e6 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 25 Dec 2024 10:50:49 -0800 Subject: [PATCH 3/5] commit page-map within one allocation --- src/page-map.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/page-map.c b/src/page-map.c index db14265b..a917175a 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -160,6 +160,7 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att #else // A 2-level page map +#define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) mi_decl_cache_align mi_page_t*** _mi_page_map; static void* mi_page_map_max_address; @@ -167,6 +168,7 @@ static mi_memid_t mi_page_map_memid; static _Atomic(mi_bfield_t) mi_page_map_commit; +static mi_page_t** mi_page_map_ensure_committed(size_t idx); static mi_page_t** mi_page_map_ensure_at(size_t idx); static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count); @@ -200,16 +202,17 @@ bool _mi_page_map_init(void) { } mi_atomic_store_release(&mi_page_map_commit, (commit ? ~MI_ZU(0) : MI_ZU(0))); - // commit the first part so NULL pointers get resolved without an access violation - mi_page_map_ensure_at(0); - - // note: for the NULL range we only commit one OS page - // mi_page_map_set_range(NULL, 0, 0, 1); - _mi_page_map[0] = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); + // note: for the NULL range we only commit one OS page (in the map and sub) if (!mi_page_map_memid.initially_committed) { - _mi_os_commit(_mi_page_map[0], os_page_size, NULL); + _mi_os_commit(&_mi_page_map[0], os_page_size, NULL); // commit first part of the map + } + _mi_page_map[0] = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved 2 subs at the end already + if (!mi_page_map_memid.initially_committed) { + _mi_os_commit(_mi_page_map[0], os_page_size, NULL); // only first OS page + } + if (!mi_page_map_memid.initially_zero) { + _mi_page_map[0][0] = NULL; } - _mi_page_map[0][0] = NULL; mi_assert_internal(_mi_ptr_page(NULL)==NULL); return true; From cfe73e9d48d6b27401f28ada8ac5d1a36b0f2747 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 25 Dec 2024 10:56:43 -0800 Subject: [PATCH 4/5] wip: merging from upstream --- include/mimalloc/types.h | 2 +- src/os.c | 47 +++++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ad948d36..f2fea3f7 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -377,7 +377,7 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) { typedef struct mi_memid_os_info { void* base; // actual base address of the block (used for offset aligned allocations) - size_t alignment; // alignment at allocation + size_t size; // full allocation size } mi_memid_os_info_t; typedef struct mi_memid_arena_info { diff --git a/src/os.c b/src/os.c index 32cb11c3..7e7dcb2c 100644 --- a/src/os.c +++ b/src/os.c @@ -157,34 +157,42 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { } #endif - /* ----------------------------------------------------------- Free memory -------------------------------------------------------------- */ static void mi_os_free_huge_os_pages(void* p, size_t size); -static void mi_os_prim_free(void* addr, size_t size, bool still_committed) { +static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) { mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) int err = _mi_prim_free(addr, size); if (err != 0) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } - if (still_committed) { _mi_stat_decrease(&os_stats->committed, size); } + if (commit_size > 0) { + _mi_stat_decrease(&os_stats->committed, commit_size); + } _mi_stat_decrease(&os_stats->reserved, size); } void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) { if (mi_memkind_is_os(memid.memkind)) { - size_t csize = _mi_os_good_alloc_size(size); + size_t csize = memid.mem.os.size; + if (csize==0) { _mi_os_good_alloc_size(size); } + size_t commit_size = (still_committed ? csize : 0); void* base = addr; // different base? (due to alignment) - if (memid.mem.os.base != NULL) { - mi_assert(memid.mem.os.base <= addr); - mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr); + if (memid.mem.os.base != base) { + mi_assert(memid.mem.os.base <= addr); base = memid.mem.os.base; - csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base); + const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base; + if (memid.mem.os.size==0) { + csize += diff; + } + if (still_committed) { + commit_size -= diff; // the (addr-base) part was already un-committed + } } // free it if (memid.memkind == MI_MEM_OS_HUGE) { @@ -192,7 +200,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me mi_os_free_huge_os_pages(base, csize); } else { - mi_os_prim_free(base, csize, still_committed); + mi_os_prim_free(base, csize, (still_committed ? commit_size : 0)); } } else { @@ -273,7 +281,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit #if !MI_TRACK_ASAN _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); #endif - mi_os_prim_free(p, size, commit); + if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); } if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -304,8 +312,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit size_t mid_size = _mi_align_up(size, _mi_os_page_size()); size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit); } - if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit); } + if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); } + if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); } // we can return the aligned pointer on `mmap` systems p = aligned_p; *base = aligned_p; // since we freed the pre part, `*base == p`. @@ -446,8 +454,13 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero) { return true; } -static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) { mi_assert_internal(needs_recommit!=NULL); - _mi_stat_decrease(&os_stats->committed, size); +bool _mi_os_commit(void* addr, size_t size, bool* is_zero) { + return _mi_os_commit_ex(addr, size, is_zero, size); +} + +static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) { + mi_assert_internal(needs_recommit!=NULL); + mi_os_stat_decrease(committed, stat_size); // page align size_t csize; @@ -466,7 +479,7 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) { bool _mi_os_decommit(void* addr, size_t size) { bool needs_recommit; - return mi_os_decommit_ex(addr, size, &needs_recommit); + return mi_os_decommit_ex(addr, size, &needs_recommit, size); } @@ -628,7 +641,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // no success, issue a warning and break if (p != NULL) { _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); - mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, true); + mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); } break; } @@ -674,7 +687,7 @@ static void mi_os_free_huge_os_pages(void* p, size_t size) { if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { - mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, true); + mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); size -= MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE; } From 587e08ccd44430bb3f5d2975653fffd18008de24 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 25 Dec 2024 11:12:26 -0800 Subject: [PATCH 5/5] improve commit stats --- include/mimalloc/internal.h | 2 +- src/arena.c | 3 +-- src/os.c | 41 ++++++++++++++++++++----------------- test/test-stress.c | 3 ++- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 8b22e1c6..48ec3f1c 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -121,7 +121,7 @@ bool _mi_os_decommit(void* addr, size_t size); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_purge(void* p, size_t size); -bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset); +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size); void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid); void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); diff --git a/src/arena.c b/src/arena.c index 0ddb2936..5952c544 100644 --- a/src/arena.c +++ b/src/arena.c @@ -474,8 +474,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) // we need to ensure we do not try to reset (as that may be invalid for uncommitted memory), // and also undo the decommit stats (as it was already adjusted) mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits)); - needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */); - if (needs_recommit) { _mi_stat_increase(&_mi_stats_main.committed, size); } + needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, 0); } // clear the purged blocks diff --git a/src/os.c b/src/os.c index 7e7dcb2c..77469775 100644 --- a/src/os.c +++ b/src/os.c @@ -9,7 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/atomic.h" #include "mimalloc/prim.h" -#define os_stats (&_mi_stats_main) +#define mi_os_stat_increase(stat,amount) _mi_stat_increase(&_mi_stats_main.stat, amount) +#define mi_os_stat_decrease(stat,amount) _mi_stat_decrease(&_mi_stats_main.stat, amount) +#define mi_os_stat_counter_increase(stat,inc) _mi_stat_counter_increase(&_mi_stats_main.stat, inc) /* ----------------------------------------------------------- Initialization. @@ -171,9 +173,9 @@ static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } if (commit_size > 0) { - _mi_stat_decrease(&os_stats->committed, commit_size); + mi_os_stat_decrease(committed, commit_size); } - _mi_stat_decrease(&os_stats->reserved, size); + mi_os_stat_decrease(reserved, size); } void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) { @@ -236,11 +238,11 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm - mi_stat_counter_increase(os_stats->mmap_calls, 1); + mi_os_stat_counter_increase(mmap_calls, 1); if (p != NULL) { - _mi_stat_increase(&os_stats->reserved, size); + mi_os_stat_increase(reserved, size); if (commit) { - _mi_stat_increase(&os_stats->committed, size); + mi_os_stat_increase(committed, size); // seems needed for asan (or `mimalloc-test-api` fails) #ifdef MI_TRACK_ASAN if (*is_zero) { mi_track_mem_defined(p,size); } @@ -357,7 +359,8 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo if (p != NULL) { *memid = _mi_memid_create_os(commit, os_is_zero, os_is_large); memid->mem.os.base = os_base; - memid->mem.os.alignment = alignment; + // memid->mem.os.alignment = alignment; + memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned } return p; } @@ -425,10 +428,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -bool _mi_os_commit(void* addr, size_t size, bool* is_zero) { +bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) { if (is_zero != NULL) { *is_zero = false; } - _mi_stat_increase(&os_stats->committed, size); // use size for precise commit vs. decommit - _mi_stat_counter_increase(&os_stats->commit_calls, 1); + mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit + mi_os_stat_counter_increase(commit_calls, 1); // page align range size_t csize; @@ -492,8 +495,8 @@ bool _mi_os_reset(void* addr, size_t size) { size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) - _mi_stat_increase(&os_stats->reset, csize); - _mi_stat_counter_increase(&os_stats->reset_calls, 1); + mi_os_stat_increase(reset, csize); + mi_os_stat_counter_increase(reset_calls, 1); #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN memset(start, 0, csize); // pretend it is eagerly reset @@ -509,17 +512,17 @@ bool _mi_os_reset(void* addr, size_t size) { // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. -bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset) +bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size) { if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? - _mi_stat_counter_increase(&os_stats->purge_calls, 1); - _mi_stat_increase(&os_stats->purged, size); + mi_os_stat_counter_increase(purge_calls, 1); + mi_os_stat_increase(purged, size); if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? !_mi_preloading()) // don't decommit during preloading (unsafe) { bool needs_recommit = true; - mi_os_decommit_ex(p, size, &needs_recommit); + mi_os_decommit_ex(p, size, &needs_recommit, stat_size); return needs_recommit; } else { @@ -533,7 +536,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset) // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. bool _mi_os_purge(void* p, size_t size) { - return _mi_os_purge_ex(p, size, true); + return _mi_os_purge_ex(p, size, true, size); } @@ -648,8 +651,8 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // success, record it page++; // increase before timeout check (see issue #711) - _mi_stat_increase(&os_stats->committed, MI_HUGE_OS_PAGE_SIZE); - _mi_stat_increase(&os_stats->reserved, MI_HUGE_OS_PAGE_SIZE); + mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE); + mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE); // check for timeout if (max_msecs > 0) { diff --git a/test/test-stress.c b/test/test-stress.c index 6284ad39..9b8eb2d0 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -64,6 +64,7 @@ static bool main_participates = false; // main thread participates as a #define custom_calloc(n,s) mi_calloc(n,s) #define custom_realloc(p,s) mi_realloc(p,s) #define custom_free(p) mi_free(p) + #ifndef NDEBUG #define HEAP_WALK // walk the heap objects? #endif @@ -223,7 +224,7 @@ static void test_stress(void) { run_os_threads(THREADS, &stress); #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) // switch between arena and OS allocation for testing - mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1); + // mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1); #endif #ifdef HEAP_WALK size_t total = 0;