From 3ef6784455479a665f1c8bfe11038237b757bc98 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 12 May 2025 22:05:15 -0700 Subject: [PATCH] destroy the page map as well if MIMALLOC_DESTROY_ON_EXIT is set; see issue #1041 --- include/mimalloc/bits.h | 1 + include/mimalloc/internal.h | 1 + src/free.c | 2 +- src/init.c | 1 + src/page-map.c | 42 +++++++++++++++++++++++++++++++++---- src/page.c | 1 + 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 0c62cd3b..371cb7ce 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -120,6 +120,7 @@ typedef int32_t mi_ssize_t; #define MI_MAX_VABITS (32) #endif + // use a flat page-map (or a 2-level one) #ifndef MI_PAGE_MAP_FLAT #if MI_MAX_VABITS <= 40 && !defined(__APPLE__) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index a5ab8162..9c5eb362 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -200,6 +200,7 @@ void _mi_page_map_register(mi_page_t* page); void _mi_page_map_unregister(mi_page_t* page); void _mi_page_map_unregister_range(void* start, size_t size); mi_page_t* _mi_safe_ptr_page(const void* p); +void _mi_page_map_unsafe_destroy(void); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; diff --git a/src/free.c b/src/free.c index 5989d43d..45ec5683 100644 --- a/src/free.c +++ b/src/free.c @@ -247,7 +247,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* if (heap != NULL && heap->allow_page_reclaim && (max_reclaim < 0 || mi_page_queue_len_is_atmost(heap, page->block_size, max_reclaim))) // we have less than N pages already { - if ((heap == page->heap) // always reclaim if we were the originating heap, + if ((heap == page->heap) // always reclaim if we were the originating heap (todo: maybe not if in a threadpool?) || // OR: (reclaim_on_free == 1 && // reclaim across heaps is allowed !mi_page_is_used_at_frac(page,8) && // and the page is not too full diff --git a/src/init.c b/src/init.c index 892f4988..c9c61dbb 100644 --- a/src/init.c +++ b/src/init.c @@ -780,6 +780,7 @@ void mi_cdecl _mi_process_done(void) { mi_heap_collect(heap, true /* force */); _mi_heap_unsafe_destroy_all(heap); // forcefully release all memory held by all heaps (of this thread only!) _mi_arenas_unsafe_destroy_all(heap->tld); + _mi_page_map_unsafe_destroy(); } if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { diff --git a/src/page-map.c b/src/page-map.c index c286d87e..0a6a2e36 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -71,6 +71,17 @@ bool _mi_page_map_init(void) { return true; } +void _mi_page_map_unsafe_destroy(void) { + mi_assert_internal(_mi_page_map != NULL); + if (_mi_page_map == NULL) return; + _mi_os_free(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, mi_page_map_memid); + _mi_page_map = NULL; + mi_page_map_commit = NULL; + mi_page_map_max_address = NULL; + mi_page_map_memid = _mi_memid_none(); +} + + static void mi_page_map_ensure_committed(size_t idx, size_t slice_count) { // is the page map area that contains the page address committed? // we always set the commit bits so we can track what ranges are in-use. @@ -163,11 +174,12 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att #define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) mi_decl_cache_align mi_page_t*** _mi_page_map; +static size_t mi_page_map_count; static void* mi_page_map_max_address; static mi_memid_t mi_page_map_memid; - static _Atomic(mi_bfield_t) mi_page_map_commit; +static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx); static mi_page_t** mi_page_map_ensure_committed(size_t idx); static mi_page_t** mi_page_map_ensure_at(size_t idx); static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count); @@ -184,10 +196,10 @@ bool _mi_page_map_init(void) { // Allocate the page map and commit bits mi_assert(MI_MAX_VABITS >= vbits); mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits)); - const size_t page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); - mi_assert(page_map_count <= MI_PAGE_MAP_COUNT); + mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT); const size_t os_page_size = _mi_os_page_size(); - const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size); + const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size); const size_t reserve_size = page_map_size + os_page_size; const bool commit = page_map_size <= 64*MI_KiB || mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); @@ -214,6 +226,28 @@ bool _mi_page_map_init(void) { return true; } +void _mi_page_map_unsafe_destroy(void) { + mi_assert_internal(_mi_page_map != NULL); + if (_mi_page_map == NULL) return; + for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 + // free all sub-maps + if (mi_page_map_is_committed(idx, NULL)) { + mi_page_t** sub = _mi_page_map[idx]; + if (sub != NULL) { + mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*), true, false, false); + _mi_os_free(memid.mem.os.base, memid.mem.os.size, memid); + _mi_page_map[idx] = NULL; + } + } + } + _mi_os_free(_mi_page_map, mi_page_map_memid.mem.os.size, mi_page_map_memid); + _mi_page_map = NULL; + mi_page_map_count = 0; + mi_page_map_memid = _mi_memid_none(); + mi_page_map_max_address = NULL; + mi_atomic_store_release(&mi_page_map_commit, 0); +} + #define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS) diff --git a/src/page.c b/src/page.c index 4b66841b..0d8e4e12 100644 --- a/src/page.c +++ b/src/page.c @@ -396,6 +396,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { // and free it mi_heap_t* heap = page->heap; mi_heap_stat_decrease(heap, page_bins[mi_page_bin(page)], 1); + mi_heap_stat_decrease(heap, pages, 1); mi_page_set_heap(page,NULL); _mi_arenas_page_free(page); _mi_arenas_collect(false, false, heap->tld); // allow purging