destroy the page map as well if MIMALLOC_DESTROY_ON_EXIT is set; see issue #1041

include/mimalloc/bits.h

@@ -120,6 +120,7 @@ typedef int32_t  mi_ssize_t;
 #define MI_MAX_VABITS     (32)
 #endif
 
+
 // use a flat page-map (or a 2-level one)
 #ifndef MI_PAGE_MAP_FLAT
 #if MI_MAX_VABITS <= 40 && !defined(__APPLE__) 

include/mimalloc/internal.h

@@ -200,6 +200,7 @@ void          _mi_page_map_register(mi_page_t* page);
 void          _mi_page_map_unregister(mi_page_t* page);
 void          _mi_page_map_unregister_range(void* start, size_t size);
 mi_page_t*    _mi_safe_ptr_page(const void* p);
+void          _mi_page_map_unsafe_destroy(void);
 
 // "page.c"
 void*         _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment)  mi_attr_noexcept mi_attr_malloc;

src/free.c

@@ -247,7 +247,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t*
       if (heap != NULL && heap->allow_page_reclaim &&
           (max_reclaim < 0 || mi_page_queue_len_is_atmost(heap, page->block_size, max_reclaim))) // we have less than N pages already
       {
-        if ((heap == page->heap) // always reclaim if we were the originating heap,
+        if ((heap == page->heap) // always reclaim if we were the originating heap (todo: maybe not if in a threadpool?)
             || // OR:
             (reclaim_on_free == 1 &&               // reclaim across heaps is allowed
               !mi_page_is_used_at_frac(page,8) &&  // and the page is not too full

src/init.c

@@ -780,6 +780,7 @@ void mi_cdecl _mi_process_done(void) {
     mi_heap_collect(heap, true /* force */);
     _mi_heap_unsafe_destroy_all(heap);     // forcefully release all memory held by all heaps (of this thread only!)
     _mi_arenas_unsafe_destroy_all(heap->tld);
+    _mi_page_map_unsafe_destroy();
   }
 
   if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {

src/page-map.c

@@ -71,6 +71,17 @@ bool _mi_page_map_init(void) {
   return true;
 }
 
+void _mi_page_map_unsafe_destroy(void) {
+  mi_assert_internal(_mi_page_map != NULL);
+  if (_mi_page_map == NULL) return;
+  _mi_os_free(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, mi_page_map_memid);
+  _mi_page_map = NULL;
+  mi_page_map_commit = NULL;
+  mi_page_map_max_address = NULL;
+  mi_page_map_memid = _mi_memid_none();  
+}
+
+
 static void mi_page_map_ensure_committed(size_t idx, size_t slice_count) {
   // is the page map area that contains the page address committed?
   // we always set the commit bits so we can track what ranges are in-use.
@@ -163,11 +174,12 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att
 #define MI_PAGE_MAP_SUB_SIZE    (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*))
 
 mi_decl_cache_align mi_page_t*** _mi_page_map;
+static size_t       mi_page_map_count;
 static void*        mi_page_map_max_address;
 static mi_memid_t   mi_page_map_memid;
-
 static _Atomic(mi_bfield_t)  mi_page_map_commit;
 
+static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx);
 static mi_page_t** mi_page_map_ensure_committed(size_t idx);
 static mi_page_t** mi_page_map_ensure_at(size_t idx);
 static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count);
@@ -184,10 +196,10 @@ bool _mi_page_map_init(void) {
   // Allocate the page map and commit bits
   mi_assert(MI_MAX_VABITS >= vbits);
   mi_page_map_max_address = (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits));
-  const size_t page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
-  mi_assert(page_map_count <= MI_PAGE_MAP_COUNT);
+  mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
+  mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT);
   const size_t os_page_size = _mi_os_page_size();
-  const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size);
+  const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size);
   const size_t reserve_size = page_map_size + os_page_size;
   const bool commit = page_map_size <= 64*MI_KiB ||
                       mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit();
@@ -214,6 +226,28 @@ bool _mi_page_map_init(void) {
   return true;
 }
 
+void _mi_page_map_unsafe_destroy(void) {
+  mi_assert_internal(_mi_page_map != NULL);
+  if (_mi_page_map == NULL) return;
+  for (size_t idx = 1; idx < mi_page_map_count; idx++) {  // skip entry 0
+    // free all sub-maps
+    if (mi_page_map_is_committed(idx, NULL)) {
+      mi_page_t** sub = _mi_page_map[idx];
+      if (sub != NULL) {
+        mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*), true, false, false);
+        _mi_os_free(memid.mem.os.base, memid.mem.os.size, memid);
+        _mi_page_map[idx] = NULL;
+      }
+    }
+  }
+  _mi_os_free(_mi_page_map, mi_page_map_memid.mem.os.size, mi_page_map_memid);
+  _mi_page_map = NULL;
+  mi_page_map_count = 0;
+  mi_page_map_memid = _mi_memid_none();
+  mi_page_map_max_address = NULL;
+  mi_atomic_store_release(&mi_page_map_commit, 0);
+}
+
 
 #define MI_PAGE_MAP_ENTRIES_PER_CBIT  (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS)
 

src/page.c

@@ -396,6 +396,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) {
   // and free it
   mi_heap_t* heap = page->heap;
   mi_heap_stat_decrease(heap, page_bins[mi_page_bin(page)], 1);
+  mi_heap_stat_decrease(heap, pages, 1);
   mi_page_set_heap(page,NULL);
   _mi_arenas_page_free(page);
   _mi_arenas_collect(false, false, heap->tld);  // allow purging