initial work on a two-level page-map

include/mimalloc/bits.h

@@ -68,6 +68,14 @@ typedef int32_t  mi_ssize_t;
 #define MI_MiB     (MI_KiB*MI_KiB)
 #define MI_GiB     (MI_MiB*MI_KiB)
 
+#if MI_INTPTR_SIZE > 4
+#define MI_MAX_VABITS     (48)
+#define MI_PAGE_MAP_FLAT  0
+#else
+#define MI_MAX_VABITS     (32)
+#define MI_PAGE_MAP_FLAT  1
+#endif
+
 
 /* --------------------------------------------------------------------------------
   Architecture

include/mimalloc/internal.h

@@ -422,6 +422,14 @@ static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
   return (heap != &_mi_heap_empty);
 }
 
+static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) {
+  mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
+  const size_t idx = _mi_wsize_from_size(size);
+  mi_assert_internal(idx < MI_PAGES_DIRECT);
+  return heap->pages_free_direct[idx];
+}
+
+
 //static inline uintptr_t _mi_ptr_cookie(const void* p) {
 //  extern mi_heap_t _mi_heap_main;
 //  mi_assert_internal(_mi_heap_main.cookie != 0);
@@ -433,14 +441,9 @@ static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
   Pages
 ----------------------------------------------------------- */
 
-static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) {
+#if MI_PAGE_MAP_FLAT
-  mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
-  const size_t idx = _mi_wsize_from_size(size);
-  mi_assert_internal(idx < MI_PAGES_DIRECT);
-  return heap->pages_free_direct[idx];
-}
-
 
+// flat page-map committed on demand
 extern uint8_t* _mi_page_map;
 
 static inline uintptr_t _mi_page_map_index(const void* p) {
@@ -465,16 +468,59 @@ static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) {
 
 static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
   bool valid;
-  mi_page_t* const page = _mi_ptr_page_ex(p,&valid);
+  mi_page_t* const page = _mi_ptr_page_ex(p, &valid);
   return (valid ? page : NULL);
 }
 
+static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
+  return _mi_ptr_page_ex(p, NULL);
+}
+
+#else 
+
+// 2-level page map
+ 
+// one page-map directory = 64 KiB => covers 2^16 * 2^16 = 2^32 = 4 GiB address space
+// the page-map needs 48-16-16 = 16 bits => 2^16 map directories = 2^16 * 2^3 = 2^19 = 512 KiB size.
+// we commit the page-map directories on-demand. (2^16 * 2^16 = 2^32 ~= 4 GiB needed to cover 256 TeB)
+
+#define MI_PAGE_MAP_SUB_SHIFT     (16)    // 64 KiB 
+#define MI_PAGE_MAP_SUB_SIZE      (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)
+#define MI_PAGE_MAP_SHIFT         (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)
+#define MI_PAGE_MAP_COUNT         (MI_ZU(1) << MI_PAGE_MAP_SHIFT)
+
+extern uint8_t** _mi_page_map;
+
+static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) {
+  const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE;
+  if (sub_idx != NULL) { *sub_idx = (uint32_t)u % MI_PAGE_MAP_SUB_SIZE; }
+  return (size_t)(u / MI_PAGE_MAP_COUNT);
+}
+
+static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
+  const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE;
+  const uint8_t* const sub = _mi_page_map[u / MI_PAGE_MAP_COUNT];
+  const uint8_t ofs = sub[(uint32_t)u % MI_PAGE_MAP_SUB_SIZE];
+  return (mi_page_t*)((u - ofs + 1) * MI_ARENA_SLICE_SIZE);
+}
+
+static inline mi_page_t* _mi_checked_ptr_page(const void* p) {
+  const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE;
+  const uint8_t* const sub = _mi_page_map[u / MI_PAGE_MAP_COUNT];
+  //if mi_unlikely(sub == NULL) { return NULL; }
+  const uint8_t ofs = sub[(uint32_t)u % MI_PAGE_MAP_SUB_SIZE];
+  //if mi_unlikely(ofs == 0) { return NULL;  }
+  return (mi_page_t*)((u - ofs + 1) * MI_ARENA_SLICE_SIZE);
+}
+
+#endif
+
 static inline mi_page_t* _mi_ptr_page(const void* p) {
   mi_assert_internal(p==NULL || mi_is_in_heap_region(p));
   #if MI_DEBUG || defined(__APPLE__)
   return _mi_checked_ptr_page(p);
   #else
-  return _mi_ptr_page_ex(p,NULL);
+  return _mi_unchecked_ptr_page(p);
   #endif
 }
 

src/page-map.c

@@ -9,6 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "mimalloc/internal.h"
 #include "bitmap.h"
 
+#if MI_PAGE_MAP_FLAT
+
 // The page-map contains a byte for each 64kb slice in the address space. 
 // For an address `a` where `n = _mi_page_map[a >> 16]`:
 // 0 = unused
@@ -17,6 +19,9 @@ terms of the MIT license. A copy of the license can be found in the file
 // 
 // 1 byte per slice => 1 GiB page map = 2^30 slices of 2^16 = 2^46 = 64 TiB address space.
 // 4 GiB virtual for 256 TiB address space (48 bit) (and 64 KiB for 4 GiB address space (on 32-bit)).
+
+// 1MiB = 2^20*2^16 = 2^36 = 64GiB address space
+// 2^12 pointers = 2^15 k = 32k
 mi_decl_cache_align uint8_t* _mi_page_map = NULL;
 static bool        mi_page_map_all_committed = false;
 static size_t      mi_page_map_entries_per_commit_bit = MI_ARENA_SLICE_SIZE;
@@ -25,7 +30,7 @@ static mi_memid_t  mi_page_map_memid;
 
 
 // (note: we need to initialize statically or otherwise C++ may run a default constructors after process initialization)
-static mi_bitmap_t mi_page_map_commit = { MI_ATOMIC_VAR_INIT(MI_BITMAP_DEFAULT_CHUNK_COUNT), MI_ATOMIC_VAR_INIT(0),
+sstatic mi_bitmap_t mi_page_map_commit = { MI_ATOMIC_VAR_INIT(MI_BITMAP_DEFAULT_CHUNK_COUNT), MI_ATOMIC_VAR_INIT(0),
                                           { 0 }, { {MI_ATOMIC_VAR_INIT(0)} }, {{{ MI_ATOMIC_VAR_INIT(0) }}} };
 
 bool _mi_page_map_init(void) {
@@ -101,7 +106,7 @@ static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t*
 
 void _mi_page_map_register(mi_page_t* page) {
   mi_assert_internal(page != NULL);
-  mi_assert_internal(_mi_is_aligned(page,MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
   mi_assert_internal(_mi_page_map != NULL);  // should be initialized before multi-thread access!
   if mi_unlikely(_mi_page_map == NULL) {
     if (!_mi_page_map_init()) return;
@@ -151,3 +156,137 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att
     return false;
   }
 }
+
+#else 
+
+mi_decl_cache_align uint8_t** _mi_page_map = NULL;
+
+static void*       mi_page_map_max_address = NULL;
+static mi_memid_t  mi_page_map_memid;
+
+bool _mi_page_map_init(void) {
+  size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS);
+  if (vbits == 0) {
+    vbits = _mi_os_virtual_address_bits();
+    mi_assert_internal(vbits <= MI_MAX_VABITS);
+  }
+
+  mi_page_map_max_address = (void*)(MI_PU(1) << vbits);
+  const size_t os_page_size = _mi_os_page_size();
+  const size_t page_map_size = _mi_align_up(MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT + MI_INTPTR_SHIFT), os_page_size);
+  const size_t reserve_size = page_map_size + (2 * MI_PAGE_MAP_SUB_SIZE);  
+  _mi_page_map = (uint8_t**)_mi_os_alloc_aligned(reserve_size, 1, true /* commit */, true, &mi_page_map_memid);
+  if (_mi_page_map==NULL) {
+    _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", reserve_size / MI_KiB);
+    return false;
+  }
+  if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
+    _mi_warning_message("the page map was committed but not zero initialized!\n");
+    _mi_memzero_aligned(_mi_page_map, reserve_size);
+  }
+
+  uint8_t* sub0 = (uint8_t*)_mi_page_map + page_map_size;
+  uint8_t* sub1 = sub0 + MI_PAGE_MAP_SUB_SIZE;
+  // initialize the first part so NULL pointers get resolved without an access violation
+  _mi_page_map[0] = sub0; 
+  sub0[0] = 1;                // so _mi_ptr_page(NULL) == NULL
+  // and initialize the 4GiB range where we were allocated 
+  _mi_page_map[_mi_page_map_index(_mi_page_map,NULL)] = sub1;
+
+  mi_assert_internal(_mi_ptr_page(NULL)==NULL);
+  return true;
+}
+
+static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* sub_idx, size_t* slice_count) {
+  size_t page_size;
+  *page_start = mi_page_area(page, &page_size);
+  if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; }  // furthest interior pointer
+  *slice_count = mi_slice_count_of_size(page_size) + (((uint8_t*)*page_start - (uint8_t*)page)/MI_ARENA_SLICE_SIZE); // add for large aligned blocks
+  return _mi_page_map_index(page,sub_idx);
+}
+
+
+static inline void mi_page_map_set_range(size_t idx, size_t sub_idx, size_t slice_count, uint8_t (*set)(uint8_t ofs)) {
+  // is the page map area that contains the page address committed?
+  uint8_t ofs = 1;
+  while (slice_count > 0) {
+    uint8_t* sub = _mi_page_map[idx];
+    if (sub == NULL) {
+      mi_memid_t memid;
+      sub = (uint8_t*)_mi_os_alloc(MI_PAGE_MAP_SUB_SIZE, &memid);
+      if (sub == NULL) {
+        _mi_error_message(EFAULT, "internal error: unable to extend the page map\n");
+        return; // abort?
+      }
+    }
+    // set the offsets for the page
+    while (sub_idx < MI_PAGE_MAP_SUB_SIZE && slice_count > 0) {
+      sub[sub_idx] = set(ofs);
+      sub_idx++;
+      ofs++;
+      slice_count--;
+    }
+    sub_idx = 0; // potentially wrap around to the next idx    
+  }  
+}
+
+static uint8_t set_ofs(uint8_t ofs) {
+  return ofs;
+}
+
+void _mi_page_map_register(mi_page_t* page) {
+  mi_assert_internal(page != NULL);
+  mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
+  mi_assert_internal(_mi_page_map != NULL);  // should be initialized before multi-thread access!
+  if mi_unlikely(_mi_page_map == NULL) {
+    if (!_mi_page_map_init()) return;
+  }
+  mi_assert(_mi_page_map!=NULL);
+  uint8_t* page_start;
+  size_t   slice_count;
+  size_t   sub_idx;
+  const size_t idx = mi_page_map_get_idx(page, &page_start, &sub_idx, &slice_count);
+  mi_page_map_set_range(idx, sub_idx, slice_count, &set_ofs);
+}
+
+static uint8_t set_zero(uint8_t ofs) {
+  MI_UNUSED(ofs);
+  return 0;
+}
+
+
+void _mi_page_map_unregister(mi_page_t* page) {
+  mi_assert_internal(_mi_page_map != NULL);
+  // get index and count
+  uint8_t* page_start;
+  size_t   slice_count;
+  size_t   sub_idx;
+  const size_t idx = mi_page_map_get_idx(page, &page_start, &sub_idx, &slice_count);
+  // unset the offsets
+  mi_page_map_set_range(idx, sub_idx, slice_count, &set_zero);
+}
+
+void _mi_page_map_unregister_range(void* start, size_t size) {
+  const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE);
+  size_t sub_idx;
+  const size_t idx = _mi_page_map_index(start, &sub_idx);
+  mi_page_map_set_range(idx, sub_idx, slice_count, &set_zero);
+}
+
+mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
+
+  if mi_unlikely(p >= mi_page_map_max_address) return false;
+  size_t sub_idx;
+  const size_t idx = _mi_page_map_index(p, &sub_idx);
+  uint8_t* sub = _mi_page_map[idx];  
+  if (sub != NULL) {
+    return (sub[sub_idx] != 0);
+  }
+  else {
+    return false;
+  }
+}
+
+
+#endif
+

test/test-stress.c

@@ -302,8 +302,8 @@ int main(int argc, char** argv) {
     mi_option_enable(mi_option_visit_abandoned);
   #endif
   #if !defined(NDEBUG) && !defined(USE_STD_MALLOC)
-    // mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */);
+    mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */);
-    mi_option_set(mi_option_purge_delay,10);
+    //mi_option_set(mi_option_purge_delay,10);
   #endif
   #ifndef USE_STD_MALLOC
     mi_stats_reset();