merge from dev-reset

src/arena.c

@@ -50,9 +50,9 @@ typedef struct mi_arena_s {
   mi_memid_t meta_memid;                  // memid of the arena structure itself (OS or static allocation)
   int      numa_node;                     // associated NUMA node
   bool     is_zero_init;                  // is the arena zero initialized?
-  bool     allow_decommit;                // is decommit allowed? if true, is_large should be false and blocks_committed != NULL
   bool     is_large;                      // large- or huge OS pages (always committed)
   bool     is_huge_alloc;                 // huge OS pages allocated by `_mi_os_alloc_huge_pages`
+  bool     allow_decommit;                // is decommit allowed? if true, is_large should be false and blocks_committed != NULL
   _Atomic(size_t) search_idx;             // optimization to start the search for free blocks
   _Atomic(mi_msecs_t) purge_expire;       // expiration time when blocks should be decommitted from `blocks_decommit`.  
   mi_bitmap_field_t* blocks_dirty;        // are the blocks potentially non-zero?
@@ -96,22 +96,19 @@ static bool mi_arena_id_is_suitable(mi_arena_id_t arena_id, bool arena_is_exclus
   memory id's
 ----------------------------------------------------------- */
 
-static mi_memid_t mi_memid_none(void) {
-  mi_memid_t memid;
-  _mi_memzero(&memid, sizeof(memid));
-  memid.memkind = MI_MEM_NONE;
-  return memid;
-}
-
 static mi_memid_t mi_memid_create(mi_memkind_t memkind) {
-  mi_memid_t memid = mi_memid_none();
+  mi_memid_t memid;
+  _mi_memzero_var(memid);
   memid.memkind = memkind;
   return memid;
 }
 
+static mi_memid_t mi_memid_none(void) {
+  return mi_memid_create(MI_MEM_NONE);
+}
+
 static mi_memid_t mi_memid_create_os(bool committed) {
-  mi_memid_t memid = mi_memid_none();
-  memid.memkind = MI_MEM_OS;
+  mi_memid_t memid = mi_memid_create(MI_MEM_OS);
   memid.was_committed = committed;
   return memid;
 }
@@ -166,11 +163,10 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit
 
 /* -----------------------------------------------------------
   Special static area for mimalloc internal structures
-  to avoid OS calls (for example, for the arena and thread
-  metadata)
+  to avoid OS calls (for example, for the arena metadata)
 ----------------------------------------------------------- */
 
-#define MI_ARENA_STATIC_MAX  (MI_INTPTR_SIZE*8*MI_KiB)  // 64 KiB on 64-bit
+#define MI_ARENA_STATIC_MAX  (MI_INTPTR_SIZE*MI_KiB)  // 8 KiB on 64-bit
 
 static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX];
 static _Atomic(size_t) mi_arena_static_top;
@@ -212,7 +208,7 @@ static void* mi_arena_meta_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* st
   p = _mi_os_alloc(size, &is_zero, stats);
   if (p != NULL) {
     *memid = mi_memid_create_os(true);
-    if (!is_zero) { _mi_memzero(p, size); }
+    if (!is_zero) { _mi_memzero_aligned(p, size); }
     return p;
   }
 
@@ -259,7 +255,6 @@ static mi_decl_noinline void* mi_arena_alloc_at(mi_arena_t* arena, size_t arena_
   // claimed it! 
   void* p = arena->start + mi_arena_block_size(mi_bitmap_index_bit(bitmap_index));
   *memid = mi_memid_create_arena(arena->id, arena->exclusive, bitmap_index);
-  memid->is_large = arena->is_large;
   memid->is_pinned = (arena->is_large || !arena->allow_decommit);
 
   // none of the claimed blocks should be scheduled for a decommit
@@ -437,13 +432,12 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset
   }
    
   // finally, fall back to the OS
-  bool os_large = allow_large;
+  bool os_is_large = false;
   bool os_is_zero = false;
-  void* p = _mi_os_alloc_aligned_offset(size, alignment, align_offset, commit, &os_large, &os_is_zero, tld->stats);
+  void* p = _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, &os_is_large, &os_is_zero, tld->stats);
   if (p != NULL) { 
     *memid = mi_memid_create_os(commit);
-    memid->is_large = os_large;
-    memid->is_pinned = os_large;
+    memid->is_pinned = os_is_large;
     memid->was_zero = os_is_zero;
     memid->mem.os.alignment = alignment;
     memid->mem.os.align_offset = align_offset;
@@ -647,7 +641,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi
       _mi_stat_decrease(&stats->committed, committed_size);
     }
     if (memid.mem.os.align_offset != 0) {
-      _mi_os_free_aligned(p, size, memid.mem.os.alignment, memid.mem.os.align_offset, all_committed, stats);
+      _mi_os_free_aligned_at_offset(p, size, memid.mem.os.alignment, memid.mem.os.align_offset, all_committed, stats);
     }
     else {
       _mi_os_free(p, size, stats);
@@ -729,7 +723,7 @@ static void mi_arenas_unsafe_destroy(void) {
       if (arena->owned && arena->start != NULL) {
         mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL);
         if (arena->is_huge_alloc) { 
-          _mi_os_free_huge_pages(arena->start, mi_arena_size(arena), &_mi_stats_main); 
+          _mi_os_free_huge_os_pages(arena->start, mi_arena_size(arena), &_mi_stats_main); 
         }
         else {
           _mi_os_free(arena->start, mi_arena_size(arena), &_mi_stats_main); 
@@ -860,16 +854,16 @@ static int mi_reserve_os_memory_ex2(size_t size, bool commit, bool allow_large,
 {
   if (arena_id != NULL) *arena_id = _mi_arena_id_none();
   size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block
-  bool large = allow_large;
-  bool is_zero;
-  void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &is_zero, &_mi_stats_main);
+  bool is_large = false;
+  bool is_zero = false;
+  void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &is_large, &is_zero, &_mi_stats_main);
   if (start==NULL) return ENOMEM;
-  if (!mi_manage_os_memory_ex2(start, size, (large || commit), large, false, is_zero, -1, exclusive, owned, arena_id)) {
+  if (!mi_manage_os_memory_ex2(start, size, (is_large || commit), is_large, false, is_zero, -1, exclusive, owned, arena_id)) {
     _mi_os_free_ex(start, size, commit, &_mi_stats_main);
     _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024));
     return ENOMEM;
   }
-  _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : "");
+  _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), is_large ? " (in large os pages)" : "");
   return 0;
 }
 
@@ -943,7 +937,7 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m
   _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);
 
   if (!mi_manage_os_memory_ex2(p, hsize, true, true, true, is_zero, numa_node, exclusive, true /* owned */, arena_id)) {
-    _mi_os_free_huge_pages(p, hsize, &_mi_stats_main);
+    _mi_os_free_huge_os_pages(p, hsize, &_mi_stats_main);
     return ENOMEM;
   }
   return 0;