Merge branch 'dev-slice-cmask' into dev-slice

include/mimalloc-internal.h

@@ -19,6 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MI_CACHE_LINE          64
 #if defined(_MSC_VER)
 #pragma warning(disable:4127)   // suppress constant conditional warning (due to MI_SECURE paths)
+#pragma warning(disable:26812)  // unscoped enum warning
 #define mi_decl_noinline        __declspec(noinline)
 #define mi_decl_thread          __declspec(thread)
 #define mi_decl_cache_align     __declspec(align(MI_CACHE_LINE))
@@ -88,7 +89,7 @@ void       _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed,
 
 // "segment-cache.c"
 void*      _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
-bool       _mi_segment_cache_push(void* start, size_t size, size_t memid, mi_commit_mask_t commit_mask, mi_commit_mask_t decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld);
+bool       _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld);
 void       _mi_segment_map_allocated_at(const mi_segment_t* segment);
 void       _mi_segment_map_freed_at(const mi_segment_t* segment);
 
@@ -499,6 +500,10 @@ static inline size_t mi_segment_size(mi_segment_t* segment) {
   return segment->segment_slices * MI_SEGMENT_SLICE_SIZE;
 }
 
+static inline uint8_t* mi_segment_end(mi_segment_t* segment) {
+  return (uint8_t*)segment + mi_segment_size(segment);
+}
+
 // Thread free access
 static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
   return (mi_block_t*)(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free) & ~3);
@@ -693,45 +698,45 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c
 // commit mask
 // -------------------------------------------------------------------
 
-#define MI_COMMIT_MASK_BITS  (sizeof(mi_commit_mask_t)*8)
+static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) {
-
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
-static inline mi_commit_mask_t mi_commit_mask_empty(void) {
+    cm->mask[i] = 0;
-  return 0;
-}
-
-static inline mi_commit_mask_t mi_commit_mask_full(void) {
-  return ~mi_commit_mask_empty();
-}
-
-static inline bool mi_commit_mask_is_empty(mi_commit_mask_t mask) {
-  return (mask == 0);
-}
-
-static inline bool mi_commit_mask_is_full(mi_commit_mask_t mask) {
-  return ((~mask) == 0);
-}
-
-size_t _mi_commit_mask_committed_size(mi_commit_mask_t mask, size_t total);
-
-#define mi_commit_mask_foreach(mask,idx,count) \
-  idx = 0; \
-  while (mask != 0) {     \
-    /* count ones */      \
-    count = 0;            \
-    while ((mask&1)==1) { \
-      mask >>= 1;         \
-      count++;            \
-    }                     \
-    /* if found, do action */ \
-    if (count > 0) {
-
-#define mi_commit_mask_foreach_end() \
-    } \
-    idx += count; \
-    /* shift out the zero */ \
-    mask >>= 1;   \
-    idx++;        \
   }
+}
+
+static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    cm->mask[i] = ~((size_t)0);
+  }
+}
+
+static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    if (cm->mask[i] != 0) return false;
+  }
+  return true;
+}
+
+static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    if (cm->mask[i] != ~((size_t)0)) return false;
+  }
+  return true;
+}
+
+// defined in `segment.c`:
+size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total);
+size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx);
+
+#define mi_commit_mask_foreach(cm,idx,count) \
+  idx = 0; \
+  while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) { 
+        
+#define mi_commit_mask_foreach_end() \
+    idx += count; \
+  }
+      
+
 
 
 // -------------------------------------------------------------------

include/mimalloc-types.h

@@ -147,13 +147,10 @@ typedef int32_t  mi_ssize_t;
 #define MI_MEDIUM_PAGE_SIZE               (MI_ZU(1)<<MI_MEDIUM_PAGE_SHIFT)
 
 #define MI_SMALL_OBJ_SIZE_MAX             (MI_SMALL_PAGE_SIZE/4)   // 8KiB on 64-bit
-
 #define MI_MEDIUM_OBJ_SIZE_MAX            (MI_MEDIUM_PAGE_SIZE/4)  // 128KiB on 64-bit
 #define MI_MEDIUM_OBJ_WSIZE_MAX           (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)   
-
 #define MI_LARGE_OBJ_SIZE_MAX             (MI_SEGMENT_SIZE/2)      // 32MiB on 64-bit
 #define MI_LARGE_OBJ_WSIZE_MAX            (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
-
 #define MI_HUGE_OBJ_SIZE_MAX              (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE)        // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)
 
 // Maximum number of size classes. (spaced exponentially in 12.5% increments)
@@ -307,17 +304,30 @@ typedef enum mi_segment_kind_e {
   MI_SEGMENT_HUGE,   // > MI_LARGE_SIZE_MAX segment with just one huge page inside.
 } mi_segment_kind_t;
 
-#define MI_COMMIT_SIZE    (MI_SEGMENT_SIZE/MI_SIZE_BITS)
+// ------------------------------------------------------
+// A segment holds a commit mask where a bit is set if
+// the corresponding MI_COMMIT_SIZE area is committed.
+// The MI_COMMIT_SIZE must be a multiple of the slice
+// size. If it is equal we have the most fine grained 
+// decommit (but in practice 2x seems to perform better).
+// ------------------------------------------------------
 
-#if (((1 << MI_SEGMENT_SHIFT)/MI_COMMIT_SIZE) > MI_SIZE_BITS)
+#define MI_COMMIT_SIZE              (MI_SEGMENT_SLICE_SIZE)   
-#error "not enough commit bits to cover the segment size"
+#define MI_COMMIT_MASK_BITS         (MI_SEGMENT_SIZE / MI_COMMIT_SIZE)  
+#define MI_COMMIT_MASK_FIELD_BITS    MI_SIZE_BITS
+#define MI_COMMIT_MASK_FIELD_COUNT  (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS)
+
+#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS))
+#error "the segment size must be exactly divisible by the (commit size * size_t bits)"
 #endif
 
-typedef mi_page_t  mi_slice_t;
+typedef struct mi_commit_mask_s {
+  size_t mask[MI_COMMIT_MASK_FIELD_COUNT];
+} mi_commit_mask_t;
 
+typedef mi_page_t  mi_slice_t;
 typedef int64_t    mi_msecs_t;
 
-typedef size_t     mi_commit_mask_t;
 
 // Segments are large allocated memory blocks (8mb on 64 bit) from
 // the OS. Inside segments we allocated fixed size _pages_ that

src/segment-cache.c

@@ -91,27 +91,26 @@ mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* comm
 
 static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats)
 {
-  if (mi_commit_mask_is_empty(*cmask)) {
+  if (mi_commit_mask_is_empty(cmask)) {
     // nothing
   }
-  else if (mi_commit_mask_is_full(*cmask)) {
+  else if (mi_commit_mask_is_full(cmask)) {
     _mi_os_decommit(p, total, stats);
   }
   else {
     // todo: one call to decommit the whole at once?
     mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
-    size_t    part = total/MI_COMMIT_MASK_BITS;
+    size_t part = total/MI_COMMIT_MASK_BITS;
-    uintptr_t idx;
+    size_t idx;
-    uintptr_t count;
+    size_t count;    
-    mi_commit_mask_t mask = *cmask;
+    mi_commit_mask_foreach(cmask, idx, count) {
-    mi_commit_mask_foreach(mask, idx, count) {
       void*  start = (uint8_t*)p + (idx*part);
       size_t size = count*part;
       _mi_os_decommit(start, size, stats);
     }
     mi_commit_mask_foreach_end()
   }
-  *cmask = mi_commit_mask_empty();
+  mi_commit_mask_create_empty(cmask);
 }
 
 #define MI_MAX_PURGE_PER_PUSH  (4)
@@ -136,12 +135,12 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld)
         if (expire != 0 && now >= expire) {  // safe read
           // still expired, decommit it
           mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
-          mi_assert_internal(!mi_commit_mask_is_empty(slot->commit_mask) && _mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx));
+          mi_assert_internal(!mi_commit_mask_is_empty(&slot->commit_mask) && _mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx));
           _mi_abandoned_await_readers();  // wait until safe to decommit
           // decommit committed parts
           // TODO: instead of decommit, we could also free to the OS?
           mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats);
-          slot->decommit_mask = mi_commit_mask_empty();
+          mi_commit_mask_create_empty(&slot->decommit_mask);
         }
         _mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop
       }
@@ -150,7 +149,7 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld)
   }
 }
 
-mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, mi_commit_mask_t commit_mask, mi_commit_mask_t decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld)
+mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld)
 {
 #ifdef MI_CACHE_DISABLE
   return false;
@@ -189,14 +188,14 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me
   slot->memid = memid;
   slot->is_pinned = is_pinned;
   mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
-  slot->commit_mask = commit_mask;
+  slot->commit_mask = *commit_mask;
-  slot->decommit_mask = decommit_mask;
+  slot->decommit_mask = *decommit_mask;
   if (!mi_commit_mask_is_empty(commit_mask) && !is_large && !is_pinned && mi_option_is_enabled(mi_option_allow_decommit)) {
     long delay = mi_option_get(mi_option_segment_decommit_delay);
     if (delay == 0) {
       _mi_abandoned_await_readers(); // wait until safe to decommit
       mi_commit_mask_decommit(&slot->commit_mask, start, MI_SEGMENT_SIZE, tld->stats);
-      slot->decommit_mask = mi_commit_mask_empty();
+      mi_commit_mask_create_empty(&slot->decommit_mask);
     }
     else {
       mi_atomic_storei64_release(&slot->expire, _mi_clock_now() + delay);

src/segment.c

@@ -15,6 +15,135 @@ terms of the MIT license. A copy of the license can be found in the file
 
 static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_stats_t* stats);
 
+
+// -------------------------------------------------------------------
+// commit mask 
+// -------------------------------------------------------------------
+
+static bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false;
+  }
+  return true;
+}
+
+static bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    if ((commit->mask[i] & cm->mask[i]) != 0) return true;
+  }
+  return false;
+}
+
+static void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    res->mask[i] = (commit->mask[i] & cm->mask[i]);
+  }
+}
+
+static void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    res->mask[i] &= ~(cm->mask[i]);
+  }
+}
+
+static void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    res->mask[i] |= cm->mask[i];
+  }
+}
+
+static void mi_commit_mask_create(size_t bitidx, size_t bitcount, mi_commit_mask_t* cm) {
+  mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
+  mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
+  if (bitcount == MI_COMMIT_MASK_BITS) {
+    mi_assert_internal(bitidx==0);
+    mi_commit_mask_create_full(cm);
+  }
+  else if (bitcount == 0) {
+    mi_commit_mask_create_empty(cm);
+  }
+  else {
+    mi_commit_mask_create_empty(cm);
+    size_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS;
+    size_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS;
+    while (bitcount > 0) {
+      mi_assert_internal(i < MI_COMMIT_MASK_FIELD_COUNT);
+      size_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs;
+      size_t count = (bitcount > avail ? avail : bitcount);
+      size_t mask = (count >= MI_COMMIT_MASK_FIELD_BITS ? ~((size_t)0) : (((size_t)1 << count) - 1) << ofs);
+      cm->mask[i] = mask;
+      bitcount -= count;
+      ofs = 0;
+      i++;
+    }
+  }
+}
+
+size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) {
+  mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
+  size_t count = 0;
+  for (size_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
+    size_t mask = cm->mask[i];
+    if (~mask == 0) {
+      count += MI_COMMIT_MASK_FIELD_BITS;
+    }
+    else {
+      for (; mask != 0; mask >>= 1) {  // todo: use popcount
+        if ((mask&1)!=0) count++;
+      }
+    }
+  }
+  // we use total since for huge segments each commit bit may represent a larger size
+  return ((total / MI_COMMIT_MASK_BITS) * count);
+}
+
+
+size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx) {
+  size_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS;
+  size_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS;
+  size_t mask = 0;
+  // find first ones
+  while (i < MI_COMMIT_MASK_FIELD_COUNT) {
+    mask = cm->mask[i];
+    mask >>= ofs;
+    if (mask != 0) {
+      while ((mask&1) == 0) {
+        mask >>= 1;
+        ofs++;
+      }
+      break;
+    }
+    i++;
+    ofs = 0;
+  }
+  if (i >= MI_COMMIT_MASK_FIELD_COUNT) {
+    // not found
+    *idx = MI_COMMIT_MASK_BITS;
+    return 0;
+  }
+  else {
+    // found, count ones
+    size_t count = 0;
+    *idx = (i*MI_COMMIT_MASK_FIELD_BITS) + ofs;
+    do {
+      mi_assert_internal(ofs < MI_COMMIT_MASK_FIELD_BITS && (mask&1) == 1);
+      do {
+        count++;
+        mask >>= 1;
+      } while ((mask&1) == 1);
+      if ((((*idx + count) % MI_COMMIT_MASK_FIELD_BITS) == 0)) {
+        i++;
+        if (i >= MI_COMMIT_MASK_FIELD_COUNT) break;
+        mask = cm->mask[i];
+        ofs = 0;
+      }
+    } while ((mask&1) == 1);
+    mi_assert_internal(count > 0);
+    return count;
+  }
+}
+
+
 /* --------------------------------------------------------------------------------
   Segment allocation
 
@@ -23,63 +152,6 @@ static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_st
   be reclaimed by still running threads, much like work-stealing.
 -------------------------------------------------------------------------------- */
 
-// -------------------------------------------------------------------
-// commit mask
-// -------------------------------------------------------------------
-
-static mi_commit_mask_t mi_commit_mask_create(uintptr_t bitidx, uintptr_t bitcount) {
-  mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
-  mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
-  if (bitcount == MI_COMMIT_MASK_BITS) {
-    mi_assert_internal(bitidx==0);
-    return mi_commit_mask_full();
-  }
-  else if (bitcount == 0) {
-    return mi_commit_mask_empty();
-  }
-  else {
-    return (((uintptr_t)1 << bitcount) - 1) << bitidx;
-  }
-}
-
-
-static bool mi_commit_mask_all_set(mi_commit_mask_t commit, mi_commit_mask_t mask) {
-  return ((commit & mask) == mask);
-}
-
-static bool mi_commit_mask_any_set(mi_commit_mask_t commit, mi_commit_mask_t mask) {
-  return ((commit & mask) != 0);
-}
-
-mi_decl_nodiscard static mi_commit_mask_t mi_commit_mask_intersect(mi_commit_mask_t commit, mi_commit_mask_t mask) {
-  return (commit & mask);
-}
-
-static void mi_commit_mask_clear(mi_commit_mask_t* commit, mi_commit_mask_t mask) {
-  *commit = (*commit) & (~mask);
-}
-
-static void mi_commit_mask_set(mi_commit_mask_t* commit, mi_commit_mask_t mask) {
-  *commit = (*commit) | mask;
-}
-
-size_t _mi_commit_mask_committed_size(mi_commit_mask_t mask, size_t total) {
-  if (mi_commit_mask_is_full(mask)) {
-    return total;
-  }
-  else if (mi_commit_mask_is_empty(mask)) {
-    return 0;
-  }
-  else {
-    size_t count = 0;
-    for (; mask != 0; mask >>= 1) {  // todo: use popcount
-      if ((mask&1)!=0) count++;
-    }
-    return (total/MI_COMMIT_MASK_BITS)*count;
-  }
-}
-
-
 
 /* -----------------------------------------------------------
    Slices
@@ -185,7 +257,7 @@ static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
   mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
   mi_assert_internal(segment->abandoned <= segment->used);
   mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id());
-  mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask)); // can only decommit committed blocks
+  mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask)); // can only decommit committed blocks
   //mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0);
   mi_slice_t* slice = &segment->slices[0];
   const mi_slice_t* end = mi_segment_slices_end(segment);
@@ -242,7 +314,7 @@ static size_t mi_segment_info_size(mi_segment_t* segment) {
 static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t xblock_size, size_t* page_size)
 {
   ptrdiff_t idx = slice - segment->slices;
-  size_t psize = slice->slice_count*MI_SEGMENT_SLICE_SIZE;
+  size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE;
   // make the start not OS page aligned for smaller blocks to avoid page/cache effects
   size_t start_offset = (xblock_size >= MI_INTPTR_SIZE && xblock_size <= 1024 ? MI_MAX_ALIGN_GUARANTEE : 0); 
   if (page_size != NULL) *page_size = psize - start_offset;
@@ -314,8 +386,8 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
   
   // _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
   const size_t size = mi_segment_size(segment);
-  if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, segment->commit_mask, segment->decommit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) {
+  if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, &segment->decommit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) {
-    const size_t csize = _mi_commit_mask_committed_size(segment->commit_mask, size);
+    const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size);
     if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize);
     _mi_abandoned_await_readers();  // wait until safe to free
     _mi_arena_free(segment, mi_segment_size(segment), segment->memid, segment->mem_is_pinned /* pretend not committed to not double count decommits */, tld->os);
@@ -389,10 +461,13 @@ void _mi_segment_thread_collect(mi_segments_tld_t* tld) {
    Span management
 ----------------------------------------------------------- */
 
-static mi_commit_mask_t mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size) {
+static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size, mi_commit_mask_t* cm) {
   mi_assert_internal(_mi_ptr_segment(p) == segment);
-  if (size == 0 || size > MI_SEGMENT_SIZE) return 0;
+  mi_assert_internal(segment->kind != MI_SEGMENT_HUGE);
-  if (p >= (uint8_t*)segment + mi_segment_size(segment)) return 0;
+  mi_commit_mask_create_empty(cm);
+  if (size == 0 || size > MI_SEGMENT_SIZE || segment->kind == MI_SEGMENT_HUGE) return;
+  const size_t segsize = mi_segment_size(segment);
+  if (p >= (uint8_t*)segment + segsize) return;
 
   size_t diff = (p - (uint8_t*)segment);
   size_t start;
@@ -405,58 +480,75 @@ static mi_commit_mask_t mi_segment_commit_mask(mi_segment_t* segment, bool conse
     start = _mi_align_down(diff, MI_COMMIT_SIZE);
     end   = _mi_align_up(diff + size, MI_COMMIT_SIZE);
   }
+  mi_assert_internal(end <= segsize);
+  if (end > segsize) {
+    end = segsize;
+  }
 
   mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0);
   *start_p   = (uint8_t*)segment + start;
   *full_size = (end > start ? end - start : 0);
-  if (*full_size == 0) return 0;
+  if (*full_size == 0) return;
 
   size_t bitidx = start / MI_COMMIT_SIZE;
   mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
   
   size_t bitcount = *full_size / MI_COMMIT_SIZE; // can be 0
-  if (bitidx + bitcount > MI_INTPTR_SIZE*8) {
+  if (bitidx + bitcount > MI_COMMIT_MASK_BITS) {
     _mi_warning_message("commit mask overflow: idx=%zu count=%zu start=%zx end=%zx p=0x%p size=%zu fullsize=%zu\n", bitidx, bitcount, start, end, p, size, *full_size);
   }
   mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
-
+  mi_commit_mask_create(bitidx, bitcount, cm);
-  return mi_commit_mask_create(bitidx, bitcount);
 }
 
-static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, size_t size, mi_stats_t* stats) {    
+#define MI_COMMIT_SIZE_BATCH  MiB
-  // commit liberal, but decommit conservative
-  uint8_t* start;
-  size_t   full_size;
-  mi_commit_mask_t mask = mi_segment_commit_mask(segment,!commit/*conservative*/,p,size,&start,&full_size);  
-  if (mi_commit_mask_is_empty(mask) || full_size==0) return true;
 
-  if (commit && !mi_commit_mask_all_set(segment->commit_mask, mask)) {
+static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, size_t size, mi_stats_t* stats) {    
+  mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask));
+
+  //if (commit && size < MI_COMMIT_SIZE_BATCH && p + MI_COMMIT_SIZE_BATCH <= mi_segment_end(segment)) {
+  //  size = MI_COMMIT_SIZE_BATCH;
+  // }
+  // commit liberal, but decommit conservative
+  uint8_t* start = NULL;
+  size_t   full_size = 0;
+  mi_commit_mask_t mask;
+  mi_segment_commit_mask(segment, !commit/*conservative*/, p, size, &start, &full_size, &mask);
+  if (mi_commit_mask_is_empty(&mask) || full_size==0) return true;
+
+  if (commit && !mi_commit_mask_all_set(&segment->commit_mask, &mask)) {
     bool is_zero = false;
-    mi_commit_mask_t cmask = mi_commit_mask_intersect(segment->commit_mask, mask);
+    mi_commit_mask_t cmask;
-    _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(cmask, MI_SEGMENT_SIZE)); // adjust for overlap
+    mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
+    _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
     if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false;    
-    mi_commit_mask_set(&segment->commit_mask,mask);     
+    mi_commit_mask_set(&segment->commit_mask, &mask);     
   }
-  else if (!commit && mi_commit_mask_any_set(segment->commit_mask,mask)) {
+  else if (!commit && mi_commit_mask_any_set(&segment->commit_mask, &mask)) {
     mi_assert_internal((void*)start != (void*)segment);
-    mi_commit_mask_t cmask = mi_commit_mask_intersect(segment->commit_mask, mask);
+    //mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &mask));
-    _mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(cmask, MI_SEGMENT_SIZE)); // adjust for overlap
+
-    if (segment->allow_decommit) { _mi_os_decommit(start, full_size, stats); } // ok if this fails
+    mi_commit_mask_t cmask;
-    mi_commit_mask_clear(&segment->commit_mask, mask);
+    mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
+    _mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
+    if (segment->allow_decommit) { 
+      _mi_os_decommit(start, full_size, stats); // ok if this fails
+    } 
+    mi_commit_mask_clear(&segment->commit_mask, &mask);
   }
   // increase expiration of reusing part of the delayed decommit
-  if (commit && mi_commit_mask_any_set(segment->decommit_mask, mask)) {
+  if (commit && mi_commit_mask_any_set(&segment->decommit_mask, &mask)) {
     segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay);
   }
   // always undo delayed decommits
-  mi_commit_mask_clear(&segment->decommit_mask, mask);   
+  mi_commit_mask_clear(&segment->decommit_mask, &mask);
-  mi_assert_internal((segment->commit_mask & segment->decommit_mask) == segment->decommit_mask);
   return true;
 }
 
 static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
-  mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask));
+  mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask));
-  if (mi_commit_mask_is_full(segment->commit_mask) && mi_commit_mask_is_empty(segment->decommit_mask)) return true; // fully committed
+  // note: assumes commit_mask is always full for huge segments as otherwise the commit mask bits can overflow
+  if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->decommit_mask)) return true; // fully committed
   return mi_segment_commitx(segment,true,p,size,stats);
 }
 
@@ -467,17 +559,21 @@ static void mi_segment_perhaps_decommit(mi_segment_t* segment, uint8_t* p, size_
   }
   else {
     // register for future decommit in the decommit mask
-    uint8_t* start;
+    uint8_t* start = NULL;
-    size_t   full_size;
+    size_t   full_size = 0;
-    mi_commit_mask_t mask = mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size);
+    mi_commit_mask_t mask; 
-    if (mi_commit_mask_is_empty(mask) || full_size==0) return;
+    mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size, &mask);
+    if (mi_commit_mask_is_empty(&mask) || full_size==0) return;
     
     // update delayed commit
-    mi_commit_mask_set(&segment->decommit_mask, mi_commit_mask_intersect(mask,segment->commit_mask));  // only decommit what is committed; span_free may try to decommit more
+    mi_commit_mask_t cmask;
+    mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);  // only decommit what is committed; span_free may try to decommit more
+    mi_commit_mask_set(&segment->decommit_mask, &cmask);
+    segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay);
     mi_msecs_t now = _mi_clock_now();
     if (segment->decommit_expire == 0) {
       // no previous decommits, initialize now
-      mi_assert_internal(mi_commit_mask_is_empty(segment->decommit_mask));
+      mi_assert_internal(mi_commit_mask_is_empty(&segment->decommit_mask));
       segment->decommit_expire = now + mi_option_get(mi_option_reset_delay);
     }
     else if (segment->decommit_expire <= now) {
@@ -487,23 +583,23 @@ static void mi_segment_perhaps_decommit(mi_segment_t* segment, uint8_t* p, size_
     }
     else {
       // previous decommit mask is not yet expired
-      // segment->decommit_expire++;
+      // segment->decommit_expire += 2; // = now + mi_option_get(mi_option_reset_delay);
     }
   }  
 }
 
 static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_stats_t* stats) {
-  if (!segment->allow_decommit || mi_commit_mask_is_empty(segment->decommit_mask)) return;
+  if (!segment->allow_decommit || mi_commit_mask_is_empty(&segment->decommit_mask)) return;
   mi_msecs_t now = _mi_clock_now();
   if (!force && now < segment->decommit_expire) return;
 
   mi_commit_mask_t mask = segment->decommit_mask;
   segment->decommit_expire = 0;
-  segment->decommit_mask = mi_commit_mask_empty();
+  mi_commit_mask_create_empty(&segment->decommit_mask);
 
   size_t idx;
   size_t count;
-  mi_commit_mask_foreach(mask, idx, count) {
+  mi_commit_mask_foreach(&mask, idx, count) {
     // if found, decommit that sequence
     if (count > 0) {
       uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE);
@@ -511,8 +607,7 @@ static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_st
       mi_segment_commitx(segment, false, p, size, stats);
     }
   }
-  mi_commit_mask_foreach_end()
+  mi_commit_mask_foreach_end()  
-  mi_assert_internal(mi_commit_mask_is_empty(segment->decommit_mask));
 }
 
 
@@ -723,8 +818,16 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
   // Try to get from our cache first
   bool is_zero = false;
   const bool commit_info_still_good = (segment != NULL);
-  mi_commit_mask_t commit_mask = (segment != NULL ? segment->commit_mask : mi_commit_mask_empty());
+  mi_commit_mask_t commit_mask;
-  mi_commit_mask_t decommit_mask = (segment != NULL ? segment->decommit_mask : mi_commit_mask_empty());
+  mi_commit_mask_t decommit_mask;
+  if (segment != NULL) {
+    commit_mask = segment->commit_mask;
+    decommit_mask = segment->decommit_mask;
+  }
+  else {
+    mi_commit_mask_create_empty(&commit_mask);
+    mi_commit_mask_create_empty(&decommit_mask);
+  }
   if (segment==NULL) {
     // Allocate the segment from the OS
     bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy    
@@ -734,23 +837,30 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
     if (segment==NULL) {
       segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld);
       if (segment == NULL) return NULL;  // failed to allocate
-      commit_mask = (commit ? mi_commit_mask_full() : mi_commit_mask_empty());
+      if (commit) {
+        mi_commit_mask_create_full(&commit_mask);
+      }
+      else {
+        mi_commit_mask_create_empty(&commit_mask);
+      }
     }    
     mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
 
     const size_t commit_needed = _mi_divide_up(info_slices*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE);
     mi_assert_internal(commit_needed>0);
-    if (!mi_commit_mask_all_set(commit_mask,mi_commit_mask_create(0, commit_needed))) {
+    mi_commit_mask_t commit_needed_mask;
+    mi_commit_mask_create(0, commit_needed, &commit_needed_mask);
+    if (!mi_commit_mask_all_set(&commit_mask, &commit_needed_mask)) {
       // at least commit the info slices
-      mi_assert_internal(commit_needed*MI_COMMIT_SIZE > info_slices*MI_SEGMENT_SLICE_SIZE);
+      mi_assert_internal(commit_needed*MI_COMMIT_SIZE >= info_slices*MI_SEGMENT_SLICE_SIZE);
       bool ok = _mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, &is_zero, tld->stats);
       if (!ok) return NULL; // failed to commit   
-      mi_commit_mask_set(&commit_mask,mi_commit_mask_create(0, commit_needed)); 
+      mi_commit_mask_set(&commit_mask, &commit_needed_mask); 
     }
     segment->memid = memid;
     segment->mem_is_pinned = is_pinned;
     segment->mem_is_large = mem_large;
-    segment->mem_is_committed = mi_commit_mask_is_full(commit_mask);
+    segment->mem_is_committed = mi_commit_mask_is_full(&commit_mask);
     mi_segments_track_size((long)(segment_size), tld);
     _mi_segment_map_allocated_at(segment);
   }
@@ -765,20 +875,22 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
 
   if (!commit_info_still_good) {
     segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed
-    segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_pinned && !segment->mem_is_large);
+    segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_pinned && !segment->mem_is_large);    
     if (segment->allow_decommit) {
       segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay);
       segment->decommit_mask = decommit_mask;
-      mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask));
+      mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask));
       #if MI_DEBUG>2
       const size_t commit_needed = _mi_divide_up(info_slices*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE);
-      mi_assert_internal(!mi_commit_mask_any_set(segment->decommit_mask, mi_commit_mask_create(0, commit_needed)));
+      mi_commit_mask_t commit_needed_mask;
+      mi_commit_mask_create(0, commit_needed, &commit_needed_mask);
+      mi_assert_internal(!mi_commit_mask_any_set(&segment->decommit_mask, &commit_needed_mask));
       #endif
     }    
     else {
-      mi_assert_internal(mi_commit_mask_is_empty(decommit_mask));
+      mi_assert_internal(mi_commit_mask_is_empty(&decommit_mask));
       segment->decommit_expire = 0;
-      segment->decommit_mask = mi_commit_mask_empty();
+      mi_commit_mask_create_empty( &segment->decommit_mask );
     }
   }
   
@@ -822,6 +934,8 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
   }
   else {
     mi_assert_internal(huge_page!=NULL);
+    mi_assert_internal(mi_commit_mask_is_empty(&segment->decommit_mask));
+    mi_assert_internal(mi_commit_mask_is_full(&segment->commit_mask));
     *huge_page = mi_segment_span_allocate(segment, info_slices, segment_slices - info_slices - guard_slices, tld);
     mi_assert_internal(*huge_page != NULL); // cannot fail as we commit in advance 
   }

test/main-override.cpp

@@ -52,7 +52,7 @@ int main() {
 
   //test_mt_shutdown();
   //fail_aslr();
-  //bench_alloc_large();
+  bench_alloc_large();
   mi_stats_print(NULL);
   return 0;
 }

test/test-stress.c

@@ -39,12 +39,12 @@ static size_t use_one_size = 0;              // use single object size of `N * s
 
 // #define USE_STD_MALLOC
 #ifdef USE_STD_MALLOC
-#define custom_calloc(n,s)    calloc(n,s)
+#define custom_calloc(n,s)    malloc(n*s)
 #define custom_realloc(p,s)   realloc(p,s)
 #define custom_free(p)        free(p)
 #else
 #include <mimalloc.h>
-#define custom_calloc(n,s)    mi_calloc(n,s)
+#define custom_calloc(n,s)    mi_malloc(n*s)
 #define custom_realloc(p,s)   mi_realloc(p,s)
 #define custom_free(p)        mi_free(p)
 #endif