bump version to v3.1.6 for further development

cmake/mimalloc-config-version.cmake

@@ -1,6 +1,6 @@
 set(mi_version_major 3)
 set(mi_version_minor 1)
-set(mi_version_patch 5)
+set(mi_version_patch 6)
 set(mi_version ${mi_version_major}.${mi_version_minor})
 
 set(PACKAGE_VERSION ${mi_version})

contrib/vcpkg/portfile.cmake

@@ -3,13 +3,13 @@ vcpkg_from_github(
   REPO microsoft/mimalloc
   HEAD_REF master
 
-  # The "REF" can be a commit hash, branch name (dev2), or a version (v2.2.1).
+  # The `REF` can be a commit hash, branch name (`dev3`), or a version (`v3.1.4`).
+  # Use `"v${VERSION}"` to use the version specified in `vcpkg.json`.
   REF "v${VERSION}"
-  # REF 6a89f8554eaab8d8d00e17b5b09f79e1d8dbf61b
   
   # The sha512 is the hash of the tar.gz bundle.
   # (To get the sha512, run `vcpkg install "mimalloc[override]" --overlay-ports=./contrib/vcpkg` and copy the sha from the error message.)
-  SHA512 1e1268292283eb3c07a1f4de1bb919ebfeb18cbd4af0ca01d197d4dc46972ea127527e95e18f73381bda03593bcca5d3e4744c3ab49876c68567abebff030690
+  SHA512 616351e549707318c1f8b164251141684a73d5bf8205b905736f48ab21fbb19bfaa4d52c4e63642fcb144345b6a5331944b6c8e0827925000553e46f2c2c31e9
 )
 
 vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS

contrib/vcpkg/vcpkg.json

@@ -1,6 +1,6 @@
 {
   "name": "mimalloc",
-  "version": "3.1.4",
+  "version": "3.1.5",
   "port-version": 2,
   "description": "Compact general purpose allocator with excellent performance",
   "homepage": "https://github.com/microsoft/mimalloc",

include/mimalloc.h

@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #ifndef MIMALLOC_H
 #define MIMALLOC_H
 
-#define MI_MALLOC_VERSION 315   // major + 2 digits minor
+#define MI_MALLOC_VERSION 316   // major + 2 digits minor
 
 // ------------------------------------------------------
 // Compiler specific attributes

include/mimalloc/bits.h

@@ -209,7 +209,7 @@ static inline size_t mi_popcount(size_t x) {
     return mi_builtinz(popcount)(x);
   #else
     #define MI_HAS_FAST_POPCOUNT  0
-    return (x<=1 ? x : _mi_popcount_generic(x));
+    return _mi_popcount_generic(x);
   #endif
 }
 

src/arena.c

@@ -189,24 +189,25 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
 
   // set commit state
   if (commit) {        
-    memid->initially_committed = true;
-
     // commit requested, but the range may not be committed as a whole: ensure it is committed now
-    if (!mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)) {
-      // not fully committed: commit the full range and set the commit bits
-      // we set the bits first since we own these slices (they are no longer free)
-      size_t already_committed_count = 0;
-      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
-      // now actually commit
+    const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count);
+    if (already_committed < slice_count) {  
+      // not all committed, try to commit now
       bool commit_zero = false;
-      if (!mi_arena_commit(arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed_count))) {
-        // if the commit fails, we roll back
-        _mi_arenas_free( p, mi_size_of_slices(slice_count), *memid);  // this will uncommit as well
+      if (!_mi_os_commit_ex(p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) {
+        // if the commit fails, release ownership, and return NULL;
+        // note: this does not roll back dirty bits but that is ok.
+        mi_bbitmap_setN(arena->slices_free, slice_index, slice_count);
         return NULL;
       }
+      if (commit_zero) { 
+        memid->initially_zero = true; 
+      }
+            
+      // set the commit bits 
+      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL);      
       
       // committed
-      if (commit_zero) { memid->initially_zero = true; }
       #if MI_DEBUG > 1
       if (memid->initially_zero) {
         if (!mi_mem_is_zero(p, mi_size_of_slices(slice_count))) {
@@ -225,6 +226,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
         mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices));
       }      
     }
+    
+    mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
+    memid->initially_committed = true;
+    
     // tool support
     if (memid->initially_zero) {
       mi_track_mem_defined(p, slice_count * MI_ARENA_SLICE_SIZE);

src/bitmap.c

@@ -256,6 +256,11 @@ static inline bool mi_bfield_atomic_is_xset_mask(mi_xset_t set, const _Atomic(mi
       else return mi_bfield_atomic_is_clear_mask(b, mask);
 }
 
+// Count bits in a mask
+static inline size_t mi_bfield_atomic_popcount_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask) {
+  const mi_bfield_t x = mi_atomic_load_relaxed(b);
+  return mi_bfield_popcount(x & mask);
+}
 
 
 /* --------------------------------------------------------------------------------
@@ -366,6 +371,38 @@ static inline bool mi_bchunk_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, b
   return mi_bchunk_xsetN_(MI_BIT_CLEAR, chunk, cidx, n, NULL, maybe_all_clear);
 }
 
+// Check if a sequence of `n` bits within a chunk are all set/cleared.
+// This can cross bfield's
+mi_decl_noinline static size_t mi_bchunk_popcountN_(mi_bchunk_t* chunk, size_t field_idx, size_t idx, size_t n) {
+  mi_assert_internal((field_idx*MI_BFIELD_BITS) + idx + n <= MI_BCHUNK_BITS);
+  size_t count = 0;
+  while (n > 0) {
+    size_t m = MI_BFIELD_BITS - idx;   // m is the bits to xset in this field
+    if (m > n) { m = n; }
+    mi_assert_internal(idx + m <= MI_BFIELD_BITS);
+    mi_assert_internal(field_idx < MI_BCHUNK_FIELDS);
+    const size_t mask = mi_bfield_mask(m, idx);
+    count += mi_bfield_atomic_popcount_mask(&chunk->bfields[field_idx], mask);
+    // next field
+    field_idx++;
+    idx = 0;
+    n -= m;
+  }
+  return count;
+}
+
+// Count set bits a sequence of `n` bits.
+static inline size_t mi_bchunk_popcountN(mi_bchunk_t* chunk, size_t cidx, size_t n) {
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);
+  mi_assert_internal(n>0);
+  if (n==0) return 0;
+  const size_t i = cidx / MI_BFIELD_BITS;
+  const size_t idx = cidx % MI_BFIELD_BITS;
+  if (n==1) { return (mi_bfield_atomic_is_set(&chunk->bfields[i], idx) ? 1 : 0); }
+  if (idx + n <= MI_BFIELD_BITS) { return mi_bfield_atomic_popcount_mask(&chunk->bfields[i], mi_bfield_mask(n, idx)); }
+  return mi_bchunk_popcountN_(chunk, i, idx, n);
+}
+
 
 // ------- mi_bchunk_is_xset ---------------------------------------
 
@@ -1110,6 +1147,20 @@ bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
   return were_allset;
 }
 
+// Count bits set in a range of `n` bits.
+// `n` cannot cross chunk boundaries (and `n <= MI_BCHUNK_BITS`)!
+size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n) {
+  mi_assert_internal(n>0);
+  mi_assert_internal(n<=MI_BCHUNK_BITS);
+
+  const size_t chunk_idx = idx / MI_BCHUNK_BITS;
+  const size_t cidx = idx % MI_BCHUNK_BITS;
+  mi_assert_internal(cidx + n <= MI_BCHUNK_BITS);  // don't cross chunks (for now)
+  mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
+  if (cidx + n > MI_BCHUNK_BITS) { n = MI_BCHUNK_BITS - cidx; }  // paranoia
+  return mi_bchunk_popcountN(&bitmap->chunks[chunk_idx], cidx, n);
+}
+
 
 // Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0)
 bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) {

src/bitmap.h

@@ -219,6 +219,9 @@ bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_a
 bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg);
 
 
+// Count all set bits in given range in the bitmap. (cannot cross chunks)
+size_t mi_bitmap_popcountN( mi_bitmap_t* bitmap, size_t idx, size_t n);
+
 /* ----------------------------------------------------------------------------
   Binned concurrent bitmap
   Assigns a size class to each chunk such that small blocks don't cause too

src/libc.c

@@ -383,6 +383,8 @@ static size_t mi_popcount_generic32(uint32_t x) {
 }
 
 mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
+  if (x<=1) return x;
+  if (~x==0) return MI_SIZE_BITS;
   return mi_popcount_generic32(x);
 }
 
@@ -407,6 +409,8 @@ static size_t mi_popcount_generic64(uint64_t x) {
 }
 
 mi_decl_noinline size_t _mi_popcount_generic(size_t x) {
+  if (x<=1) return x;
+  if (~x==0) return MI_SIZE_BITS;
   return mi_popcount_generic64(x);
 }
 #endif

src/os.c

@@ -471,7 +471,6 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
 
 bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
   if (is_zero != NULL) { *is_zero = false; }
-  mi_os_stat_increase(committed, stat_size);  // use size for precise commit vs. decommit
   mi_os_stat_counter_increase(commit_calls, 1);
 
   // page align range
@@ -495,6 +494,7 @@ bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size)
   if (os_is_zero) { mi_track_mem_defined(start,csize); }
              else { mi_track_mem_undefined(start,csize); }
   #endif
+  mi_os_stat_increase(committed, stat_size);  // use size for precise commit vs. decommit
   return true;
 }