Merge branch 'dev' into dev-exp

CMakeLists.txt

@@ -91,7 +91,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU")
   if(CMAKE_C_COMPILER_ID MATCHES "GNU")
     list(APPEND mi_cflags -Wno-invalid-memory-model)
     list(APPEND mi_cflags -fvisibility=hidden)
-    list(APPEND mi_cflags -fbranch-target-load-optimize )
+    list(APPEND mi_cflags -fbranch-target-load-optimize)
   endif()
 endif()
 

include/mimalloc-types.h

@@ -167,9 +167,9 @@ typedef struct mi_page_s {
   #if MI_SECURE
   uintptr_t             cookie;            // random cookie to encode the free lists
   #endif
-  size_t                used;              // number of blocks in use (including blocks in `local_free` and `thread_free`)
   mi_page_flags_t       flags;             // threadid:62 | has_aligned:1 | in_full:1
-
+  size_t                used;              // number of blocks in use (including blocks in `local_free` and `thread_free`)
+  
   mi_block_t*           local_free;        // list of deferred free blocks by this thread (migrates to `free`)
   volatile uintptr_t    thread_freed;      // at least this number of blocks are in `thread_free`
   volatile mi_thread_free_t thread_free;   // list of deferred free blocks freed by other threads
@@ -383,7 +383,8 @@ typedef struct mi_segments_tld_s {
 } mi_segments_tld_t;
 
 // OS thread local data
-typedef struct mi_os_tld_s {  
+typedef struct mi_os_tld_s {
+  size_t              region_idx;   // start point for next allocation
   mi_stats_t*         stats;        // points to tld stats
 } mi_os_tld_t;
 

src/alloc.c

@@ -72,7 +72,7 @@ extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcep
   void* p;
   if (mi_likely(size <= MI_SMALL_SIZE_MAX)) {
     p = mi_heap_malloc_small(heap, size);
-  }  
+  }
   else {
     p = _mi_malloc_generic(heap, size);
   }
@@ -200,16 +200,16 @@ static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, mi_pag
 
 // Free a block
 void mi_free(void* p) mi_attr_noexcept
-{  
+{
 #if (MI_DEBUG>0)
   if (mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)) {
     _mi_error_message("trying to free an invalid (unaligned) pointer: %p\n", p);
     return;
   }
 #endif
-  
+
   const mi_segment_t* const segment = _mi_ptr_segment(p);
-  if (segment == NULL) return;  // checks for (p==NULL) 
+  if (segment == NULL) return;  // checks for (p==NULL)
 
 #if (MI_DEBUG>0)
   if (mi_unlikely(!mi_is_in_heap_region(p))) {
@@ -224,8 +224,8 @@ void mi_free(void* p) mi_attr_noexcept
     return;
   }
 #endif
-  
-  mi_page_t* page = _mi_segment_page_of(segment, p);
+
+  mi_page_t* const page = _mi_segment_page_of(segment, p);
 
 #if (MI_STAT>1)
   mi_heap_t* heap = mi_heap_get_default();
@@ -236,11 +236,11 @@ void mi_free(void* p) mi_attr_noexcept
   // huge page stat is accounted for in `_mi_page_retire`
 #endif
 
-  uintptr_t tid = _mi_thread_id();
+  const uintptr_t tid = _mi_thread_id();
   if (mi_likely(tid == page->flags)) {  // if equal, the thread id matches and it is not a full page, nor has aligned blocks
     // local, and not full or aligned
     mi_block_t* block = (mi_block_t*)p;
-    mi_block_set_next(page, block, page->local_free);  
+    mi_block_set_next(page, block, page->local_free);
     page->local_free = block;
     page->used--;
     if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); }

src/init.c

@@ -12,16 +12,16 @@ terms of the MIT license. A copy of the license can be found in the file
 
 // Empty page used to initialize the small free pages array
 const mi_page_t _mi_page_empty = {
-  0, false, false, false, 0, 0, 
+  0, false, false, false, 0, 0,
   NULL,    // free
   #if MI_SECURE
   0,
   #endif
-  0, {0}, // used, flags
+  0, 0, // flags, used
   NULL, 0, 0,
   0, NULL, NULL, NULL
   #if (MI_INTPTR_SIZE==8 && MI_SECURE==0)
-  , { NULL } 
+  , { NULL }
   #endif
 };
 
@@ -97,7 +97,7 @@ static mi_tld_t tld_main = {
   0,
   &_mi_heap_main,
   { { NULL, NULL }, {NULL ,NULL}, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats }, // segments
-  { tld_main_stats },          // os
+  { 0, tld_main_stats },       // os
   { MI_STATS_NULL }            // stats
 };
 
@@ -421,7 +421,7 @@ static void mi_process_load(void) {
   // show message from the redirector (if present)
   const char* msg = NULL;
   mi_allocator_init(&msg);
-  if (msg != NULL) _mi_verbose_message(msg);  
+  if (msg != NULL) _mi_verbose_message(msg);
 }
 
 // Initialize the process; called by thread_init or the process loader
@@ -433,7 +433,7 @@ void mi_process_init(void) mi_attr_noexcept {
   // when using dynamic linking with interpose.
   mi_heap_t* h = _mi_heap_default;
   _mi_process_is_initialized = true;
-  
+
   _mi_heap_main.thread_id = _mi_thread_id();
   _mi_verbose_message("process init: 0x%zx\n", _mi_heap_main.thread_id);
   uintptr_t random = _mi_random_init(_mi_heap_main.thread_id)  ^ (uintptr_t)h;
@@ -442,7 +442,7 @@ void mi_process_init(void) mi_attr_noexcept {
   #endif
   _mi_heap_main.random = _mi_random_shuffle(random);
   mi_process_setup_auto_thread_done();
-  _mi_os_init();  
+  _mi_os_init();
   #if (MI_DEBUG)
   _mi_verbose_message("debug level : %d\n", MI_DEBUG);
   #endif
@@ -478,7 +478,7 @@ static void mi_process_done(void) {
   __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
     UNUSED(reserved);
     UNUSED(inst);
-    if (reason==DLL_PROCESS_ATTACH) {     
+    if (reason==DLL_PROCESS_ATTACH) {
       mi_process_load();
     }
     else if (reason==DLL_THREAD_DETACH) {

src/memory.c

@@ -79,7 +79,6 @@ typedef struct mem_region_s {
 static mem_region_t regions[MI_REGION_MAX];
 
 static volatile size_t regions_count = 0;        // allocated regions
-static volatile uintptr_t region_next_idx = 0;   // good place to start searching
 
 
 /* ----------------------------------------------------------------------------
@@ -180,12 +179,45 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
   }
 
   // and return the allocation
-  mi_atomic_write(&region_next_idx,idx);  // next search from here
   *p  = blocks_start;
   *id = (idx*MI_REGION_MAP_BITS) + bitidx;
   return true;
 }
 
+// Use bit scan forward to quickly find the first zero bit if it is available
+#if defined(_MSC_VER)
+#define MI_HAVE_BITSCAN
+#include <intrin.h>
+static inline size_t mi_bsf(uintptr_t x) {
+  if (x==0) return 8*MI_INTPTR_SIZE;
+  DWORD idx;
+  #if (MI_INTPTR_SIZE==8)
+  _BitScanForward64(&idx, x);
+  #else
+  _BitScanForward(&idx, x);
+  #endif
+  return idx;
+}
+static inline size_t mi_bsr(uintptr_t x) {
+  if (x==0) return 8*MI_INTPTR_SIZE;
+  DWORD idx;
+  #if (MI_INTPTR_SIZE==8)
+  _BitScanReverse64(&idx, x);
+  #else
+  _BitScanReverse(&idx, x);
+  #endif
+  return idx;
+}
+#elif defined(__GNUC__) || defined(__clang__)
+#define MI_HAVE_BITSCAN
+static inline size_t mi_bsf(uintptr_t x) {
+  return (x==0 ? 8*MI_INTPTR_SIZE : __builtin_ctzl(x));
+}
+static inline size_t mi_bsr(uintptr_t x) {
+  return (x==0 ? 8*MI_INTPTR_SIZE : (8*MI_INTPTR_SIZE - 1) - __builtin_clzl(x));
+}
+#endif
+
 // Allocate `blocks` in a `region` at `idx` of a given `size`. 
 // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
 // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. 
@@ -194,21 +226,28 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
 {
   mi_assert_internal(p != NULL && id != NULL);
   mi_assert_internal(blocks < MI_REGION_MAP_BITS);
-
-  const uintptr_t mask = mi_region_block_mask(blocks,0);
+  
+  const uintptr_t mask = mi_region_block_mask(blocks, 0);
   const size_t bitidx_max = MI_REGION_MAP_BITS - blocks;
-
-  // scan linearly for a free range of zero bits
   uintptr_t map = mi_atomic_read(&region->map);
-  uintptr_t m   = mask;    // the mask shifted by bitidx
-  for(size_t bitidx = 0; bitidx <= bitidx_max; bitidx++, m <<= 1) {
+
+  #ifdef MI_HAVE_BITSCAN
+  size_t bitidx = mi_bsf(~map);    // quickly find the first zero bit if possible
+  #else
+  size_t bitidx = 0;               // otherwise start at 0
+  #endif
+  uintptr_t m = (mask << bitidx);     // invariant: m == mask shifted by bitidx
+  
+  // scan linearly for a free range of zero bits
+  while(bitidx <= bitidx_max) {
     if ((map & m) == 0) {  // are the mask bits free at bitidx?
       mi_assert_internal((m >> bitidx) == mask); // no overflow?      
       uintptr_t newmap = map | m;
       mi_assert_internal((newmap^map) >> bitidx == mask);
       if (!mi_atomic_compare_exchange(&region->map, newmap, map)) {
         // no success, another thread claimed concurrently.. keep going
-        map = mi_atomic_read(&region->map);        
+        map = mi_atomic_read(&region->map);   
+        continue;
       }
       else {
         // success, we claimed the bits
@@ -216,6 +255,17 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
         return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, p, id, tld);
       }
     }
+    else {
+      // on to the next bit range
+      #ifdef MI_HAVE_BITSCAN
+      size_t shift = (blocks == 1 ? 1 : mi_bsr(map & m) - bitidx + 1);
+      mi_assert_internal(shift > 0 && shift <= blocks);
+      #else
+      size_t shift = 1;      
+      #endif
+      bitidx += shift;
+      m <<= shift;    
+    }
   }
   // no error, but also no bits found
   return true;  
@@ -267,7 +317,7 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t*
   // find a range of free blocks
   void* p = NULL;
   size_t count = mi_atomic_read(&regions_count);
-  size_t idx = mi_atomic_read(&region_next_idx);
+  size_t idx = tld->region_idx; // start index is per-thread to reduce contention
   for (size_t visited = 0; visited < count; visited++, idx++) {
     if (idx >= count) idx = 0;  // wrap around
     if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error
@@ -286,6 +336,9 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t*
     // we could not find a place to allocate, fall back to the os directly
     p = _mi_os_alloc_aligned(size, alignment, commit, tld);
   }
+  else {
+    tld->region_idx = idx;  // next start of search
+  }
 
   mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0);
   return p;