space out threads when searching for free pages

CMakeLists.txt

@@ -15,6 +15,7 @@ option(MI_TRACK_ASAN        "Compile with address sanitizer support (adds a smal
 option(MI_TRACK_ETW         "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF)
 option(MI_USE_CXX           "Use the C++ compiler to compile the library (instead of the C compiler)" OFF)
 option(MI_OPT_ARCH          "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" ON)
+option(MI_OPT_SIMD          "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF)
 option(MI_SEE_ASM           "Generate assembly files" OFF)
 option(MI_OSX_INTERPOSE     "Use interpose to override standard malloc on macOS" ON)
 option(MI_OSX_ZONE          "Use malloc zone to override standard malloc on macOS" ON)
@@ -227,7 +228,7 @@ endif()
 if(MI_SEE_ASM)
   message(STATUS "Generate assembly listings (MI_SEE_ASM=ON)")
   list(APPEND mi_cflags -save-temps)
-  if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang")
+  if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER 14)
     message(STATUS "No GNU Line marker")
     list(APPEND mi_cflags -Wno-gnu-line-marker)
   endif()
@@ -330,10 +331,10 @@ endif()
 # Determine architecture
 set(MI_OPT_ARCH_FLAGS "")
 set(MI_ARCH "")
-if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR 
+if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR
    CMAKE_GENERATOR_PLATFORM STREQUAL "x64")          # msvc
   set(MI_ARCH "x64")
-elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR   
+elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR
        CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64" OR    # apple
        CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64")    # msvc
   set(MI_ARCH "arm64")
@@ -419,6 +420,12 @@ endif()
 if(MI_OPT_ARCH_FLAGS)
   list(APPEND mi_cflags ${MI_OPT_ARCH_FLAGS})
   message(STATUS "Architecture specific optimization is enabled (with ${MI_OPT_ARCH_FLAGS}) (MI_OPT_ARCH=ON)")
+  if (MI_OPT_SIMD)
+    list(APPEND mi_defines "MI_OPT_SIMD=1")
+    message(STATUS "SIMD instructions are enabled (MI_OPT_SIMD=ON)")
+  endif()
+elseif(MI_OPT_SIMD)
+  message(STATUS "SIMD instructions are not enabled (either MI_OPT_ARCH=OFF or this architecture has no SIMD support)")
 endif()
 
 # extra needed libraries

src/arena.c

@@ -36,7 +36,7 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
 typedef struct mi_arena_s {
   mi_memid_t          memid;                // memid of the memory area
   mi_arena_id_t       id;                   // arena id (> 0 where `arena == arenas[arena->id - 1]`)
-  
+
   size_t              slice_count;          // size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
   size_t              info_slices;          // initial slices reserved for the arena bitmaps
   int                 numa_node;            // associated NUMA node
@@ -165,7 +165,7 @@ static mi_memid_t mi_memid_create_arena(mi_arena_t* arena, size_t slice_index, s
   mi_memid_t memid = _mi_memid_create(MI_MEM_ARENA);
   memid.mem.arena.arena = arena;
   memid.mem.arena.slice_index = (uint32_t)slice_index;
-  memid.mem.arena.slice_count = (uint32_t)slice_count;  
+  memid.mem.arena.slice_count = (uint32_t)slice_count;
   return memid;
 }
 
@@ -562,7 +562,7 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
       mi_atomic_decrement_relaxed(&subproc->abandoned_count[bin]);
       _mi_stat_decrease(&_mi_stats_main.pages_abandoned, 1);
       _mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_alloc, 1);
-      
+
       _mi_page_free_collect(page, false);  // update `used` count
       mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
       mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
@@ -770,7 +770,7 @@ void _mi_arena_page_free(mi_page_t* page) {
     mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
     mi_assert_internal(mi_bitmap_is_clearN(arena->slices_purge, slice_index, slice_count));
     mi_assert_internal(mi_bitmap_is_clearN(arena->pages_abandoned[bin], slice_index, 1));
-    mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1));    
+    mi_assert_internal(mi_bitmap_is_setN(page->memid.mem.arena.arena->pages, page->memid.mem.arena.slice_index, 1));
     // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
     // be (temporarily) not true if the free happens while trying to reclaim
     // see `mi_arana_try_claim_abandoned`
@@ -891,10 +891,9 @@ static void mi_arena_schedule_purge(mi_arena_t* arena, size_t slice_index, size_
 static void mi_arenas_try_purge(bool force, bool visit_all);
 
 static void mi_arena_free(void* p, size_t size, mi_memid_t memid) {
-  mi_assert_internal(size >= 0);
   if (p==NULL) return;
   if (size==0) return;
-  
+
   // need to set all memory to undefined as some parts may still be marked as no_access (like padding etc.)
   mi_track_mem_undefined(p, size);
 
@@ -981,7 +980,7 @@ static void mi_arenas_unsafe_destroy(void) {
     if (arena != NULL) {
       // mi_lock_done(&arena->abandoned_visit_lock);
       mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL);
-      if (mi_memkind_is_os(arena->memid.memkind)) {        
+      if (mi_memkind_is_os(arena->memid.memkind)) {
         _mi_os_free(mi_arena_start(arena), mi_arena_size(arena), arena->memid);
       }
     }
@@ -1457,12 +1456,12 @@ mi_decl_export bool mi_arena_unload(mi_arena_id_t arena_id, void** base, size_t*
     asize = mi_arena_info_slices(arena) * MI_ARENA_SLICE_SIZE;
   }
   if (base != NULL) { *base = (void*)arena; }
-  if (full_size != NULL) { *full_size = arena->memid.mem.os.size;  }  
+  if (full_size != NULL) { *full_size = arena->memid.mem.os.size;  }
   if (accessed_size != NULL) { *accessed_size = asize; }
 
-  // unregister the pages 
+  // unregister the pages
   _mi_page_map_unregister_range(arena, asize);
-  
+
   // set the entry to NULL
   mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[arena_idx], NULL);
   if (arena_idx + 1 == count) { // try adjust the count?

src/bitmap.c

@@ -14,7 +14,9 @@ Concurrent bitmap that can set/reset sequences of bits atomically
 #include "mimalloc/bits.h"
 #include "bitmap.h"
 
-#define MI_USE_SIMD   0
+#ifndef MI_OPT_SIMD
+#define MI_OPT_SIMD   0
+#endif
 
 /* --------------------------------------------------------------------------------
   bfields
@@ -24,11 +26,15 @@ static inline size_t mi_bfield_ctz(mi_bfield_t x) {
   return mi_ctz(x);
 }
 
-
 static inline size_t mi_bfield_popcount(mi_bfield_t x) {
   return mi_popcount(x);
 }
 
+static inline mi_bfield_t mi_bfield_clear_least_bit(mi_bfield_t x) {
+  return (x & (x-1));
+}
+
+
 // find the least significant bit that is set (i.e. count trailing zero's)
 // return false if `x==0` (with `*idx` undefined) and true otherwise,
 // with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
@@ -156,16 +162,6 @@ static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b) {
 // ------- mi_bfield_atomic_try_xset ---------------------------------------
 
 
-// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0.
-// `all_clear` is set to true if the new bfield is zero (and false otherwise)
-static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
-  mi_assert_internal(idx < MI_BFIELD_BITS);
-  const mi_bfield_t mask = mi_bfield_one()<<idx;
-  const mi_bfield_t old = mi_atomic_and_acq_rel(b, ~mask);
-  if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
-  return ((old&mask) == mask);
-}
-
 // Tries to  set a mask atomically, and returns true if the mask bits atomically transitioned from 0 to mask
 // and false otherwise (leaving the bit field as is).
 static inline bool mi_bfield_atomic_try_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask) {
@@ -194,6 +190,17 @@ static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)*b, mi_bf
 }
 
 
+// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0.
+// `all_clear` is set to true if the new bfield is zero (and false otherwise)
+static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)*b, size_t idx, bool* all_clear) {
+  mi_assert_internal(idx < MI_BFIELD_BITS);
+  const mi_bfield_t mask = mi_bfield_one()<<idx;
+  return mi_bfield_atomic_try_clear_mask(b,mask,all_clear);
+  // const mi_bfield_t old = mi_atomic_and_acq_rel(b, ~mask);
+  // if (all_clear != NULL) { *all_clear = ((old&~mask)==0); }
+  // return ((old&mask) == mask);
+}
+
 // Tries to (un)set a mask atomically, and returns true if the mask bits atomically transitioned from 0 to mask (or mask to 0)
 // and false otherwise (leaving the bit field as is).
 static inline bool mi_bfield_atomic_try_xset_mask(mi_xset_t set, _Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear ) {
@@ -458,7 +465,7 @@ static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t
 
 // ------- mi_bchunk_try_find_and_clear ---------------------------------------
 
-#if MI_USE_SIMD && defined(__AVX2__) 
+#if MI_OPT_SIMD && defined(__AVX2__)
 static inline __m256i mi_mm256_zero(void) {
   return _mm256_setzero_si256();
 }
@@ -493,7 +500,7 @@ static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t ch
 // This is used to find free slices and abandoned pages and should be efficient.
 // todo: try neon version
 static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx) {
-  #if MI_USE_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
   while (true) {
     const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
     const __m256i vcmp = _mm256_cmpeq_epi64(vec, mi_mm256_zero()); // (elem64 == 0 ? 0xFF  : 0)
@@ -502,10 +509,10 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx
     if (mask==0) return false;
     mi_assert_internal((_tzcnt_u32(mask)%8) == 0); // tzcnt == 0, 8, 16, or 24
     const size_t chunk_idx = _tzcnt_u32(mask) / 8;
-    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;    
+    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx, true)) return true;
     // try again
   }
-  #elif MI_USE_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  #elif MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
   while (true) {
     size_t chunk_idx = 0;
     #if 0
@@ -534,9 +541,9 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx
     // mask is inverted, so each 8-bits is 0xFF iff the corresponding elem64 has a bit set (and thus can be cleared)
     if (mask==0) return false;
     mi_assert_internal((_tzcnt_u64(mask)%8) == 0); // tzcnt == 0, 8, 16, 24 , ..
-    chunk_idx = _tzcnt_u64(mask) / 8;
+    chunk_idx = mi_ctz(mask) / 8;
     #endif
-    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx)) return true;
+    if (mi_bchunk_try_find_and_clear_at(chunk, chunk_idx, pidx, true)) return true;
     // try again
   }
   #else
@@ -551,12 +558,17 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx
   #endif
 }
 
+static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  mi_assert_internal(n==1); MI_UNUSED(n);
+  return mi_bchunk_try_find_and_clear(chunk, pidx);
+}
 
+#if !MI_OPT_SIMD
 static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool allow_all_set) {
   const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]);
   if (!allow_all_set && (~b == 0)) return false;
   // has_set8 has low bit in each byte set if the byte in x == 0xFF
-  const mi_bfield_t has_set8 = 
+  const mi_bfield_t has_set8 =
     ((~b - MI_BFIELD_LO_BIT8) &      // high bit set if byte in x is 0xFF or < 0x7F
      (b  & MI_BFIELD_HI_BIT8))       // high bit set if byte in x is >= 0x80
      >> 7;                           // shift high bit to low bit
@@ -573,13 +585,14 @@ static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t c
   }
   return false;
 }
+#endif
 
 // find least byte in a chunk with all bits set, and try unset it atomically
 // set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success.
 // Used to find medium size pages in the free blocks.
 // todo: try neon version
 static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) {
-  #if MI_USE_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
   while (true) {
     // since a cache-line is 64b, load all at once
     const __m256i vec1  = _mm256_load_si256((const __m256i*)chunk->bfields);
@@ -615,6 +628,10 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, s
   #endif
 }
 
+static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  mi_assert_internal(n==8); MI_UNUSED(n);
+  return mi_bchunk_try_find_and_clear8(chunk, pidx);
+}
 
 
 // find least bfield in a chunk with all bits set, and try unset it atomically
@@ -622,7 +639,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, s
 // Used to find large size pages in the free blocks.
 // todo: try neon version
 static mi_decl_noinline  bool mi_bchunk_try_find_and_clearX(mi_bchunk_t* chunk, size_t* pidx) {
-  #if MI_USE_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512)
   while (true) {
     // since a cache-line is 64b, load all at once
     const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
@@ -658,6 +675,10 @@ static mi_decl_noinline  bool mi_bchunk_try_find_and_clearX(mi_bchunk_t* chunk,
 #endif
 }
 
+static inline bool mi_bchunk_try_find_and_clear_X(mi_bchunk_t* chunk, size_t n, size_t* pidx) {
+  mi_assert_internal(n==MI_BFIELD_BITS); MI_UNUSED(n);
+  return mi_bchunk_try_find_and_clearX(chunk, pidx);
+}
 
 // find a sequence of `n` bits in a chunk with `n < MI_BFIELD_BITS` with all bits set,
 // and try to clear them atomically.
@@ -783,10 +804,10 @@ static inline bool mi_bchunk_all_are_clear(mi_bchunk_t* chunk) {
 
 // are all bits in a bitmap chunk clear?
 static inline bool mi_bchunk_all_are_clear_relaxed(mi_bchunk_t* chunk) {
-  #if MI_USE_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
+  #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==256)
   const __m256i vec = _mm256_load_si256((const __m256i*)chunk->bfields);
   return mi_mm256_is_zero(vec);
-  #elif MI_USE_SIMD &&  defined(__AVX2__) && (MI_BCHUNK_BITS==512)
+  #elif MI_OPT_SIMD &&  defined(__AVX2__) && (MI_BCHUNK_BITS==512)
   // a 64b cache-line contains the entire chunk anyway so load both at once
   const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields);
   const __m256i vec2 = _mm256_load_si256(((const __m256i*)chunk->bfields)+1);
@@ -835,7 +856,7 @@ static bool mi_bitmap_chunkmap_try_clear(mi_bitmap_t* bitmap, size_t chunk_idx)
   mi_bchunk_clear(&bitmap->chunkmap, chunk_idx, NULL);
   // .. but a concurrent set may have happened in between our all-clear test and the clearing of the
   // bit in the mask. We check again to catch this situation.
-  if (!mi_bchunk_all_are_clear(&bitmap->chunks[chunk_idx])) {
+  if (!mi_bchunk_all_are_clear_relaxed(&bitmap->chunks[chunk_idx])) {
     mi_bchunk_set(&bitmap->chunkmap, chunk_idx);
     return false;
   }
@@ -1043,11 +1064,129 @@ bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n
 
 
 
+/* --------------------------------------------------------------------------------
+  Iterate through a bfield
+-------------------------------------------------------------------------------- */
+
+// Cycle iteration through a bitfield. This is used to space out threads
+// so there is less chance of contention. When searching for a free page we
+// like to first search only the accessed part (so we reuse better). This
+// high point is called the `cycle`.
+//
+// We then iterate through the bitfield as:
+// first: [start, cycle>
+// then : [0, start>
+// then : [cycle, MI_BFIELD_BITS>
+//
+// The start is determined usually as `tseq % cycle` to have each thread
+// start at a different spot.
+// - We use `popcount` to improve branch prediction`
+// - The `cycle_mask` is the part `[start, cycle>`.
+#define mi_bfield_iterate(bfield,start,cycle,name_idx,SUF) { \
+  mi_assert_internal(start <= cycle); \
+  mi_assert_internal(start < MI_BFIELD_BITS); \
+  mi_assert_internal(cycle < MI_BFIELD_BITS); \
+  mi_bfield_t _cycle_mask##SUF = mi_bfield_mask(cycle - start, start); \
+  size_t _bcount##SUF = mi_bfield_popcount(bfield); \
+  mi_bfield_t _b##SUF = bfield & _cycle_mask##SUF; /* process [start, cycle> first*/\
+  while(_bcount##SUF > 0) { \
+    _bcount##SUF--;\
+    if (_b##SUF==0) { _b##SUF = bfield & ~_cycle_mask##SUF; } /* process [0,start> + [cycle, MI_BFIELD_BITS> next */ \
+    size_t name_idx; \
+    bool _found##SUF = mi_bfield_find_least_bit(_b##SUF,&name_idx); \
+    mi_assert_internal(_found##SUF); MI_UNUSED(_found##SUF); \
+    { \
+
+#define mi_bfield_iterate_end(SUF) \
+    } \
+    _b##SUF = mi_bfield_clear_least_bit(_b##SUF); \
+  } \
+}
+
+#define mi_bfield_cycle_iterate(bfield,tseq,cycle,name_idx,SUF) { \
+  const size_t _start##SUF = (uint32_t)(tseq) % (uint32_t)(cycle); \
+  mi_bfield_iterate(bfield,_start##SUF,cycle,name_idx,SUF)
+
+#define mi_bfield_cycle_iterate_end(SUF) \
+  mi_bfield_iterate_end(SUF); }
+
+
 /* --------------------------------------------------------------------------------
   bitmap  try_find_and_clear
   (used to find free pages)
 -------------------------------------------------------------------------------- */
 
+
+typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx);
+
+static inline bool mi_bitmap_try_find_and_clear_generic(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* try_find_and_clear)
+{
+  // we space out threads to reduce contention
+  const size_t cmap_max_count  = _mi_divide_up(mi_bitmap_chunk_count(bitmap),MI_BFIELD_BITS);
+  const size_t chunk_acc       = mi_atomic_load_relaxed(&bitmap->chunk_max_accessed);
+  const size_t cmap_acc        = chunk_acc / MI_BFIELD_BITS;
+  const size_t cmap_acc_bits   = 1 + (chunk_acc % MI_BFIELD_BITS);
+
+  // create a mask over the chunkmap entries to iterate over them efficiently
+  mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS);
+  const mi_bfield_t cmap_mask  = mi_bfield_mask(cmap_max_count,0);
+  const size_t cmap_cycle      = cmap_acc+1;
+  mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X)
+  {
+    // and for each chunkmap entry we iterate over its bits to find the chunks
+    mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[cmap_idx]);
+    size_t cmap_entry_cycle = (cmap_idx != cmap_acc ? MI_BFIELD_BITS : cmap_acc_bits);
+    mi_bfield_cycle_iterate(cmap_entry, tseq, cmap_entry_cycle, eidx, Y)
+    {
+      mi_assert_internal(eidx <= MI_BFIELD_BITS);
+      const size_t chunk_idx = cmap_idx*MI_BFIELD_BITS + eidx;
+      mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap));
+      size_t cidx;
+      // if we find a spot in the chunk we are done
+      if ((*try_find_and_clear)(&bitmap->chunks[chunk_idx], n, &cidx)) {
+        *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+        mi_assert_internal(*pidx + n <= mi_bitmap_max_bits(bitmap));
+        return true;
+      }
+      else {
+        /* we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation. */
+        mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx);
+      }
+    }
+    mi_bfield_cycle_iterate_end(Y);
+  }
+  mi_bfield_cycle_iterate_end(X);
+  return false;
+}
+
+mi_decl_nodiscard bool mi_bitmap_try_find_and_clear(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) {
+  return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1);
+}
+
+mi_decl_nodiscard bool mi_bitmap_try_find_and_clear8(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) {
+  return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8);
+}
+
+mi_decl_nodiscard bool mi_bitmap_try_find_and_clearX(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) {
+  return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X);
+}
+
+mi_decl_nodiscard bool mi_bitmap_try_find_and_clearNX(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx) {
+  mi_assert_internal(n<=MI_BFIELD_BITS);
+  return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX);
+}
+
+mi_decl_nodiscard bool mi_bitmap_try_find_and_clearN_(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx) {
+  mi_assert_internal(n<=MI_BCHUNK_BITS);
+  return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearN_);
+}
+
+
+/* --------------------------------------------------------------------------------
+  bitmap  try_find_and_claim
+  (used to allocate abandoned pages)
+-------------------------------------------------------------------------------- */
+
 #define mi_bitmap_forall_chunks(bitmap, tseq, name_chunk_idx) \
   { \
   /* start chunk index -- todo: can depend on the tseq to decrease contention between threads */ \
@@ -1084,53 +1223,6 @@ bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n
     } \
   }}
 
-
-#define mi_bitmap_forall_chunks_try_find_and_clear(bitmap, tseq, pidx, NSUF, NPAR) { \
-  mi_bitmap_forall_chunks(bitmap, tseq, _chunk_idx) { \
-    size_t _cidx; \
-    if mi_likely(mi_bchunk_try_find_and_clear##NSUF(&bitmap->chunks[_chunk_idx] NPAR, &_cidx)) { \
-      *pidx = (_chunk_idx * MI_BCHUNK_BITS) + _cidx; \
-      return true; \
-    } \
-    else { \
-      /* we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation. */ \
-      mi_bitmap_chunkmap_try_clear(bitmap, _chunk_idx); \
-    } \
-  } \
-  mi_bitmap_forall_chunks_end(); \
-  return false; \
-}
-
-#define COMMA ,
-
-mi_decl_nodiscard bool mi_bitmap_try_find_and_clear(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) {
-  mi_bitmap_forall_chunks_try_find_and_clear(bitmap, tseq, pidx, , );
-}
-
-mi_decl_nodiscard bool mi_bitmap_try_find_and_clear8(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) {
-  mi_bitmap_forall_chunks_try_find_and_clear(bitmap, tseq, pidx, 8, );
-}
-
-mi_decl_nodiscard bool mi_bitmap_try_find_and_clearX(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) {
-  mi_bitmap_forall_chunks_try_find_and_clear(bitmap, tseq, pidx, X, );
-}
-
-mi_decl_nodiscard bool mi_bitmap_try_find_and_clearNX(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx) {
-  mi_assert_internal(n<=MI_BFIELD_BITS);
-  mi_bitmap_forall_chunks_try_find_and_clear(bitmap, tseq, pidx, NX, COMMA n);
-}
-
-mi_decl_nodiscard bool mi_bitmap_try_find_and_clearN_(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx) {
-  mi_assert_internal(n<=MI_BCHUNK_BITS);
-  mi_bitmap_forall_chunks_try_find_and_clear(bitmap, tseq, pidx, N_, COMMA n);
-}
-
-
-/* --------------------------------------------------------------------------------
-  bitmap  try_find_and_claim
-  (used to allocate abandoned pages)
--------------------------------------------------------------------------------- */
-
 // Find a set bit in the bitmap and try to atomically clear it and claim it.
 // (Used to find pages in the pages_abandoned bitmaps.)
 mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx,
@@ -1177,7 +1269,7 @@ bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx) {
     if (mi_bsr(cmap,&cmap_idx)) {
       // highest chunk
       const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
-      size_t cidx; 
+      size_t cidx;
       if (mi_bchunk_bsr(&bitmap->chunks[chunk_idx], &cidx)) {
         *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
         return true;