Merge branch 'dev3' into dev3-bin

include/mimalloc/internal.h

@@ -215,8 +215,8 @@ void          _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head);
 void          _mi_page_init(mi_heap_t* heap, mi_page_t* page);
 bool          _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq);
 
-size_t        _mi_bin_size(uint8_t bin); // for stats
+size_t        _mi_bin_size(size_t bin);            // for stats
-uint8_t       _mi_bin(size_t size);      // for stats
+size_t        _mi_bin(size_t size);                // for stats
 
 // "heap.c"
 mi_heap_t*    _mi_heap_create(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id, mi_tld_t* tld);

src/heap.c

@@ -592,7 +592,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
 
 static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
   mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
-  *shift = MI_INTPTR_BITS - mi_clz(divisor - 1);
+  *shift = MI_SIZE_BITS - mi_clz(divisor - 1);
   *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
 }
 

src/page-queue.c

@@ -61,45 +61,39 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) {
 // Returns MI_BIN_HUGE if the size is too large.
 // We use `wsize` for the size in "machine word sizes",
 // i.e. byte size == `wsize*sizeof(void*)`.
-static inline uint8_t mi_bin(size_t size) {
+static mi_decl_noinline size_t mi_bin(size_t size) {
   size_t wsize = _mi_wsize_from_size(size);
-  uint8_t bin;
-  if (wsize <= 1) {
-    bin = 1;
-  }
 #if defined(MI_ALIGN4W)
-  else if (wsize <= 4) {
+  if mi_likely(wsize <= 4) {
-    bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
+    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
   }
 #elif defined(MI_ALIGN2W)
-  else if (wsize <= 8) {
+  if mi_likely(wsize <= 8) {
-    bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
+    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
   }
 #else
-  else if (wsize <= 8) {
+  if mi_likely(wsize <= 8) {
-    bin = (uint8_t)wsize;
+    return (wsize == 0 ? 1 : wsize);
   }
 #endif
-  else if (wsize > MI_LARGE_MAX_OBJ_WSIZE) {
+  else if mi_unlikely(wsize > MI_LARGE_MAX_OBJ_WSIZE) {
-    bin = MI_BIN_HUGE;
+    return MI_BIN_HUGE;
   }
   else {
     #if defined(MI_ALIGN4W)
     if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes
     #endif
     wsize--;
-    mi_assert_internal(wsize!=0);
+    // find the highest bit
-    // find the highest bit position
+    const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize));  // note: wsize != 0
-    uint8_t b = (uint8_t)(MI_SIZE_BITS - 1 - mi_clz(wsize));
     // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
     // - adjust with 3 because we use do not round the first 8 sizes
     //   which each get an exact bin
-    bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3;
+    const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
-    mi_assert_internal(bin < MI_BIN_HUGE);
+    mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);
-  }
-  mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE);
     return bin;
   }
+}
 
 
 
@@ -107,11 +101,11 @@ static inline uint8_t mi_bin(size_t size) {
   Queue of pages with free blocks
 ----------------------------------------------------------- */
 
-uint8_t _mi_bin(size_t size) {
+size_t _mi_bin(size_t size) {
   return mi_bin(size);
 }
 
-size_t _mi_bin_size(uint8_t bin) {
+size_t _mi_bin_size(size_t bin) {
   return _mi_heap_empty.pages[bin].block_size;
 }
 
@@ -167,7 +161,7 @@ bool _mi_page_queue_is_valid(mi_heap_t* heap, const mi_page_queue_t* pq) {
 
 static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) {
   mi_assert_internal(heap!=NULL);
-  uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
+  size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
   mi_assert_internal(bin <= MI_BIN_FULL);
   mi_page_queue_t* pq = &heap->pages[bin];
   mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||
@@ -209,7 +203,7 @@ static inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_que
   }
   else {
     // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped
-    uint8_t bin = mi_bin(size);
+    size_t bin = mi_bin(size);
     const mi_page_queue_t* prev = pq - 1;
     while( bin == mi_bin(prev->block_size) && prev > &heap->pages[0]) {
       prev--;

src/prim/windows/prim.c

@@ -142,7 +142,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
   }
   // get virtual address bits
   if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {
-    const size_t vbits = MI_INTPTR_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
+    const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
     config->virtual_address_bits = vbits;
   }
 

test/main-override-static.c

@@ -32,7 +32,10 @@ static void test_manage_os_memory(void);
 int main() {
   mi_version();
   mi_stats_reset();
-  test_manage_os_memory();
+
+  // mi_bins();
+
+  // test_manage_os_memory();
   // test_large_pages();
   // detect double frees and heap corruption
   // double_free1();
@@ -40,7 +43,7 @@ int main() {
   // corrupt_free();
   // block_overflow1();
   // block_overflow2();
-  test_canary_leak();
+  // test_canary_leak();
   // test_aslr();
   // invalid_free();
   // test_reserved();
@@ -48,7 +51,6 @@ int main() {
   // test_heap_walk();
   // alloc_huge();
 
-  // mi_bins();
 
   void* p1 = malloc(78);
   void* p2 = malloc(24);
@@ -191,7 +193,7 @@ static void test_reserved(void) {
 #define KiB 1024ULL
 #define MiB (KiB*KiB)
 #define GiB (MiB*KiB)
-  mi_reserve_os_memory(4*GiB, false, true);
+  mi_reserve_os_memory(3*GiB, false, true);
   void* p1 = malloc(100);
   void* p2 = malloc(100000);
   void* p3 = malloc(2*GiB);
@@ -373,31 +375,34 @@ static inline size_t _mi_wsize_from_size(size_t size) {
   return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
 }
 
+// #define MI_ALIGN2W
+
 // Return the bin for a given field size.
 // Returns MI_BIN_HUGE if the size is too large.
 // We use `wsize` for the size in "machine word sizes",
 // i.e. byte size == `wsize*sizeof(void*)`.
-extern inline uint8_t _mi_bin8(size_t size) {
+static inline size_t mi_bin(size_t wsize) {
-  size_t wsize = _mi_wsize_from_size(size);
+  // size_t wsize = _mi_wsize_from_size(size);
-  uint8_t bin;
+  // size_t bin;
-  if (wsize <= 1) {
+  /*if (wsize <= 1) {
     bin = 1;
   }
+  */
 #if defined(MI_ALIGN4W)
-  else if (wsize <= 4) {
+  if (wsize <= 4) {
-    bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
+    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
   }
 #elif defined(MI_ALIGN2W)
-  else if (wsize <= 8) {
+  if (wsize <= 8) {
-    bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
+    return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
   }
 #else
-  else if (wsize <= 8) {
+  if (wsize <= 8) {
-    bin = (uint8_t)wsize;
+    return (wsize == 0 ? 1 : wsize);
   }
 #endif
   else if (wsize > MI_LARGE_WSIZE_MAX) {
-    bin = MI_BIN_HUGE;
+    return MI_BIN_HUGE;
   }
   else {
 #if defined(MI_ALIGN4W)
@@ -411,10 +416,12 @@ extern inline uint8_t _mi_bin8(size_t size) {
     // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
     // - adjust with 3 because we use do not round the first 8 sizes
     //   which each get an exact bin
-    bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3;
+    const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
-  }
+    assert(bin > 0 && bin < MI_BIN_HUGE);
     return bin;
   }
+}
+
 
 static inline uint8_t _mi_bin4(size_t size) {
   size_t wsize = _mi_wsize_from_size(size);
@@ -477,7 +484,7 @@ static size_t _mi_binx8(size_t bsize) {
 }
 
 
-static inline size_t mi_bin(size_t wsize) {
+static inline size_t mi_binx(size_t wsize) {
   uint8_t bin;
   if (wsize <= 1) {
     bin = 1;