merge from dev

include/mimalloc-atomic.h

@@ -27,10 +27,6 @@ terms of the MIT license. A copy of the license can be found in the file
 // Atomic operations specialized for mimalloc
 // ------------------------------------------------------
 
-// Atomically add a 64-bit value; returns the previous value.
-// Note: not using _Atomic(int64_t) as it is only used for statistics.
-static inline void mi_atomic_addi64(volatile int64_t* p, int64_t add);
-
 // Atomically add a value; returns the previous value. Memory ordering is relaxed.
 static inline uintptr_t mi_atomic_add(volatile _Atomic(uintptr_t)* p, uintptr_t add);
 
@@ -65,6 +61,17 @@ static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x);
 // Yield
 static inline void mi_atomic_yield(void);
 
+// Atomically add a 64-bit value; returns the previous value.
+// Note: not using _Atomic(int64_t) as it is only used for statistics.
+static inline void mi_atomic_addi64(volatile int64_t* p, int64_t add);
+
+// Atomically update `*p` with the maximum of `*p` and `x` as a 64-bit value.
+// Returns the previous value. Note: not using _Atomic(int64_t) as it is only used for statistics.
+static inline void mi_atomic_maxi64(volatile int64_t* p, int64_t x);
+
+// Atomically read a 64-bit value
+// Note: not using _Atomic(int64_t) as it is only used for statistics.
+static inline int64_t mi_atomic_readi64(volatile int64_t* p);
 
 // Atomically subtract a value; returns the previous value.
 static inline uintptr_t mi_atomic_sub(volatile _Atomic(uintptr_t)* p, uintptr_t sub) {
@@ -177,35 +184,50 @@ static inline void mi_atomic_addi64(volatile _Atomic(int64_t)* p, int64_t add) {
   #endif
 }
 
+static inline void mi_atomic_maxi64(volatile _Atomic(int64_t)*p, int64_t x) {
+  int64_t current;
+  do {
+    current = *p;
+  } while (current < x && _InterlockedCompareExchange64(p, x, current) != current);
+}
+
+static inline int64_t mi_atomic_readi64(volatile _Atomic(int64_t)*p) {
+  #ifdef _WIN64
+  return *p;
+  #else
+  int64_t current;
+  do {
+    current = *p;
+  } while (_InterlockedCompareExchange64(p, current, current) != current);
+  return current;
+  #endif
+}
+
 #else
 #ifdef __cplusplus
 #define  MI_USING_STD   using namespace std;
 #else
 #define  MI_USING_STD
 #endif
-static inline void mi_atomic_addi64(volatile int64_t* p, int64_t add) {
-  MI_USING_STD
-  atomic_fetch_add_explicit((volatile _Atomic(int64_t)*)p, add, memory_order_relaxed);
-}
 static inline uintptr_t mi_atomic_add(volatile _Atomic(uintptr_t)* p, uintptr_t add) {
   MI_USING_STD
   return atomic_fetch_add_explicit(p, add, memory_order_relaxed);
 }
 static inline uintptr_t mi_atomic_and(volatile _Atomic(uintptr_t)* p, uintptr_t x) {
   MI_USING_STD
-  return atomic_fetch_and_explicit(p, x, memory_order_relaxed);
+  return atomic_fetch_and_explicit(p, x, memory_order_acq_rel);
 }
 static inline uintptr_t mi_atomic_or(volatile _Atomic(uintptr_t)* p, uintptr_t x) {
   MI_USING_STD
-  return atomic_fetch_or_explicit(p, x, memory_order_relaxed);
+  return atomic_fetch_or_explicit(p, x, memory_order_acq_rel);
 }
 static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) {
   MI_USING_STD
-  return atomic_compare_exchange_weak_explicit(p, &expected, desired, memory_order_release, memory_order_relaxed);
+  return atomic_compare_exchange_weak_explicit(p, &expected, desired, memory_order_acq_rel, memory_order_acquire);
 }
 static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) {
   MI_USING_STD
-  return atomic_compare_exchange_strong_explicit(p, &expected, desired, memory_order_acq_rel, memory_order_relaxed);
+  return atomic_compare_exchange_strong_explicit(p, &expected, desired, memory_order_acq_rel, memory_order_acquire);
 }
 static inline uintptr_t mi_atomic_exchange(volatile _Atomic(uintptr_t)* p, uintptr_t exchange) {
   MI_USING_STD
@@ -223,6 +245,21 @@ static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x)
   MI_USING_STD
   return atomic_store_explicit(p, x, memory_order_release);
 }
+static inline void mi_atomic_addi64(volatile int64_t* p, int64_t add) {
+  MI_USING_STD
+  atomic_fetch_add_explicit((volatile _Atomic(int64_t)*)p, add, memory_order_relaxed);
+}
+static inline int64_t mi_atomic_readi64(volatile int64_t* p) {
+  MI_USING_STD
+  return atomic_load_explicit((volatile _Atomic(int64_t)*) p, memory_order_relaxed);
+}
+static inline void mi_atomic_maxi64(volatile int64_t* p, int64_t x) {
+  MI_USING_STD
+  int64_t current;
+  do {
+    current = mi_atomic_readi64(p);
+  } while (current < x && !atomic_compare_exchange_weak_explicit((volatile _Atomic(int64_t)*)p, &current, x, memory_order_acq_rel, memory_order_relaxed));
+}
 
 #if defined(__cplusplus)
   #include <thread>
@@ -233,11 +270,11 @@ static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x)
       (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))
 #if defined(__x86_64__) || defined(__i386__)
   static inline void mi_atomic_yield(void) {
-    asm volatile ("pause" ::: "memory");
+    __asm__ volatile ("pause" ::: "memory");
   }
 #elif defined(__arm__) || defined(__aarch64__)
   static inline void mi_atomic_yield(void) {
-    asm volatile("yield");
+    __asm__ volatile("yield");
   }
 #endif
 #elif defined(__wasi__)

include/mimalloc-internal.h

@@ -245,23 +245,28 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size)
 }
 
 // Overflow detecting multiply
-static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
 #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5
-#include <limits.h>   // UINT_MAX, ULONG_MAX
-#if (SIZE_MAX == UINT_MAX)
-  return __builtin_umul_overflow(count, size, total);
-#elif (SIZE_MAX == ULONG_MAX)
-  return __builtin_umull_overflow(count, size, total);
-#else
-  return __builtin_umulll_overflow(count, size, total);
+#include <limits.h>      // UINT_MAX, ULONG_MAX
+#if defined(_CLOCK_T)    // for Illumos
+#undef _CLOCK_T
 #endif
+static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
+  #if (SIZE_MAX == UINT_MAX)
+    return __builtin_umul_overflow(count, size, total);
+  #elif (SIZE_MAX == ULONG_MAX)
+    return __builtin_umull_overflow(count, size, total);
+  #else
+    return __builtin_umulll_overflow(count, size, total);
+  #endif
+}
 #else /* __builtin_umul_overflow is unavailable */
+static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
   #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t)))  // sqrt(SIZE_MAX)
   *total = count * size;
   return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW)
-          && size > 0 && (SIZE_MAX / size) < count);
-#endif
+    && size > 0 && (SIZE_MAX / size) < count);
 }
+#endif
 
 // Safe multiply `count*size` into `total`; return `true` on overflow.
 static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {
@@ -585,11 +590,11 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) {
 
 static inline uintptr_t mi_rotl(uintptr_t x, uintptr_t shift) {
   shift %= MI_INTPTR_BITS;
-  return ((x << shift) | (x >> (MI_INTPTR_BITS - shift)));
+  return (shift==0 ? x : ((x << shift) | (x >> (MI_INTPTR_BITS - shift))));
 }
 static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) {
   shift %= MI_INTPTR_BITS;
-  return ((x >> shift) | (x << (MI_INTPTR_BITS - shift)));
+  return (shift==0 ? x : ((x >> shift) | (x << (MI_INTPTR_BITS - shift))));
 }
 
 static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
@@ -714,11 +719,11 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept {
   __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : );  // x86_64 Linux, BSD uses FS
 #elif defined(__arm__)
   void** tcb; UNUSED(ofs);
-  asm volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
+  __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
   res = tcb[slot];
 #elif defined(__aarch64__)
   void** tcb; UNUSED(ofs);
-  asm volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
+  __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
   res = tcb[slot];
 #endif
   return res;
@@ -735,11 +740,11 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
   __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : );  // x86_64 Linux, BSD uses FS
 #elif defined(__arm__)
   void** tcb; UNUSED(ofs);
-  asm volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
+  __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
   tcb[slot] = value;
 #elif defined(__aarch64__)
   void** tcb; UNUSED(ofs);
-  asm volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
+  __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
   tcb[slot] = value;
 #endif
 }

include/mimalloc.h

@@ -1,5 +1,5 @@
 /* ----------------------------------------------------------------------------
-Copyright (c) 2018, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
 This is free software; you can redistribute it and/or modify it under the
 terms of the MIT license. A copy of the license can be found in the file
 "LICENSE" at the root of this distribution.