Merge branch 'dev' into dev-exp

ide/vs2017/mimalloc-override.vcxproj

@@ -150,7 +150,6 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -159,6 +158,7 @@
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <CompileAs>Default</CompileAs>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
@@ -182,7 +182,6 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -191,6 +190,7 @@
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <CompileAs>Default</CompileAs>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>

ide/vs2017/mimalloc.vcxproj

@@ -141,8 +141,6 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -150,11 +148,9 @@
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <BufferSecurityCheck>false</BufferSecurityCheck>
-      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <InlineFunctionExpansion>Default</InlineFunctionExpansion>
-      <FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
-      <OmitFramePointers>false</OmitFramePointers>
-      <EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
       <CompileAs>CompileAsCpp</CompileAs>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
@@ -172,8 +168,6 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -181,11 +175,9 @@
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <BufferSecurityCheck>false</BufferSecurityCheck>
-      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <InlineFunctionExpansion>Default</InlineFunctionExpansion>
-      <FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
-      <OmitFramePointers>false</OmitFramePointers>
-      <EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
       <CompileAs>CompileAsCpp</CompileAs>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>

include/mimalloc-internal.h

@@ -151,15 +151,15 @@ bool        _mi_page_is_valid(mi_page_t* page);
 
 // Overflow detecting multiply
 #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t)))  // sqrt(SIZE_MAX)
-static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) {
+static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
 #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5
 #if (MI_INTPTR_SIZE == 4)
-  return __builtin_umul_overflow(size, count, total);
+  return __builtin_umul_overflow(count, size, total);
 #else
-  return __builtin_umull_overflow(size, count, total);
+  return __builtin_umull_overflow(count, size, total);
 #endif
 #else /* __builtin_umul_overflow is unavailable */
-  *total = size * count;
+  *total = count * size;
   return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW)
           && size > 0 && (SIZE_MAX / size) < count);
 #endif

include/mimalloc-types.h

@@ -93,7 +93,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 #define MI_SMALL_OBJ_SIZE_MAX             (MI_SMALL_PAGE_SIZE/4)
 #define MI_MEDIUM_OBJ_SIZE_MAX            (MI_MEDIUM_PAGE_SIZE/4)   // 128kb on 64-bit
-#define MI_LARGE_OBJ_SIZE_MAX             (MI_LARGE_PAGE_SIZE/4)    // 1Mb on 64-bit
+#define MI_LARGE_OBJ_SIZE_MAX             (MI_LARGE_PAGE_SIZE/2)    // 2Mb on 64-bit
 #define MI_LARGE_OBJ_WSIZE_MAX            (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT)
 #define MI_HUGE_OBJ_SIZE_MAX              (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE)  // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)
 

include/mimalloc.h

@@ -220,8 +220,8 @@ mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
 
 typedef enum mi_option_e {
   // stable options
-  mi_option_show_stats,
   mi_option_show_errors,
+  mi_option_show_stats,
   mi_option_verbose,
   // the following options are experimental
   mi_option_secure,
@@ -231,14 +231,15 @@ typedef enum mi_option_e {
   mi_option_page_reset,
   mi_option_cache_reset,
   mi_option_reset_decommits,
-  mi_option_reset_discards,
   _mi_option_last
 } mi_option_t;
 
 
 mi_decl_export bool  mi_option_is_enabled(mi_option_t option);
-mi_decl_export void  mi_option_enable(mi_option_t option, bool enable);
+mi_decl_export void  mi_option_enable(mi_option_t option);
-mi_decl_export void  mi_option_enable_default(mi_option_t option, bool enable);
+mi_decl_export void  mi_option_disable(mi_option_t option);
+mi_decl_export void  mi_option_set_enabled(mi_option_t option, bool enable);
+mi_decl_export void  mi_option_set_enabled_default(mi_option_t option, bool enable);
 
 mi_decl_export long  mi_option_get(mi_option_t option);
 mi_decl_export void  mi_option_set(mi_option_t option, long value);

src/alloc-override-win.c

@@ -666,7 +666,7 @@ static void mi_patches_at_quick_exit(void) {
   mi_patches_enable_term();             // enter termination phase and patch realloc/free with a no-op
 }
 
-__declspec(dllexport) BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
+BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
   if (reason == DLL_PROCESS_ATTACH) {
     __security_init_cookie();
   }

src/heap.c

@@ -303,14 +303,14 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
   mi_assert_internal(heap!=NULL);
   if (from==NULL || from->page_count == 0) return;
 
-  // unfull all full pages
+  // unfull all full pages in the `from` heap
-  mi_page_t* page = heap->pages[MI_BIN_FULL].first; 
+  mi_page_t* page = from->pages[MI_BIN_FULL].first; 
   while (page != NULL) {
     mi_page_t* next = page->next;
     _mi_page_unfull(page);
     page = next;
   }
-  mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL);
+  mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL);
 
   // free outstanding thread delayed free blocks
   _mi_heap_delayed_free(from);

src/init.c

@@ -106,14 +106,14 @@ mi_heap_t _mi_heap_main = {
   MI_SMALL_PAGES_EMPTY,
   MI_PAGE_QUEUES_EMPTY,
   NULL,
-  0,
+  0,      // thread id
-  0,
 #if MI_INTPTR_SIZE==8   // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!)
   0xCDCDCDCDCDCDCDCDUL,
 #else
   0xCDCDCDCDUL,
 #endif
-  0,
+  0,      // random
+  0,      // page count
   false   // can reclaim
 };
 

src/memory.c

@@ -128,6 +128,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
   size_t mask = mi_region_block_mask(blocks,bitidx);
   mi_assert_internal(mask != 0);
   mi_assert_internal((mask & mi_atomic_read(&region->map)) == mask);
+  mi_assert_internal(&regions[idx] == region);
 
   // ensure the region is reserved
   void* start = mi_atomic_read_ptr(&region->start);
@@ -149,9 +150,23 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
       mi_atomic_increment(&regions_count);
     }
     else {
-      // failed, another thread allocated just before us, free our allocated memory
+      // failed, another thread allocated just before us!
-      // TODO: should we keep the allocated memory and assign it to some other region?
+      // we assign it to a later slot instead (up to 4 tries).
-      _mi_os_free(start, MI_REGION_SIZE, tld->stats);
+      // note: we don't need to increment the region count, this will happen on another allocation
+      for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) {
+        void* s = mi_atomic_read_ptr(&regions[idx+i].start);
+        if (s == NULL) { // quick test
+          if (mi_atomic_compare_exchange_ptr(&regions[idx+i].start, start, s)) {
+            start = NULL;
+            break;
+          }
+        }
+      }
+      if (start != NULL) {
+        // free it if we didn't succeed to save it to some other region
+        _mi_os_free(start, MI_REGION_SIZE, tld->stats);
+      }
+      // and continue with the memory at our index
       start = mi_atomic_read_ptr(&region->start);
     }
   }

src/options.c

@@ -34,34 +34,37 @@ typedef enum mi_init_e {
 typedef struct mi_option_desc_s {
   long        value;  // the value
   mi_init_t   init;   // is it initialized yet? (from the environment)
+  mi_option_t option; // for debugging: the option index should match the option
   const char* name;   // option name without `mimalloc_` prefix
 } mi_option_desc_t;
 
+#define MI_OPTION(opt)        mi_option_##opt, #opt
+#define MI_OPTION_DESC(opt)   {0, UNINIT, MI_OPTION(opt) }
+
 static mi_option_desc_t options[_mi_option_last] =
 {
   // stable options
-  { 0, UNINIT, "show_stats" },
+  { MI_DEBUG, UNINIT, MI_OPTION(show_errors) },
-  { MI_DEBUG, UNINIT, "show_errors" },
+  { 0, UNINIT, MI_OPTION(show_stats) },
-  { 0, UNINIT, "verbose" },
+  { 0, UNINIT, MI_OPTION(verbose) },
 
   #if MI_SECURE
-  { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored
+  { MI_SECURE, INITIALIZED, MI_OPTION(secure) }, // in a secure build the environment setting is ignored
   #else
-  { 0, UNINIT, "secure" },
+  { 0, UNINIT, MI_OPTION(secure) },
   #endif
 
   // the following options are experimental and not all combinations make sense.
-  { 1, UNINIT, "eager_commit" },        // note: if eager_region_commit is on, this should be on too.
+  { 1, UNINIT, MI_OPTION(eager_commit) },        // note: if eager_region_commit is on, this should be on too.
   #ifdef _WIN32   // and BSD?
-  { 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...)
+  { 0, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...)
   #else
-  { 1, UNINIT, "eager_region_commit" },
+  { 1, UNINIT, MI_OPTION(eager_region_commit) },
   #endif
-  { 0, UNINIT, "large_os_pages" },      // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
+  { 0, UNINIT, MI_OPTION(large_os_pages) },      // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
-  { 0, UNINIT, "page_reset" },
+  { 0, UNINIT, MI_OPTION(page_reset) },
-  { 0, UNINIT, "cache_reset" },
+  { 0, UNINIT, MI_OPTION(cache_reset) },
-  { 0, UNINIT, "reset_decommits" },     // note: cannot enable this if secure is on
+  { 0, UNINIT, MI_OPTION(reset_decommits) }      // note: cannot enable this if secure is on
-  { 0, UNINIT, "reset_discards" }       // note: cannot enable this if secure is on
 };
 
 static void mi_option_init(mi_option_desc_t* desc);
@@ -69,6 +72,7 @@ static void mi_option_init(mi_option_desc_t* desc);
 long mi_option_get(mi_option_t option) {
   mi_assert(option >= 0 && option < _mi_option_last);
   mi_option_desc_t* desc = &options[option];
+  mi_assert(desc->option == option);  // index should match the option
   if (mi_unlikely(desc->init == UNINIT)) {
     mi_option_init(desc);
     if (option != mi_option_verbose) {
@@ -81,6 +85,7 @@ long mi_option_get(mi_option_t option) {
 void mi_option_set(mi_option_t option, long value) {
   mi_assert(option >= 0 && option < _mi_option_last);
   mi_option_desc_t* desc = &options[option];
+  mi_assert(desc->option == option);  // index should match the option
   desc->value = value;
   desc->init = INITIALIZED;
 }
@@ -97,14 +102,23 @@ bool mi_option_is_enabled(mi_option_t option) {
   return (mi_option_get(option) != 0);
 }
 
-void mi_option_enable(mi_option_t option, bool enable) {
+void mi_option_set_enabled(mi_option_t option, bool enable) {
   mi_option_set(option, (enable ? 1 : 0));
 }
 
-void mi_option_enable_default(mi_option_t option, bool enable) {
+void mi_option_set_enabled_default(mi_option_t option, bool enable) {
   mi_option_set_default(option, (enable ? 1 : 0));
 }
 
+void mi_option_enable(mi_option_t option) {
+  mi_option_set_enabled(option,true);
+}
+
+void mi_option_disable(mi_option_t option) {
+  mi_option_set_enabled(option,false);
+}
+
+
 // --------------------------------------------------------
 // Messages
 // --------------------------------------------------------

src/os.c

@@ -22,6 +22,9 @@ terms of the MIT license. A copy of the license can be found in the file
 #else
 #include <sys/mman.h>  // mmap
 #include <unistd.h>    // sysconf
+#if defined(__linux__)
+#include <linux/mman.h> // linux mmap flags
+#endif
 #if defined(__APPLE__)
 #include <mach/vm_statistics.h>
 #endif
@@ -67,7 +70,7 @@ size_t _mi_os_large_page_size() {
 
 static bool use_large_os_page(size_t size, size_t alignment) {
   // if we have access, check the size and alignment requirements
-  if (large_os_page_size == 0) return false;
+  if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false;
   return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0);
 }
 
@@ -81,11 +84,9 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
 #if defined(_WIN32)
 // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
 // So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
-// Same for DiscardVirtualMemory. (hide MEM_EXTENDED_PARAMETER to compile with older SDK's)
+// (hide MEM_EXTENDED_PARAMETER to compile with older SDK's)
 typedef PVOID(__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
-typedef DWORD(__stdcall *PDiscardVirtualMemory)(PVOID,SIZE_T);
 static PVirtualAlloc2 pVirtualAlloc2 = NULL;
-static PDiscardVirtualMemory pDiscardVirtualMemory = NULL;
 
 void _mi_os_init(void) {
   // get the page size
@@ -100,7 +101,6 @@ void _mi_os_init(void) {
     // use VirtualAlloc2FromApp if possible as it is available to Windows store apps
     pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2FromApp");
     if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2");
-    pDiscardVirtualMemory = (PDiscardVirtualMemory)GetProcAddress(hDll, "DiscardVirtualMemory");
     FreeLibrary(hDll);
   }
   // Try to see if large OS pages are supported
@@ -555,17 +555,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
 
 #if defined(_WIN32)
   // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
-  // (but this is for an access pattern that immediately reuses the memory)
+  void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
-  if (mi_option_is_enabled(mi_option_reset_discards) && pDiscardVirtualMemory != NULL) {
+  mi_assert_internal(p == start);
-    DWORD ok = (*pDiscardVirtualMemory)(start, csize);
+  if (p != start) return false;
-    mi_assert_internal(ok == ERROR_SUCCESS);
-    if (ok != ERROR_SUCCESS) return false;
-  }
-  else {
-    void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
-    mi_assert_internal(p == start);
-    if (p != start) return false;
-  }
 #else
 #if defined(MADV_FREE)
   static int advice = MADV_FREE;

src/page.c

@@ -741,7 +741,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
   // huge allocation?
   mi_page_t* page;
   if (mi_unlikely(size > MI_LARGE_OBJ_SIZE_MAX)) {
-    if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) {
+    if (mi_unlikely(size > PTRDIFF_MAX)) {  // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
       page = NULL;
     }
     else {

test/test-api.c

@@ -66,7 +66,7 @@ bool test_heap2();
 // Main testing
 // ---------------------------------------------------------------------------
 int main() {
-  mi_option_enable(mi_option_verbose,false);
+  mi_option_disable(mi_option_verbose);
 
   // ---------------------------------------------------
   // Malloc