merge dev-win

ide/vs2017/mimalloc-override.vcxproj

@@ -146,7 +146,6 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -155,6 +154,7 @@
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <CompileAs>Default</CompileAs>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
@@ -173,7 +173,6 @@
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -182,6 +181,7 @@
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <CompileAs>Default</CompileAs>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>

ide/vs2017/mimalloc.vcxproj

@@ -141,8 +141,6 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -150,11 +148,9 @@
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <BufferSecurityCheck>false</BufferSecurityCheck>
-      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <InlineFunctionExpansion>Default</InlineFunctionExpansion>
-      <FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
-      <OmitFramePointers>false</OmitFramePointers>
-      <EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
       <CompileAs>CompileAsCpp</CompileAs>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
@@ -172,8 +168,6 @@
       <WarningLevel>Level3</WarningLevel>
       <Optimization>MaxSpeed</Optimization>
       <FunctionLevelLinking>true</FunctionLevelLinking>
-      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <SDLCheck>true</SDLCheck>
       <ConformanceMode>true</ConformanceMode>
       <AdditionalIncludeDirectories>../../include</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>%(PreprocessorDefinitions);NDEBUG</PreprocessorDefinitions>
@@ -181,11 +175,9 @@
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <BufferSecurityCheck>false</BufferSecurityCheck>
-      <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
+      <InlineFunctionExpansion>Default</InlineFunctionExpansion>
-      <FavorSizeOrSpeed>Neither</FavorSizeOrSpeed>
-      <OmitFramePointers>false</OmitFramePointers>
-      <EnableFiberSafeOptimizations>false</EnableFiberSafeOptimizations>
       <CompileAs>CompileAsCpp</CompileAs>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
     </ClCompile>
     <Link>
       <EnableCOMDATFolding>true</EnableCOMDATFolding>

include/mimalloc-atomic.h

@@ -68,6 +68,9 @@ static inline void* mi_atomic_exchange_ptr(volatile void** p, void* exchange) {
   return (void*)mi_atomic_exchange((volatile uintptr_t*)p, (uintptr_t)exchange);
 }
 
+static inline intptr_t mi_atomic_iread(volatile intptr_t* p) {
+  return (intptr_t)mi_atomic_read( (volatile uintptr_t*)p );
+}
 
 #ifdef _MSC_VER
 #define WIN32_LEAN_AND_MEAN

include/mimalloc-internal.h

@@ -102,6 +102,8 @@ uintptr_t  _mi_heap_random(mi_heap_t* heap);
 
 // "stats.c"
 void       _mi_stats_done(mi_stats_t* stats);
+double     _mi_clock_end(double start);
+double     _mi_clock_start(void);
 
 // "alloc.c"
 void*       _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept;  // called from `_mi_malloc_generic`
@@ -161,15 +163,15 @@ bool        _mi_page_is_valid(mi_page_t* page);
 
 // Overflow detecting multiply
 #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t)))  // sqrt(SIZE_MAX)
-static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) {
+static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
 #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5
 #if (MI_INTPTR_SIZE == 4)
-  return __builtin_umul_overflow(size, count, total);
+  return __builtin_umul_overflow(count, size, total);
 #else
-  return __builtin_umull_overflow(size, count, total);
+  return __builtin_umull_overflow(count, size, total);
 #endif
 #else /* __builtin_umul_overflow is unavailable */
-  *total = size * count;
+  *total = count * size;
   return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW)
           && size > 0 && (SIZE_MAX / size) < count);
 #endif

include/mimalloc-types.h

@@ -90,12 +90,13 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MI_SMALL_PAGE_SIZE                (1<<MI_SMALL_PAGE_SHIFT)
 #define MI_MEDIUM_PAGE_SIZE               (1<<MI_MEDIUM_PAGE_SHIFT)
 
-#define MI_MEDIUM_SIZE_MAX                (MI_MEDIUM_PAGE_SIZE/4)   // 128kb on 64-bit
+#define MI_SMALL_OBJ_SIZE_MAX             (MI_SMALL_PAGE_SIZE/4)   // 16kb on 64-bit
-#define MI_MEDIUM_WSIZE_MAX               (MI_MEDIUM_SIZE_MAX/MI_INTPTR_SIZE)   // 64kb on 64-bit
 
-#define MI_LARGE_SIZE_MAX                 (MI_SEGMENT_SIZE/4)       // 16mb on 64-bit
+#define MI_MEDIUM_OBJ_SIZE_MAX            (MI_MEDIUM_PAGE_SIZE/4)   // 128kb on 64-bit
-#define MI_LARGE_WSIZE_MAX                (MI_LARGE_SIZE_MAX/MI_INTPTR_SIZE)
+#define MI_MEDIUM_OBJ_WSIZE_MAX           (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)   // 64kb on 64-bit
 
+#define MI_LARGE_OBJ_SIZE_MAX             (MI_SEGMENT_SIZE/4)       // 16mb on 64-bit
+#define MI_LARGE_OBJ_WSIZE_MAX            (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
 
 // Minimal alignment necessary. On most platforms 16 bytes are needed
 // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE`
@@ -104,7 +105,8 @@ terms of the MIT license. A copy of the license can be found in the file
 // Maximum number of size classes. (spaced exponentially in 12.5% increments)
 #define MI_BIN_HUGE  (73U)
 
-#if (MI_MEDIUM_WSIZE_MAX >= 655360)
+
+#if (MI_MEDIUM_OBJ_WSIZE_MAX >= 655360)
 #error "define more bins"
 #endif
 

include/mimalloc.h

@@ -195,7 +195,7 @@ typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_
 mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
 
 mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
-
+mi_decl_export int  mi_reserve_huge_os_pages(size_t pages, double max_secs) mi_attr_noexcept;
 
 // ------------------------------------------------------
 // Convenience
@@ -220,25 +220,27 @@ mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept;
 
 typedef enum mi_option_e {
   // stable options
-  mi_option_show_stats,
   mi_option_show_errors,
+  mi_option_show_stats,
   mi_option_verbose,
   // the following options are experimental
   mi_option_secure,
   mi_option_eager_commit,
   mi_option_eager_region_commit,
   mi_option_large_os_pages,         // implies eager commit
+  mi_option_reserve_huge_os_pages,
   mi_option_page_reset,
   mi_option_cache_reset,
   mi_option_reset_decommits,
-  mi_option_reset_discards,
   _mi_option_last
 } mi_option_t;
 
 
 mi_decl_export bool  mi_option_is_enabled(mi_option_t option);
-mi_decl_export void  mi_option_enable(mi_option_t option, bool enable);
+mi_decl_export void  mi_option_enable(mi_option_t option);
-mi_decl_export void  mi_option_enable_default(mi_option_t option, bool enable);
+mi_decl_export void  mi_option_disable(mi_option_t option);
+mi_decl_export void  mi_option_set_enabled(mi_option_t option, bool enable);
+mi_decl_export void  mi_option_set_enabled_default(mi_option_t option, bool enable);
 
 mi_decl_export long  mi_option_get(mi_option_t option);
 mi_decl_export void  mi_option_set(mi_option_t option, long value);

src/alloc-override-win.c

@@ -666,7 +666,7 @@ static void mi_patches_at_quick_exit(void) {
   mi_patches_enable_term();             // enter termination phase and patch realloc/free with a no-op
 }
 
-__declspec(dllexport) BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
+BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
   if (reason == DLL_PROCESS_ATTACH) {
     __security_init_cookie();
   }

src/alloc.c

@@ -38,7 +38,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
   block->next = 0;
 #endif
 #if (MI_STAT>1)
-  if(size <= MI_LARGE_SIZE_MAX) {
+  if(size <= MI_LARGE_OBJ_SIZE_MAX) {
     size_t bin = _mi_bin(size);
     mi_heap_stat_increase(heap,normal[bin], 1);
   }
@@ -230,7 +230,7 @@ void mi_free(void* p) mi_attr_noexcept
 #if (MI_STAT>1)
   mi_heap_t* heap = mi_heap_get_default();
   mi_heap_stat_decrease( heap, malloc, mi_usable_size(p));
-  if (page->block_size <= MI_LARGE_SIZE_MAX) {
+  if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
     mi_heap_stat_decrease( heap, normal[_mi_bin(page->block_size)], 1);
   }
   // huge page stat is accounted for in `_mi_page_retire`

src/heap.c

@@ -245,8 +245,8 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
   _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE);
 
   // stats
-  if (page->block_size > MI_MEDIUM_SIZE_MAX) {
+  if (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX) {
-    if (page->block_size <= MI_LARGE_SIZE_MAX) {
+    if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
       _mi_stat_decrease(&heap->tld->stats.large,page->block_size);
     }
     else {
@@ -255,7 +255,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
   }
   #if (MI_STAT>1)
   size_t inuse = page->used - page->thread_freed;
-  if (page->block_size <= MI_LARGE_SIZE_MAX)  {
+  if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX)  {
     mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse);
   }
   mi_heap_stat_decrease(heap,malloc, page->block_size * inuse);  // todo: off for aligned blocks...
@@ -303,14 +303,14 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
   mi_assert_internal(heap!=NULL);
   if (from==NULL || from->page_count == 0) return;
 
-  // unfull all full pages
+  // unfull all full pages in the `from` heap
-  mi_page_t* page = heap->pages[MI_BIN_FULL].first; 
+  mi_page_t* page = from->pages[MI_BIN_FULL].first;
   while (page != NULL) {
     mi_page_t* next = page->next;
     _mi_page_unfull(page);
     page = next;
   }
-  mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL);
+  mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL);
 
   // free outstanding thread delayed free blocks
   _mi_heap_delayed_free(from);
@@ -525,4 +525,3 @@ bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_vis
   mi_visit_blocks_args_t args = { visit_blocks, visitor, arg };
   return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);
 }
-

src/init.c

@@ -43,8 +43,8 @@ const mi_page_t _mi_page_empty = {
     QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \
     QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \
     QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
-    QNULL(MI_MEDIUM_WSIZE_MAX + 1  /* 655360, Huge queue */), \
+    QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 1  /* 655360, Huge queue */), \
-    QNULL(MI_MEDIUM_WSIZE_MAX + 2) /* Full queue */ }
+    QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 2) /* Full queue */ }
 
 #define MI_STAT_COUNT_NULL()  {0,0,0,0}
 
@@ -116,14 +116,14 @@ mi_heap_t _mi_heap_main = {
   MI_SMALL_PAGES_EMPTY,
   MI_PAGE_QUEUES_EMPTY,
   NULL,
-  0,
+  0,      // thread id
-  0,
 #if MI_INTPTR_SIZE==8   // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!)
   0xCDCDCDCDCDCDCDCDUL,
 #else
   0xCDCDCDCDUL,
 #endif
-  0,
+  0,      // random
+  0,      // page count
   false   // can reclaim
 };
 
@@ -432,6 +432,12 @@ static void mi_process_load(void) {
   const char* msg = NULL;
   mi_allocator_init(&msg);
   if (msg != NULL) _mi_verbose_message(msg);
+
+  if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
+    size_t pages     = mi_option_get(mi_option_reserve_huge_os_pages);
+    double max_secs = (double)pages / 5.0; // 0.2s per page
+    mi_reserve_huge_os_pages(pages, max_secs);
+  }
 }
 
 // Initialize the process; called by thread_init or the process loader

src/memory.c

@@ -128,6 +128,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
   size_t mask = mi_region_block_mask(blocks,bitidx);
   mi_assert_internal(mask != 0);
   mi_assert_internal((mask & mi_atomic_read(&region->map)) == mask);
+  mi_assert_internal(&regions[idx] == region);
 
   // ensure the region is reserved
   void* start = mi_atomic_read_ptr(&region->start);
@@ -149,9 +150,23 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
       mi_atomic_increment(&regions_count);
     }
     else {
-      // failed, another thread allocated just before us, free our allocated memory
+      // failed, another thread allocated just before us!
-      // TODO: should we keep the allocated memory and assign it to some other region?
+      // we assign it to a later slot instead (up to 4 tries).
+      // note: we don't need to increment the region count, this will happen on another allocation
+      for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) {
+        void* s = mi_atomic_read_ptr(&regions[idx+i].start);
+        if (s == NULL) { // quick test
+          if (mi_atomic_compare_exchange_ptr(&regions[idx+i].start, start, s)) {
+            start = NULL;
+            break;
+          }
+        }
+      }
+      if (start != NULL) {
+        // free it if we didn't succeed to save it to some other region
         _mi_os_free(start, MI_REGION_SIZE, tld->stats);
+      }
+      // and continue with the memory at our index
       start = mi_atomic_read_ptr(&region->start);
     }
   }

src/options.c

@@ -34,34 +34,38 @@ typedef enum mi_init_e {
 typedef struct mi_option_desc_s {
   long        value;  // the value
   mi_init_t   init;   // is it initialized yet? (from the environment)
+  mi_option_t option; // for debugging: the option index should match the option
   const char* name;   // option name without `mimalloc_` prefix
 } mi_option_desc_t;
 
+#define MI_OPTION(opt)        mi_option_##opt, #opt
+#define MI_OPTION_DESC(opt)   {0, UNINIT, MI_OPTION(opt) }
+
 static mi_option_desc_t options[_mi_option_last] =
 {
   // stable options
-  { 0, UNINIT, "show_stats" },
+  { MI_DEBUG, UNINIT, MI_OPTION(show_errors) },
-  { MI_DEBUG, UNINIT, "show_errors" },
+  { 0, UNINIT, MI_OPTION(show_stats) },
-  { 0, UNINIT, "verbose" },
+  { 0, UNINIT, MI_OPTION(verbose) },
 
   #if MI_SECURE
-  { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored
+  { MI_SECURE, INITIALIZED, MI_OPTION(secure) }, // in a secure build the environment setting is ignored
   #else
-  { 0, UNINIT, "secure" },
+  { 0, UNINIT, MI_OPTION(secure) },
   #endif
 
   // the following options are experimental and not all combinations make sense.
-  { 1, UNINIT, "eager_commit" },        // note: if eager_region_commit is on, this should be on too.
+  { 1, UNINIT, MI_OPTION(eager_commit) },        // note: if eager_region_commit is on, this should be on too.
   #ifdef _WIN32   // and BSD?
-  { 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...)
+  { 1, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...)
   #else
-  { 1, UNINIT, "eager_region_commit" },
+  { 1, UNINIT, MI_OPTION(eager_region_commit) },
   #endif
-  { 0, UNINIT, "large_os_pages" },      // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
+  { 0, UNINIT, MI_OPTION(large_os_pages) },      // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
-  { 0, UNINIT, "page_reset" },
+  { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
-  { 0, UNINIT, "cache_reset" },
+  { 0, UNINIT, MI_OPTION(page_reset) },
-  { 0, UNINIT, "reset_decommits" },     // note: cannot enable this if secure is on
+  { 0, UNINIT, MI_OPTION(cache_reset) },
-  { 0, UNINIT, "reset_discards" }       // note: cannot enable this if secure is on
+  { 0, UNINIT, MI_OPTION(reset_decommits) }      // note: cannot enable this if secure is on
 };
 
 static void mi_option_init(mi_option_desc_t* desc);
@@ -69,6 +73,7 @@ static void mi_option_init(mi_option_desc_t* desc);
 long mi_option_get(mi_option_t option) {
   mi_assert(option >= 0 && option < _mi_option_last);
   mi_option_desc_t* desc = &options[option];
+  mi_assert(desc->option == option);  // index should match the option
   if (mi_unlikely(desc->init == UNINIT)) {
     mi_option_init(desc);
     if (option != mi_option_verbose) {
@@ -81,6 +86,7 @@ long mi_option_get(mi_option_t option) {
 void mi_option_set(mi_option_t option, long value) {
   mi_assert(option >= 0 && option < _mi_option_last);
   mi_option_desc_t* desc = &options[option];
+  mi_assert(desc->option == option);  // index should match the option
   desc->value = value;
   desc->init = INITIALIZED;
 }
@@ -97,14 +103,23 @@ bool mi_option_is_enabled(mi_option_t option) {
   return (mi_option_get(option) != 0);
 }
 
-void mi_option_enable(mi_option_t option, bool enable) {
+void mi_option_set_enabled(mi_option_t option, bool enable) {
   mi_option_set(option, (enable ? 1 : 0));
 }
 
-void mi_option_enable_default(mi_option_t option, bool enable) {
+void mi_option_set_enabled_default(mi_option_t option, bool enable) {
   mi_option_set_default(option, (enable ? 1 : 0));
 }
 
+void mi_option_enable(mi_option_t option) {
+  mi_option_set_enabled(option,true);
+}
+
+void mi_option_disable(mi_option_t option) {
+  mi_option_set_enabled(option,false);
+}
+
+
 // --------------------------------------------------------
 // Messages
 // --------------------------------------------------------

src/os.c

@@ -22,6 +22,9 @@ terms of the MIT license. A copy of the license can be found in the file
 #else
 #include <sys/mman.h>  // mmap
 #include <unistd.h>    // sysconf
+#if defined(__linux__)
+#include <linux/mman.h> // linux mmap flags
+#endif
 #if defined(__APPLE__)
 #include <mach/vm_statistics.h>
 #endif
@@ -34,6 +37,9 @@ terms of the MIT license. A copy of the license can be found in the file
 ----------------------------------------------------------- */
 bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
 
+static bool  mi_os_is_huge_reserved(void* p);
+static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit);
+
 static void* mi_align_up_ptr(void* p, size_t alignment) {
   return (void*)_mi_align_up((uintptr_t)p, alignment);
 }
@@ -67,7 +73,7 @@ size_t _mi_os_large_page_size() {
 
 static bool use_large_os_page(size_t size, size_t alignment) {
   // if we have access, check the size and alignment requirements
-  if (large_os_page_size == 0) return false;
+  if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false;
   return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0);
 }
 
@@ -81,11 +87,13 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
 #if defined(_WIN32)
 // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
 // So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
-// Same for DiscardVirtualMemory. (hide MEM_EXTENDED_PARAMETER to compile with older SDK's)
+// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
-typedef PVOID(__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
+// We hide MEM_EXTENDED_PARAMETER to compile with older SDK's.
-typedef DWORD(__stdcall *PDiscardVirtualMemory)(PVOID,SIZE_T);
+#include <winternl.h>
+typedef PVOID    (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
+typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ PVOID, ULONG);
 static PVirtualAlloc2 pVirtualAlloc2 = NULL;
-static PDiscardVirtualMemory pDiscardVirtualMemory = NULL;
+static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
 
 void _mi_os_init(void) {
   // get the page size
@@ -100,12 +108,16 @@ void _mi_os_init(void) {
     // use VirtualAlloc2FromApp if possible as it is available to Windows store apps
     pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2FromApp");
     if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2");
-    pDiscardVirtualMemory = (PDiscardVirtualMemory)GetProcAddress(hDll, "DiscardVirtualMemory");
+    FreeLibrary(hDll);
+  }
+  hDll = LoadLibrary(TEXT("ntdll.dll"));
+  if (hDll != NULL) {    
+    pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
     FreeLibrary(hDll);
   }
   // Try to see if large OS pages are supported
   unsigned long err = 0;
-  bool ok = mi_option_is_enabled(mi_option_large_os_pages);
+  bool ok = mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages);
   if (ok) {
     // To use large pages on Windows, we first need access permission
     // Set "Lock pages in memory" permission in the group policy editor
@@ -161,7 +173,7 @@ void _mi_os_init() {
 
 static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
 {
-  if (addr == NULL || size == 0) return true;
+  if (addr == NULL || size == 0 || mi_os_is_huge_reserved(addr)) return true;
   bool err = false;
 #if defined(_WIN32)
   err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
@@ -185,25 +197,49 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
 #ifdef _WIN32
 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
 #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
-  if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
+  // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
+  if ((size % (uintptr_t)1 << 30) == 0 /* 1GiB multiple */
+    && (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0 
+    && (addr != NULL || try_alignment == 0 || try_alignment % _mi_os_page_size() == 0)
+    && pNtAllocateVirtualMemoryEx != NULL)
+  {
+    #ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
+    #define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE  (0x10)
+    #endif
+    MEM_EXTENDED_PARAMETER param = { 0, 0 };
+    param.Type = 5; // == MemExtendedParameterAttributeFlags;
+    param.ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
+    SIZE_T psize = size;
+    void*  base  = addr;
+    NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags | MEM_RESERVE, PAGE_READWRITE, &param, 1);
+    if (err == 0) {
+      return base;
+    }
+    else {
+      // else fall back to regular large OS pages
+      _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) page instead (error %lx)\n", err);
+    }
+  }
+  
   // on modern Windows try use VirtualAlloc2 for aligned allocation
+  if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
     MEM_ADDRESS_REQUIREMENTS reqs = { 0 };
     reqs.Alignment = try_alignment;
     MEM_EXTENDED_PARAMETER param = { 0 };
     param.Type = MemExtendedParameterAddressRequirements;
     param.Pointer = &reqs;
-    return (*pVirtualAlloc2)(addr, NULL, size, flags, PAGE_READWRITE, &param, 1);
+    return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, &param, 1);
   }
 #endif
   return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
 }
 
-static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) {
+static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only) {
   static volatile uintptr_t large_page_try_ok = 0;
   void* p = NULL;
-  if (use_large_os_page(size, try_alignment)) {
+  if (large_only || use_large_os_page(size, try_alignment)) {
     uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
-    if (try_ok > 0) {
+    if (!large_only && try_ok > 0) {
       // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
       // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times.
       mi_atomic_compare_exchange(&large_page_try_ok, try_ok - 1, try_ok);
@@ -211,6 +247,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
     else {
       // large OS pages must always reserve and commit.
       p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE | flags);
+      if (large_only) return p;
       // fall back to non-large page allocation on error (`p == NULL`).
       if (p == NULL) {
         mi_atomic_write(&large_page_try_ok,10);  // on error, don't try again for the next N allocations
@@ -237,12 +274,13 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
   return (void*)aligned_base;
 }
 #else
-static void* mi_unix_mmapx(size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
+#define MI_OS_USE_MMAP
+static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
   void* p = NULL;
   #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
   // on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations
-  static volatile intptr_t aligned_base = ((intptr_t)1 << 42); // starting at 4TiB
+  static volatile intptr_t aligned_base = ((intptr_t)4 << 40); // starting at 4TiB
-  if (try_alignment <= MI_SEGMENT_SIZE && (size%MI_SEGMENT_SIZE)==0) {
+  if (addr==NULL && try_alignment <= MI_SEGMENT_SIZE && (size%MI_SEGMENT_SIZE)==0) {
     intptr_t hint = mi_atomic_add(&aligned_base,size) - size;
     if (hint%try_alignment == 0) {
       p = mmap((void*)hint,size,protect_flags,flags,fd,0);
@@ -251,12 +289,13 @@ static void* mi_unix_mmapx(size_t size, size_t try_alignment, int protect_flags,
   }
   #endif
   if (p==NULL) {
-    p = mmap(NULL,size,protect_flags,flags,fd,0);
+    p = mmap(addr,size,protect_flags,flags,fd,0);
+    if (p==MAP_FAILED) p = NULL;
   }
   return p;
 }
 
-static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) {
+static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only) {
   void* p = NULL;
   #if !defined(MAP_ANONYMOUS)
   #define MAP_ANONYMOUS  MAP_ANON
@@ -278,10 +317,10 @@ static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags)
   // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
   fd = VM_MAKE_TAG(100);
   #endif
-  if (use_large_os_page(size, try_alignment)) {
+  if (large_only || use_large_os_page(size, try_alignment)) {
     static volatile uintptr_t large_page_try_ok = 0;
     uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
-    if (try_ok > 0) {
+    if (!large_only && try_ok > 0) {
       // If the OS is not configured for large OS pages, or the user does not have
       // enough permission, the `mmap` will always fail (but it might also fail for other reasons).
       // Therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times
@@ -297,27 +336,32 @@ static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags)
       #ifdef MAP_HUGETLB
       lflags |= MAP_HUGETLB;
       #endif
+      #ifdef MAP_HUGE_1GB
+      if ((size % (uintptr_t)1 << 20) == 0) {
+        lflags |= MAP_HUGE_1GB;
+      }
+      else
+      #endif
+      {
         #ifdef MAP_HUGE_2MB
         lflags |= MAP_HUGE_2MB;
         #endif
+      }
       #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
       lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
       #endif
-      if (lflags != flags) {
+      if (large_only || lflags != flags) {
         // try large OS page allocation
-        p = mi_unix_mmapx(size, try_alignment, protect_flags, lflags, lfd);
+        p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
-        if (p == MAP_FAILED) {
+        if (large_only) return p;
+        if (p == NULL) {
           mi_atomic_write(&large_page_try_ok, 10);  // on error, don't try again for the next N allocations
-          p = NULL; // and fall back to regular mmap
         }
       }
     }
   }
   if (p == NULL) {
-    p = mi_unix_mmapx(size, try_alignment, protect_flags, flags, fd);
+    p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd);    
-    if (p == MAP_FAILED) {
-      p = NULL;
-    }
     #if defined(MADV_HUGEPAGE)
     // Many Linux systems don't allow MAP_HUGETLB but they support instead
     // transparent huge pages (TPH). It is not required to call `madvise` with MADV_HUGE
@@ -325,7 +369,7 @@ static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags)
     // in that case -- in particular for our large regions (in `memory.c`).
     // However, some systems only allow TPH if called with explicit `madvise`, so
     // when large OS pages are enabled for mimalloc, we call `madvice` anyways.
-    else if (use_large_os_page(size, try_alignment)) {
+    if (use_large_os_page(size, try_alignment)) {
       madvise(p, size, MADV_HUGEPAGE);
     }
     #endif
@@ -340,17 +384,19 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_
   mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
   if (size == 0) return NULL;
 
-  void* p = NULL;
+  void* p = mi_os_alloc_from_huge_reserved(size, try_alignment, commit);
-#if defined(_WIN32)
+  if (p != NULL) return p;
+
+  #if defined(_WIN32)
     int flags = MEM_RESERVE;
     if (commit) flags |= MEM_COMMIT;
-  p = mi_win_virtual_alloc(NULL, size, try_alignment, flags);
+    p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false);
-#elif defined(__wasi__)
+  #elif defined(__wasi__)
     p = mi_wasm_heap_grow(size, try_alignment);
-#else
+  #else
     int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
-  p = mi_unix_mmap(size, try_alignment, protect_flags);
+    p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false);
-#endif
+  #endif
   _mi_stat_increase(&stats->mmap_calls, 1);
   if (p != NULL) {
     _mi_stat_increase(&stats->reserved, size);
@@ -399,7 +445,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
         // otherwise free and allocate at an aligned address in there
         mi_os_mem_free(p, over_size, stats);
         void* aligned_p = mi_align_up_ptr(p, alignment);
-        p = mi_win_virtual_alloc(aligned_p, size, alignment, flags);
+        p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false);
         if (p == aligned_p) break; // success!
         if (p != NULL) { // should not happen?
           mi_os_mem_free(p, size, stats);
@@ -555,17 +601,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
 
 #if defined(_WIN32)
   // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
-  // (but this is for an access pattern that immediately reuses the memory)
-  if (mi_option_is_enabled(mi_option_reset_discards) && pDiscardVirtualMemory != NULL) {
-    DWORD ok = (*pDiscardVirtualMemory)(start, csize);
-    mi_assert_internal(ok == ERROR_SUCCESS);
-    if (ok != ERROR_SUCCESS) return false;
-  }
-  else {
   void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
   mi_assert_internal(p == start);
   if (p != start) return false;
-  }
 #else
 #if defined(MADV_FREE)
   static int advice = MADV_FREE;
@@ -664,3 +702,127 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
   return mi_os_mem_free(start, size, stats);
 #endif
 }
+
+
+/* ----------------------------------------------------------------------------
+
+-----------------------------------------------------------------------------*/
+#define MI_HUGE_OS_PAGE_SIZE ((size_t)1 << 30)  // 1GiB
+
+typedef struct mi_huge_info_s {
+  uint8_t*           start;
+  ptrdiff_t          reserved;
+  volatile ptrdiff_t used;
+} mi_huge_info_t;
+
+static mi_huge_info_t os_huge_reserved = { NULL, 0, 0 };
+
+static bool mi_os_is_huge_reserved(void* p) {
+  return (os_huge_reserved.start != NULL && 
+          (uint8_t*)p >= os_huge_reserved.start && 
+          (uint8_t*)p < os_huge_reserved.start + os_huge_reserved.reserved);
+}
+
+static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit) 
+{
+  // only allow large aligned allocations
+  if (size < MI_SEGMENT_SIZE || (size % MI_SEGMENT_SIZE) != 0) return NULL;
+  if (try_alignment > MI_SEGMENT_SIZE) return NULL;
+  if (!commit) return NULL;
+  if (os_huge_reserved.start==NULL) return NULL;
+  if (mi_atomic_iread(&os_huge_reserved.used) >= os_huge_reserved.reserved) return NULL; // already full
+
+  // always aligned
+  mi_assert_internal( os_huge_reserved.used % MI_SEGMENT_SIZE == 0 );
+  mi_assert_internal( (uintptr_t)os_huge_reserved.start % MI_SEGMENT_SIZE == 0 );
+  
+  // try to reserve space
+  ptrdiff_t next = mi_atomic_add( &os_huge_reserved.used, (ptrdiff_t)size );
+  if (next > os_huge_reserved.reserved) {
+    // "free" our over-allocation
+    mi_atomic_add( &os_huge_reserved.used, -((ptrdiff_t)size) );
+    return NULL;
+  }
+
+  // success!
+  uint8_t* p = os_huge_reserved.start + next - (ptrdiff_t)size;
+  mi_assert_internal( (uintptr_t)p % MI_SEGMENT_SIZE == 0 );
+  return p;
+}
+
+/*
+static void mi_os_free_huge_reserved() {
+  uint8_t* addr = os_huge_reserved.start;
+  size_t total  = os_huge_reserved.reserved;
+  os_huge_reserved.reserved = 0;
+  os_huge_reserved.start = NULL;
+  for( size_t current = 0; current < total; current += MI_HUGE_OS_PAGE_SIZE) {
+    _mi_os_free(addr + current, MI_HUGE_OS_PAGE_SIZE, &_mi_stats_main);
+  }
+}
+*/
+
+#if !(MI_INTPTR_SIZE >= 8 && (defined(_WIN32) || defined(MI_OS_USE_MMAP)))
+int mi_reserve_huge_os_pages(size_t pages, size_t max_secs) {
+  return -2; // cannot allocate
+}
+#else
+int mi_reserve_huge_os_pages( size_t pages, double max_secs ) mi_attr_noexcept
+{
+  if (max_secs==0) return -1; // timeout 
+  if (pages==0) return 0;     // ok
+  if (os_huge_reserved.start != NULL) return -2; // already reserved
+
+  // Allocate one page at the time but try to place them contiguously
+  // We allocate one page at the time to be able to abort if it takes too long
+  double start_t = _mi_clock_start();
+  uint8_t* start = (uint8_t*)((uintptr_t)8 << 40); // 8TiB virtual start address
+  uint8_t* addr = start;  // current top of the allocations
+  for (size_t page = 0; page < pages; page++, addr += MI_HUGE_OS_PAGE_SIZE ) {
+    // allocate lorgu pages
+    void* p = NULL; 
+    #ifdef _WIN32
+    p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true);
+    #elif defined(MI_OS_USE_MMAP)
+    p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true);
+    #else 
+    // always fail
+    #endif  
+    
+    // Did we succeed at a contiguous address?
+    if (p != addr) {
+      if (p != NULL) {
+        _mi_warning_message("could not allocate contiguous huge page %zu at 0x%p\n", page, addr); 
+        _mi_os_free(p, MI_HUGE_OS_PAGE_SIZE, &_mi_stats_main );
+      }
+      else {
+        #ifdef _WIN32
+        int err = GetLastError();
+        #else
+        int err = errno;
+        #endif
+        _mi_warning_message("could not allocate huge page %zu at 0x%p, error: %i\n", page, addr, err);
+      }
+      return -2;
+    }
+    // success, record it
+    if (page==0) {
+      os_huge_reserved.start = addr;
+    }
+    os_huge_reserved.reserved += MI_HUGE_OS_PAGE_SIZE;
+    _mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE); 
+    _mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE);
+    
+    // check for timeout
+    double elapsed = _mi_clock_end(start_t);
+    if (elapsed > max_secs) return (-1); // timeout
+    if (page >= 1) {
+      double estimate = ((elapsed / (double)(page+1)) * (double)pages);
+      if (estimate > 1.5*max_secs) return (-1); // seems like we are going to timeout
+    }
+  }  
+  _mi_verbose_message("reserved %zu huge pages\n", pages);
+  return 0;
+}
+#endif
+

src/page-queue.c

@@ -34,15 +34,15 @@ terms of the MIT license. A copy of the license can be found in the file
 
 
 static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {
-  return (pq->block_size == (MI_MEDIUM_SIZE_MAX+sizeof(uintptr_t)));
+  return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+sizeof(uintptr_t)));
 }
 
 static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {
-  return (pq->block_size == (MI_MEDIUM_SIZE_MAX+(2*sizeof(uintptr_t))));
+  return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+(2*sizeof(uintptr_t))));
 }
 
 static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
-  return (pq->block_size > MI_MEDIUM_SIZE_MAX);
+  return (pq->block_size > MI_MEDIUM_OBJ_SIZE_MAX);
 }
 
 /* -----------------------------------------------------------
@@ -116,7 +116,7 @@ extern inline uint8_t _mi_bin(size_t size) {
     bin = (uint8_t)wsize;
   }
   #endif
-  else if (wsize > MI_MEDIUM_WSIZE_MAX) {
+  else if (wsize > MI_MEDIUM_OBJ_WSIZE_MAX) {
     bin = MI_BIN_HUGE;
   }
   else {
@@ -147,7 +147,7 @@ size_t _mi_bin_size(uint8_t bin) {
 
 // Good size for allocation
 size_t mi_good_size(size_t size) mi_attr_noexcept {
-  if (size <= MI_MEDIUM_SIZE_MAX) {
+  if (size <= MI_MEDIUM_OBJ_SIZE_MAX) {
     return _mi_bin_size(_mi_bin(size));
   }
   else {
@@ -245,7 +245,7 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) {
 static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
   mi_assert_internal(page != NULL);
   mi_assert_expensive(mi_page_queue_contains(queue, page));
-  mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_huge(queue))  || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
+  mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue))  || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
   if (page->prev != NULL) page->prev->next = page->next;
   if (page->next != NULL) page->next->prev = page->prev;
   if (page == queue->last)  queue->last = page->prev;
@@ -268,7 +268,7 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_
   mi_assert_internal(page->heap == NULL);
   mi_assert_internal(!mi_page_queue_contains(queue, page));
   mi_assert_internal(page->block_size == queue->block_size ||
-                      (page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_huge(queue)) ||
+                      (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) ||
                         (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
 
   mi_page_set_in_full(page, mi_page_queue_is_full(queue));
@@ -297,8 +297,8 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro
   mi_assert_internal((page->block_size == to->block_size && page->block_size == from->block_size) ||
                      (page->block_size == to->block_size && mi_page_queue_is_full(from)) ||
                      (page->block_size == from->block_size && mi_page_queue_is_full(to)) ||
-                     (page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_huge(to)) ||
+                     (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) ||
-                     (page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_full(to)));
+                     (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_full(to)));
 
   if (page->prev != NULL) page->prev->next = page->next;
   if (page->next != NULL) page->next->prev = page->prev;

src/page.c

@@ -102,7 +102,7 @@ bool _mi_page_is_valid(mi_page_t* page) {
     mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == page->heap->thread_id);
     mi_page_queue_t* pq = mi_page_queue_of(page);
     mi_assert_internal(mi_page_queue_contains(pq, page));
-    mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_MEDIUM_SIZE_MAX || mi_page_is_in_full(page));
+    mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_in_full(page));
     mi_assert_internal(mi_heap_contains_queue(page->heap,pq));
   }
   return true;
@@ -356,8 +356,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
   mi_page_set_has_aligned(page, false);
 
   // account for huge pages here
-  if (page->block_size > MI_MEDIUM_SIZE_MAX) {
+  if (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX) {
-    if (page->block_size <= MI_LARGE_SIZE_MAX) {
+    if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
       _mi_stat_decrease(&page->heap->tld->stats.large, page->block_size);
     }
     else {
@@ -394,7 +394,7 @@ void _mi_page_retire(mi_page_t* page) {
   // is the only page left with free blocks. It is not clear
   // how to check this efficiently though... for now we just check
   // if its neighbours are almost fully used.
-  if (mi_likely(page->block_size <= MI_MEDIUM_SIZE_MAX)) {
+  if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) {
     if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) {
       _mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1);
       return; // dont't retire after all
@@ -713,7 +713,7 @@ static mi_page_t* mi_large_page_alloc(mi_heap_t* heap, size_t size) {
   if (page != NULL) {
     mi_assert_internal(mi_page_immediate_available(page));
     mi_assert_internal(page->block_size == block_size);
-    if (page->block_size <= MI_LARGE_SIZE_MAX) {
+    if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
       _mi_stat_increase(&heap->tld->stats.large, block_size);
       _mi_stat_counter_increase(&heap->tld->stats.large_count, 1);
     }
@@ -746,8 +746,8 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
 
   // huge allocation?
   mi_page_t* page;
-  if (mi_unlikely(size > MI_MEDIUM_SIZE_MAX)) {
+  if (mi_unlikely(size > MI_MEDIUM_OBJ_SIZE_MAX)) {
-    if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) {
+    if (mi_unlikely(size > PTRDIFF_MAX)) {
       page = NULL;
     }
     else {

src/segment.c

@@ -22,7 +22,6 @@ static void mi_segment_map_freed_at(const mi_segment_t* segment);
 /* -----------------------------------------------------------
   Segment allocation
 
-
   In any case the memory for a segment is virtual and only
   committed on demand (i.e. we are careful to not touch the memory
   until we actually allocate a block there)
@@ -873,13 +872,13 @@ static bool mi_is_good_fit(size_t bsize, size_t size) {
 
 mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
   mi_page_t* page;
-  if (block_size <= MI_SMALL_SIZE_MAX) {// || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) {
+  if (block_size <= MI_SMALL_OBJ_SIZE_MAX) {// || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) {
     page = mi_segment_page_alloc(MI_PAGE_SMALL,block_size,tld,os_tld);
   }
-  else if (block_size <= MI_MEDIUM_SIZE_MAX) {// || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) {
+  else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) {// || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) {
     page = mi_segment_page_alloc(MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,tld, os_tld);
   }
-  else if (block_size <= MI_LARGE_SIZE_MAX) {
+  else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) {
     page = mi_segment_page_alloc(MI_PAGE_LARGE,block_size,tld, os_tld);
   }
   else {
@@ -1006,5 +1005,3 @@ static void* mi_segment_range_of(const void* p, size_t* size) {
   }
 }
 */
-
-

src/stats.c

@@ -28,11 +28,14 @@ void _mi_stats_done(mi_stats_t* stats) {
   Statistics operations
 ----------------------------------------------------------- */
 
+static bool mi_is_in_main(void* stat) {
+  return ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main
+         && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));  
+}
+
 static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
   if (amount == 0) return;
-  bool in_main = ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main
+  if (mi_is_in_main(stat))
-                  && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t)));
-  if (in_main)
   {
     // add atomically (for abandoned pages)
     int64_t current = mi_atomic_add(&stat->current,amount);
@@ -58,11 +61,16 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
 }
 
 void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {  
+  if (mi_is_in_main(stat)) {
     mi_atomic_add( &stat->count, 1 );
     mi_atomic_add( &stat->total, (int64_t)amount );
+  }
+  else {
+    stat->count++;
+    stat->total += amount;
+  }
 }
 
-
 void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) {
   mi_stat_update(stat, (int64_t)amount);
 }
@@ -276,8 +284,8 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n
   _mi_fprintf(out,"\n");
 }
 
-static double mi_clock_end(double start);
+double _mi_clock_end(double start);
-static double mi_clock_start(void);
+double _mi_clock_start(void);
 static double mi_time_start = 0.0;
 
 static mi_stats_t* mi_stats_get_default(void) {
@@ -289,7 +297,7 @@ void mi_stats_reset(void) mi_attr_noexcept {
   mi_stats_t* stats = mi_stats_get_default();
   if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); }
   memset(&_mi_stats_main, 0, sizeof(mi_stats_t));
-  mi_time_start = mi_clock_start();
+  mi_time_start = _mi_clock_start();
 }
 
 static void mi_stats_print_ex(mi_stats_t* stats, double secs, FILE* out) {
@@ -301,11 +309,11 @@ static void mi_stats_print_ex(mi_stats_t* stats, double secs, FILE* out) {
 }
 
 void mi_stats_print(FILE* out) mi_attr_noexcept {
-  mi_stats_print_ex(mi_stats_get_default(),mi_clock_end(mi_time_start),out);
+  mi_stats_print_ex(mi_stats_get_default(),_mi_clock_end(mi_time_start),out);
 }
 
 void mi_thread_stats_print(FILE* out) mi_attr_noexcept {
-  _mi_stats_print(mi_stats_get_default(), mi_clock_end(mi_time_start), out);
+  _mi_stats_print(mi_stats_get_default(), _mi_clock_end(mi_time_start), out);
 }
 
 
@@ -350,7 +358,7 @@ static double mi_clock_now(void) {
 
 static double mi_clock_diff = 0.0;
 
-static double mi_clock_start(void) {
+double _mi_clock_start(void) {
   if (mi_clock_diff == 0.0) {
     double t0 = mi_clock_now();
     mi_clock_diff = mi_clock_now() - t0;
@@ -358,7 +366,7 @@ static double mi_clock_start(void) {
   return mi_clock_now();
 }
 
-static double mi_clock_end(double start) {
+double _mi_clock_end(double start) {
   double end = mi_clock_now();
   return (end - start - mi_clock_diff);
 }

test/test-api.c

@@ -66,7 +66,7 @@ bool test_heap2();
 // Main testing
 // ---------------------------------------------------------------------------
 int main() {
-  mi_option_enable(mi_option_verbose,false);
+  mi_option_disable(mi_option_verbose);
 
   // ---------------------------------------------------
   // Malloc

test/test-stress.c

@@ -154,6 +154,9 @@ int main(int argc, char** argv) {
     if (n > 0) N = n;
   }
   printf("start with %i threads with a %i%% load-per-thread\n", THREADS, N);  
+  //int res = mi_reserve_huge_os_pages(4,1);
+  //printf("(reserve huge: %i\n)", res);
+
   //bench_start_program();
   memset((void*)transfer, 0, TRANSFERS*sizeof(void*));
   run_os_threads(THREADS);