From 10a29f17c89be84e6ada232f4d41f5c4da2e4c47 Mon Sep 17 00:00:00 2001
From: daan <daan@microsoft.com>
Date: Tue, 2 Jul 2019 22:49:12 -0700
Subject: [PATCH 1/5] more conservative shrinking of memory

---
 src/alloc-override.c |  6 ++---
 src/os.c             | 59 ++++++++++++++++++++++++++------------------
 src/segment.c        |  2 +-
 3 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/src/alloc-override.c b/src/alloc-override.c
index 42c9f3d6..7c052690 100644
--- a/src/alloc-override.c
+++ b/src/alloc-override.c
@@ -52,7 +52,7 @@ terms of the MIT license. A copy of the license can be found in the file
     MI_INTERPOSE_MI(realloc),
     MI_INTERPOSE_MI(free),
     MI_INTERPOSE_MI(strdup),
-    MI_INTERPOSE_MI(strndup)    
+    MI_INTERPOSE_MI(strndup)
   };
 #else
   // On all other systems forward to our API
@@ -60,8 +60,8 @@ terms of the MIT license. A copy of the license can be found in the file
   void* calloc(size_t size, size_t n)    mi_attr_noexcept  MI_FORWARD2(mi_calloc, size, n);
   void* realloc(void* p, size_t newsize) mi_attr_noexcept  MI_FORWARD2(mi_realloc, p, newsize);
   void  free(void* p)                    mi_attr_noexcept  MI_FORWARD0(mi_free, p);
-  char* strdup(const char* s)                              MI_FORWARD1(mi_strdup, s);
-  char* strndup(const char* s, size_t n)                   MI_FORWARD2(mi_strndup, s, n);  
+  //char* strdup(const char* s)                              MI_FORWARD1(mi_strdup, s);
+  //char* strndup(const char* s, size_t n)                   MI_FORWARD2(mi_strndup, s, n);  
 #endif
 
 #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__)
diff --git a/src/os.c b/src/os.c
index b866d1b2..9bf576a1 100644
--- a/src/os.c
+++ b/src/os.c
@@ -16,12 +16,12 @@ terms of the MIT license. A copy of the license can be found in the file
 
 /* -----------------------------------------------------------
   Initialization.
-  On windows initializes support for aligned allocation and 
+  On windows initializes support for aligned allocation and
   large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
 ----------------------------------------------------------- */
 
 #if defined(_WIN32)
-  #include <windows.h>  
+  #include <windows.h>
 #else
   #include <sys/mman.h>  // mmap
   #include <unistd.h>    // sysconf
@@ -56,7 +56,7 @@ static bool use_large_os_page(size_t size, size_t alignment) {
 static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
   UNUSED(alignment);
   if (size >= (SIZE_MAX - os_alloc_granularity)) return size; // possible overflow?
-  return _mi_align_up(size, os_alloc_granularity);  
+  return _mi_align_up(size, os_alloc_granularity);
 }
 
 #if defined(_WIN32)
@@ -76,7 +76,7 @@ void _mi_os_init(void) {
   hDll = LoadLibrary("kernelbase.dll");
   if (hDll!=NULL) {
     // use VirtualAlloc2FromApp as it is available to Windows store apps
-    pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp");  
+    pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp");
     FreeLibrary(hDll);
   }
   // Try to see if large OS pages are supported
@@ -109,7 +109,7 @@ void _mi_os_init(void) {
       if (err==0) err = GetLastError();
       _mi_warning_message("cannot enable large OS page support, error %lu\n", err);
     }
-  }  
+  }
 }
 #else
 void _mi_os_init() {
@@ -250,11 +250,11 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
     reqs.Alignment = alignment;
     MEM_EXTENDED_PARAMETER param = { 0 };
     param.Type = MemExtendedParameterAddressRequirements;
-    param.Pointer = &reqs; 
+    param.Pointer = &reqs;
     DWORD flags = MEM_RESERVE;
     if (commit) flags |= MEM_COMMIT;
     if (use_large_os_page(size, alignment)) flags |= MEM_LARGE_PAGES;
-    p = (*pVirtualAlloc2)(NULL, NULL, size, flags, PAGE_READWRITE, &param, 1);  
+    p = (*pVirtualAlloc2)(NULL, NULL, size, flags, PAGE_READWRITE, &param, 1);
   }
   #elif defined(MAP_ALIGNED)
   // on BSD, use the aligned mmap api
@@ -276,15 +276,19 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
   return p;
 }
 
-// Conservatively OS page align within a given area
-static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) {
+// OS page align within a given area,
+// either conservative (pages inside the area only),
+// or not (straddling pages outside the area is possible)
+static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
   mi_assert(addr != NULL && size > 0);
   if (newsize != NULL) *newsize = 0;
   if (size == 0 || addr == NULL) return NULL;
 
   // page align conservatively within the range
-  void* start = mi_align_up_ptr(addr, _mi_os_page_size());
-  void* end = mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size());
+  void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
+                              : mi_align_down_ptr(addr, _mi_os_page_size()));
+  void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
+                            : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
   ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
   if (diff <= 0) return NULL;
 
@@ -293,6 +297,12 @@ static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) {
   return start;
 }
 
+static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
+  return mi_os_page_align_areax(true,addr,size,newsize);
+}
+
+
+
 // Signal to the OS that the address range is no longer in use
 // but may be used later again. This will release physical memory
 // pages and reduce swapping while keeping the memory committed.
@@ -300,7 +310,7 @@ static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) {
 bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
   // page align conservatively within the range
   size_t csize;
-  void* start = mi_os_page_align_area(addr,size,&csize);
+  void* start = mi_os_page_align_area_conservative(addr,size,&csize);
   if (csize==0) return true;
   UNUSED(stats); // if !STATS
   mi_stat_increase(stats->reset, csize);
@@ -318,10 +328,10 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
   /*
   // VirtualUnlock removes the memory eagerly from the current working set (which MEM_RESET does lazily on demand)
   // TODO: put this behind an option?
-  DWORD ok = VirtualUnlock(start, csize); 
+  DWORD ok = VirtualUnlock(start, csize);
   if (ok != 0) return false;
   */
-  return true;  
+  return true;
 #else
   #if defined(MADV_FREE)
     static int advice = MADV_FREE;
@@ -346,7 +356,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
 static  bool mi_os_protectx(void* addr, size_t size, bool protect) {
   // page align conservatively within the range
   size_t csize = 0;
-  void* start = mi_os_page_align_area(addr, size, &csize);
+  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
   if (csize==0) return false;
 
   int err = 0;
@@ -374,9 +384,9 @@ bool _mi_os_unprotect(void* addr, size_t size) {
 // Commit/Decommit memory.
 // We page align to a conservative area inside the range to reset.
 static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) {
-  // page align conservatively within the range
+  // page align in the range, commit liberally, decommit conservative
   size_t csize;
-  void* start = mi_os_page_align_area(addr, size, &csize);
+  void* start = mi_os_page_align_areax(!commit, addr, size, &csize);
   if (csize == 0) return true;
   int err = 0;
   UNUSED(stats); // if !STATS
@@ -424,12 +434,12 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
   // oldsize and newsize should be page aligned or we cannot shrink precisely
   void* addr = (uint8_t*)p + newsize;
   size_t size = 0;
-  void* start = mi_os_page_align_area(addr, oldsize - newsize, &size);
+  void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size);
   if (size==0 || start != addr) return false;
 
   #ifdef _WIN32
-  // we cannot shrink on windows
-  return false;
+  // we cannot shrink on windows, but we can decommit
+  return mi_os_decommit(start, size, stats);
   #else
   return mi_os_mem_free(start, size, stats);
   #endif
@@ -448,7 +458,8 @@ void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
 }
 
 void  _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
-  UNUSED(stats);
+  if (size==0) return;
+  size = mi_os_good_alloc_size(size, 0);
   mi_os_mem_free(p, size, stats);
 }
 
@@ -505,7 +516,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld
   void* suggest = NULL;
   void* p = mi_os_mem_alloc_aligned(size,alignment,commit,tld->stats);
 
-  // Fall back 
+  // Fall back
   if (p==NULL && (tld->mmap_next_probable % alignment) == 0) {
     // if the next probable address is aligned,
     // then try to just allocate `size` and hope it is aligned...
@@ -523,8 +534,8 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld
     //fprintf(stderr, "mimalloc: slow mmap 0x%lx\n", _mi_thread_id());
     p = mi_os_alloc_aligned_ensured(size, alignment,commit,0,tld->stats);
   }
-  if (p != NULL) {    
-    // next probable address is the page-aligned address just after the newly allocated area.    
+  if (p != NULL) {
+    // next probable address is the page-aligned address just after the newly allocated area.
     size_t probable_size = MI_SEGMENT_SIZE;
     if (tld->mmap_previous > p) {
       // Linux tends to allocate downward
diff --git a/src/segment.c b/src/segment.c
index addb3bb3..14f893e8 100644
--- a/src/segment.c
+++ b/src/segment.c
@@ -147,7 +147,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
     // secure >  1: every page has an os guard page
     psize -= _mi_os_page_size();
   }
-  
+
   if (page_size != NULL) *page_size = psize;
   mi_assert_internal(_mi_ptr_page(p) == page);
   mi_assert_internal(_mi_ptr_segment(p) == segment);

From 158705815e01dac9a7d9bb6c85841b25393c7fd2 Mon Sep 17 00:00:00 2001
From: daan <daan@microsoft.com>
Date: Wed, 3 Jul 2019 00:16:27 -0700
Subject: [PATCH 2/5] fix  aligned_ensured bug, trie inc bug, stats in the OS
 module

---
 src/os.c   | 44 +++++++++++++++++++-------------------------
 src/page.c |  3 +++
 2 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/src/os.c b/src/os.c
index 9bf576a1..d952726a 100644
--- a/src/os.c
+++ b/src/os.c
@@ -173,7 +173,6 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
 }
 
 static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_flags, mi_stats_t* stats) {
-  UNUSED(stats);
   if (size == 0) return NULL;
   void* p = NULL;
 #if defined(_WIN32)
@@ -230,12 +229,11 @@ static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_fla
     p = NULL;
   }
 #endif
-  UNUSED(stats);
   mi_assert(p == NULL || (addr == NULL && p != addr) || (addr != NULL && p == addr));
   if (p != NULL) {
-    mi_stat_increase(stats->mmap_calls, 1);
-    mi_stat_increase(stats->reserved, size);
-    if (commit) mi_stat_increase(stats->committed, size);
+    _mi_stat_increase(&stats->mmap_calls, 1);
+    _mi_stat_increase(&stats->reserved, size);
+    if (commit) _mi_stat_increase(&stats->committed, size);
   }
   return p;
 }
@@ -266,12 +264,11 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
   UNUSED(size);
   UNUSED(alignment);
   #endif
-  UNUSED(stats); // if !STATS
   mi_assert(p == NULL || (uintptr_t)p % alignment == 0);
   if (p != NULL) {
-    mi_stat_increase(stats->mmap_calls, 1);
-    mi_stat_increase(stats->reserved, size);
-    if (commit) mi_stat_increase(stats->committed, size);
+    _mi_stat_increase(&stats->mmap_calls, 1);
+    _mi_stat_increase(&stats->reserved, size);
+    if (commit) _mi_stat_increase(&stats->committed, size);
   }
   return p;
 }
@@ -312,8 +309,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
   size_t csize;
   void* start = mi_os_page_align_area_conservative(addr,size,&csize);
   if (csize==0) return true;
-  UNUSED(stats); // if !STATS
-  mi_stat_increase(stats->reset, csize);
+  _mi_stat_increase(&stats->reset, csize);
 
 #if defined(_WIN32)
   // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
@@ -389,13 +385,12 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stat
   void* start = mi_os_page_align_areax(!commit, addr, size, &csize);
   if (csize == 0) return true;
   int err = 0;
-  UNUSED(stats); // if !STATS
   if (commit) {
-    mi_stat_increase(stats->committed, csize);
-    mi_stat_increase(stats->commit_calls,1);
+    _mi_stat_increase(&stats->committed, csize);
+    _mi_stat_increase(&stats->commit_calls,1);
   }
   else {
-    mi_stat_decrease(stats->committed, csize);
+    _mi_stat_decrease(&stats->committed, csize);
   }
 
 #if defined(_WIN32)
@@ -439,7 +434,7 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
 
   #ifdef _WIN32
   // we cannot shrink on windows, but we can decommit
-  return mi_os_decommit(start, size, stats);
+  return _mi_os_decommit(start, size, stats);
   #else
   return mi_os_mem_free(start, size, stats);
   #endif
@@ -469,17 +464,16 @@ void  _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
 static void* mi_os_alloc_aligned_ensured(size_t size, size_t alignment, bool commit, size_t trie, mi_stats_t* stats)
 {
   if (trie >= 3) return NULL; // stop recursion (only on Windows)
-  size_t alloc_size = size + alignment;
-  mi_assert(alloc_size >= size); // overflow?
-  if (alloc_size < size) return NULL;
-
+  if (size > SIZE_MAX - alignment) return NULL; // overflow
+  size_t alloc_size = size + alignment; // no need for -1 as we need to be page aligned anyways
+  
   // allocate a chunk that includes the alignment
   void* p = mi_os_mem_alloc(NULL, alloc_size, commit, 0, stats);
   if (p == NULL) return NULL;
   // create an aligned pointer in the allocated area
   void* aligned_p = mi_align_up_ptr(p, alignment);
   mi_assert(aligned_p != NULL);
-
+#if _WIN32
   // free it and try to allocate `size` at exactly `aligned_p`
   // note: this may fail in case another thread happens to allocate
   // concurrently at that spot. We try up to 3 times to mitigate this.
@@ -487,9 +481,9 @@ static void* mi_os_alloc_aligned_ensured(size_t size, size_t alignment, bool com
   p = mi_os_mem_alloc(aligned_p, size, commit, 0, stats);
   if (p != aligned_p) {
     if (p != NULL) mi_os_mem_free(p, size, stats);
-    return mi_os_alloc_aligned_ensured(size, alignment, commit, trie++, stats);
+    return mi_os_alloc_aligned_ensured(size, alignment, commit, trie+1, stats);
   }
-#if 0  // could use this on mmap systems
+#else  
   // we selectively unmap parts around the over-allocated area.
   size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
   size_t mid_size = _mi_align_up(size, _mi_os_page_size());
@@ -522,7 +516,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld
     // then try to just allocate `size` and hope it is aligned...
     p = mi_os_mem_alloc(suggest, size, commit, 0, tld->stats);
     if (p == NULL) return NULL;
-    if (((uintptr_t)p % alignment) == 0) mi_stat_increase(tld->stats->mmap_right_align, 1);
+    if (((uintptr_t)p % alignment) == 0) _mi_stat_increase(&tld->stats->mmap_right_align, 1);
   }
   //fprintf(stderr, "segment address guess: %s, p=%lxu, guess:%lxu\n", (p != NULL && (uintptr_t)p % alignment ==0 ? "correct" : "incorrect"), (uintptr_t)p, next_probable);
 
@@ -530,7 +524,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld
     // if `p` is not yet aligned after all, free the block and use a slower
     // but guaranteed way to allocate an aligned block
     if (p != NULL) mi_os_mem_free(p, size, tld->stats);
-    mi_stat_increase( tld->stats->mmap_ensure_aligned, 1);
+    _mi_stat_increase( &tld->stats->mmap_ensure_aligned, 1);
     //fprintf(stderr, "mimalloc: slow mmap 0x%lx\n", _mi_thread_id());
     p = mi_os_alloc_aligned_ensured(size, alignment,commit,0,tld->stats);
   }
diff --git a/src/page.c b/src/page.c
index fae67ef2..ea86c138 100644
--- a/src/page.c
+++ b/src/page.c
@@ -391,6 +391,9 @@ void _mi_page_retire(mi_page_t* page) {
 static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats)
 {
   UNUSED(stats);
+  mi_assert_internal(page->free == NULL);
+  mi_assert_internal(page->local_free == NULL);
+  mi_assert_internal(page->capacity + extend <= page->reserved);
   void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL );
   size_t bsize = page->block_size;
   mi_block_t* start = mi_page_block_at(page, page_area, page->capacity);

From 93906428798ff6f4e1b71bf2afc087f6920dddf6 Mon Sep 17 00:00:00 2001
From: daan <daan@microsoft.com>
Date: Wed, 3 Jul 2019 13:00:43 -0700
Subject: [PATCH 3/5] fix windows dynamic malloc overried when both ucrtbase
 and msvcrt are loaded; also fix virtualalloc2 on 32-bit

---
 src/alloc-override-win.c | 12 +++++++-----
 src/os.c                 |  2 +-
 test/main-override.cpp   |  2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c
index 7158e920..9c091389 100644
--- a/src/alloc-override-win.c
+++ b/src/alloc-override-win.c
@@ -526,7 +526,7 @@ static void mi_module_resolve(HMODULE mod) {
   // see if any patches apply
   for (size_t i = 0; patches[i].name != NULL; i++) {
     mi_patch_t* patch = &patches[i];
-    if (!patch->applied && patch->original==NULL) {
+    if (!patch->applied && patch->original==NULL) { // we apply only the first found dll
       void* addr = GetProcAddress(mod, patch->name);
       if (addr != NULL) {
         // found it! set the address
@@ -554,9 +554,11 @@ static bool mi_patches_resolve(void) {
   size_t count = needed / sizeof(HMODULE);
   size_t ucrtbase_index = 0;
   size_t mimalloc_index = 0;
-  // iterate through the loaded modules
-  for (size_t i = 0; i < count; i++) {
-    HMODULE mod = modules[i];
+  // iterate through the loaded modules; do this from the end so we prefer the
+  // first loaded DLL as sometimes both "msvcr" and "ucrt" are both loaded and we should
+  // override "ucrt" in that situation.
+  for (size_t i = count; i > 0; i--) {
+    HMODULE mod = modules[i-1];
     char filename[MAX_PATH] = { 0 };
     DWORD slen = GetModuleFileName(mod, filename, MAX_PATH);
     if (slen > 0 && slen < MAX_PATH) {
@@ -566,7 +568,7 @@ static bool mi_patches_resolve(void) {
       const char* basename = (lastsep==NULL ? filename : lastsep+1);
       if (i==0                                    // main module to allow static crt linking
         || _strnicmp(basename, "ucrt", 4) == 0    // new ucrtbase.dll in windows 10
-        || _strnicmp(basename, "msvcr", 5) == 0)  // older runtimes
+        || _strnicmp(basename, "msvcr", 5) == 0)  // older runtimes      
       {
         // remember indices so we can check load order (in debug mode)
         if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i;
diff --git a/src/os.c b/src/os.c
index d952726a..2957e0f9 100644
--- a/src/os.c
+++ b/src/os.c
@@ -62,7 +62,7 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
 #if defined(_WIN32)
 // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
 // So, we need to look it up dynamically to run on older systems.
-typedef PVOID (*VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG );
+typedef PVOID (__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG );
 static VirtualAlloc2Ptr pVirtualAlloc2 = NULL;
 
 void _mi_os_init(void) {
diff --git a/test/main-override.cpp b/test/main-override.cpp
index 406d3bea..e24ed6d3 100644
--- a/test/main-override.cpp
+++ b/test/main-override.cpp
@@ -13,7 +13,7 @@ void free_p() {
 }
 
 int main() {
-  mi_stats_reset();
+  mi_stats_reset();  
   atexit(free_p);
   void* p1 = malloc(78);
   void* p2 = malloc(24);

From 7b4f3591f00341763f02981be105041d1e5dc672 Mon Sep 17 00:00:00 2001
From: daan <daan@microsoft.com>
Date: Wed, 3 Jul 2019 14:23:30 -0700
Subject: [PATCH 4/5] fix dynamic override when both msvcrt and ucrtbase are
 loaded in any order using priorities

---
 src/alloc-override-win.c | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c
index 9c091389..cd1953c8 100644
--- a/src/alloc-override-win.c
+++ b/src/alloc-override-win.c
@@ -377,6 +377,7 @@ typedef enum patch_apply_e {
 
 typedef struct mi_patch_s {
   const char*   name;       // name of the function to patch
+  int           priority;   // priority to patch this one (used to prioritize over multiple entries in various dll's)
   void*         original;   // the resolved address of the function (or NULL)
   void*         target;     // the address of the new target (never NULL)
   void*         target_term;// the address of the target during termination (or NULL)
@@ -384,8 +385,8 @@ typedef struct mi_patch_s {
   mi_jump_t     save;       // the saved instructions in case it was applied
 } mi_patch_t;
 
-#define MI_PATCH_NAME3(name,target,term)  { name, NULL, &target, &term, false }
-#define MI_PATCH_NAME2(name,target)       { name, NULL, &target, NULL, false }
+#define MI_PATCH_NAME3(name,target,term)  { name, 0, NULL, &target, &term, PATCH_NONE }
+#define MI_PATCH_NAME2(name,target)       { name, 0, NULL, &target, NULL, PATCH_NONE }
 #define MI_PATCH3(name,target,term)       MI_PATCH_NAME3(#name, target, term)
 #define MI_PATCH2(name,target)            MI_PATCH_NAME2(#name, target)
 #define MI_PATCH1(name)                   MI_PATCH2(name,mi_##name)
@@ -447,7 +448,7 @@ static mi_patch_t patches[] = {
   MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free, mi_free_term),
 #endif
 
-  { NULL, NULL, NULL, false }
+  { NULL, 0, NULL, NULL, NULL, PATCH_NONE }
 };
 
 
@@ -522,15 +523,16 @@ static int __cdecl mi_setmaxstdio(int newmax) {
 // ------------------------------------------------------
 
 // Try to resolve patches for a given module (DLL)
-static void mi_module_resolve(HMODULE mod) {
+static void mi_module_resolve(HMODULE mod, int priority) {
   // see if any patches apply
   for (size_t i = 0; patches[i].name != NULL; i++) {
     mi_patch_t* patch = &patches[i];
-    if (!patch->applied && patch->original==NULL) { // we apply only the first found dll
+    if (!patch->applied && patch->priority < priority) { 
       void* addr = GetProcAddress(mod, patch->name);
       if (addr != NULL) {
         // found it! set the address
         patch->original = addr;
+        patch->priority = priority;
       }
     }
   }
@@ -566,16 +568,19 @@ static bool mi_patches_resolve(void) {
       filename[slen] = 0;
       const char* lastsep = strrchr(filename, '\\');
       const char* basename = (lastsep==NULL ? filename : lastsep+1);
-      if (i==0                                    // main module to allow static crt linking
-        || _strnicmp(basename, "ucrt", 4) == 0    // new ucrtbase.dll in windows 10
-        || _strnicmp(basename, "msvcr", 5) == 0)  // older runtimes      
-      {
+
+      int priority = 0; 
+      if (i == 0) priority = 2; // main module to allow static crt linking
+      else if (_strnicmp(basename, "ucrt", 4) == 0) priority = 3;   // new ucrtbase.dll in windows 10
+      else if (_strnicmp(basename, "msvcr", 5) == 0) priority = 1;  // older runtimes      
+      
+      if (priority > 0) {
         // remember indices so we can check load order (in debug mode)
         if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i;
         if (_stricmp(basename, UCRTBASE_NAME) == 0) ucrtbase_index = i;
 
         // probably found a crt module, try to patch it
-        mi_module_resolve(mod);
+        mi_module_resolve(mod,priority);
 
         // try to find the atexit functions for the main process (in `ucrtbase.dll`)
         if (crt_atexit==NULL) crt_atexit = (atexit_fun_t*)GetProcAddress(mod, "_crt_atexit");

From c3a5f84ad3a6e9a4e77c3d0d67b1067cb1e2c868 Mon Sep 17 00:00:00 2001
From: daan <daan@microsoft.com>
Date: Wed, 3 Jul 2019 14:52:32 -0700
Subject: [PATCH 5/5] rewrite of low-level OS (aligned) allocation to better
 handle large pages and aligned allocation

---
 src/init.c |   6 +-
 src/os.c   | 451 ++++++++++++++++++++++++++---------------------------
 2 files changed, 225 insertions(+), 232 deletions(-)

diff --git a/src/init.c b/src/init.c
index a36d45f2..8ceb8d21 100644
--- a/src/init.c
+++ b/src/init.c
@@ -103,7 +103,11 @@ mi_heap_t _mi_heap_main = {
   NULL,
   0,
   0,
-  0xCDCDCDCDCDCDCDL,
+#if MI_INTPTR_SIZE==8   // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!)
+  0xCDCDCDCDCDCDCDCDUL,
+#else
+  0xCDCDCDCDUL,
+#endif
   0,
   false   // can reclaim
 };
diff --git a/src/os.c b/src/os.c
index 2957e0f9..0678b524 100644
--- a/src/os.c
+++ b/src/os.c
@@ -14,18 +14,38 @@ terms of the MIT license. A copy of the license can be found in the file
 #include <string.h>  // memset
 #include <errno.h>
 
+#if defined(_WIN32)
+#include <windows.h>
+#else
+#include <sys/mman.h>  // mmap
+#include <unistd.h>    // sysconf
+#endif
+
 /* -----------------------------------------------------------
   Initialization.
   On windows initializes support for aligned allocation and
   large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
 ----------------------------------------------------------- */
+bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
 
-#if defined(_WIN32)
-  #include <windows.h>
-#else
-  #include <sys/mman.h>  // mmap
-  #include <unistd.h>    // sysconf
-#endif
+uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
+  uintptr_t x = (sz / alignment) * alignment;
+  if (x < sz) x += alignment;
+  if (x < sz) return 0; // overflow
+  return x;
+}
+
+static void* mi_align_up_ptr(void* p, size_t alignment) {
+  return (void*)_mi_align_up((uintptr_t)p, alignment);
+}
+
+static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
+  return (sz / alignment) * alignment;
+}
+
+static void* mi_align_down_ptr(void* p, size_t alignment) {
+  return (void*)_mi_align_down((uintptr_t)p, alignment);
+}
 
 // page size (initialized properly in `os_init`)
 static size_t os_page_size = 4096;
@@ -61,8 +81,8 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
 
 #if defined(_WIN32)
 // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
-// So, we need to look it up dynamically to run on older systems.
-typedef PVOID (__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG );
+// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
+typedef PVOID(__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
 static VirtualAlloc2Ptr pVirtualAlloc2 = NULL;
 
 void _mi_os_init(void) {
@@ -74,7 +94,7 @@ void _mi_os_init(void) {
   // get the VirtualAlloc2 function
   HINSTANCE  hDll;
   hDll = LoadLibrary("kernelbase.dll");
-  if (hDll!=NULL) {
+  if (hDll != NULL) {
     // use VirtualAlloc2FromApp as it is available to Windows store apps
     pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp");
     FreeLibrary(hDll);
@@ -106,7 +126,7 @@ void _mi_os_init(void) {
       CloseHandle(token);
     }
     if (!ok) {
-      if (err==0) err = GetLastError();
+      if (err == 0) err = GetLastError();
       _mi_warning_message("cannot enable large OS page support, error %lu\n", err);
     }
   }
@@ -120,37 +140,16 @@ void _mi_os_init() {
     os_alloc_granularity = os_page_size;
   }
   if (mi_option_is_enabled(mi_option_large_os_pages)) {
-    large_os_page_size = (1UL<<21); // 2MiB
+    large_os_page_size = (1UL << 21); // 2MiB
   }
 }
 #endif
 
 
 /* -----------------------------------------------------------
-  Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).
-  Defines a portable `mmap`, `munmap` and `mmap_trim`.
+  Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).  
 ----------------------------------------------------------- */
 
-uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
-  uintptr_t x = (sz / alignment) * alignment;
-  if (x < sz) x += alignment;
-  if (x < sz) return 0; // overflow
-  return x;
-}
-
-static void* mi_align_up_ptr(void* p, size_t alignment) {
-  return (void*)_mi_align_up((uintptr_t)p, alignment);
-}
-
-static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
-  return (sz / alignment) * alignment;
-}
-
-static void* mi_align_down_ptr(void* p, size_t alignment) {
-  return (void*)_mi_align_down((uintptr_t)p, alignment);
-}
-
-
 static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
 {
   if (addr == NULL || size == 0) return true;
@@ -163,7 +162,7 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
   _mi_stat_decrease(&stats->committed, size); // TODO: what if never committed?
   _mi_stat_decrease(&stats->reserved, size);
   if (err) {
-    #pragma warning(suppress:4996)
+#pragma warning(suppress:4996)
     _mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size);
     return false;
   }
@@ -172,38 +171,53 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
   }
 }
 
-static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_flags, mi_stats_t* stats) {
-  if (size == 0) return NULL;
+#ifdef _WIN32
+static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
+#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
+  if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
+    // on modern Windows try use VirtualAlloc2
+    MEM_ADDRESS_REQUIREMENTS reqs = { 0 };
+    reqs.Alignment = try_alignment;
+    MEM_EXTENDED_PARAMETER param = { 0 };
+    param.Type = MemExtendedParameterAddressRequirements;
+    param.Pointer = &reqs;
+    return (*pVirtualAlloc2)(addr, NULL, size, flags, PAGE_READWRITE, &param, 1);
+  }
+#endif
+  return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
+}
+
+static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) {
   void* p = NULL;
-#if defined(_WIN32)
-  int flags = MEM_RESERVE | extra_flags;
-  if (commit) flags |= MEM_COMMIT;
-  if (use_large_os_page(size, 0)) {
-    p = VirtualAlloc(addr, size, MEM_LARGE_PAGES | flags, PAGE_READWRITE);
+  if (use_large_os_page(size, try_alignment)) {
+    p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | flags);
+    // fall back to non-large page allocation on error (`p == NULL`).
   }
   if (p == NULL) {
-    p = VirtualAlloc(addr, size, flags, PAGE_READWRITE);
+    p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
   }
+  return p;
+}
+
 #else
+static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) {
+  void* p = NULL;
   #if !defined(MAP_ANONYMOUS)
   #define MAP_ANONYMOUS  MAP_ANON
   #endif
-  int flags = MAP_PRIVATE | MAP_ANONYMOUS | extra_flags;
-  if (addr != NULL) {
-    #if defined(MAP_EXCL)
-      flags |= MAP_FIXED | MAP_EXCL;  // BSD
-    #elif defined(MAP_FIXED_NOREPLACE)
-      flags |= MAP_FIXED_NOREPLACE;   // Linux
-    #elif defined(MAP_FIXED)
-      flags |= MAP_FIXED;
-    #endif
+  int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+  #if defined(MAP_ALIGNED)  // BSD
+  if (try_alignment > 0) {
+    size_t n = _mi_bsr(try_alignment);
+    if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) {  // alignment is a power of 2 and 4096 <= alignment <= 1GiB
+      flags |= MAP_ALIGNED(n);
+    }
   }
-  int pflags = (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE);
-  #if defined(PROT_MAX)
-  pflags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
   #endif
-
-  if (large_os_page_size > 0 && use_large_os_page(size, 0) && ((uintptr_t)addr % large_os_page_size) == 0) {
+  #if defined(PROT_MAX)
+  protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
+  #endif
+  if (large_os_page_size > 0 && use_large_os_page(size, try_alignment)) {
     int lflags = flags;
     #ifdef MAP_ALIGNED_SUPER
     lflags |= MAP_ALIGNED_SUPER;
@@ -215,66 +229,145 @@ static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_fla
     lflags |= MAP_HUGE_2MB;
     #endif
     if (lflags != flags) {
-      // try large page allocation
-      p = mmap(addr, size, pflags, lflags, -1, 0);
+      // try large page allocation 
+      // TODO: if always failing due to permissions or no huge pages, try to avoid repeatedly trying? 
+      // Should we check this in _mi_os_init? (as on Windows)
+      p = mmap(NULL, size, protect_flags, lflags, -1, 0);
       if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission
     }
   }
   if (p == NULL) {
-    p = mmap(addr, size, pflags, flags, -1, 0);
+    p = mmap(NULL, size, protect_flags, flags, -1, 0);
     if (p == MAP_FAILED) p = NULL;
   }
-  if (addr != NULL && p != addr) {
-    mi_os_mem_free(p, size, stats);
-    p = NULL;
-  }
+  return p;
+}
 #endif
-  mi_assert(p == NULL || (addr == NULL && p != addr) || (addr != NULL && p == addr));
-  if (p != NULL) {
-    _mi_stat_increase(&stats->mmap_calls, 1);
-    _mi_stat_increase(&stats->reserved, size);
-    if (commit) _mi_stat_increase(&stats->committed, size);
-  }
-  return p;
-}
 
-static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) {
-  if (alignment < _mi_os_page_size() || ((alignment & (~alignment + 1)) != alignment)) return NULL;
+// Primitive allocation from the OS.
+// Note: the `alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
+static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) {
+  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  if (size == 0) return NULL;
+
   void* p = NULL;
-  #if defined(_WIN32) && defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
-  if (pVirtualAlloc2 != NULL) {
-    // on modern Windows try use VirtualAlloc2
-    MEM_ADDRESS_REQUIREMENTS reqs = {0};
-    reqs.Alignment = alignment;
-    MEM_EXTENDED_PARAMETER param = { 0 };
-    param.Type = MemExtendedParameterAddressRequirements;
-    param.Pointer = &reqs;
-    DWORD flags = MEM_RESERVE;
-    if (commit) flags |= MEM_COMMIT;
-    if (use_large_os_page(size, alignment)) flags |= MEM_LARGE_PAGES;
-    p = (*pVirtualAlloc2)(NULL, NULL, size, flags, PAGE_READWRITE, &param, 1);
-  }
-  #elif defined(MAP_ALIGNED)
-  // on BSD, use the aligned mmap api
-  size_t n = _mi_bsr(alignment);
-  if (((size_t)1 << n) == alignment && n >= 12) {  // alignment is a power of 2 and >= 4096
-    p = mi_os_mem_alloc(suggest, size, commit, MAP_ALIGNED(n), tld->stats);     // use the NetBSD/freeBSD aligned flags
-  }
-  #else
-  UNUSED(size);
-  UNUSED(alignment);
-  #endif
-  mi_assert(p == NULL || (uintptr_t)p % alignment == 0);
+#if defined(_WIN32)
+  int flags = MEM_RESERVE;
+  if (commit) flags |= MEM_COMMIT;
+  p = mi_win_virtual_alloc(NULL, size, try_alignment, flags);
+#else
+  int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
+  p = mi_unix_mmap(size, try_alignment, protect_flags);
+#endif  
+  _mi_stat_increase(&stats->mmap_calls, 1);
   if (p != NULL) {
-    _mi_stat_increase(&stats->mmap_calls, 1);
     _mi_stat_increase(&stats->reserved, size);
     if (commit) _mi_stat_increase(&stats->committed, size);
   }
   return p;
 }
 
-// OS page align within a given area,
-// either conservative (pages inside the area only),
+
+// Primitive aligned allocation from the OS.
+// This function guarantees the allocated memory is aligned.
+static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) {
+  mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
+  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
+  size = _mi_align_up(size, _mi_os_page_size());
+  
+  // try first with a hint (this will be aligned directly on Win 10+ or BSD)
+  void* p = mi_os_mem_alloc(size, alignment, commit, stats);
+  if (p == NULL) return NULL;
+
+  // if not aligned, free it, overallocate, and unmap around it
+  if (((uintptr_t)p % alignment != 0)) {
+    mi_os_mem_free(p, size, stats);
+    if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
+    size_t over_size = size + alignment;
+
+#if _WIN32
+    // over-allocate and than re-allocate exactly at an aligned address in there.
+    // this may fail due to threads allocating at the same time so we
+    // retry this at most 3 times before giving up. 
+    // (we can not decommit around the overallocation on Windows, because we can only
+    //  free the original pointer, not one pointing inside the area)
+    int flags = MEM_RESERVE;
+    if (commit) flags |= MEM_COMMIT;
+    for (int tries = 0; tries < 3; tries++) {
+      // over-allocate to determine a virtual memory range
+      p = mi_os_mem_alloc(over_size, alignment, commit, stats);
+      if (p == NULL) return NULL; // error
+      if (((uintptr_t)p % alignment) == 0) {
+        // if p happens to be aligned, just decommit the left-over area
+        _mi_os_decommit((uint8_t*)p + size, over_size - size, stats);
+        break;
+      }
+      else {
+        // otherwise free and allocate at an aligned address in there
+        mi_os_mem_free(p, over_size, stats);
+        void* aligned_p = mi_align_up_ptr(p, alignment);
+        p = mi_win_virtual_alloc(aligned_p, size, alignment, flags);
+        if (p == aligned_p) break; // success!
+        if (p != NULL) { // should not happen?
+          mi_os_mem_free(p, size, stats);  
+          p = NULL;
+        }
+      }
+    }
+#else
+    // overallocate...
+    p = mi_os_mem_alloc(over_size, alignment, commit, stats);
+    if (p == NULL) return NULL;
+    // and selectively unmap parts around the over-allocated area.
+    void* aligned_p = mi_align_up_ptr(p, alignment);
+    size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
+    size_t mid_size = _mi_align_up(size, _mi_os_page_size());
+    size_t post_size = over_size - pre_size - mid_size;
+    mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
+    if (pre_size > 0)  mi_os_mem_free(p, pre_size, stats);
+    if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
+    // we can return the aligned pointer on `mmap` systems
+    p = aligned_p;
+#endif
+  }
+
+  mi_assert_internal(p == NULL || (p != NULL && ((uintptr_t)p % alignment) == 0));
+  return p;
+}
+
+/* -----------------------------------------------------------
+  OS API: alloc, free, alloc_aligned
+----------------------------------------------------------- */
+
+void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
+  if (size == 0) return NULL;
+  size = mi_os_good_alloc_size(size, 0);
+  return mi_os_mem_alloc(size, 0, true, stats);
+}
+
+void  _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
+  if (size == 0 || p == NULL) return;
+  size = mi_os_good_alloc_size(size, 0);
+  mi_os_mem_free(p, size, stats);
+}
+
+void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
+{
+  if (size == 0) return NULL;
+  size = mi_os_good_alloc_size(size, alignment);
+  alignment = _mi_align_up(alignment, _mi_os_page_size());
+  return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats);
+}
+
+
+
+/* -----------------------------------------------------------
+  OS memory API: reset, commit, decommit, protect, unprotect.
+----------------------------------------------------------- */
+
+
+// OS page align within a given area, either conservative (pages inside the area only),
 // or not (straddling pages outside the area is possible)
 static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
   mi_assert(addr != NULL && size > 0);
@@ -283,9 +376,9 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size,
 
   // page align conservatively within the range
   void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
-                              : mi_align_down_ptr(addr, _mi_os_page_size()));
+    : mi_align_down_ptr(addr, _mi_os_page_size()));
   void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
-                            : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
+    : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
   ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
   if (diff <= 0) return NULL;
 
@@ -295,7 +388,7 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size,
 }
 
 static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
-  return mi_os_page_align_areax(true,addr,size,newsize);
+  return mi_os_page_align_areax(true, addr, size, newsize);
 }
 
 
@@ -307,8 +400,8 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
 bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
   // page align conservatively within the range
   size_t csize;
-  void* start = mi_os_page_align_area_conservative(addr,size,&csize);
-  if (csize==0) return true;
+  void* start = mi_os_page_align_area_conservative(addr, size, &csize);
+  if (csize == 0) return true;
   _mi_stat_increase(&stats->reset, csize);
 
 #if defined(_WIN32)
@@ -329,17 +422,17 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
   */
   return true;
 #else
-  #if defined(MADV_FREE)
-    static int advice = MADV_FREE;
-    int err = madvise(start, csize, advice);
-    if (err!=0 && errno==EINVAL && advice==MADV_FREE) {
-      // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
-      advice = MADV_DONTNEED;
-      err = madvise(start, csize, advice);
-    }
-  #else
-    int err = madvise(start, csize, MADV_DONTNEED);
-  #endif
+#if defined(MADV_FREE)
+  static int advice = MADV_FREE;
+  int err = madvise(start, csize, advice);
+  if (err != 0 && errno == EINVAL && advice == MADV_FREE) {
+    // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
+    advice = MADV_DONTNEED;
+    err = madvise(start, csize, advice);
+  }
+#else
+  int err = madvise(start, csize, MADV_DONTNEED);
+#endif
   if (err != 0) {
     _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno);
   }
@@ -353,32 +446,31 @@ static  bool mi_os_protectx(void* addr, size_t size, bool protect) {
   // page align conservatively within the range
   size_t csize = 0;
   void* start = mi_os_page_align_area_conservative(addr, size, &csize);
-  if (csize==0) return false;
+  if (csize == 0) return false;
 
   int err = 0;
 #ifdef _WIN32
   DWORD oldprotect = 0;
-  BOOL ok = VirtualProtect(start,csize,protect ? PAGE_NOACCESS : PAGE_READWRITE,&oldprotect);
+  BOOL ok = VirtualProtect(start, csize, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);
   err = (ok ? 0 : GetLastError());
 #else
-  err = mprotect(start,csize,protect ? PROT_NONE : (PROT_READ|PROT_WRITE));
+  err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
 #endif
   if (err != 0) {
     _mi_warning_message("mprotect error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err);
   }
-  return (err==0);
+  return (err == 0);
 }
 
 bool _mi_os_protect(void* addr, size_t size) {
-  return mi_os_protectx(addr,size,true);
+  return mi_os_protectx(addr, size, true);
 }
 
 bool _mi_os_unprotect(void* addr, size_t size) {
   return mi_os_protectx(addr, size, false);
 }
 
-// Commit/Decommit memory.
-// We page align to a conservative area inside the range to reset.
+// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative.
 static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) {
   // page align in the range, commit liberally, decommit conservative
   size_t csize;
@@ -387,7 +479,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stat
   int err = 0;
   if (commit) {
     _mi_stat_increase(&stats->committed, csize);
-    _mi_stat_increase(&stats->commit_calls,1);
+    _mi_stat_increase(&stats->commit_calls, 1);
   }
   else {
     _mi_stat_decrease(&stats->committed, csize);
@@ -423,123 +515,20 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
 bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
   // page align conservatively within the range
   mi_assert_internal(oldsize > newsize && p != NULL);
-  if (oldsize < newsize || p==NULL) return false;
+  if (oldsize < newsize || p == NULL) return false;
   if (oldsize == newsize) return true;
 
   // oldsize and newsize should be page aligned or we cannot shrink precisely
   void* addr = (uint8_t*)p + newsize;
   size_t size = 0;
   void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size);
-  if (size==0 || start != addr) return false;
+  if (size == 0 || start != addr) return false;
 
-  #ifdef _WIN32
+#ifdef _WIN32
   // we cannot shrink on windows, but we can decommit
   return _mi_os_decommit(start, size, stats);
-  #else
+#else
   return mi_os_mem_free(start, size, stats);
-  #endif
-}
-
-/* -----------------------------------------------------------
-  OS allocation using mmap/munmap
------------------------------------------------------------ */
-
-void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
-  if (size == 0) return NULL;
-  size = mi_os_good_alloc_size(size, 0);
-  void* p = mi_os_mem_alloc(NULL, size, true, 0, stats);
-  mi_assert(p!=NULL);
-  return p;
-}
-
-void  _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
-  if (size==0) return;
-  size = mi_os_good_alloc_size(size, 0);
-  mi_os_mem_free(p, size, stats);
-}
-
-// Slow but guaranteed way to allocated aligned memory
-// by over-allocating and then reallocating at a fixed aligned
-// address that should be available then.
-static void* mi_os_alloc_aligned_ensured(size_t size, size_t alignment, bool commit, size_t trie, mi_stats_t* stats)
-{
-  if (trie >= 3) return NULL; // stop recursion (only on Windows)
-  if (size > SIZE_MAX - alignment) return NULL; // overflow
-  size_t alloc_size = size + alignment; // no need for -1 as we need to be page aligned anyways
-  
-  // allocate a chunk that includes the alignment
-  void* p = mi_os_mem_alloc(NULL, alloc_size, commit, 0, stats);
-  if (p == NULL) return NULL;
-  // create an aligned pointer in the allocated area
-  void* aligned_p = mi_align_up_ptr(p, alignment);
-  mi_assert(aligned_p != NULL);
-#if _WIN32
-  // free it and try to allocate `size` at exactly `aligned_p`
-  // note: this may fail in case another thread happens to allocate
-  // concurrently at that spot. We try up to 3 times to mitigate this.
-  mi_os_mem_free(p, alloc_size, stats);
-  p = mi_os_mem_alloc(aligned_p, size, commit, 0, stats);
-  if (p != aligned_p) {
-    if (p != NULL) mi_os_mem_free(p, size, stats);
-    return mi_os_alloc_aligned_ensured(size, alignment, commit, trie+1, stats);
-  }
-#else  
-  // we selectively unmap parts around the over-allocated area.
-  size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
-  size_t mid_size = _mi_align_up(size, _mi_os_page_size());
-  size_t post_size = alloc_size - pre_size - mid_size;
-  if (pre_size > 0)  mi_os_mem_free(p, pre_size, stats);
-  if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
 #endif
-
-  mi_assert(((uintptr_t)aligned_p) % alignment == 0);
-  return aligned_p;
 }
 
-// Allocate an aligned block.
-// Since `mi_mmap` is relatively slow we try to allocate directly at first and
-// hope to get an aligned address; only when that fails we fall back
-// to a guaranteed method by overallocating at first and adjusting.
-void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
-{
-  if (size == 0) return NULL;
-  size = mi_os_good_alloc_size(size,alignment);
-  if (alignment < 1024) return mi_os_mem_alloc(NULL, size, commit, 0, tld->stats);
-
-  // try direct OS aligned allocation; only supported on BSD and Windows 10+
-  void* suggest = NULL;
-  void* p = mi_os_mem_alloc_aligned(size,alignment,commit,tld->stats);
-
-  // Fall back
-  if (p==NULL && (tld->mmap_next_probable % alignment) == 0) {
-    // if the next probable address is aligned,
-    // then try to just allocate `size` and hope it is aligned...
-    p = mi_os_mem_alloc(suggest, size, commit, 0, tld->stats);
-    if (p == NULL) return NULL;
-    if (((uintptr_t)p % alignment) == 0) _mi_stat_increase(&tld->stats->mmap_right_align, 1);
-  }
-  //fprintf(stderr, "segment address guess: %s, p=%lxu, guess:%lxu\n", (p != NULL && (uintptr_t)p % alignment ==0 ? "correct" : "incorrect"), (uintptr_t)p, next_probable);
-
-  if (p==NULL || ((uintptr_t)p % alignment) != 0) {
-    // if `p` is not yet aligned after all, free the block and use a slower
-    // but guaranteed way to allocate an aligned block
-    if (p != NULL) mi_os_mem_free(p, size, tld->stats);
-    _mi_stat_increase( &tld->stats->mmap_ensure_aligned, 1);
-    //fprintf(stderr, "mimalloc: slow mmap 0x%lx\n", _mi_thread_id());
-    p = mi_os_alloc_aligned_ensured(size, alignment,commit,0,tld->stats);
-  }
-  if (p != NULL) {
-    // next probable address is the page-aligned address just after the newly allocated area.
-    size_t probable_size = MI_SEGMENT_SIZE;
-    if (tld->mmap_previous > p) {
-      // Linux tends to allocate downward
-      tld->mmap_next_probable = _mi_align_down((uintptr_t)p - probable_size, os_alloc_granularity); // ((uintptr_t)previous - (uintptr_t)p);
-    }
-    else {
-      // Otherwise, guess the next address is page aligned `size` from current pointer
-      tld->mmap_next_probable = _mi_align_up((uintptr_t)p + probable_size, os_alloc_granularity);
-    }
-    tld->mmap_previous = p;
-  }
-  return p;
-}