merge from dev; add mi_malloc_expandable

CMakeLists.txt

@@ -27,7 +27,7 @@ option(MI_DEBUG_TSAN        "Build with thread sanitizer (needs clang)" OFF)
 option(MI_DEBUG_UBSAN       "Build with undefined-behavior sanitizer (needs clang++)" OFF)
 option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF)
 option(MI_NO_PADDING        "Force no use of padding even in DEBUG mode etc." OFF)
-option(MI_INSTALL_TOPLEVEL  "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version (deprecated)" OFF)
+option(MI_INSTALL_TOPLEVEL  "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF)
 
 # deprecated options
 option(MI_CHECK_FULL        "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF)
@@ -45,6 +45,7 @@ set(mi_sources
     src/bitmap.c
     src/heap.c
     src/init.c
+    src/libc.c
     src/options.c
     src/os.c
     src/page.c

ide/vs2017/mimalloc-override.vcxproj

@@ -238,6 +238,7 @@
     <ClCompile Include="..\..\src\bitmap.c" />
     <ClCompile Include="..\..\src\heap.c" />
     <ClCompile Include="..\..\src\init.c" />
+    <ClCompile Include="..\..\src\libc.c" />
     <ClCompile Include="..\..\src\prim\prim.c" />
     <ClCompile Include="..\..\src\options.c" />
     <ClCompile Include="..\..\src\os.c" />

ide/vs2017/mimalloc-override.vcxproj.filters

@@ -91,5 +91,8 @@
     <ClCompile Include="..\..\src\segment-map.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\libc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

ide/vs2017/mimalloc.vcxproj

@@ -227,6 +227,7 @@
     <ClCompile Include="..\..\src\bitmap.c" />
     <ClCompile Include="..\..\src\heap.c" />
     <ClCompile Include="..\..\src\init.c" />
+    <ClCompile Include="..\..\src\libc.c" />
     <ClCompile Include="..\..\src\prim\prim.c" />
     <ClCompile Include="..\..\src\options.c" />
     <ClCompile Include="..\..\src\page-queue.c">

ide/vs2017/mimalloc.vcxproj.filters

@@ -62,6 +62,9 @@
     <ClCompile Include="..\..\src\segment-map.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\libc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(ProjectDir)..\..\include\mimalloc.h">

ide/vs2019/mimalloc-override.vcxproj

@@ -238,6 +238,7 @@
     <ClCompile Include="..\..\src\bitmap.c" />
     <ClCompile Include="..\..\src\heap.c" />
     <ClCompile Include="..\..\src\init.c" />
+    <ClCompile Include="..\..\src\libc.c" />
     <ClCompile Include="..\..\src\prim\prim.c" />
     <ClCompile Include="..\..\src\options.c" />
     <ClCompile Include="..\..\src\os.c" />

ide/vs2019/mimalloc-override.vcxproj.filters

@@ -52,6 +52,9 @@
     <ClCompile Include="..\..\src\segment-map.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\libc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(ProjectDir)..\..\include\mimalloc.h">

ide/vs2019/mimalloc.vcxproj

@@ -219,6 +219,7 @@
     </ClCompile>
     <ClCompile Include="..\..\src\heap.c" />
     <ClCompile Include="..\..\src\init.c" />
+    <ClCompile Include="..\..\src\libc.c" />
     <ClCompile Include="..\..\src\prim\prim.c" />
     <ClCompile Include="..\..\src\prim\windows\prim.c">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>

ide/vs2019/mimalloc.vcxproj.filters

@@ -55,6 +55,9 @@
     <ClCompile Include="..\..\src\segment-map.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\libc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="$(ProjectDir)..\..\include\mimalloc.h">

ide/vs2022/mimalloc-override.vcxproj

@@ -240,6 +240,7 @@
     <ClCompile Include="..\..\src\bitmap.c" />
     <ClCompile Include="..\..\src\heap.c" />
     <ClCompile Include="..\..\src\init.c" />
+    <ClCompile Include="..\..\src\libc.c" />
     <ClCompile Include="..\..\src\prim\prim.c" />
     <ClCompile Include="..\..\src\prim\windows\prim.c">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

ide/vs2022/mimalloc-override.vcxproj.filters

@@ -55,6 +55,9 @@
     <ClCompile Include="..\..\src\stats.c">
       <Filter>Sources</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\libc.c">
+      <Filter>Sources</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\include\mimalloc\atomic.h">

ide/vs2022/mimalloc.vcxproj

@@ -219,6 +219,7 @@
     </ClCompile>
     <ClCompile Include="..\..\src\heap.c" />
     <ClCompile Include="..\..\src\init.c" />
+    <ClCompile Include="..\..\src\libc.c" />
     <ClCompile Include="..\..\src\prim\prim.c" />
     <ClCompile Include="..\..\src\prim\windows\prim.c">
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

ide/vs2022/mimalloc.vcxproj.filters

@@ -55,6 +55,9 @@
     <ClCompile Include="..\..\src\stats.c">
       <Filter>Sources</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\libc.c">
+      <Filter>Sources</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\bitmap.h">

include/mimalloc.h

@@ -256,6 +256,17 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_zalloc_remappable(mi_heap_t* heap
 
 // mi_decl_nodiscard mi_decl_export void* mi_remap(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
 
+// ------------------------------------------------------
+// Expandable memory reserves a virtual address range of `max_expand_size` (capped at 1TiB)
+// and commits this on-demand through `mi_realloc` and `mi_expand`. Both of those will 
+// expand inplace for blocks allocated as expandable up to `max_expand_size`.
+// ------------------------------------------------------
+
+mi_decl_nodiscard void* mi_heap_malloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept;
+mi_decl_nodiscard void* mi_heap_zalloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept;
+mi_decl_nodiscard void* mi_malloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept;
+mi_decl_nodiscard void* mi_zalloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept;
+
 
 // ------------------------------------------------------
 // Analysis

include/mimalloc/internal.h

@@ -202,14 +202,17 @@ bool        _mi_free_delayed_block(mi_block_t* block);
 void        _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept;  // for runtime integration
 void        _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);
 
-// option.c, c primitives
+// "libc.c"
+#include    <stdarg.h>
+void        _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
+void        _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
 char        _mi_toupper(char c);
 int         _mi_strnicmp(const char* s, const char* t, size_t n);
 void        _mi_strlcpy(char* dest, const char* src, size_t dest_size);
 void        _mi_strlcat(char* dest, const char* src, size_t dest_size);
 size_t      _mi_strlen(const char* s);
 size_t      _mi_strnlen(const char* s, size_t max_len);
-
+bool        _mi_getenv(const char* name, char* result, size_t result_size);
 
 #if MI_DEBUG>1
 bool        _mi_page_is_valid(mi_page_t* page);

src/alloc.c

@@ -690,53 +690,78 @@ mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) m
   return mi_heap_mallocn(mi_prim_get_default_heap(),count,size);
 }
 
+static void* mi_heap_try_expand_zero(mi_heap_t* heap, mi_segment_t* segment, void* p, size_t size, size_t newsize, bool zero);
+
 // Expand (or shrink) in place (or fail)
-void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
+void* mi_expand(void* p, size_t newsize) mi_attr_noexcept 
-  #if MI_PADDING
+{
-  // we do not shrink/expand with padding enabled
-  MI_UNUSED(p); MI_UNUSED(newsize);
-  return NULL;
-  #else
   if (p == NULL) return NULL;
+  
   const size_t size = _mi_usable_size(p,"mi_expand");
+  if (size == newsize) return p;
+
+  // try OS expand
+  mi_segment_t* const segment = mi_checked_ptr_segment(p, "mi_expand");
+  if mi_unlikely(segment->memid.memkind == MI_MEM_OS_EXPAND) {
+    void* newp = mi_heap_try_expand_zero(mi_prim_get_default_heap(), segment, p, size, newsize, false);
+    mi_assert_internal(newp == NULL || newp == p);
+    if (newp != NULL) return newp;
+  }
+
+  // otherwise we cannot grow inplace
   if (newsize > size) return NULL;
-  return p; // it fits
+
-  #endif
+  // shrink padding
+  mi_page_t* page = _mi_segment_page_of(segment, p);
+  mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+  mi_padding_init(page, block, newsize);
+  mi_track_resize(p, size, newsize);
+
+  return p; // it still fits  
 }
 
 
 static void* mi_heap_try_remap_zero(mi_heap_t* heap, mi_segment_t* segment, void* p, size_t size, size_t newsize, bool zero);
 
+static void mi_padding_init_ptr(void* p, size_t size) {
+  mi_segment_t* segment = mi_checked_ptr_segment(p, "_mi_padding_init_ptr");
+  mi_page_t* page = _mi_segment_page_of(segment, p);
+  mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+  mi_padding_init(page, block, size);
+}
+
 void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept {
   // if p == NULL then behave as malloc.
   if mi_unlikely(p==NULL) return _mi_heap_malloc_zero(heap,newsize,zero);
 
-  // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
+  // else if newsize == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
   // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
   const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0)
-  if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) {  // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
+  if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0 && newsize < 1024) {  // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
     mi_assert_internal(p!=NULL);
-    // todo: do not track as the usable size is still the same in the free; adjust potential padding?
+    mi_padding_init_ptr(p, newsize);
-    // mi_track_resize(p,size,newsize)
+    mi_track_resize(p,size,newsize)
-    // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
     return p;  // reallocation still fits and not more than 50% waste
   }
-  // use OS remap for large reallocations
+
+  // try OS expand
   mi_segment_t* const segment = mi_checked_ptr_segment(p, "mi_realloc");
+  if mi_unlikely(segment->memid.memkind == MI_MEM_OS_EXPAND) {
+    void* newp = mi_heap_try_expand_zero(heap, segment, p, size, newsize, zero);
+    mi_assert_internal(newp == NULL || newp == p);
+    if (newp != NULL) return newp;
+  }
+
+  // try OS remap for large reallocations
   const size_t remap_threshold = mi_option_get_size(mi_option_remap_threshold);
   const bool use_remap = (segment->memid.memkind == MI_MEM_OS_REMAP) || (remap_threshold > 0 && newsize >= remap_threshold);
   if mi_unlikely(use_remap) {
     void* newp = mi_heap_try_remap_zero(heap, segment, p, size, newsize, zero);
     if (newp != NULL) return newp;
   }
+
   // otherwise copy into a new area
-  void* newp;
+  void* newp = (use_remap ? mi_heap_malloc_remappable(heap, newsize) : mi_heap_malloc(heap, newsize));
-  if mi_unlikely(use_remap) {
-    newp = mi_heap_malloc_remappable(heap, newsize);
-  }
-  else {
-    newp = mi_heap_malloc(heap, newsize);
-  }
   if mi_likely(newp != NULL) {
     if (zero && newsize > size) {
       // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
@@ -806,6 +831,76 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_
   return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size);
 }
 
+// ------------------------------------------------------
+// expand
+// ------------------------------------------------------
+
+static void* mi_heap_malloc_zero_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size, bool zero) mi_attr_noexcept {
+  if (max_expand_size <= size) { 
+    return _mi_heap_malloc_zero(heap, size, zero);
+  }
+  size_t increments = _mi_divide_up(max_expand_size, MI_EXPAND_INCREMENT);
+  if (increments > MI_ALIGN_EXPAND_MAX) increments = MI_ALIGN_EXPAND_MAX;
+  else if (increments < MI_ALIGN_EXPAND_MIN) increments = MI_ALIGN_EXPAND_MIN;
+  return _mi_heap_malloc_zero_ex(heap, size, zero, increments);
+}
+
+mi_decl_nodiscard void* mi_heap_malloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept {
+  return mi_heap_malloc_zero_expandable(heap, size, max_expand_size, false);
+}
+
+mi_decl_nodiscard void* mi_heap_zalloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept {
+  return mi_heap_malloc_zero_expandable(heap, size, max_expand_size, true);
+}
+
+mi_decl_nodiscard void* mi_malloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept {
+  return mi_heap_malloc_expandable(mi_prim_get_default_heap(), size, max_expand_size);
+}
+
+mi_decl_nodiscard void* mi_zalloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept {
+  return mi_heap_zalloc_expandable(mi_prim_get_default_heap(), size, max_expand_size);
+}
+
+
+static void* mi_heap_try_expand_zero(mi_heap_t* heap, mi_segment_t* segment, void* p, size_t size, size_t newsize, bool zero)
+{
+  if (newsize == 0) return NULL;
+  if (p == NULL) {
+    return mi_heap_malloc_zero_expandable(heap, newsize, zero, (newsize < PTRDIFF_MAX/2 ? 2*newsize : newsize));
+  }
+
+  // expandable memory?
+  if (segment->memid.memkind != MI_MEM_OS_EXPAND) return NULL;
+
+  // we can only expand from an owning thread (as the segment is modified temporarily)
+  const mi_threadid_t tid = _mi_prim_thread_id();
+  mi_assert(heap->thread_id == tid);
+  if (segment->thread_id != tid) return NULL;
+
+  // check size
+  const size_t padsize = newsize + MI_PADDING_SIZE;
+  mi_assert_internal(segment != NULL);
+  mi_page_t* page = _mi_segment_page_of(segment, p);
+  mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+  
+  // try to use OS expand
+  mi_assert_internal((void*)block == p);
+  block = _mi_segment_huge_page_expand(segment, page, block, padsize, &heap->tld->segments);
+  mi_assert_internal(block == NULL || (void*)block == p);
+  if (block == NULL) return NULL;
+  
+  _mi_trace_message("expanded inplace (address: %p to %zu bytes)\n", p, newsize);
+  mi_padding_init(page, block, newsize);
+  mi_track_resize(p, size, newsize);
+  if (zero) {
+    // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+    const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
+    _mi_memzero((uint8_t*)p + start, newsize - start);
+  }
+  return block;
+}
+
+
 // ------------------------------------------------------
 // remap
 // ------------------------------------------------------
@@ -871,6 +966,7 @@ static void* mi_heap_try_remap_zero(mi_heap_t* heap, mi_segment_t* segment, void
   if (bsize >= padsize && 9*(bsize/10) <= padsize) {  // if smaller and not more than 10% waste, keep it
     _mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
     mi_padding_init(page, block, newsize);
+    mi_track_resize(p, size, newsize);
     return p;
   }
 
@@ -879,7 +975,7 @@ static void* mi_heap_try_remap_zero(mi_heap_t* heap, mi_segment_t* segment, void
   block = _mi_segment_huge_page_remap(segment, page, block, padsize, &heap->tld->segments);
   if (block != NULL) {     
     // succes! re-establish the pointers to the potentially relocated memory
-    _mi_verbose_message("used remap (address: %p to %zu bytes)\n", p, newsize);
+    _mi_trace_message("used remap (address: %p to %zu bytes)\n", p, newsize);
     segment = mi_checked_ptr_segment(block, "mi_remap");
     page = _mi_segment_page_of(segment, block);
     mi_padding_init(page, block, newsize);

src/arena.c

@@ -812,7 +812,7 @@ int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exc
   const bool is_large = memid.is_pinned; // todo: use separate is_large field?
   if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) {
     _mi_os_free_ex(start, size, commit, memid, &_mi_stats_main);
-    _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size, 1024));
+    _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
     return ENOMEM;
   }
   _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), is_large ? " (in large os pages)" : "");

src/init.c

@@ -175,9 +175,9 @@ mi_heap_t* _mi_heap_main_get(void) {
 
 // note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size).
 typedef struct mi_thread_data_s {
-  mi_heap_t  heap;  // must come first due to cast in `_mi_heap_done`
+  mi_heap_t  heap;   // must come first due to cast in `_mi_heap_done`
   mi_tld_t   tld;
-  mi_memid_t memid;
+  mi_memid_t memid;  // must come last due to zero'ing
 } mi_thread_data_t;
 
 
@@ -223,7 +223,7 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) {
   }
   
   if (td != NULL && !is_zero) {
-    _mi_memzero_aligned(td, sizeof(*td));
+    _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid));
   }
   return td;
 }

src/libc.c

@@ -0,0 +1,273 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// --------------------------------------------------------
+// This module defines various std libc functions to reduce
+// the dependency on libc, and also prevent errors caused 
+// by some libc implementations when called before `main`
+// executes (due to malloc redirection)
+// --------------------------------------------------------
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h"      // mi_prim_getenv
+
+char _mi_toupper(char c) {
+  if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
+                       else return c;
+}
+
+int _mi_strnicmp(const char* s, const char* t, size_t n) {
+  if (n == 0) return 0;
+  for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
+    if (_mi_toupper(*s) != _mi_toupper(*t)) break;
+  }
+  return (n == 0 ? 0 : *s - *t);
+}
+
+void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
+  if (dest==NULL || src==NULL || dest_size == 0) return;
+  // copy until end of src, or when dest is (almost) full
+  while (*src != 0 && dest_size > 1) {
+    *dest++ = *src++;
+    dest_size--;
+  }
+  // always zero terminate
+  *dest = 0;
+}
+
+void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
+  if (dest==NULL || src==NULL || dest_size == 0) return;
+  // find end of string in the dest buffer
+  while (*dest != 0 && dest_size > 1) {
+    dest++;
+    dest_size--;
+  }
+  // and catenate
+  _mi_strlcpy(dest, src, dest_size);
+}
+
+size_t _mi_strlen(const char* s) {
+  if (s==NULL) return 0;
+  size_t len = 0;
+  while(s[len] != 0) { len++; }
+  return len;
+}
+
+size_t _mi_strnlen(const char* s, size_t max_len) {
+  if (s==NULL) return 0;
+  size_t len = 0;
+  while(s[len] != 0 && len < max_len) { len++; }
+  return len;
+}
+
+#ifdef MI_NO_GETENV
+bool _mi_getenv(const char* name, char* result, size_t result_size) {
+  MI_UNUSED(name);
+  MI_UNUSED(result);
+  MI_UNUSED(result_size);
+  return false;
+}
+#else
+bool _mi_getenv(const char* name, char* result, size_t result_size) {
+  if (name==NULL || result == NULL || result_size < 64) return false;
+  return _mi_prim_getenv(name,result,result_size);
+}
+#endif
+
+// --------------------------------------------------------
+// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`
+// This is mostly to avoid calling these when libc is not yet
+// initialized (and to reduce dependencies)
+// 
+// format:      d i, p x u, s
+// prec:        z l ll L
+// width:       10
+// align-left:  -
+// fill:        0
+// plus:        +
+// --------------------------------------------------------
+
+static void mi_outc(char c, char** out, char* end) {
+  char* p = *out;
+  if (p >= end) return;
+  *p = c;
+  *out = p + 1;
+}
+
+static void mi_outs(const char* s, char** out, char* end) {
+  if (s == NULL) return;
+  char* p = *out;
+  while (*s != 0 && p < end) {
+    *p++ = *s++;
+  }
+  *out = p;
+}
+
+static void mi_out_fill(char fill, size_t len, char** out, char* end) {
+  char* p = *out;
+  for (size_t i = 0; i < len && p < end; i++) {
+    *p++ = fill;
+  }
+  *out = p;
+}
+
+static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {
+  if (len == 0 || extra == 0) return;
+  if (start + len + extra >= end) return;
+  // move `len` characters to the right (in reverse since it can overlap)
+  for (size_t i = 1; i <= len; i++) {
+    start[len + extra - i] = start[len - i];
+  }
+  // and fill the start
+  for (size_t i = 0; i < extra; i++) {
+    start[i] = fill;
+  }
+}
+
+
+static void mi_out_num(uintptr_t x, size_t base, char prefix, char** out, char* end) 
+{
+  if (x == 0 || base == 0 || base > 16) {
+    if (prefix != 0) { mi_outc(prefix, out, end); }
+    mi_outc('0',out,end);
+  }
+  else {
+    // output digits in reverse
+    char* start = *out;
+    while (x > 0) {
+      char digit = (char)(x % base);
+      mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);
+      x = x / base;
+    }
+    if (prefix != 0) { 
+      mi_outc(prefix, out, end); 
+    }
+    size_t len = *out - start;
+    // and reverse in-place
+    for (size_t i = 0; i < (len / 2); i++) {
+      char c = start[len - i - 1];
+      start[len - i - 1] = start[i];
+      start[i] = c;
+    }
+  }
+}
+
+
+#define MI_NEXTC()  c = *in; if (c==0) break; in++;
+
+void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
+  if (buf == NULL || bufsize == 0 || fmt == NULL) return;
+  buf[bufsize - 1] = 0;
+  char* const end = buf + (bufsize - 1);
+  const char* in = fmt;
+  char* out = buf;
+  while (true) {
+    if (out >= end) break;
+    char c;
+    MI_NEXTC();
+    if (c != '%') {
+      if ((c >= ' ' && c <= '~') || c=='\n' || c=='\r' || c=='\t') { // output visible ascii or standard control only
+        mi_outc(c, &out, end);
+      }
+    }
+    else {
+      MI_NEXTC();
+      char   fill = ' ';
+      size_t width = 0;
+      char   numtype = 'd';
+      char   numplus = 0;
+      bool   alignright = true; 
+      if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }
+      if (c == '-') { alignright = false; MI_NEXTC(); }
+      if (c == '0') { fill = '0'; MI_NEXTC(); }
+      if (c >= '1' && c <= '9') {
+        width = (c - '0'); MI_NEXTC();
+        while (c >= '0' && c <= '9') {
+          width = (10 * width) + (c - '0'); MI_NEXTC();
+        }
+        if (c == 0) break;  // extra check due to while
+      }      
+      if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }
+      else if (c == 'l') {
+        numtype = c; MI_NEXTC();
+        if (c == 'l') { numtype = 'L'; MI_NEXTC(); }
+      }
+
+      char* start = out;
+      if (c == 's') {
+        // string
+        const char* s = va_arg(args, const char*);
+        mi_outs(s, &out, end);
+      }
+      else if (c == 'p' || c == 'x' || c == 'u') {
+        // unsigned
+        uintptr_t x = 0;
+        if (c == 'x' || c == 'u') {
+          if (numtype == 'z')       x = va_arg(args, size_t);
+          else if (numtype == 't')  x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
+          else if (numtype == 'L')  x = va_arg(args, unsigned long long);
+                               else x = va_arg(args, unsigned long);
+        }
+        else if (c == 'p') {
+          x = va_arg(args, uintptr_t);
+          mi_outs("0x", &out, end);
+          start = out;
+          width = (width >= 2 ? width - 2 : 0);
+        }
+        if (width == 0 && (c == 'x' || c == 'p')) {
+          if (c == 'p')   { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }
+          if (width == 0) { width = 2; }
+          fill = '0';
+        }
+        mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);
+      }
+      else if (c == 'i' || c == 'd') {
+        // signed
+        intptr_t x = 0;
+        if (numtype == 'z')       x = va_arg(args, intptr_t );
+        else if (numtype == 't')  x = va_arg(args, ptrdiff_t);
+        else if (numtype == 'L')  x = va_arg(args, long long);
+                             else x = va_arg(args, long);
+        char pre = 0;
+        if (x < 0) {
+          pre = '-';
+          if (x > INTPTR_MIN) { x = -x; }
+        }
+        else if (numplus != 0) {
+          pre = numplus;
+        }
+        mi_out_num((uintptr_t)x, 10, pre, &out, end);
+      }
+      else if (c >= ' ' && c <= '~') {
+        // unknown format
+        mi_outc('%', &out, end);
+        mi_outc(c, &out, end);
+      }
+
+      // fill & align
+      mi_assert_internal(out <= end);
+      mi_assert_internal(out >= start);
+      const size_t len = out - start;
+      if (len < width) {
+        mi_out_fill(fill, width - len, &out, end);
+        if (alignright && out <= end) {
+          mi_out_alignright(fill, start, len, width - len, end);
+        }
+      }
+    }
+  }
+  mi_assert_internal(out <= end);
+  *out = 0;
+}
+
+void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  _mi_vsnprintf(buf, buflen, fmt, args);
+  va_end(args);
+}

src/options.c

@@ -1,5 +1,5 @@
 /* ----------------------------------------------------------------------------
-Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
 This is free software; you can redistribute it and/or modify it under the
 terms of the MIT license. A copy of the license can be found in the file
 "LICENSE" at the root of this distribution.
@@ -9,9 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "mimalloc/atomic.h"
 #include "mimalloc/prim.h"  // mi_prim_out_stderr
 
-#include <stdio.h>      // FILE
+#include <stdio.h>      // stdin/stdout
 #include <stdlib.h>     // abort
-#include <stdarg.h>
+
 
 
 static long mi_max_error_count   = 16; // stop outputting errors after this (use < 0 for no limit)
@@ -94,12 +94,13 @@ static mi_option_desc_t options[_mi_option_last] =
   { 1024,UNINIT, MI_OPTION(remap_threshold) },         // size in KiB after which realloc starts using OS remap (0 to disable auto remap)
 };
 
-static bool mi_option_is_size_in_kib(mi_option_t option) {
-  return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || option == mi_option_remap_threshold);
-}
 
 static void mi_option_init(mi_option_desc_t* desc);
 
+static bool mi_option_has_size_in_kib(mi_option_t option) {
+  return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || option == mi_option_remap_threshold);
+}
+
 void _mi_options_init(void) {
   // called on process load; should not be called before the CRT is initialized!
   // (e.g. do not call this from process_init as that may run before CRT initialization)
@@ -110,7 +111,7 @@ void _mi_options_init(void) {
     // if (option != mi_option_verbose)
     {
       mi_option_desc_t* desc = &options[option];
-      _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value);
+      _mi_verbose_message("option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));
     }
   }
   mi_max_error_count = mi_option_get(mi_option_max_errors);
@@ -134,7 +135,7 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma
 }
 
 mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
-  mi_assert_internal(mi_option_is_size_in_kib(option));
+  mi_assert_internal(mi_option_has_size_in_kib(option));
   long x = mi_option_get(option);
   return (x < 0 ? 0 : (size_t)x * MI_KiB);
 }
@@ -317,12 +318,12 @@ void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* me
 }
 
 // Define our own limited `fprintf` that avoids memory allocation.
-// We do this using `snprintf` with a limited buffer.
+// We do this using `_mi_vsnprintf` with a limited buffer.
 static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {
   char buf[512];
   if (fmt==NULL) return;
   if (!mi_recurse_enter()) return;
-  vsnprintf(buf,sizeof(buf)-1,fmt,args);
+  _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);
   mi_recurse_exit();
   _mi_fputs(out,arg,prefix,buf);
 }
@@ -337,7 +338,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
 static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
   if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {
     char tprefix[64];
-    snprintf(tprefix, sizeof(tprefix), "%sthread 0x%llx: ", prefix, (unsigned long long)_mi_thread_id());
+    _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());
     mi_vfprintf(out, arg, tprefix, fmt, args);
   }
   else {
@@ -440,68 +441,6 @@ void _mi_error_message(int err, const char* fmt, ...) {
 // --------------------------------------------------------
 // Initialize options by checking the environment
 // --------------------------------------------------------
-char _mi_toupper(char c) {
-  if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
-                       else return c;
-}
-
-int _mi_strnicmp(const char* s, const char* t, size_t n) {
-  if (n == 0) return 0;
-  for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
-    if (_mi_toupper(*s) != _mi_toupper(*t)) break;
-  }
-  return (n == 0 ? 0 : *s - *t);
-}
-
-void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
-  if (dest==NULL || src==NULL || dest_size == 0) return;
-  // copy until end of src, or when dest is (almost) full
-  while (*src != 0 && dest_size > 1) {
-    *dest++ = *src++;
-    dest_size--;
-  }
-  // always zero terminate
-  *dest = 0;
-}
-
-void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
-  if (dest==NULL || src==NULL || dest_size == 0) return;
-  // find end of string in the dest buffer
-  while (*dest != 0 && dest_size > 1) {
-    dest++;
-    dest_size--;
-  }
-  // and catenate
-  _mi_strlcpy(dest, src, dest_size);
-}
-
-size_t _mi_strlen(const char* s) {
-  if (s==NULL) return 0;
-  size_t len = 0;
-  while(s[len] != 0) { len++; }
-  return len;
-}
-
-size_t _mi_strnlen(const char* s, size_t max_len) {
-  if (s==NULL) return 0;
-  size_t len = 0;
-  while(s[len] != 0 && len < max_len) { len++; }
-  return len;
-}
-
-#ifdef MI_NO_GETENV
-static bool mi_getenv(const char* name, char* result, size_t result_size) {
-  MI_UNUSED(name);
-  MI_UNUSED(result);
-  MI_UNUSED(result_size);
-  return false;
-}
-#else
-static bool mi_getenv(const char* name, char* result, size_t result_size) {
-  if (name==NULL || result == NULL || result_size < 64) return false;
-  return _mi_prim_getenv(name,result,result_size);
-}
-#endif
 
 // TODO: implement ourselves to reduce dependencies on the C runtime
 #include <stdlib.h> // strtol
@@ -514,11 +453,11 @@ static void mi_option_init(mi_option_desc_t* desc) {
   char buf[64+1];
   _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
   _mi_strlcat(buf, desc->name, sizeof(buf));
-  bool found = mi_getenv(buf, s, sizeof(s));
+  bool found = _mi_getenv(buf, s, sizeof(s));
   if (!found && desc->legacy_name != NULL) {
     _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
     _mi_strlcat(buf, desc->legacy_name, sizeof(buf));
-    found = mi_getenv(buf, s, sizeof(s));
+    found = _mi_getenv(buf, s, sizeof(s));
     if (found) {
       _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
     }
@@ -541,7 +480,7 @@ static void mi_option_init(mi_option_desc_t* desc) {
     else {
       char* end = buf;
       long value = strtol(buf, &end, 10);
-      if (mi_option_is_size_in_kib(desc->option)) {
+      if (mi_option_has_size_in_kib(desc->option)) {
         // this option is interpreted in KiB to prevent overflow of `long`
         if (*end == 'K') { end++; }
         else if (*end == 'M') { value *= MI_KiB; end++; }

src/os.c

@@ -421,11 +421,15 @@ bool  _mi_os_expand(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_
   if (p == NULL) return false;
   if (memid->memkind != MI_MEM_OS_EXPAND) return false;
   if (newsize > size) {
-    mi_assert(memid->mem.os.size <= newsize);
+    if (memid->mem.os.size < newsize) {
-    return _mi_os_commit((uint8_t*)p + size, newsize - size, NULL, stats);
+      return false;
+    }
+    else {
+      return _mi_os_commit((uint8_t*)p + size, newsize - size, NULL, stats);
+    }
   }
   else if (newsize < size) {
-    mi_assert(memid->mem.os.size <= size);    
+    mi_assert(memid->mem.os.size >= size);    
     return _mi_os_decommit((uint8_t*)p + newsize, size - newsize, stats);
   }
   else {

src/prim/unix/prim.c

@@ -483,8 +483,6 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo
 
 #if defined(__linux__)
 
-#include <stdio.h>    // snprintf
-
 size_t _mi_prim_numa_node(void) {
   #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
     unsigned long node = 0;
@@ -502,7 +500,7 @@ size_t _mi_prim_numa_node_count(void) {
   unsigned node = 0;
   for(node = 0; node < 256; node++) {
     // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
-    snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
+    _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
     if (mi_prim_access(buf,R_OK) != 0) break;
   }
   return (node+1);

src/static.c

@@ -31,6 +31,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "bitmap.c"
 #include "heap.c"
 #include "init.c"
+#include "libc.c"
 #include "options.c"
 #include "os.c"
 #include "page.c"           // includes page-queue.c

src/stats.c

@@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file
 #include "mimalloc/atomic.h"
 #include "mimalloc/prim.h"
 
-#include <stdio.h>  // snprintf
 #include <string.h> // memset
 
 #if defined(_MSC_VER) && (_MSC_VER < 1920)
@@ -146,7 +145,7 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*
   const int64_t pos = (n < 0 ? -n : n);
   if (pos < base) {
     if (n!=1 || suffix[0] != 'B') {  // skip printing 1 B for the unit column
-      snprintf(buf, len, "%d   %-3s", (int)n, (n==0 ? "" : suffix));
+      _mi_snprintf(buf, len, "%lld   %-3s", (long long)n, (n==0 ? "" : suffix));
     }
   }
   else {
@@ -158,8 +157,8 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*
     const long whole = (long)(tens/10);
     const long frac1 = (long)(tens%10);
     char unitdesc[8];
-    snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
+    _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
-    snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
+    _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
   }
   _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);
 }
@@ -255,7 +254,7 @@ static void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const c
     if (bins[i].allocated > 0) {
       found = true;
       int64_t unit = _mi_bin_size((uint8_t)i);
-      snprintf(buf, 64, "%s %3lu", fmt, (long)i);
+      _mi_snprintf(buf, 64, "%s %3lu", fmt, (long)i);
       mi_stat_print(&bins[i], buf, unit, out, arg);
     }
   }

test/main-override-static.c

@@ -21,14 +21,16 @@ static void test_heap_walk(void);
 static void test_remap(bool start_remappable);
 static void test_remap2(void);
 static void test_remap3(void);
+static void test_expandable(bool use_realloc);
 
 int main() {
   mi_version();
   mi_stats_reset();
 
   //test_remap2();
-  test_remap3();
+  //test_remap3();
-  //test_remap(true);
+  //test_remap(false);
+  test_expandable(true);
   
   // detect double frees and heap corruption
   // double_free1();
@@ -247,15 +249,21 @@ static void test_remap3(void) {    // by Jason Gibson
   mi_free(x);
 }
 
-static void test_remap(bool start_remappable) {
+static void test_remap_expand(bool start_remappable, bool start_expandable, bool use_expand) {
   const size_t iterN = 100;
   const size_t size0 = 64 * 1024 * 1024;
   const size_t inc   = 1024 * 1024;
+  const size_t expand_size = size0 + 2 * inc; // (iterN * inc);
   size_t size = size0;
-  uint8_t* p = (uint8_t*)(start_remappable ? mi_malloc_remappable(size) : mi_malloc(size));
+  uint8_t* p = (uint8_t*)(start_remappable ? mi_malloc_remappable(size) : (start_expandable ? mi_malloc_expandable(size, expand_size) : mi_malloc(size)));
   memset(p, 1, size);
   for (int i = 2; i < iterN; i++) {
-    p = mi_realloc(p, size + inc);
+    void* newp = (use_expand ? mi_expand(p, size + inc) : mi_realloc(p, size + inc));
+    if (use_expand && newp != p) {
+      printf("error: could not expand in place: i=%i, size %zu\n", i, size + inc);
+      abort();
+    }
+    p = newp;
     memset(p + size, i, inc);
     size += inc;
     printf("%3d: increased to size %zu\n", i, size);
@@ -271,6 +279,14 @@ static void test_remap(bool start_remappable) {
   mi_free(p);
 }
 
+static void test_remap(bool start_remappable) {
+  test_remap_expand(start_remappable, false, false);
+}
+
+static void test_expandable(bool use_realloc) {
+  test_remap_expand(false, true, !use_realloc);
+}
+
 
 // ----------------------------
 // bin size experiments