Merge branch 'dev' of https://github.com/microsoft/mimalloc into dev

.gitattributes

@@ -9,3 +9,4 @@
 *.patch binary
 *.dll binary
 *.lib binary
+*.exe binary

CMakeLists.txt

@@ -231,7 +231,7 @@ endif()
 # -----------------------------------------------------------------------------
 
 # dynamic/shared library and symlinks always go to /usr/local/lib equivalent
-set(mi_install_libdir   "${CMAKE_INSTALL_LIBDIR}") 
+set(mi_install_libdir   "${CMAKE_INSTALL_LIBDIR}")
 
 # static libraries and object files, includes, and cmake config files
 # are either installed at top level, or use versioned directories for side-by-side installation (default)
@@ -337,6 +337,7 @@ if (MI_BUILD_STATIC)
   endif()
 
   install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_objdir} LIBRARY)
+  install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir})
 endif()
 
 # install include files
@@ -359,7 +360,7 @@ if (MI_BUILD_OBJECT)
   )
 
   # the following seems to lead to cmake warnings/errors on some systems, disable for now :-(
-  # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_libdir})
+  # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_objdir})
 
   # the FILES expression can also be: $<TARGET_OBJECTS:mimalloc-obj>
   # but that fails cmake versions less than 3.10 so we leave it as is for now
@@ -373,21 +374,17 @@ endif()
 # -----------------------------------------------------------------------------
 
 if (MI_BUILD_TESTS)
-  add_executable(mimalloc-test-api test/test-api.c)
-  target_compile_definitions(mimalloc-test-api PRIVATE ${mi_defines})
-  target_compile_options(mimalloc-test-api PRIVATE ${mi_cflags})
-  target_include_directories(mimalloc-test-api PRIVATE include)
-  target_link_libraries(mimalloc-test-api PRIVATE mimalloc ${mi_libraries})
-
-  add_executable(mimalloc-test-stress test/test-stress.c)
-  target_compile_definitions(mimalloc-test-stress PRIVATE ${mi_defines})
-  target_compile_options(mimalloc-test-stress PRIVATE ${mi_cflags})
-  target_include_directories(mimalloc-test-stress PRIVATE include)
-  target_link_libraries(mimalloc-test-stress PRIVATE mimalloc ${mi_libraries})
-
   enable_testing()
-  add_test(test_api, mimalloc-test-api)
-  add_test(test_stress, mimalloc-test-stress)
+
+  foreach(TEST_NAME api api-fill stress)
+    add_executable(mimalloc-test-${TEST_NAME} test/test-${TEST_NAME}.c)
+    target_compile_definitions(mimalloc-test-${TEST_NAME} PRIVATE ${mi_defines})
+    target_compile_options(mimalloc-test-${TEST_NAME} PRIVATE ${mi_cflags})
+    target_include_directories(mimalloc-test-${TEST_NAME} PRIVATE include)
+    target_link_libraries(mimalloc-test-${TEST_NAME} PRIVATE mimalloc ${mi_libraries})
+
+    add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME})
+  endforeach()
 endif()
 
 # -----------------------------------------------------------------------------

azure-pipelines.yml

@@ -43,7 +43,7 @@ jobs:
       solution: $(BuildType)/libmimalloc.sln
       configuration: '$(MSBuildConfiguration)'
       msbuildArguments: -m
-  - script: ctest --verbose --timeout 120
+  - script: ctest --verbose --timeout 120 -C $(MSBuildConfiguration)
     workingDirectory: $(BuildType)
     displayName: CTest
   #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress

include/mimalloc-types.h

@@ -393,9 +393,15 @@ struct mi_heap_s {
 // Debug
 // ------------------------------------------------------
 
+#if !defined(MI_DEBUG_UNINIT)
 #define MI_DEBUG_UNINIT     (0xD0)
+#endif
+#if !defined(MI_DEBUG_FREED)
 #define MI_DEBUG_FREED      (0xDF)
+#endif
+#if !defined(MI_DEBUG_PADDING)
 #define MI_DEBUG_PADDING    (0xDE)
+#endif
 
 #if (MI_DEBUG)
 // use our own assertion to print without memory allocation

src/init.c

@@ -446,7 +446,9 @@ static void mi_process_load(void) {
   MI_UNUSED(dummy);
   #endif
   os_preloading = false;
-  atexit(&mi_process_done);
+  #if !(defined(_WIN32) && defined(MI_SHARED_LIB))  // use Dll process detach (see below) instead of atexit (issue #521)
+  atexit(&mi_process_done);  
+  #endif
   _mi_options_init();
   mi_process_init();
   //mi_stats_reset();-
@@ -493,6 +495,14 @@ void mi_process_init(void) mi_attr_noexcept {
   #endif
   _mi_verbose_message("secure level: %d\n", MI_SECURE);
   mi_thread_init();
+
+  #if defined(_WIN32) && !defined(MI_SHARED_LIB)
+  // When building as a static lib the FLS cleanup happens to early for the main thread.
+  // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup
+  // will not call _mi_thread_done on the (still executing) main thread. See issue #508.
+  FlsSetValue(mi_fls_key, NULL);
+  #endif
+
   mi_stats_reset();  // only call stat reset *after* thread init (or the heap tld == NULL)
 
   if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
@@ -522,8 +532,7 @@ static void mi_process_done(void) {
   process_done = true;
 
   #if defined(_WIN32) && !defined(MI_SHARED_LIB)
-  FlsSetValue(mi_fls_key, NULL);  // don't call main-thread callback
-  FlsFree(mi_fls_key);            // call thread-done on all threads to prevent dangling callback pointer if statically linked with a DLL; Issue #208
+  FlsFree(mi_fls_key);  // call thread-done on all threads (except the main thread) to prevent dangling callback pointer if statically linked with a DLL; Issue #208
   #endif
   
   #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB)  
@@ -551,12 +560,35 @@ static void mi_process_done(void) {
     if (reason==DLL_PROCESS_ATTACH) {
       mi_process_load();
     }
-    else if (reason==DLL_THREAD_DETACH) {
-      if (!mi_is_redirected()) mi_thread_done();
+    else if (reason==DLL_PROCESS_DETACH) {
+      mi_process_done();
     }
+    else if (reason==DLL_THREAD_DETACH) {
+      if (!mi_is_redirected()) {
+        mi_thread_done();
+      }
+    }    
     return TRUE;
   }
 
+#elif defined(_MSC_VER)
+  // MSVC: use data section magic for static libraries
+  // See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
+  static int _mi_process_init(void) {
+    mi_process_load();
+    return 0;
+  }
+  typedef int(*_mi_crt_callback_t)(void);
+  #if defined(_M_X64) || defined(_M_ARM64)
+    __pragma(comment(linker, "/include:" "_mi_msvc_initu"))
+    #pragma section(".CRT$XIU", long, read)
+  #else
+    __pragma(comment(linker, "/include:" "__mi_msvc_initu"))
+  #endif
+  #pragma data_seg(".CRT$XIU")
+  mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init };
+  #pragma data_seg()
+
 #elif defined(__cplusplus)
   // C++: use static initialization to detect process start
   static bool _mi_process_init(void) {
@@ -571,24 +603,6 @@ static void mi_process_done(void) {
     mi_process_load();
   }
 
-#elif defined(_MSC_VER)
-  // MSVC: use data section magic for static libraries
-  // See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
-  static int _mi_process_init(void) {
-    mi_process_load();
-    return 0;
-  }
-  typedef int(*_crt_cb)(void);
-  #if defined(_M_X64) || defined(_M_ARM64)
-    __pragma(comment(linker, "/include:" "_mi_msvc_initu"))
-    #pragma section(".CRT$XIU", long, read)
-  #else
-    __pragma(comment(linker, "/include:" "__mi_msvc_initu"))
-  #endif
-  #pragma data_seg(".CRT$XIU")
-  _crt_cb _mi_msvc_initu[] = { &_mi_process_init };
-  #pragma data_seg()
-
 #else
 #pragma message("define a way to call mi_process_load on your platform")
 #endif

src/os.c

@@ -242,7 +242,7 @@ static void os_detect_overcommit(void) {
 #if defined(__linux__)
   int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
 	if (fd < 0) return;
-  char buf[128];
+  char buf[32];
   ssize_t nread = read(fd, &buf, sizeof(buf));
 	close(fd);
   // <https://www.kernel.org/doc/Documentation/vm/overcommit-accounting>
@@ -274,6 +274,17 @@ void _mi_os_init() {
 #endif
 
 
+#if defined(MADV_NORMAL)
+static int mi_madvise(void* addr, size_t length, int advice) {
+  #if defined(__sun)
+  return madvise((caddr_t)addr, length, advice);  // Solaris needs cast (issue #520)
+  #else
+  return madvise(addr, length, advice);
+  #endif
+}
+#endif
+
+
 /* -----------------------------------------------------------
   free memory
 -------------------------------------------------------------- */
@@ -477,7 +488,7 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr
   }
   #elif defined(MAP_ALIGN)  // Solaris
   if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
-    void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0);
+    void* p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0);  // addr parameter is the required alignment
     if (p!=MAP_FAILED) return p;
     // fall back to regular mmap
   }
@@ -589,7 +600,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
       // However, some systems only allow THP if called with explicit `madvise`, so
       // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
       if (allow_large && use_large_os_page(size, try_alignment)) {
-        if (madvise(p, size, MADV_HUGEPAGE) == 0) {
+        if (mi_madvise(p, size, MADV_HUGEPAGE) == 0) {
           *is_large = true; // possibly
         };
       }
@@ -598,7 +609,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
         struct memcntl_mha cmd = {0};
         cmd.mha_pagesize = large_os_page_size;
         cmd.mha_cmd = MHA_MAPSIZE_VA;
-        if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
+        if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
           *is_large = true;
         }
       }      
@@ -878,8 +889,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
 
   #if defined(_WIN32)
   if (commit) {
-    // if the memory was already committed, the call succeeds but it is not zero'd
-    // *is_zero = true;
+    // *is_zero = true;  // note: if the memory was already committed, the call succeeds but the memory is not zero'd
     void* p = VirtualAlloc(start, csize, MEM_COMMIT, PAGE_READWRITE);
     err = (p == start ? 0 : GetLastError());
   }
@@ -889,23 +899,40 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
   }
   #elif defined(__wasi__)
   // WebAssembly guests can't control memory protection
-  #elif defined(MAP_FIXED)
-  if (!commit) {
-    // use mmap with MAP_FIXED to discard the existing memory (and reduce commit charge)
-    void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
-    if (p != start) { err = errno; }
-  }
-  else {
-    // for commit, just change the protection
+  #elif 0 && defined(MAP_FIXED) && !defined(__APPLE__)
+  // Linux: disabled for now as mmap fixed seems much more expensive than MADV_DONTNEED (and splits VMA's?)
+  if (commit) {
+    // commit: just change the protection
     err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
     if (err != 0) { err = errno; }
-    //#if defined(MADV_FREE_REUSE)
-    //  while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; }
-    //#endif
+  } 
+  else {
+    // decommit: use mmap with MAP_FIXED to discard the existing memory (and reduce rss)
+    const int fd = mi_unix_mmap_fd();
+    void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
+    if (p != start) { err = errno; }
   }
   #else
-  err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
-  if (err != 0) { err = errno; }
+  // Linux, macOSX and others.
+  if (commit) {
+    // commit: ensure we can access the area    
+    err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
+    if (err != 0) { err = errno; }
+  } 
+  else {
+    #if defined(MADV_DONTNEED) && MI_DEBUG == 0 && MI_SECURE == 0
+    // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
+    // (on the other hand, MADV_FREE would be good enough.. it is just not reflected in the stats :-( )
+    err = madvise(start, csize, MADV_DONTNEED);
+    #else
+    // decommit: just disable access (also used in debug and secure mode to trap on illegal access)
+    err = mprotect(start, csize, PROT_NONE);
+    if (err != 0) { err = errno; }
+    #endif
+    //#if defined(MADV_FREE_REUSE)
+    //  while ((err = mi_madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; }
+    //#endif
+  }
   #endif
   if (err != 0) {
     _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err);
@@ -966,16 +993,16 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
   static _Atomic(size_t) advice = ATOMIC_VAR_INIT(MADV_FREE);
   int oadvice = (int)mi_atomic_load_relaxed(&advice);
   int err;
-  while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0;  };
+  while ((err = mi_madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0;  };
   if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) {  
     // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
     mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED);
-    err = madvise(start, csize, MADV_DONTNEED);
+    err = mi_madvise(start, csize, MADV_DONTNEED);
   }
 #elif defined(__wasi__)
   int err = 0;
 #else
-  int err = madvise(start, csize, MADV_DONTNEED);
+  int err = mi_madvise(start, csize, MADV_DONTNEED);
 #endif
   if (err != 0) {
     _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, errno);

src/region.c

@@ -94,7 +94,7 @@ typedef struct mem_region_s {
   mi_bitmap_field_t         commit;      // track if committed per block
   mi_bitmap_field_t         reset;       // track if reset per block
   _Atomic(size_t)           arena_memid; // if allocated from a (huge page) arena
-  size_t                    padding;     // round to 8 fields
+  _Atomic(size_t)           padding;     // round to 8 fields (needs to be atomic for msvc, see issue #508)
 } mem_region_t;
 
 // The region map

test/main-override.cpp

@@ -46,7 +46,7 @@ int main() {
   tsan_numa_test();
   strdup_test();
 
-  //test_mt_shutdown();
+  test_mt_shutdown();
   //fail_aslr();
   mi_stats_print(NULL);
   return 0;
@@ -71,7 +71,7 @@ public:
 static void various_tests() {
   atexit(free_p);
   void* p1 = malloc(78);
-  void* p2 = mi_malloc_aligned(16, 24);
+  void* p2 = mi_malloc_aligned(24, 16);
   free(p1);
   p1 = malloc(8);
   char* s = mi_strdup("hello\n");

test/test-api-fill.c

@@ -0,0 +1,332 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#include "mimalloc.h"
+#include "mimalloc-types.h"
+
+#include "testhelper.h"
+
+// ---------------------------------------------------------------------------
+// Helper functions
+// ---------------------------------------------------------------------------
+bool check_zero_init(uint8_t* p, size_t size);
+#if MI_DEBUG >= 2
+bool check_debug_fill_uninit(uint8_t* p, size_t size);
+bool check_debug_fill_freed(uint8_t* p, size_t size);
+#endif
+
+// ---------------------------------------------------------------------------
+// Main testing
+// ---------------------------------------------------------------------------
+int main(void) {
+  mi_option_disable(mi_option_verbose);
+
+  // ---------------------------------------------------
+  // Zeroing allocation
+  // ---------------------------------------------------
+  CHECK_BODY("zeroinit-zalloc-small", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_zalloc(zalloc_size);
+    result = check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-zalloc-large", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_zalloc(zalloc_size);
+    result = check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-zalloc_small", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_zalloc_small(zalloc_size);
+    result = check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("zeroinit-calloc-small", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_calloc(calloc_size, 1);
+    result = check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-calloc-large", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_calloc(calloc_size, 1);
+    result = check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("zeroinit-rezalloc-small", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_zalloc(zalloc_size);
+    result = check_zero_init(p, zalloc_size);
+    zalloc_size *= 3;
+    p = (uint8_t*)mi_rezalloc(p, zalloc_size);
+    result &= check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-rezalloc-large", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_zalloc(zalloc_size);
+    result = check_zero_init(p, zalloc_size);
+    zalloc_size *= 3;
+    p = (uint8_t*)mi_rezalloc(p, zalloc_size);
+    result &= check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("zeroinit-recalloc-small", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_calloc(calloc_size, 1);
+    result = check_zero_init(p, calloc_size);
+    calloc_size *= 3;
+    p = (uint8_t*)mi_recalloc(p, calloc_size, 1);
+    result &= check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-recalloc-large", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_calloc(calloc_size, 1);
+    result = check_zero_init(p, calloc_size);
+    calloc_size *= 3;
+    p = (uint8_t*)mi_recalloc(p, calloc_size, 1);
+    result &= check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+
+  // ---------------------------------------------------
+  // Zeroing in aligned API
+  // ---------------------------------------------------
+  CHECK_BODY("zeroinit-zalloc_aligned-small", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_zalloc_aligned(zalloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-zalloc_aligned-large", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_zalloc_aligned(zalloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("zeroinit-calloc_aligned-small", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_calloc_aligned(calloc_size, 1, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-calloc_aligned-large", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_calloc_aligned(calloc_size, 1, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("zeroinit-rezalloc_aligned-small", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_zalloc_aligned(zalloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, zalloc_size);
+    zalloc_size *= 3;
+    p = (uint8_t*)mi_rezalloc_aligned(p, zalloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result &= check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-rezalloc_aligned-large", {
+    size_t zalloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_zalloc_aligned(zalloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, zalloc_size);
+    zalloc_size *= 3;
+    p = (uint8_t*)mi_rezalloc_aligned(p, zalloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result &= check_zero_init(p, zalloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("zeroinit-recalloc_aligned-small", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_calloc_aligned(calloc_size, 1, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, calloc_size);
+    calloc_size *= 3;
+    p = (uint8_t*)mi_recalloc_aligned(p, calloc_size, 1, MI_MAX_ALIGN_SIZE * 2);
+    result &= check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("zeroinit-recalloc_aligned-large", {
+    size_t calloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_calloc_aligned(calloc_size, 1, MI_MAX_ALIGN_SIZE * 2);
+    result = check_zero_init(p, calloc_size);
+    calloc_size *= 3;
+    p = (uint8_t*)mi_recalloc_aligned(p, calloc_size, 1, MI_MAX_ALIGN_SIZE * 2);
+    result &= check_zero_init(p, calloc_size);
+    mi_free(p);
+  });
+
+#if MI_DEBUG >= 2
+  // ---------------------------------------------------
+  // Debug filling
+  // ---------------------------------------------------
+  CHECK_BODY("uninit-malloc-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_malloc(malloc_size);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("uninit-malloc-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_malloc(malloc_size);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("uninit-malloc_small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_malloc_small(malloc_size);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("uninit-realloc-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_malloc(malloc_size);
+    result = check_debug_fill_uninit(p, malloc_size);
+    malloc_size *= 3;
+    p = (uint8_t*)mi_realloc(p, malloc_size);
+    result &= check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("uninit-realloc-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_malloc(malloc_size);
+    result = check_debug_fill_uninit(p, malloc_size);
+    malloc_size *= 3;
+    p = (uint8_t*)mi_realloc(p, malloc_size);
+    result &= check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("uninit-mallocn-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_mallocn(malloc_size, 1);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("uninit-mallocn-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_mallocn(malloc_size, 1);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("uninit-reallocn-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_mallocn(malloc_size, 1);
+    result = check_debug_fill_uninit(p, malloc_size);
+    malloc_size *= 3;
+    p = (uint8_t*)mi_reallocn(p, malloc_size, 1);
+    result &= check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("uninit-reallocn-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_mallocn(malloc_size, 1);
+    result = check_debug_fill_uninit(p, malloc_size);
+    malloc_size *= 3;
+    p = (uint8_t*)mi_reallocn(p, malloc_size, 1);
+    result &= check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("uninit-malloc_aligned-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_malloc_aligned(malloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("uninit-malloc_aligned-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_malloc_aligned(malloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+  CHECK_BODY("uninit-realloc_aligned-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_malloc_aligned(malloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_debug_fill_uninit(p, malloc_size);
+    malloc_size *= 3;
+    p = (uint8_t*)mi_realloc_aligned(p, malloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result &= check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+  CHECK_BODY("uninit-realloc_aligned-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_malloc_aligned(malloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result = check_debug_fill_uninit(p, malloc_size);
+    malloc_size *= 3;
+    p = (uint8_t*)mi_realloc_aligned(p, malloc_size, MI_MAX_ALIGN_SIZE * 2);
+    result &= check_debug_fill_uninit(p, malloc_size);
+    mi_free(p);
+  });
+
+
+  CHECK_BODY("fill-freed-small", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX / 2;
+    uint8_t* p = (uint8_t*)mi_malloc(malloc_size);
+    mi_free(p);
+    // First sizeof(void*) bytes will contain housekeeping data, skip these
+    result = check_debug_fill_freed(p + sizeof(void*), malloc_size - sizeof(void*));
+  });
+  CHECK_BODY("fill-freed-large", {
+    size_t malloc_size = MI_SMALL_SIZE_MAX * 2;
+    uint8_t* p = (uint8_t*)mi_malloc(malloc_size);
+    mi_free(p);
+    // First sizeof(void*) bytes will contain housekeeping data, skip these
+    result = check_debug_fill_freed(p + sizeof(void*), malloc_size - sizeof(void*));
+  });
+#endif
+
+  // ---------------------------------------------------
+  // Done
+  // ---------------------------------------------------[]
+  return print_test_summary();
+}
+
+// ---------------------------------------------------------------------------
+// Helper functions
+// ---------------------------------------------------------------------------
+bool check_zero_init(uint8_t* p, size_t size) {
+  if(!p)
+    return false;
+  bool result = true;
+  for (size_t i = 0; i < size; ++i) {
+    result &= p[i] == 0;
+  }
+  return result;
+}
+
+#if MI_DEBUG >= 2
+bool check_debug_fill_uninit(uint8_t* p, size_t size) {
+  if(!p)
+    return false;
+
+  bool result = true;
+  for (size_t i = 0; i < size; ++i) {
+    result &= p[i] == MI_DEBUG_UNINIT;
+  }
+  return result;
+}
+
+bool check_debug_fill_freed(uint8_t* p, size_t size) {
+  if(!p)
+    return false;
+
+  bool result = true;
+  for (size_t i = 0; i < size; ++i) {
+    result &= p[i] == MI_DEBUG_FREED;
+  }
+  return result;
+}
+#endif

test/test-api.c

@@ -23,7 +23,6 @@ we therefore test the API over various inputs. Please add more tests :-)
 [1] https://github.com/daanx/mimalloc-bench
 */
 
-#include <stdio.h>
 #include <assert.h>
 #include <stdbool.h>
 #include <stdint.h>
@@ -35,34 +34,9 @@ we therefore test the API over various inputs. Please add more tests :-)
 
 #include "mimalloc.h"
 // #include "mimalloc-internal.h"
+#include "mimalloc-types.h" // for MI_DEBUG
 
-// ---------------------------------------------------------------------------
-// Test macros: CHECK(name,predicate) and CHECK_BODY(name,body)
-// ---------------------------------------------------------------------------
-static int ok = 0;
-static int failed = 0;
-
-#define CHECK_BODY(name,body) \
- do { \
-  fprintf(stderr,"test: %s...  ", name ); \
-  bool result = true;                                     \
-  do { body } while(false);                                \
-  if (!(result)) {                                        \
-    failed++; \
-    fprintf(stderr,                                       \
-            "\n  FAILED: %s:%d:\n  %s\n",                 \
-            __FILE__,                                     \
-            __LINE__,                                     \
-            #body);                                       \
-    /* exit(1); */ \
-  } \
-  else { \
-    ok++;                               \
-    fprintf(stderr,"ok.\n");                    \
-  }                                             \
- } while (false)
-
-#define CHECK(name,expr)      CHECK_BODY(name,{ result = (expr); })
+#include "testhelper.h"
 
 // ---------------------------------------------------------------------------
 // Test functions
@@ -219,10 +193,7 @@ int main(void) {
   // ---------------------------------------------------
   // Done
   // ---------------------------------------------------[]
-  fprintf(stderr,"\n\n---------------------------------------------\n"
-                 "succeeded: %i\n"
-                 "failed   : %i\n\n", ok, failed);
-  return failed;
+  return print_test_summary();
 }
 
 // ---------------------------------------------------

test/testhelper.h

@@ -0,0 +1,49 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2020, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+#ifndef TESTHELPER_H_
+#define TESTHELPER_H_
+
+#include <stdio.h>
+
+// ---------------------------------------------------------------------------
+// Test macros: CHECK(name,predicate) and CHECK_BODY(name,body)
+// ---------------------------------------------------------------------------
+static int ok = 0;
+static int failed = 0;
+
+#define CHECK_BODY(name,body) \
+ do { \
+  fprintf(stderr,"test: %s...  ", name ); \
+  bool result = true;                                     \
+  do { body } while(false);                                \
+  if (!(result)) {                                        \
+    failed++; \
+    fprintf(stderr,                                       \
+            "\n  FAILED: %s:%d:\n  %s\n",                 \
+            __FILE__,                                     \
+            __LINE__,                                     \
+            #body);                                       \
+    /* exit(1); */ \
+  } \
+  else { \
+    ok++;                               \
+    fprintf(stderr,"ok.\n");                    \
+  }                                             \
+ } while (false)
+
+#define CHECK(name,expr)      CHECK_BODY(name,{ result = (expr); })
+
+// Print summary of test. Return value can be directly use as a return value for main().
+static inline int print_test_summary(void)
+{
+  fprintf(stderr,"\n\n---------------------------------------------\n"
+                 "succeeded: %i\n"
+                 "failed   : %i\n\n", ok, failed);
+  return failed;
+}
+
+#endif // TESTHELPER_H_