Merge branch 'dev3' of https://github.com/microsoft/mimalloc into dev3

src/free.c

@@ -115,6 +115,7 @@ static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, vo
 
 // free a local pointer  (page parameter comes first for better codegen)
 static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
+  mi_assert_internal(p!=NULL && page != NULL);
   mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p);
   mi_block_check_unguard(page, block, p);
   mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */);
@@ -122,11 +123,7 @@ static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_
 
 // free a pointer owned by another thread (page parameter comes first for better codegen)
 static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {
-  if (p==NULL) return;  // a NULL pointer is seen as abandoned (tid==0) with a full flag set
-  #if !MI_PAGE_MAP_FLAT
-  if (page==&_mi_page_empty) return;  // an invalid pointer may lead to using the empty page
-  #endif
-  mi_assert_internal(p!=NULL && page != NULL && page != &_mi_page_empty);
+  mi_assert_internal(p!=NULL && page != NULL);
   mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865)
   mi_block_check_unguard(page, block, p);
   mi_free_block_mt(page, block);
@@ -150,13 +147,8 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
     return NULL;
   }
   mi_page_t* page = _mi_safe_ptr_page(p);
-  if (page == NULL) {
-    if (p != NULL) {
-      _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);
-    }
-    #if !MI_PAGE_MAP_FLAT
-    page = (mi_page_t*)&_mi_page_empty;
-    #endif
+  if (p != NULL && page == NULL) {
+    _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p);    
   }
   return page;
   #else
@@ -169,12 +161,9 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
 void mi_free(void* p) mi_attr_noexcept
 {
   mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");
-
-  #if MI_PAGE_MAP_FLAT  // if not flat, p==NULL leads to `_mi_page_empty` which leads to `mi_free_generic_mt`
   if mi_unlikely(page==NULL) return;
-  #endif
-  mi_assert_internal(page!=NULL);
-
+  mi_assert_internal(p!=NULL && page!=NULL);
+  
   const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page));
   if mi_likely(xtid == 0) {                        // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`
     // thread-local, aligned, and not a full page

src/init.c

@@ -16,7 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file
 
 // Empty page used to initialize the small free pages array
 const mi_page_t _mi_page_empty = {
-  MI_ATOMIC_VAR_INIT(MI_PAGE_IN_FULL_QUEUE),  // xthread_id  (must set flag to catch NULL on a free)
+  MI_ATOMIC_VAR_INIT(0),  // xthread_id 
   NULL,                   // free
   0,                      // used
   0,                      // capacity

src/options.c

@@ -98,8 +98,8 @@ int mi_version(void) mi_attr_noexcept {
 #endif
 #endif
 
-// Static options (only exposed for the debugger)
-mi_option_desc_t mi_options[_mi_option_last] =
+// Static options 
+static mi_option_desc_t mi_options[_mi_option_last] =
 {
   // stable options
 #if MI_DEBUG || defined(MI_SHOW_ERRORS)
@@ -329,7 +329,7 @@ static void mi_cdecl mi_out_stderr(const char* msg, void* arg) {
 #ifndef MI_MAX_DELAY_OUTPUT
 #define MI_MAX_DELAY_OUTPUT ((size_t)(16*1024))
 #endif
-static char out_buf[MI_MAX_DELAY_OUTPUT+1];
+static char mi_output_buffer[MI_MAX_DELAY_OUTPUT+1];
 static _Atomic(size_t) out_len;
 
 static void mi_cdecl mi_out_buf(const char* msg, void* arg) {
@@ -345,7 +345,8 @@ static void mi_cdecl mi_out_buf(const char* msg, void* arg) {
   if (start+n >= MI_MAX_DELAY_OUTPUT) {
     n = MI_MAX_DELAY_OUTPUT-start-1;
   }
-  _mi_memcpy(&out_buf[start], msg, n);
+  mi_assert_internal(start + n <= MI_MAX_DELAY_OUTPUT);
+  _mi_memcpy(&mi_output_buffer[start], msg, n);
 }
 
 static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
@@ -354,10 +355,10 @@ static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) {
   size_t count = mi_atomic_add_acq_rel(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1));
   // and output the current contents
   if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
-  out_buf[count] = 0;
-  out(out_buf,arg);
+  mi_output_buffer[count] = 0;
+  out(mi_output_buffer,arg);
   if (!no_more_buf) {
-    out_buf[count] = '\n'; // if continue with the buffer, insert a newline
+    mi_output_buffer[count] = '\n'; // if continue with the buffer, insert a newline
   }
 }
 

src/page-map.c

@@ -210,9 +210,7 @@ bool _mi_page_map_init(void) {
   if (!mi_page_map_memid.initially_committed) {
     _mi_os_commit(_mi_page_map[0], os_page_size, NULL);   // only first OS page
   }
-  _mi_page_map[0][0] = (mi_page_t*)&_mi_page_empty;       // caught in `mi_free`
-  
-  mi_assert_internal(_mi_ptr_page(NULL)==&_mi_page_empty);
+  mi_assert_internal(_mi_ptr_page(NULL)==NULL);
   return true;
 }
 

src/stats.c

@@ -227,12 +227,14 @@ static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int
   _mi_fprintf(out, arg, "\n");
 }
 
+#if MI_STAT>1
 static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
   _mi_fprintf(out, arg, "%10s:", msg);
   _mi_fprintf(out, arg, "%12s", " ");  // no peak
   mi_print_amount(stat->total, unit, out, arg);
   _mi_fprintf(out, arg, "\n");
 }
+#endif
 
 static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
   _mi_fprintf(out, arg, "%10s:", msg);