merge from dev-reset

include/mimalloc/internal.h

@@ -88,8 +88,10 @@ void       _mi_thread_data_collect(void);
 
 // os.c
 void       _mi_os_init(void);                                            // called from process init
-void*      _mi_os_alloc(size_t size, bool* is_zero, mi_stats_t* stats);  // to allocate thread local data
-void       _mi_os_free(void* p, size_t size, mi_stats_t* stats);         // to free thread local data
+void*      _mi_os_alloc(size_t size, bool* is_zero, mi_stats_t* stats);  
+void       _mi_os_free(void* p, size_t size, mi_stats_t* stats);
+void       _mi_os_free_ex(void* p, size_t size, bool is_committed, mi_stats_t* stats);
+
 size_t     _mi_os_page_size(void);
 size_t     _mi_os_good_alloc_size(size_t size);
 bool       _mi_os_has_overcommit(void);
@@ -104,16 +106,16 @@ bool       _mi_os_unprotect(void* addr, size_t size);
 bool       _mi_os_purge(void* p, size_t size, mi_stats_t* stats);
 bool       _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats);
 
-void*      _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, bool* is_zero, mi_stats_t* stats);
-void*      _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool* large, bool* is_zero, mi_stats_t* tld_stats);
-void       _mi_os_free_aligned(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats);
+void*      _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats);
+void*      _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats);
+void       _mi_os_free_aligned_at_offset(void* p, size_t size, size_t alignment, size_t align_offset, bool was_committed, mi_stats_t* tld_stats);
+
 void*      _mi_os_get_aligned_hint(size_t try_alignment, size_t size);
 bool       _mi_os_use_large_page(size_t size, size_t alignment);
 size_t     _mi_os_large_page_size(void);
 
-void       _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats);
 void*      _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, bool* is_zero);
-void       _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats);
+void       _mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats);
 
 // arena.c
 mi_arena_id_t _mi_arena_id_none(void);
@@ -918,6 +920,8 @@ static inline void _mi_memzero(void* dst, size_t n) {
 }
 #endif
 
+// initialize a local variable to zero; use memset as compilers optimize constant sized memset's
+#define _mi_memzero_var(x)  memset(&x,0,sizeof(x))
 
 // -------------------------------------------------------------------------------
 // The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned

include/mimalloc/prim.h

@@ -26,8 +26,8 @@ typedef struct mi_os_mem_config_s {
   size_t  large_page_size;      // 2MiB
   size_t  alloc_granularity;    // smallest allocation size (on Windows 64KiB)
   bool    has_overcommit;       // can we reserve more memory than can be actually committed?
-  bool    must_free_whole;      // must allocated blocks free as a whole (false for mmap, true for VirtualAlloc)
-  bool    has_virtual_reserve;  // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
+  bool    must_free_whole;      // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc)
+  bool    has_virtual_reserve;  // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
 } mi_os_mem_config_t;
 
 // Initialize

include/mimalloc/types.h

@@ -325,6 +325,10 @@ typedef struct mi_page_s {
 
 
 
+// ------------------------------------------------------
+// Mimalloc segments contain mimalloc pages
+// ------------------------------------------------------
+
 typedef enum mi_page_kind_e {
   MI_PAGE_SMALL,    // small blocks go into 64KiB pages inside a segment
   MI_PAGE_MEDIUM,   // medium blocks go into medium pages inside a segment
@@ -393,7 +397,6 @@ typedef struct mi_memid_s {
     mi_memid_arena_info_t arena;// only used for MI_MEM_ARENA
   } mem;
   bool          is_pinned;      // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large OS pages)
-  bool          is_large;       // `true` if the memory is in OS large (2MiB) or huge (1GiB) pages. (`is_pinned` will be true)
   bool          was_committed;  // `true` if the memory was originally allocated as committed
   bool          was_zero;       // `true` if the memory was originally zero initialized
   mi_memkind_t  memkind;