Initial commit of separate memory region layer and improved large OS pages support, see 'memory.c'

include/mimalloc-atomic.h

@@ -39,6 +39,15 @@ static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t e
 // Atomically exchange a value.
 static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange);
 
+// Atomically read a value
+static inline uintptr_t mi_atomic_read(volatile uintptr_t* p);
+
+// Atomically read a pointer
+static inline void* mi_atomic_read_ptr(volatile void** p);
+
+// Atomically write a value
+static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x);
+
 static inline void mi_atomic_yield(void);
 
 // Atomically compare and exchange a pointer; returns `true` if successful.
@@ -85,6 +94,15 @@ static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t e
 static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) {
   return (uintptr_t)RC64(_InterlockedExchange)((volatile intptr_t*)p, (intptr_t)exchange);
 }
+static inline uintptr_t mi_atomic_read(volatile uintptr_t* p) {
+  return *p;
+}
+static inline void* mi_atomic_read_ptr(volatile void** p) {
+  return (void*)(*p);
+}
+static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x) {
+  *p = x;
+}
 static inline void mi_atomic_yield(void) {
   YieldProcessor();
 }
@@ -147,6 +165,18 @@ static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exch
   MI_USING_STD
   return atomic_exchange_explicit((volatile atomic_uintptr_t*)p, exchange, memory_order_relaxed);
 }
+static inline uintptr_t mi_atomic_read(volatile uintptr_t* p) {
+  MI_USING_STD
+  return atomic_load_explicit((volatile atomic_uintptr_t*)p, memory_order_relaxed);
+}
+static inline void* mi_atomic_read_ptr(volatile void** p) {
+  MI_USING_STD
+  return atomic_load_explicit((volatile _Atomic(void*)*)p, memory_order_relaxed);
+}
+static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x) {
+  MI_USING_STD
+  return atomic_store_explicit((volatile atomic_uintptr_t*)p, x, memory_order_relaxed);
+}
 
 #if defined(__cplusplus)
   #include <thread>

include/mimalloc-internal.h

@@ -29,18 +29,21 @@ uintptr_t  _mi_ptr_cookie(const void* p);
 uintptr_t  _mi_random_shuffle(uintptr_t x);
 uintptr_t  _mi_random_init(uintptr_t seed /* can be zero */);
 
-// "os.c"
-bool       _mi_os_reset(void* p, size_t size);
-void*      _mi_os_alloc(size_t size, mi_stats_t* stats);
-bool       _mi_os_shrink(void* p, size_t oldsize, size_t newsize);
-void       _mi_os_free(void* p, size_t size, mi_stats_t* stats);
-bool       _mi_os_protect(void* addr, size_t size);
-bool       _mi_os_unprotect(void* addr, size_t size);
-void       _mi_os_init(void);  // called from process init
-
-void*      _mi_os_alloc_aligned(size_t size, size_t alignment, mi_os_tld_t* tld);
+// os.c
 size_t     _mi_os_page_size(void);
 uintptr_t  _mi_align_up(uintptr_t sz, size_t alignment);
+void       _mi_os_init(void);                                      // called from process init
+void*      _mi_os_alloc(size_t size, mi_stats_t* stats);           // to allocate thread local data
+void       _mi_os_free(void* p, size_t size, mi_stats_t* stats);   // to free thread local data
+
+// memory.c
+void*      _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t* id, mi_os_tld_t* tld);
+void*      _mi_mem_alloc(size_t size, size_t* id, mi_os_tld_t* tld);
+void       _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats);
+
+bool       _mi_mem_reset(void* p, size_t size, mi_stats_t* stats);
+bool       _mi_mem_protect(void* addr, size_t size);
+bool       _mi_mem_unprotect(void* addr, size_t size);
 
 // "segment.c"
 mi_page_t* _mi_segment_page_alloc(size_t block_wsize, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);

include/mimalloc-types.h

@@ -89,7 +89,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MI_SMALL_PAGES_PER_SEGMENT        (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE)
 #define MI_LARGE_PAGES_PER_SEGMENT        (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE)
 
-#define MI_LARGE_SIZE_MAX                 (MI_LARGE_PAGE_SIZE/8)   // 512kb on 64-bit
+#define MI_LARGE_SIZE_MAX                 (MI_LARGE_PAGE_SIZE/4)   // 1MiB on 64-bit
 #define MI_LARGE_WSIZE_MAX                (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT)
 
 
@@ -215,6 +215,7 @@ typedef struct mi_segment_s {
   size_t          segment_size;// for huge pages this may be different from `MI_SEGMENT_SIZE`
   size_t          segment_info_size;  // space we are using from the first page for segment meta-data and possible guard pages.
   uintptr_t       cookie;      // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie`
+  size_t          memid;       // id for the os-level memory manager
 
   // layout like this to optimize access in `mi_free`
   size_t          page_shift;  // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`).
@@ -322,12 +323,14 @@ typedef struct mi_stats_s {
   mi_stat_count_t reserved;
   mi_stat_count_t committed;
   mi_stat_count_t reset;
+  mi_stat_count_t page_committed;
   mi_stat_count_t segments_abandoned;
   mi_stat_count_t pages_abandoned;
   mi_stat_count_t pages_extended;
   mi_stat_count_t mmap_calls;
   mi_stat_count_t mmap_right_align;
   mi_stat_count_t mmap_ensure_aligned;
+  mi_stat_count_t commit_calls;
   mi_stat_count_t threads;
   mi_stat_count_t huge;
   mi_stat_count_t malloc;
@@ -370,11 +373,13 @@ typedef struct mi_segment_queue_s {
 // Segments thread local data
 typedef struct mi_segments_tld_s {
   mi_segment_queue_t  small_free;   // queue of segments with free small pages
+  size_t              count;        // current number of segments;
+  size_t              peak_count;   // peak number of segments
   size_t              current_size; // current size of all segments
   size_t              peak_size;    // peak size of all segments
   size_t              cache_count;  // number of segments in the cache
   size_t              cache_size;   // total size of all segments in the cache
-  mi_segment_queue_t  cache;        // (small) cache of segments for small and large pages (to avoid repeated mmap calls)
+  mi_segment_t*       cache;        // (small) cache of segments
   mi_stats_t*         stats;        // points to tld stats
 } mi_segments_tld_t;
 

include/mimalloc.h

@@ -215,8 +215,8 @@ mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_b
 typedef enum mi_option_e {
   mi_option_page_reset,
   mi_option_cache_reset,
-  mi_option_pool_commit,
-  mi_option_large_os_pages,
+  mi_option_eager_commit,
+  mi_option_large_os_pages,  // implies eager commit
   mi_option_secure,
   mi_option_show_stats,
   mi_option_show_errors,