wip: add expandable primitives

include/mimalloc/internal.h

@@ -119,6 +119,9 @@ void*      _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t ma
 void*      _mi_os_alloc_remappable(size_t size, size_t alignment, mi_memid_t* memid, mi_stats_t* stats);
 void*      _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats);
 
+void*      _mi_os_alloc_expandable(size_t size, size_t alignment, size_t future_reserve, mi_memid_t* memid, mi_stats_t* stats);
+bool       _mi_os_expand(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats);
+
 
 // arena.c
 mi_arena_id_t _mi_arena_id_none(void);
@@ -687,14 +690,14 @@ static inline mi_memid_t _mi_memid_none(void) {
   return _mi_memid_create(MI_MEM_NONE);
 }
 
-static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, size_t alignment, bool committed, bool is_zero, bool is_large) {
+static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, size_t alignment, bool committed, bool is_large_or_pinned, bool is_zero ) {
   mi_memid_t memid = _mi_memid_create(MI_MEM_OS);
   memid.mem.os.base = base;
   memid.mem.os.size = size;
   memid.mem.os.alignment = alignment;
   memid.initially_committed = committed;
   memid.initially_zero = is_zero;
-  memid.is_pinned = is_large;
+  memid.is_pinned = is_large_or_pinned;
   return memid;
 }
 

include/mimalloc/types.h

@@ -331,6 +331,7 @@ typedef enum mi_memkind_e {
   MI_MEM_STATIC,    // allocated in a static area and should not be freed (for arena meta data for example)
   MI_MEM_OS,        // allocated from the OS
   MI_MEM_OS_HUGE,   // allocated as huge os pages
+  MI_MEM_OS_EXPAND, // allocated in an expandable area
   MI_MEM_OS_REMAP,  // allocated in a remapable area (i.e. using `mremap`)
   MI_MEM_ARENA      // allocated from an arena (the usual case)
 } mi_memkind_t;

src/os.c

@@ -202,6 +202,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
       mi_os_prim_free_remappable(base, csize, still_committed, memid.mem.os.prim_info, tld_stats);
     }
     else {
+      mi_assert_internal(memid.memkind == MI_MEM_OS || memid.memkind == MI_MEM_OS_EXPAND);
       mi_os_prim_free(base, csize, still_committed, tld_stats);
     }
   }
@@ -301,7 +302,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
 
   // aligned already?
   if (((uintptr_t)p % alignment) == 0) {
-    *memid = _mi_memid_create_os(p, size, alignment, commit, os_is_zero, os_is_large);
+    *memid = _mi_memid_create_os(p, size, alignment, commit, os_is_large, os_is_zero);
   }
   else {
     // if not aligned, free the original allocation, overallocate, and unmap around it
@@ -313,7 +314,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
     p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, &os_is_large, &os_is_zero, stats);
     if (p == NULL) return NULL;
 
-    *memid = _mi_memid_create_os(p, oversize, 1, commit, os_is_zero, os_is_large);
+    *memid = _mi_memid_create_os(p, oversize, 1, commit, os_is_large, os_is_zero);
     p = mi_os_align_within(memid, alignment, size, stats);
   }
 
@@ -336,7 +337,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* tld_stats) {
   bool os_is_zero  = false;
   void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats);
   if (p != NULL) {
-    *memid = _mi_memid_create_os(p, size, 0, true, os_is_zero, os_is_large);
+    *memid = _mi_memid_create_os(p, size, 0, true, os_is_large, os_is_zero);
   }  
   return p;
 }
@@ -388,6 +389,40 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse
 }
 
 
+/* -----------------------------------------------------------
+  Expandable memory
+----------------------------------------------------------- */
+
+void* _mi_os_alloc_expandable(size_t size, size_t alignment, size_t future_reserve, mi_memid_t* memid, mi_stats_t* stats) {
+  size = mi_os_get_alloc_size(size);
+  if (future_reserve < 2*size) { future_reserve = 2*size; }
+  void* p = _mi_os_alloc_aligned(future_reserve, alignment, false, false, memid, stats);
+  if (p == NULL) return NULL;
+  memid->memkind = MI_MEM_OS_EXPAND;
+  if (!_mi_os_expand(p, 0, size, memid, stats)) {
+    _mi_os_free(p, future_reserve, *memid, stats);
+    return NULL;
+  }
+  return p;
+}
+
+bool  _mi_os_expand(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) {
+  if (p == NULL) return false;
+  if (memid->memkind != MI_MEM_OS_EXPAND) return false;
+  if (newsize > size) {
+    mi_assert(memid->mem.os.size <= newsize);
+    return _mi_os_commit((uint8_t*)p + size, newsize - size, NULL, stats);
+  }
+  else if (newsize < size) {
+    mi_assert(memid->mem.os.size <= size);    
+    return _mi_os_decommit((uint8_t*)p + newsize, size - newsize, stats);
+  }
+  else {
+    return true;
+  }
+}
+
+
 /* -----------------------------------------------------------
   Remappable memory
 ----------------------------------------------------------- */
@@ -446,7 +481,7 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s
   }
 
   // create an aligned pointer within
-  mi_memid_t newmemid = _mi_memid_create_os(base, oversize, 1, false /* commit */, false /* iszero */, os_is_pinned);
+  mi_memid_t newmemid = _mi_memid_create_os(base, oversize, 1, false /* commit */, os_is_pinned, false /* iszero */);
   newmemid.memkind = MI_MEM_OS_REMAP;
   newmemid.mem.os.prim_info = remap_info;
   void* newp = mi_os_align_within(&newmemid, alignment, newsize, stats);
@@ -745,7 +780,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
   if (psize != NULL) { *psize = alloc_size; }
   if (page != 0) {
     mi_assert(start != NULL);
-    *memid = _mi_memid_create_os(start, alloc_size, _mi_os_page_size(), true /* is committed */, all_zero, true /* is_large */);
+    *memid = _mi_memid_create_os(start, alloc_size, _mi_os_page_size(), true /* is committed */, true /* is_large */, all_zero);
     memid->memkind = MI_MEM_OS_HUGE;
     mi_assert(memid->is_pinned);
     #ifdef MI_TRACK_ASAN

src/prim/unix/prim.c

@@ -912,7 +912,8 @@ int _mi_prim_remap_free(void* base, size_t size, void* remap_info) {
   return _mi_prim_free(base,size);
 }
 
-#else // no mremap
+#else // no mremap (e.g. macOS)
+
 int _mi_prim_remap_reserve(size_t size, bool* is_pinned, void** base, void** remap_info) {
   MI_UNUSED(size); MI_UNUSED(is_pinned); MI_UNUSED(base); MI_UNUSED(remap_info);
   return EINVAL;
@@ -927,5 +928,6 @@ int _mi_prim_remap_free(void* base, size_t size, void* remap_info) {
   MI_UNUSED(base); MI_UNUSED(size); MI_UNUSED(remap_info);
   return EINVAL;
 }
+
 #endif