wip: remappable memory

include/mimalloc.h

@@ -242,6 +242,17 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap,
 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4);
 
 
+// ------------------------------------------------------
+// Remappable memory (uses `mremap` if possible)
+// ------------------------------------------------------
+
+mi_decl_nodiscard mi_decl_export void* mi_malloc_remappable(size_t size) mi_attr_noexcept mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export void* mi_zalloc_remappable(size_t size) mi_attr_noexcept mi_attr_alloc_size(1);
+mi_decl_nodiscard mi_decl_export void* mi_remap(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export void* mi_heap_malloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_alloc_size(2);
+mi_decl_nodiscard mi_decl_export void* mi_heap_zalloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_alloc_size(2);
+
+
 // ------------------------------------------------------
 // Analysis
 // ------------------------------------------------------

include/mimalloc/internal.h

@@ -56,6 +56,8 @@ terms of the MIT license. A copy of the license can be found in the file
 #include <pthread.h>
 #endif
 
+#define MI_PAGE_ALIGN_REMAPPABLE (1)
+
 // "options.c"
 void       _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
 void       _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
@@ -114,6 +116,10 @@ size_t     _mi_os_large_page_size(void);
 
 void*      _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
 
+void*      _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats);
+void*      _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats);
+
+
 // arena.c
 mi_arena_id_t _mi_arena_id_none(void);
 void       _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats);
@@ -144,6 +150,8 @@ void       _mi_segment_thread_collect(mi_segments_tld_t* tld);
 void       _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
 void       _mi_abandoned_await_readers(void);
 
+mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld);
+
 // "page.c"
 void*      _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment)  mi_attr_noexcept mi_attr_malloc;
 
@@ -671,8 +679,11 @@ static inline mi_memid_t _mi_memid_none(void) {
   return _mi_memid_create(MI_MEM_NONE);
 }
 
-static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool is_large) {
+static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, size_t alignment, bool committed, bool is_zero, bool is_large) {
   mi_memid_t memid = _mi_memid_create(MI_MEM_OS);
+  memid.mem.os.base = base;
+  memid.mem.os.size = size;
+  memid.mem.os.alignment = alignment;
   memid.initially_committed = committed;
   memid.initially_zero = is_zero;
   memid.is_pinned = is_large;

include/mimalloc/prim.h

@@ -69,6 +69,24 @@ int _mi_prim_protect(void* addr, size_t size, bool protect);
 //      numa_node is either negative (don't care), or a numa node number.
 int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
 
+
+// Allocate remappable memory that can be used with `_mi_prim_remap`.
+// Return `EINVAL` if this is not supported.
+// The returned memory is always committed.
+// If `is_pinned` is `true` the memory cannot be decommitted or reset.
+// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_realloc_remappable` and `_mi_prim_free_remappable`.
+int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info );
+
+// Remap remappable memory. Return `EINVAL` if this is not supported.
+// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`.
+//      `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`.
+int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info );
+
+// Free remappable memory. Return `EINVAL` if this is not supported.
+// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`.
+int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info );
+
+
 // Return the current NUMA node
 size_t _mi_prim_numa_node(void);
 

include/mimalloc/types.h

@@ -341,7 +341,9 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
 
 typedef struct mi_memid_os_info {
   void*         base;               // actual base address of the block (used for offset aligned allocations)
-  size_t        alignment;          // alignment at allocation
+  size_t        size;               // allocated size (the full extent from base)
+  size_t        alignment;          // requested alignment at allocation (may be offset aligned, so base may not be aligned at this value)
+  void*         prim_info;          // potential OS dependent information (used for remappable memory on Windows)
 } mi_memid_os_info_t;
 
 typedef struct mi_memid_arena_info {

src/alloc.c

@@ -780,7 +780,56 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_
   return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size);
 }
 
+// ------------------------------------------------------
+// remap
+// ------------------------------------------------------
 
+static void* mi_heap_malloc_zero_remappable(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
+  return _mi_heap_malloc_zero_ex(heap, size, zero, MI_PAGE_ALIGN_REMAPPABLE);
+}
+
+mi_decl_nodiscard void* mi_heap_malloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+  return mi_heap_malloc_zero_remappable(heap, size, false);
+}
+
+mi_decl_nodiscard void* mi_heap_zalloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept {
+  return mi_heap_malloc_zero_remappable(heap, size, true);
+}
+
+mi_decl_nodiscard void* mi_malloc_remappable(size_t size) mi_attr_noexcept {
+  return mi_heap_malloc_remappable(mi_prim_get_default_heap(), size);
+}
+
+mi_decl_nodiscard void* mi_zalloc_remappable(size_t size) mi_attr_noexcept {
+  return mi_heap_zalloc_remappable(mi_prim_get_default_heap(), size);
+}
+
+mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept {
+  if (p == NULL) return mi_malloc(newsize);
+
+  mi_segment_t* segment = mi_checked_ptr_segment(p, "mi_remap");
+  mi_assert_internal(segment != NULL);
+  mi_page_t* const page = _mi_segment_page_of(segment, p);
+  const size_t bsize = mi_page_usable_block_size(page);
+  if (bsize >= newsize) {
+    // TODO: adjust padding
+    return p;
+  }
+
+  mi_heap_t* heap = mi_prim_get_default_heap();
+  if (segment->thread_id == heap->thread_id &&
+      segment->memid.memkind == MI_MEM_OS_REMAP) 
+  {
+    mi_heap_t* heap = mi_prim_get_default_heap();
+    mi_block_t* block = _mi_segment_huge_page_remap(segment, page, (mi_block_t*)p, newsize, &heap->tld->segments);
+    if (block != NULL) {
+      // TODO: adjust padding?
+      return block;
+    }
+  }
+  _mi_warning_message("unable to remap block, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, 0, newsize);
+  return mi_realloc(p, newsize);
+}
 
 // ------------------------------------------------------
 // strdup, strndup, and realpath

src/os.c

@@ -50,6 +50,10 @@ bool _mi_os_use_large_page(size_t size, size_t alignment) {
   return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);
 }
 
+static size_t mi_os_alloc_size(size_t size) {
+  return _mi_align_up(size, mi_os_mem_config.alloc_granularity);
+}
+
 // round to a good OS allocation size (bounded by max 12.5% waste)
 size_t _mi_os_good_alloc_size(size_t size) {
   size_t align_size;
@@ -58,12 +62,16 @@ size_t _mi_os_good_alloc_size(size_t size) {
   else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
   else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
   else align_size = 4*MI_MiB;
+  if (align_size < mi_os_mem_config.alloc_granularity) align_size = mi_os_mem_config.alloc_granularity;
   if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?
   return _mi_align_up(size, align_size);
 }
 
 void _mi_os_init(void) {
   _mi_prim_mem_init(&mi_os_mem_config);
+  if (mi_os_mem_config.alloc_granularity < mi_os_mem_config.page_size) {
+    mi_os_mem_config.alloc_granularity = mi_os_mem_config.page_size;
+  }
 }
 
 
@@ -117,23 +125,23 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
 {
   if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;
   size = _mi_align_up(size, MI_SEGMENT_SIZE);
-  if (size > 1*MI_GiB) return NULL;  // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
+  if (size > 1 * MI_GiB) return NULL;  // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
-  #if (MI_SECURE>0)
+#if (MI_SECURE>0)
   size += MI_SEGMENT_SIZE;        // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
-  #endif
+#endif
 
   uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
   if (hint == 0 || hint > MI_HINT_MAX) {   // wrap or initialize
     uintptr_t init = MI_HINT_BASE;
-    #if (MI_SECURE>0 || MI_DEBUG==0)       // security: randomize start of aligned allocations unless in debug mode
+#if (MI_SECURE>0 || MI_DEBUG==0)       // security: randomize start of aligned allocations unless in debug mode
     uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
-    init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA);  // (randomly 20 bits)*4MiB == 0 to 4TiB
+    init = init + ((MI_SEGMENT_SIZE * ((r >> 17) & 0xFFFFF)) % MI_HINT_AREA);  // (randomly 20 bits)*4MiB == 0 to 4TiB
-    #endif
+#endif
     uintptr_t expected = hint + size;
     mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
     hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
   }
-  if (hint%try_alignment != 0) return NULL;
+  if (hint % try_alignment != 0) return NULL;
   return (void*)hint;
 }
 #else
@@ -163,22 +171,39 @@ static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_st
   _mi_stat_decrease(&stats->reserved, size);
 }
 
+static void mi_os_prim_free_remappable(void* addr, size_t size, bool still_committed, void* remap_info, mi_stats_t* tld_stats) {
+  MI_UNUSED(tld_stats);
+  mi_assert_internal((size % _mi_os_page_size()) == 0);
+  if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
+  int err = _mi_prim_free_remappable(addr, size, remap_info);
+  if (err != 0) {
+    _mi_warning_message("unable to free remappable OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
+  }
+  mi_stats_t* stats = &_mi_stats_main;
+  if (still_committed) { _mi_stat_decrease(&stats->committed, size); }
+  _mi_stat_decrease(&stats->reserved, size);
+}
+
 void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) {
   if (mi_memkind_is_os(memid.memkind)) {
-    size_t csize = _mi_os_good_alloc_size(size);
+    size_t csize = mi_os_alloc_size(size);
     void* base = addr;
     // different base? (due to alignment)
     if (memid.mem.os.base != NULL) {
       mi_assert(memid.mem.os.base <= addr);
       mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr);
+      mi_assert(memid.mem.os.size >= csize);
       base = memid.mem.os.base;
-      csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base);
+      csize = memid.mem.os.size;
     }
     // free it
     if (memid.memkind == MI_MEM_OS_HUGE) {
       mi_assert(memid.is_pinned);
       mi_os_free_huge_os_pages(base, csize, tld_stats);
     }
+    else if (memid.memkind == MI_MEM_OS_REMAP) {
+      mi_os_prim_free_remappable(base, csize, still_committed, memid.mem.os.prim_info, tld_stats);
+    }
     else {
       mi_os_prim_free(base, csize, still_committed, tld_stats);
     }
@@ -200,7 +225,7 @@ void  _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats)
 
 // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
 static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) {
-  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  mi_assert_internal(size > 0 && size == mi_os_alloc_size(size));
   mi_assert_internal(is_zero != NULL);
   mi_assert_internal(is_large != NULL);
   if (size == 0) return NULL;
@@ -228,18 +253,44 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
   return p;
 }
 
+static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats)
+{
+  //mi_assert_internal((size + alignment - 1) <= over_size);
+  void* p = mi_align_up_ptr(base, alignment);
+  mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size);
+  if (!is_pinned) {
+    size_t pre_size = (uint8_t*)p - (uint8_t*)base;
+    size_t mid_size = _mi_align_up(size, _mi_os_page_size());
+    size_t post_size = over_size - pre_size - mid_size;
+    mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
+    if (mi_os_mem_config.must_free_whole) { 
+      // decommit the pre- and post part (if needed)
+      if (committed) {
+        if (pre_size > 0)  { _mi_os_decommit(base, pre_size, stats); }
+        if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); }
+      }
+    }
+    else {
+      // free the pre- and post part
+      if (pre_size > 0)  { mi_os_prim_free(base, pre_size, committed, stats); }
+      if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); }
+    }
+  }
+  mi_assert_internal(_mi_is_aligned(p, alignment));
+  return p;
+}
 
 // Primitive aligned allocation from the OS.
 // This function guarantees the allocated memory is aligned.
-static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) {
+static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) {
   mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
-  mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
+  mi_assert_internal(size > 0 && size == mi_os_alloc_size(size));
   mi_assert_internal(is_large != NULL);
   mi_assert_internal(is_zero != NULL);
   mi_assert_internal(base != NULL);
   if (!commit) allow_large = false;
   if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
-  size = _mi_align_up(size, _mi_os_page_size());
+  size = mi_os_alloc_size(size);
 
   // try first with a hint (this will be aligned directly on Win 10+ or BSD)
   void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats);
@@ -248,47 +299,21 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
   // aligned already?
   if (((uintptr_t)p % alignment) == 0) {
     *base = p;
+    *fullsize = size;
+    return p;
   }
   else {
-    // if not aligned, free it, overallocate, and unmap around it
+    // if not aligned, free the original allocation, overallocate, and unmap around it
     _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
     mi_os_prim_free(p, size, commit, stats);
     if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
-    const size_t over_size = size + alignment;
+    const size_t oversize = size + alignment - 1;
     
-    if (mi_os_mem_config.must_free_whole) {  // win32 virtualAlloc cannot free parts of an allocate block
+    p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, is_large, is_zero, stats);
-      // over-allocate uncommitted (virtual) memory
+    if (p == NULL) return NULL;
-      p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats);
+    *base = p;
-      if (p == NULL) return NULL;
+    *fullsize = oversize;
-      
+    return mi_os_align_within(base, oversize, alignment, size, commit, *is_large, stats);
-      // set p to the aligned part in the full region
-      // note: this is dangerous on Windows as VirtualFree needs the actual base pointer
-      // this is handled though by having the `base` field in the memid's
-      *base = p; // remember the base
-      p = mi_align_up_ptr(p, alignment);
-
-      // explicitly commit only the aligned part
-      if (commit) {
-        _mi_os_commit(p, size, NULL, stats);
-      }
-    }
-    else  { // mmap can free inside an allocation
-      // overallocate...
-      p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats);
-      if (p == NULL) return NULL;
-      
-      // and selectively unmap parts around the over-allocated area. (noop on sbrk)
-      void* aligned_p = mi_align_up_ptr(p, alignment);
-      size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
-      size_t mid_size = _mi_align_up(size, _mi_os_page_size());
-      size_t post_size = over_size - pre_size - mid_size;
-      mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
-      if (pre_size > 0)  { mi_os_prim_free(p, pre_size, commit, stats); }
-      if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); }
-      // we can return the aligned pointer on `mmap` (and sbrk) systems
-      p = aligned_p;
-      *base = aligned_p; // since we freed the pre part, `*base == p`.      
-    }
   }
 
   mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0));
@@ -310,7 +335,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* tld_stats) {
   bool os_is_zero  = false;
   void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats);
   if (p != NULL) {
-    *memid = _mi_memid_create_os(true, os_is_zero, os_is_large);
+    *memid = _mi_memid_create_os(p, size, 0, true, os_is_zero, os_is_large);
   }  
   return p;
 }
@@ -327,11 +352,10 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
   bool os_is_large = false;
   bool os_is_zero  = false;
   void* os_base = NULL;
-  void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &_mi_stats_main /*tld->stats*/ );
+  size_t os_size = 0;
+  void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &os_size, &_mi_stats_main /*tld->stats*/ );
   if (p != NULL) {
-    *memid = _mi_memid_create_os(commit, os_is_zero, os_is_large);
+    *memid = _mi_memid_create_os(os_base, os_size, alignment, commit, os_is_zero, os_is_large);
-    memid->mem.os.base = os_base;
-    memid->mem.os.alignment = alignment;
   }
   return p;
 }
@@ -357,20 +381,86 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse
   else {
     // overallocate to align at an offset
     const size_t extra = _mi_align_up(offset, alignment) - offset;
-    const size_t oversize = size + extra;
+    const size_t oversize = mi_os_alloc_size(size + extra);
     void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, tld_stats);
     if (start == NULL) return NULL;
 
     void* const p = (uint8_t*)start + extra;
     mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
     // decommit the overallocation at the start
-    if (commit && extra > _mi_os_page_size()) {
+    if (memid->initially_committed && !memid->is_pinned && (extra > _mi_os_page_size())) {
       _mi_os_decommit(start, extra, tld_stats);
     }
     return p;
   }
 }
 
+
+/* -----------------------------------------------------------
+  Remappable memory
+----------------------------------------------------------- */
+
+void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) {
+  mi_assert_internal(size > 0);
+  mi_assert_internal(memid != NULL);
+  *memid = _mi_memid_none();
+  if (alignment == 0) { alignment = 1;  }
+  const size_t oversize = mi_os_alloc_size(size + alignment - 1);
+  if (future_reserve < oversize) { future_reserve = oversize; }
+  bool os_is_pinned = true;
+  bool os_is_zero = false;
+  void* base = NULL;
+  void* remap_info = NULL;
+  int err = _mi_prim_alloc_remappable(oversize, future_reserve, &os_is_pinned, &os_is_zero, &base, &remap_info);
+  if (err != 0 || base == NULL) {
+    // fall back to regular allocation
+    return _mi_os_alloc_aligned(size, alignment, true /* commit */, true /* allow_large */, memid, stats);
+  }
+  *memid = _mi_memid_create_os(base, oversize, alignment, true, os_is_zero, os_is_pinned);
+  memid->memkind = MI_MEM_OS_REMAP; 
+  memid->mem.os.prim_info = remap_info;
+  return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats);
+}
+
+void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) {
+  mi_assert_internal(size > 0);
+  mi_assert_internal(newsize > 0);
+  mi_assert_internal(p != NULL && memid != NULL);
+  mi_assert_internal(mi_memkind_is_os(memid->memkind));
+  if (p == NULL) return NULL;
+  if (!mi_memkind_is_os(memid->memkind)) return NULL;
+
+  newsize = mi_os_alloc_size(newsize);
+  const size_t oversize = newsize + memid->mem.os.alignment - 1;
+
+  if (memid->memkind == MI_MEM_OS_REMAP) {
+    bool extend_is_zero = false;
+    void* newp = NULL;
+    int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info);
+    if (err == 0 && newp != NULL) {
+      memid->initially_committed = true;      
+      memid->mem.os.base = newp;
+      memid->mem.os.size = oversize;
+      return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats);
+    }
+    else {
+      _mi_warning_message("failed to remap OS re-allocation (error %d (0x%x) at %p of size %zu to size %zu)\n", err, err, p, size, newsize);
+    }
+  }
+  
+  // fall back to regular realloc
+  mi_memid_t newmemid = _mi_memid_none();
+  void* newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, memid->initially_committed, true /* allow large? */, &newmemid, stats);
+  if (newp == NULL) return NULL;
+
+  size_t csize = (size > newsize ? newsize : size);
+  _mi_memcpy_aligned(newp, p, csize);
+  _mi_os_free(p, size, *memid, stats);
+  *memid = newmemid;
+  return newp;
+}
+
+
 /* -----------------------------------------------------------
   OS memory API: reset, commit, decommit, protect, unprotect.
 ----------------------------------------------------------- */
@@ -640,16 +730,17 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
       }
     }
   }
-  mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size);
+  const size_t alloc_size = page * MI_HUGE_OS_PAGE_SIZE;
+  mi_assert_internal(alloc_size <= size);
   if (pages_reserved != NULL) { *pages_reserved = page; }
-  if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; }
+  if (psize != NULL) { *psize = alloc_size; }
   if (page != 0) {
     mi_assert(start != NULL);
-    *memid = _mi_memid_create_os(true /* is committed */, all_zero, true /* is_large */);
+    *memid = _mi_memid_create_os(start, alloc_size, _mi_os_page_size(), true /* is committed */, all_zero, true /* is_large */);
     memid->memkind = MI_MEM_OS_HUGE;
     mi_assert(memid->is_pinned);
     #ifdef MI_TRACK_ASAN
-    if (all_zero) { mi_track_mem_defined(start,size); }
+    if (all_zero) { mi_track_mem_defined(start,alloc_size); }
     #endif
   }
   return (page == 0 ? NULL : start);

src/page.c

@@ -814,11 +814,10 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex
   General allocation
 ----------------------------------------------------------- */
 
-// A huge page is allocated directly without being in a queue.
+// A huge page always occupies a single segment.
-// Because huge pages contain just one block, and the segment contains
+// It is used for large allocations, (very) large alignments (> MI_ALIGNMENT_MAX), or remappable blocks.
-// just that page, we always treat them as abandoned and any thread
+// When a huge page is freed from another thread, it is immediately reset to reduce memory pressure.
-// that frees the block can free the whole page and segment directly.
+// We use a page_alignment of 1 for remappable memory.
-// Huge pages are also use if the requested alignment is very large (> MI_ALIGNMENT_MAX).
 static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) {
   size_t block_size = _mi_os_good_alloc_size(size);
   mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0);
@@ -918,7 +917,9 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al
     // note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case.
     void* p = _mi_page_malloc(heap, page, size, false);
     mi_assert_internal(p != NULL);
-    _mi_memzero_aligned(p, mi_page_usable_block_size(page));
+    if (!page->free_is_zero) {
+      _mi_memzero_aligned(p, mi_page_usable_block_size(page));
+    }
     return p;
   }
   else {

src/prim/unix/prim.c

@@ -11,6 +11,10 @@ terms of the MIT license. A copy of the license can be found in the file
 #define _DEFAULT_SOURCE   // ensure mmap flags and syscall are defined
 #endif
 
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE       // ensure mremap is defined
+#endif
+
 #if defined(__sun)
 // illumos provides new mman.h api when any of these are defined
 // otherwise the old api based on caddr_t which predates the void pointers one.
@@ -861,3 +865,41 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
 }
 
 #endif
+
+
+
+
+//----------------------------------------------------------------
+// Remappable memory
+//----------------------------------------------------------------
+
+int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
+  #if !defined(MREMAP_MAYMOVE)
+    MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
+    return EINVAL;
+  #else
+    MI_UNUSED(future_reserve);
+    *remap_info = NULL;
+    return _mi_prim_alloc(size,1,true /* commit? */, true /* allow_large */, is_pinned /* is_large? */, is_zero, addr);
+  #endif
+}
+
+int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
+  #if !defined(MREMAP_MAYMOVE)
+    MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
+    return EINVAL;
+  #else
+    mi_assert_internal(*remap_info == NULL); MI_UNUSED(remap_info);  
+    void* p = mremap(addr,size,newsize,MREMAP_MAYMOVE);
+    if (p == MAP_FAILED) { return errno; }
+    *extend_is_zero = true;
+    *newaddr = p;
+    return 0;
+  #endif
+}
+
+int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
+  MI_UNUSED(remap_info);
+  mi_assert_internal(remap_info == NULL);
+  return _mi_prim_free(addr,size);
+}

src/prim/wasi/prim.c

@@ -273,3 +273,23 @@ void _mi_prim_thread_done_auto_done(void) {
 void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
   MI_UNUSED(heap);
 }
+
+
+//----------------------------------------------------------------
+// Remappable memory
+//----------------------------------------------------------------
+
+int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
+  MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
+  return EINVAL;
+}
+
+int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
+  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
+  return EINVAL;
+}
+
+int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
+  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info);
+  return EINVAL;
+}

src/prim/windows/prim.c

@@ -620,3 +620,24 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
 }
 
 #endif
+
+
+
+//----------------------------------------------------------------
+// Remappable memory
+//----------------------------------------------------------------
+
+int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
+  MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
+  return EINVAL;
+}
+
+int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
+  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
+  return EINVAL;
+}
+
+int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
+  MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info);
+  return EINVAL;
+}

src/segment.c

@@ -517,13 +517,19 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme
   bool   allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy
   size_t align_offset = 0;
   size_t alignment = MI_SEGMENT_SIZE;
-  if (page_alignment > 0) {
+  if (page_alignment > MI_PAGE_ALIGN_REMAPPABLE) {
     alignment = page_alignment;
     align_offset = _mi_align_up(pre_size, MI_SEGMENT_SIZE);
     segment_size = segment_size + (align_offset - pre_size);  // adjust the segment size
   }
 
-  mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os);
+  mi_segment_t* segment = NULL;
+  if (page_alignment == MI_PAGE_ALIGN_REMAPPABLE) {
+    segment = (mi_segment_t*)_mi_os_alloc_remappable(segment_size, 0, alignment, &memid, tld_os->stats);
+  }
+  else {
+    segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os);
+  }
   if (segment == NULL) {
     return NULL;  // failed to allocate
   }
@@ -1224,7 +1230,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment,
   page->xblock_size = (psize > MI_HUGE_BLOCK_SIZE ? MI_HUGE_BLOCK_SIZE : (uint32_t)psize);
 
   // reset the part of the page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE)
-  if (page_alignment > 0 && segment->allow_decommit && page->is_committed) {
+  if (page_alignment > MI_PAGE_ALIGN_REMAPPABLE && segment->allow_decommit && page->is_committed) {
     uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment);
     mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment));
     mi_assert_internal(psize - (aligned_p - start) >= size);
@@ -1283,17 +1289,52 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc
 }
 #endif
 
+mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld) {
+  mi_assert_internal(segment == _mi_page_segment(page));
+  mi_assert_internal(page->used == 1);
+  mi_assert_internal(page->free == NULL);
+  mi_assert_internal(page->local_free == NULL);
+  mi_assert_internal(mi_page_thread_free(page) == NULL);
+  mi_assert_internal(segment->next == NULL && segment->prev == NULL);
+  mi_assert_internal(page->next == NULL && page->prev == NULL);
+  const size_t bsize = mi_page_block_size(page);
+  const size_t newssize = _mi_align_up(_mi_align_up(newsize, _mi_os_page_size()) + (mi_segment_size(segment) - bsize), MI_SEGMENT_SIZE);
+  mi_memid_t memid = segment->memid;
+  const ptrdiff_t block_ofs = (uint8_t*)block - (uint8_t*)segment;
+
+  mi_segment_protect(segment, false, tld->os);
+  if (segment->allow_decommit && !page->is_committed) {  // TODO: always commit fully regardless of the page?
+    _mi_os_commit(segment, mi_segment_size(segment), NULL, tld->stats);
+    page->is_committed = true;
+  }
+  mi_segment_t* newsegment = (mi_segment_t*)_mi_os_realloc(segment, mi_segment_size(segment), newssize, &memid, tld->stats);
+  if (newsegment == NULL) {
+    mi_segment_protect(segment, true, tld->os);
+    return NULL;
+  }
+  else {
+    newsegment->memid = memid;
+    newsegment->segment_size = newssize;
+    newsegment->cookie = _mi_ptr_cookie(newsegment);
+    mi_segment_protect(newsegment, true, tld->os);
+  }
+
+  mi_block_t* newblock = (mi_block_t*)((uint8_t*)newsegment + block_ofs);
+  mi_assert_internal(_mi_ptr_segment(newblock) == newsegment);
+  mi_page_t* newpage = _mi_ptr_page(newblock);
+  mi_assert_internal(mi_page_block_size(newpage) >= newsize); MI_UNUSED(newpage);
+  return newblock;
+}
+
 /* -----------------------------------------------------------
    Page allocation
 ----------------------------------------------------------- */
 
 mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
   mi_page_t* page;
-  if mi_unlikely(page_alignment > MI_ALIGNMENT_MAX) {
+  if mi_unlikely(page_alignment > 0) {
-    mi_assert_internal(_mi_is_power_of_two(page_alignment));
+    mi_assert_internal(page_alignment == MI_PAGE_ALIGN_REMAPPABLE || (_mi_is_power_of_two(page_alignment) && page_alignment >= MI_SEGMENT_SIZE));
-    mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE);
+    if (page_alignment != MI_PAGE_ALIGN_REMAPPABLE && page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; }
-    //mi_assert_internal((MI_SEGMENT_SIZE % page_alignment) == 0);
-    if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; }
     page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld);
   }
   else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) {

src/static.c

@@ -5,8 +5,12 @@ terms of the MIT license. A copy of the license can be found in the file
 "LICENSE" at the root of this distribution.
 -----------------------------------------------------------------------------*/
 #ifndef _DEFAULT_SOURCE
-#define _DEFAULT_SOURCE
+#define _DEFAULT_SOURCE   // ensure mmap is defined
 #endif
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE       // ensure mremap is defined
+#endif
+
 #if defined(__sun)
 // same remarks as os.c for the static's context.
 #undef _XOPEN_SOURCE

test/main-override-static.c

@@ -18,11 +18,14 @@ static void test_reserved(void);
 static void negative_stat(void);
 static void alloc_huge(void);
 static void test_heap_walk(void);
-
+static void test_remap(void);
 
 int main() {
   mi_version();
   mi_stats_reset();
+
+  test_remap();
+
   // detect double frees and heap corruption
   // double_free1();
   // double_free2();
@@ -216,6 +219,20 @@ static void test_heap_walk(void) {
   mi_heap_visit_blocks(heap, true, &test_visit, NULL);
 }
 
+
+static void test_remap(void) {
+  size_t size = 64 * 1024 * 1024;
+  size_t inc  = 1024 * 1024;
+  uint8_t* p = (uint8_t*)mi_malloc_remappable(size);
+  memset(p, 1, size);
+  for (int i = 2; i < 100; i++) {
+    p = mi_remap(p, size + inc);
+    memset(p + size, i, inc);
+    size += inc;
+  }
+  mi_free(p);
+}
+
 // ----------------------------
 // bin size experiments
 // ------------------------------