wip: further improvement for remap

include/mimalloc/prim.h

@@ -74,16 +74,16 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo
 // Return `EINVAL` if this is not supported.
 // The returned memory is always committed.
 // If `is_pinned` is `true` the memory cannot be decommitted or reset.
-// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_realloc_remappable` and `_mi_prim_free_remappable`.
+// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_remap` and `_mi_prim_free_remappable`.
 int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info );
 
 // Remap remappable memory. Return `EINVAL` if this is not supported.
-// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`.
+// pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`.
 //      `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`.
-int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info );
+int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info );
 
 // Free remappable memory. Return `EINVAL` if this is not supported.
-// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`.
+// pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`.
 int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info );
 
 

src/alloc.c

@@ -541,10 +541,9 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms
 
 #if (MI_DEBUG>0)
   if mi_unlikely(!mi_is_in_heap_region(p)) {
-    _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n"
+    _mi_trace_message("%s: pointer might not point to a valid heap region: %p\n" "(this may still be a valid very large allocation (over 64MiB))\n", msg, p);
-      "(this may still be a valid very large allocation (over 64MiB))\n", msg, p);
     if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) {
-      _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p);
+      _mi_trace_message("(yes, the previous pointer %p was valid after all)\n", p);
     }
   }
 #endif
@@ -827,7 +826,7 @@ mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept {
   mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
   const size_t bsize = mi_page_usable_block_size(page);
   if (bsize >= padsize && 9*(bsize/10) <= padsize) {  // if smaller and not more than 10% waste, keep it
-    _mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
+    //_mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
     mi_padding_init(page, block, newsize);
     return p;
   }

src/os.c

@@ -253,37 +253,33 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
   return p;
 }
 
-static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats)
+// aligns within an already allocated area; may modify `memid` with a new base and size.
+static void* mi_os_align_within(mi_memid_t* memid, size_t alignment, size_t size, mi_stats_t* stats)
 {
   mi_assert_internal(alignment >= _mi_os_page_size());
-  mi_assert_internal((size + alignment - 1) <= over_size);
+  mi_assert_internal((size + alignment - 1) <= memid->mem.os.size);
-  void* p = mi_align_up_ptr(base, alignment);
+  void* p = mi_align_up_ptr(memid->mem.os.base, alignment);
-  mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size);
+  mi_assert_internal((uintptr_t)p + size <= (uintptr_t)memid->mem.os.base + memid->mem.os.size);
-  if (!is_pinned) {
+  if (!memid->is_pinned) {
-    size_t pre_size = (uint8_t*)p - (uint8_t*)base;
+    size_t pre_size = (uint8_t*)p - (uint8_t*)memid->mem.os.base;
-    size_t mid_size = _mi_align_up(size, _mi_os_page_size());
+    size_t mid_size = mi_os_get_alloc_size(size);
-    size_t post_size = over_size - pre_size - mid_size;
+    size_t post_size = memid->mem.os.size - pre_size - mid_size;
-    mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
+    mi_assert_internal(pre_size < memid->mem.os.size && post_size < memid->mem.os.size && mid_size >= size);
-    // decommit the pre- and post part (if needed)
-    if (committed) {
-      if (pre_size > 0) { _mi_os_decommit(base, pre_size, stats); }
-      if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); }
-    }
-    /*
     if (mi_os_mem_config.must_free_whole) {
       // decommit the pre- and post part (if needed)
-      if (committed) {
+      if (memid->initially_committed) {
-        if (pre_size > 0)  { _mi_os_decommit(base, pre_size, stats); }
+        if (pre_size > 0)  { _mi_os_decommit(memid->mem.os.base, pre_size, stats); }
         if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); }
       }
     }
     else {
-      // TODO: we can only free parts if we also return an adjusted base and fullsize.
+      // free the pre- and post part and adjust the base and size
-      // free the pre- and post part
+      if (pre_size > 0)  { mi_os_prim_free(memid->mem.os.base, pre_size, memid->initially_committed, stats); }
-      if (pre_size > 0)  { mi_os_prim_free(base, pre_size, committed, stats); }
+      if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, memid->initially_committed, stats); }
-      if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); }
+      memid->mem.os.base = p;
+      memid->mem.os.size = mid_size;
+      memid->mem.os.alignment = alignment;
     }
-    */
   }
   mi_assert_internal(_mi_is_aligned(p, alignment));
   return p;
@@ -291,41 +287,40 @@ static void* mi_os_align_within(void* base, size_t over_size, size_t alignment,
 
 // Primitive aligned allocation from the OS.
 // This function guarantees the allocated memory is aligned.
-static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) {
+static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats) {
   mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
   mi_assert_internal(size > 0 && size == mi_os_get_alloc_size(size));
-  mi_assert_internal(is_large != NULL);
+  mi_assert_internal(memid != NULL);
-  mi_assert_internal(is_zero != NULL);
+  *memid = _mi_memid_none();
-  mi_assert_internal(base != NULL);
   if (!commit) allow_large = false;
   if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
   size = mi_os_get_alloc_size(size);
 
   // try first with a hint (this will be aligned directly on Win 10+ or BSD)
-  void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats);
+  bool os_is_zero = false;
+  bool os_is_large = false;
+  void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, stats);
   if (p == NULL) return NULL;
 
   // aligned already?
   if (((uintptr_t)p % alignment) == 0) {
-    *base = p;
+    *memid = _mi_memid_create_os(p, size, alignment, commit, os_is_zero, os_is_large);
-    *fullsize = size;
-    return p;
   }
   else {
     // if not aligned, free the original allocation, overallocate, and unmap around it
     _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
     mi_os_prim_free(p, size, commit, stats);
     if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
-    const size_t oversize = size + alignment - 1;
+    const size_t oversize = mi_os_get_alloc_size(size + alignment - 1);
     
-    p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, is_large, is_zero, stats);
+    p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, &os_is_large, &os_is_zero, stats);
     if (p == NULL) return NULL;
-    *base = p;
+
-    *fullsize = oversize;
+    *memid = _mi_memid_create_os(p, oversize, 1, commit, os_is_zero, os_is_large);
-    return mi_os_align_within(p, oversize, alignment, size, commit, *is_large, stats);
+    p = mi_os_align_within(memid, alignment, size, stats);
   }
 
-  mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0));
+  mi_assert_internal(p != NULL && memid->mem.os.base != NULL && _mi_is_aligned(p, alignment));;
   return p;
 }
 
@@ -357,16 +352,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
   if (size == 0) return NULL;
   size = _mi_os_good_alloc_size(size);
   alignment = _mi_align_up(alignment, _mi_os_page_size());
-  
+  return mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid, &_mi_stats_main /*tld->stats*/ );
-  bool os_is_large = false;
-  bool os_is_zero  = false;
-  void* os_base = NULL;
-  size_t os_size = 0;
-  void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &os_size, &_mi_stats_main /*tld->stats*/ );
-  if (p != NULL) {
-    *memid = _mi_memid_create_os(os_base, os_size, alignment, commit, os_is_zero, os_is_large);
-  }
-  return p;
 }
 
 /* -----------------------------------------------------------
@@ -428,7 +414,7 @@ void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignme
   *memid = _mi_memid_create_os(base, oversize, alignment, true, os_is_zero, os_is_pinned);
   memid->memkind = MI_MEM_OS_REMAP; 
   memid->mem.os.prim_info = remap_info;
-  return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats);
+  return mi_os_align_within(memid,alignment,size,stats);
 }
 
 void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) {
@@ -449,12 +435,12 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s
       // if parts may have been decommitted, ensure it is committed now (or we get EFAULT from mremap)
       _mi_os_commit(memid->mem.os.base, memid->mem.os.size, NULL, stats);
     }
-    int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info);
+    int err = _mi_prim_remap(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info);
     if (err == 0 && newp != NULL) {
-      memid->initially_committed = true;      
+      const size_t alignment = memid->mem.os.alignment;
-      memid->mem.os.base = newp;
+      *memid = _mi_memid_create_os(newp, oversize, 1, true /* committed */, false /* iszero */, false /* islarge */);
-      memid->mem.os.size = oversize;
+      memid->memkind = MI_MEM_OS_REMAP;
-      return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats);
+      return mi_os_align_within(memid, alignment, newsize, stats);
     }
     else {
       _mi_warning_message("failed to remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, size, newsize);
@@ -464,7 +450,10 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s
   // fall back to copy (but in remappable memory if possible)
   mi_memid_t newmemid = _mi_memid_none();
   void* newp = _mi_os_alloc_remappable(newsize, 0, memid->mem.os.alignment, &newmemid, stats);
+  if (newp == NULL) {
+    newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, true /* commit */, false /* allow_large */, &newmemid, stats);
     if (newp == NULL) return NULL;
+  }
 
   size_t csize = (size > newsize ? newsize : size);
   _mi_memcpy_aligned(newp, p, csize);

src/prim/unix/prim.c

@@ -884,7 +884,7 @@ int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinne
   #endif
 }
 
-int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
+int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
   #if !defined(MREMAP_MAYMOVE)
     MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
     return EINVAL;

src/prim/wasi/prim.c

@@ -284,7 +284,7 @@ int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinne
   return EINVAL;
 }
 
-int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
+int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
   MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
   return EINVAL;
 }

src/prim/windows/prim.c

@@ -633,7 +633,7 @@ int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinne
   return _mi_prim_alloc(size, 1, true, true, is_pinned, is_zero, addr);
 }
 
-int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
+int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
   MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
   return EINVAL;
 }

test/main-override-static.c

@@ -237,6 +237,7 @@ static void test_remap(void) {
     uint8_t v = p[idx];
     if (v != i) {
       printf("error: corrupted memory in remap: i=%d, index=0x%zx, value=%u \n", i, idx,v);
+      abort();
     };
   }
   mi_free(p);