diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index db1c7ea6..71187294 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -74,16 +74,16 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo // Return `EINVAL` if this is not supported. // The returned memory is always committed. // If `is_pinned` is `true` the memory cannot be decommitted or reset. -// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_realloc_remappable` and `_mi_prim_free_remappable`. +// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_remap` and `_mi_prim_free_remappable`. int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ); // Remap remappable memory. Return `EINVAL` if this is not supported. -// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`. +// pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`. // `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`. -int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ); +int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ); // Free remappable memory. Return `EINVAL` if this is not supported. -// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`. +// pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`. int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ); diff --git a/src/alloc.c b/src/alloc.c index efb03d7b..75889aea 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -541,10 +541,9 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #if (MI_DEBUG>0) if mi_unlikely(!mi_is_in_heap_region(p)) { - _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" - "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); + _mi_trace_message("%s: pointer might not point to a valid heap region: %p\n" "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); if mi_likely(_mi_ptr_cookie(segment) == segment->cookie) { - _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); + _mi_trace_message("(yes, the previous pointer %p was valid after all)\n", p); } } #endif @@ -827,7 +826,7 @@ mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept { mi_block_t* block = _mi_page_ptr_unalign(segment, page, p); const size_t bsize = mi_page_usable_block_size(page); if (bsize >= padsize && 9*(bsize/10) <= padsize) { // if smaller and not more than 10% waste, keep it - _mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize); + //_mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize); mi_padding_init(page, block, newsize); return p; } diff --git a/src/os.c b/src/os.c index 8e42675d..3addcd50 100644 --- a/src/os.c +++ b/src/os.c @@ -253,37 +253,33 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo return p; } -static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats) +// aligns within an already allocated area; may modify `memid` with a new base and size. +static void* mi_os_align_within(mi_memid_t* memid, size_t alignment, size_t size, mi_stats_t* stats) { mi_assert_internal(alignment >= _mi_os_page_size()); - mi_assert_internal((size + alignment - 1) <= over_size); - void* p = mi_align_up_ptr(base, alignment); - mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size); - if (!is_pinned) { - size_t pre_size = (uint8_t*)p - (uint8_t*)base; - size_t mid_size = _mi_align_up(size, _mi_os_page_size()); - size_t post_size = over_size - pre_size - mid_size; - mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size); - // decommit the pre- and post part (if needed) - if (committed) { - if (pre_size > 0) { _mi_os_decommit(base, pre_size, stats); } - if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); } - } - /* - if (mi_os_mem_config.must_free_whole) { + mi_assert_internal((size + alignment - 1) <= memid->mem.os.size); + void* p = mi_align_up_ptr(memid->mem.os.base, alignment); + mi_assert_internal((uintptr_t)p + size <= (uintptr_t)memid->mem.os.base + memid->mem.os.size); + if (!memid->is_pinned) { + size_t pre_size = (uint8_t*)p - (uint8_t*)memid->mem.os.base; + size_t mid_size = mi_os_get_alloc_size(size); + size_t post_size = memid->mem.os.size - pre_size - mid_size; + mi_assert_internal(pre_size < memid->mem.os.size && post_size < memid->mem.os.size && mid_size >= size); + if (mi_os_mem_config.must_free_whole) { // decommit the pre- and post part (if needed) - if (committed) { - if (pre_size > 0) { _mi_os_decommit(base, pre_size, stats); } + if (memid->initially_committed) { + if (pre_size > 0) { _mi_os_decommit(memid->mem.os.base, pre_size, stats); } if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); } } } else { - // TODO: we can only free parts if we also return an adjusted base and fullsize. - // free the pre- and post part - if (pre_size > 0) { mi_os_prim_free(base, pre_size, committed, stats); } - if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); } + // free the pre- and post part and adjust the base and size + if (pre_size > 0) { mi_os_prim_free(memid->mem.os.base, pre_size, memid->initially_committed, stats); } + if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, memid->initially_committed, stats); } + memid->mem.os.base = p; + memid->mem.os.size = mid_size; + memid->mem.os.alignment = alignment; } - */ } mi_assert_internal(_mi_is_aligned(p, alignment)); return p; @@ -291,41 +287,40 @@ static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. -static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) { +static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats) { mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); mi_assert_internal(size > 0 && size == mi_os_get_alloc_size(size)); - mi_assert_internal(is_large != NULL); - mi_assert_internal(is_zero != NULL); - mi_assert_internal(base != NULL); + mi_assert_internal(memid != NULL); + *memid = _mi_memid_none(); if (!commit) allow_large = false; if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = mi_os_get_alloc_size(size); // try first with a hint (this will be aligned directly on Win 10+ or BSD) - void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats); + bool os_is_zero = false; + bool os_is_large = false; + void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, stats); if (p == NULL) return NULL; // aligned already? if (((uintptr_t)p % alignment) == 0) { - *base = p; - *fullsize = size; - return p; + *memid = _mi_memid_create_os(p, size, alignment, commit, os_is_zero, os_is_large); } else { // if not aligned, free the original allocation, overallocate, and unmap around it _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); mi_os_prim_free(p, size, commit, stats); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow - const size_t oversize = size + alignment - 1; + const size_t oversize = mi_os_get_alloc_size(size + alignment - 1); - p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, is_large, is_zero, stats); + p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, &os_is_large, &os_is_zero, stats); if (p == NULL) return NULL; - *base = p; - *fullsize = oversize; - return mi_os_align_within(p, oversize, alignment, size, commit, *is_large, stats); + + *memid = _mi_memid_create_os(p, oversize, 1, commit, os_is_zero, os_is_large); + p = mi_os_align_within(memid, alignment, size, stats); } - mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0)); + mi_assert_internal(p != NULL && memid->mem.os.base != NULL && _mi_is_aligned(p, alignment));; return p; } @@ -357,16 +352,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); - - bool os_is_large = false; - bool os_is_zero = false; - void* os_base = NULL; - size_t os_size = 0; - void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &os_size, &_mi_stats_main /*tld->stats*/ ); - if (p != NULL) { - *memid = _mi_memid_create_os(os_base, os_size, alignment, commit, os_is_zero, os_is_large); - } - return p; + return mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, memid, &_mi_stats_main /*tld->stats*/ ); } /* ----------------------------------------------------------- @@ -428,7 +414,7 @@ void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignme *memid = _mi_memid_create_os(base, oversize, alignment, true, os_is_zero, os_is_pinned); memid->memkind = MI_MEM_OS_REMAP; memid->mem.os.prim_info = remap_info; - return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats); + return mi_os_align_within(memid,alignment,size,stats); } void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) { @@ -449,12 +435,12 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s // if parts may have been decommitted, ensure it is committed now (or we get EFAULT from mremap) _mi_os_commit(memid->mem.os.base, memid->mem.os.size, NULL, stats); } - int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info); + int err = _mi_prim_remap(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info); if (err == 0 && newp != NULL) { - memid->initially_committed = true; - memid->mem.os.base = newp; - memid->mem.os.size = oversize; - return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats); + const size_t alignment = memid->mem.os.alignment; + *memid = _mi_memid_create_os(newp, oversize, 1, true /* committed */, false /* iszero */, false /* islarge */); + memid->memkind = MI_MEM_OS_REMAP; + return mi_os_align_within(memid, alignment, newsize, stats); } else { _mi_warning_message("failed to remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, size, newsize); @@ -464,7 +450,10 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s // fall back to copy (but in remappable memory if possible) mi_memid_t newmemid = _mi_memid_none(); void* newp = _mi_os_alloc_remappable(newsize, 0, memid->mem.os.alignment, &newmemid, stats); - if (newp == NULL) return NULL; + if (newp == NULL) { + newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, true /* commit */, false /* allow_large */, &newmemid, stats); + if (newp == NULL) return NULL; + } size_t csize = (size > newsize ? newsize : size); _mi_memcpy_aligned(newp, p, csize); diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 76e006ed..8660a54b 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -884,7 +884,7 @@ int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinne #endif } -int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { +int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { #if !defined(MREMAP_MAYMOVE) MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); return EINVAL; diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 0960f683..208203ff 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -284,7 +284,7 @@ int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinne return EINVAL; } -int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { +int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); return EINVAL; } diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index f4d9c39b..57afaef7 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -633,7 +633,7 @@ int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinne return _mi_prim_alloc(size, 1, true, true, is_pinned, is_zero, addr); } -int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { +int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); return EINVAL; } diff --git a/test/main-override-static.c b/test/main-override-static.c index eba9132d..3e3257a2 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -237,6 +237,7 @@ static void test_remap(void) { uint8_t v = p[idx]; if (v != i) { printf("error: corrupted memory in remap: i=%d, index=0x%zx, value=%u \n", i, idx,v); + abort(); }; } mi_free(p);