From be2f35641af4e4760833d552dbe2154bae96c811 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 26 Apr 2023 12:23:38 -0700 Subject: [PATCH] wip: remappable memory --- include/mimalloc.h | 11 ++ include/mimalloc/internal.h | 13 ++- include/mimalloc/prim.h | 18 +++ include/mimalloc/types.h | 4 +- src/alloc.c | 49 +++++++++ src/os.c | 211 ++++++++++++++++++++++++++---------- src/page.c | 13 ++- src/prim/unix/prim.c | 42 +++++++ src/prim/wasi/prim.c | 20 ++++ src/prim/windows/prim.c | 21 ++++ src/segment.c | 57 ++++++++-- src/static.c | 6 +- test/main-override-static.c | 19 +++- 13 files changed, 406 insertions(+), 78 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 368c22cc..243919f0 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -242,6 +242,17 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4); +// ------------------------------------------------------ +// Remappable memory (uses `mremap` if possible) +// ------------------------------------------------------ + +mi_decl_nodiscard mi_decl_export void* mi_malloc_remappable(size_t size) mi_attr_noexcept mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export void* mi_zalloc_remappable(size_t size) mi_attr_noexcept mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export void* mi_remap(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_heap_malloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_nodiscard mi_decl_export void* mi_heap_zalloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_alloc_size(2); + + // ------------------------------------------------------ // Analysis // ------------------------------------------------------ diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 4dabe8ba..14544b60 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -56,6 +56,8 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif +#define MI_PAGE_ALIGN_REMAPPABLE (1) + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -114,6 +116,10 @@ size_t _mi_os_large_page_size(void); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); +void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats); +void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats); + + // arena.c mi_arena_id_t _mi_arena_id_none(void); void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats); @@ -144,6 +150,8 @@ void _mi_segment_thread_collect(mi_segments_tld_t* tld); void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); void _mi_abandoned_await_readers(void); +mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld); + // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; @@ -671,8 +679,11 @@ static inline mi_memid_t _mi_memid_none(void) { return _mi_memid_create(MI_MEM_NONE); } -static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool is_large) { +static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, size_t alignment, bool committed, bool is_zero, bool is_large) { mi_memid_t memid = _mi_memid_create(MI_MEM_OS); + memid.mem.os.base = base; + memid.mem.os.size = size; + memid.mem.os.alignment = alignment; memid.initially_committed = committed; memid.initially_zero = is_zero; memid.is_pinned = is_large; diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 9e560696..db1c7ea6 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -69,6 +69,24 @@ int _mi_prim_protect(void* addr, size_t size, bool protect); // numa_node is either negative (don't care), or a numa node number. int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr); + +// Allocate remappable memory that can be used with `_mi_prim_remap`. +// Return `EINVAL` if this is not supported. +// The returned memory is always committed. +// If `is_pinned` is `true` the memory cannot be decommitted or reset. +// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_realloc_remappable` and `_mi_prim_free_remappable`. +int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ); + +// Remap remappable memory. Return `EINVAL` if this is not supported. +// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`. +// `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`. +int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ); + +// Free remappable memory. Return `EINVAL` if this is not supported. +// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`. +int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ); + + // Return the current NUMA node size_t _mi_prim_numa_node(void); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 06b96587..d760a961 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -341,7 +341,9 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) { typedef struct mi_memid_os_info { void* base; // actual base address of the block (used for offset aligned allocations) - size_t alignment; // alignment at allocation + size_t size; // allocated size (the full extent from base) + size_t alignment; // requested alignment at allocation (may be offset aligned, so base may not be aligned at this value) + void* prim_info; // potential OS dependent information (used for remappable memory on Windows) } mi_memid_os_info_t; typedef struct mi_memid_arena_info { diff --git a/src/alloc.c b/src/alloc.c index b17fdbdc..900c2f71 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -780,7 +780,56 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_ return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size); } +// ------------------------------------------------------ +// remap +// ------------------------------------------------------ +static void* mi_heap_malloc_zero_remappable(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { + return _mi_heap_malloc_zero_ex(heap, size, zero, MI_PAGE_ALIGN_REMAPPABLE); +} + +mi_decl_nodiscard void* mi_heap_malloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_heap_malloc_zero_remappable(heap, size, false); +} + +mi_decl_nodiscard void* mi_heap_zalloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_heap_malloc_zero_remappable(heap, size, true); +} + +mi_decl_nodiscard void* mi_malloc_remappable(size_t size) mi_attr_noexcept { + return mi_heap_malloc_remappable(mi_prim_get_default_heap(), size); +} + +mi_decl_nodiscard void* mi_zalloc_remappable(size_t size) mi_attr_noexcept { + return mi_heap_zalloc_remappable(mi_prim_get_default_heap(), size); +} + +mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept { + if (p == NULL) return mi_malloc(newsize); + + mi_segment_t* segment = mi_checked_ptr_segment(p, "mi_remap"); + mi_assert_internal(segment != NULL); + mi_page_t* const page = _mi_segment_page_of(segment, p); + const size_t bsize = mi_page_usable_block_size(page); + if (bsize >= newsize) { + // TODO: adjust padding + return p; + } + + mi_heap_t* heap = mi_prim_get_default_heap(); + if (segment->thread_id == heap->thread_id && + segment->memid.memkind == MI_MEM_OS_REMAP) + { + mi_heap_t* heap = mi_prim_get_default_heap(); + mi_block_t* block = _mi_segment_huge_page_remap(segment, page, (mi_block_t*)p, newsize, &heap->tld->segments); + if (block != NULL) { + // TODO: adjust padding? + return block; + } + } + _mi_warning_message("unable to remap block, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, 0, newsize); + return mi_realloc(p, newsize); +} // ------------------------------------------------------ // strdup, strndup, and realpath diff --git a/src/os.c b/src/os.c index 69ad2bf9..6a50c5c6 100644 --- a/src/os.c +++ b/src/os.c @@ -50,6 +50,10 @@ bool _mi_os_use_large_page(size_t size, size_t alignment) { return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); } +static size_t mi_os_alloc_size(size_t size) { + return _mi_align_up(size, mi_os_mem_config.alloc_granularity); +} + // round to a good OS allocation size (bounded by max 12.5% waste) size_t _mi_os_good_alloc_size(size_t size) { size_t align_size; @@ -58,12 +62,16 @@ size_t _mi_os_good_alloc_size(size_t size) { else if (size < 8*MI_MiB) align_size = 256*MI_KiB; else if (size < 32*MI_MiB) align_size = 1*MI_MiB; else align_size = 4*MI_MiB; + if (align_size < mi_os_mem_config.alloc_granularity) align_size = mi_os_mem_config.alloc_granularity; if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow? return _mi_align_up(size, align_size); } void _mi_os_init(void) { _mi_prim_mem_init(&mi_os_mem_config); + if (mi_os_mem_config.alloc_granularity < mi_os_mem_config.page_size) { + mi_os_mem_config.alloc_granularity = mi_os_mem_config.page_size; + } } @@ -117,23 +125,23 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; size = _mi_align_up(size, MI_SEGMENT_SIZE); - if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. - #if (MI_SECURE>0) + if (size > 1 * MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. +#if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. - #endif +#endif uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode +#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB - #endif + init = init + ((MI_SEGMENT_SIZE * ((r >> 17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB +#endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all } - if (hint%try_alignment != 0) return NULL; + if (hint % try_alignment != 0) return NULL; return (void*)hint; } #else @@ -163,22 +171,39 @@ static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_st _mi_stat_decrease(&stats->reserved, size); } +static void mi_os_prim_free_remappable(void* addr, size_t size, bool still_committed, void* remap_info, mi_stats_t* tld_stats) { + MI_UNUSED(tld_stats); + mi_assert_internal((size % _mi_os_page_size()) == 0); + if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) + int err = _mi_prim_free_remappable(addr, size, remap_info); + if (err != 0) { + _mi_warning_message("unable to free remappable OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); + } + mi_stats_t* stats = &_mi_stats_main; + if (still_committed) { _mi_stat_decrease(&stats->committed, size); } + _mi_stat_decrease(&stats->reserved, size); +} + void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) { if (mi_memkind_is_os(memid.memkind)) { - size_t csize = _mi_os_good_alloc_size(size); + size_t csize = mi_os_alloc_size(size); void* base = addr; // different base? (due to alignment) if (memid.mem.os.base != NULL) { mi_assert(memid.mem.os.base <= addr); mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr); + mi_assert(memid.mem.os.size >= csize); base = memid.mem.os.base; - csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base); + csize = memid.mem.os.size; } // free it if (memid.memkind == MI_MEM_OS_HUGE) { mi_assert(memid.is_pinned); mi_os_free_huge_os_pages(base, csize, tld_stats); } + else if (memid.memkind == MI_MEM_OS_REMAP) { + mi_os_prim_free_remappable(base, csize, still_committed, memid.mem.os.prim_info, tld_stats); + } else { mi_os_prim_free(base, csize, still_committed, tld_stats); } @@ -200,7 +225,7 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats) // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) { - mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(size > 0 && size == mi_os_alloc_size(size)); mi_assert_internal(is_zero != NULL); mi_assert_internal(is_large != NULL); if (size == 0) return NULL; @@ -228,18 +253,44 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo return p; } +static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats) +{ + //mi_assert_internal((size + alignment - 1) <= over_size); + void* p = mi_align_up_ptr(base, alignment); + mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size); + if (!is_pinned) { + size_t pre_size = (uint8_t*)p - (uint8_t*)base; + size_t mid_size = _mi_align_up(size, _mi_os_page_size()); + size_t post_size = over_size - pre_size - mid_size; + mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size); + if (mi_os_mem_config.must_free_whole) { + // decommit the pre- and post part (if needed) + if (committed) { + if (pre_size > 0) { _mi_os_decommit(base, pre_size, stats); } + if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); } + } + } + else { + // free the pre- and post part + if (pre_size > 0) { mi_os_prim_free(base, pre_size, committed, stats); } + if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); } + } + } + mi_assert_internal(_mi_is_aligned(p, alignment)); + return p; +} // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. -static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) { +static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) { mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); - mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + mi_assert_internal(size > 0 && size == mi_os_alloc_size(size)); mi_assert_internal(is_large != NULL); mi_assert_internal(is_zero != NULL); mi_assert_internal(base != NULL); if (!commit) allow_large = false; if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; - size = _mi_align_up(size, _mi_os_page_size()); + size = mi_os_alloc_size(size); // try first with a hint (this will be aligned directly on Win 10+ or BSD) void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats); @@ -248,47 +299,21 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit // aligned already? if (((uintptr_t)p % alignment) == 0) { *base = p; + *fullsize = size; + return p; } else { - // if not aligned, free it, overallocate, and unmap around it + // if not aligned, free the original allocation, overallocate, and unmap around it _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); mi_os_prim_free(p, size, commit, stats); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow - const size_t over_size = size + alignment; - - if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block - // over-allocate uncommitted (virtual) memory - p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats); - if (p == NULL) return NULL; - - // set p to the aligned part in the full region - // note: this is dangerous on Windows as VirtualFree needs the actual base pointer - // this is handled though by having the `base` field in the memid's - *base = p; // remember the base - p = mi_align_up_ptr(p, alignment); - - // explicitly commit only the aligned part - if (commit) { - _mi_os_commit(p, size, NULL, stats); - } - } - else { // mmap can free inside an allocation - // overallocate... - p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats); - if (p == NULL) return NULL; - - // and selectively unmap parts around the over-allocated area. (noop on sbrk) - void* aligned_p = mi_align_up_ptr(p, alignment); - size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; - size_t mid_size = _mi_align_up(size, _mi_os_page_size()); - size_t post_size = over_size - pre_size - mid_size; - mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); } - if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); } - // we can return the aligned pointer on `mmap` (and sbrk) systems - p = aligned_p; - *base = aligned_p; // since we freed the pre part, `*base == p`. - } + const size_t oversize = size + alignment - 1; + + p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, is_large, is_zero, stats); + if (p == NULL) return NULL; + *base = p; + *fullsize = oversize; + return mi_os_align_within(base, oversize, alignment, size, commit, *is_large, stats); } mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0)); @@ -310,7 +335,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* tld_stats) { bool os_is_zero = false; void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats); if (p != NULL) { - *memid = _mi_memid_create_os(true, os_is_zero, os_is_large); + *memid = _mi_memid_create_os(p, size, 0, true, os_is_zero, os_is_large); } return p; } @@ -327,11 +352,10 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo bool os_is_large = false; bool os_is_zero = false; void* os_base = NULL; - void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &_mi_stats_main /*tld->stats*/ ); + size_t os_size = 0; + void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &os_size, &_mi_stats_main /*tld->stats*/ ); if (p != NULL) { - *memid = _mi_memid_create_os(commit, os_is_zero, os_is_large); - memid->mem.os.base = os_base; - memid->mem.os.alignment = alignment; + *memid = _mi_memid_create_os(os_base, os_size, alignment, commit, os_is_zero, os_is_large); } return p; } @@ -357,20 +381,86 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse else { // overallocate to align at an offset const size_t extra = _mi_align_up(offset, alignment) - offset; - const size_t oversize = size + extra; + const size_t oversize = mi_os_alloc_size(size + extra); void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, tld_stats); if (start == NULL) return NULL; void* const p = (uint8_t*)start + extra; mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); // decommit the overallocation at the start - if (commit && extra > _mi_os_page_size()) { + if (memid->initially_committed && !memid->is_pinned && (extra > _mi_os_page_size())) { _mi_os_decommit(start, extra, tld_stats); } return p; } } + +/* ----------------------------------------------------------- + Remappable memory +----------------------------------------------------------- */ + +void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) { + mi_assert_internal(size > 0); + mi_assert_internal(memid != NULL); + *memid = _mi_memid_none(); + if (alignment == 0) { alignment = 1; } + const size_t oversize = mi_os_alloc_size(size + alignment - 1); + if (future_reserve < oversize) { future_reserve = oversize; } + bool os_is_pinned = true; + bool os_is_zero = false; + void* base = NULL; + void* remap_info = NULL; + int err = _mi_prim_alloc_remappable(oversize, future_reserve, &os_is_pinned, &os_is_zero, &base, &remap_info); + if (err != 0 || base == NULL) { + // fall back to regular allocation + return _mi_os_alloc_aligned(size, alignment, true /* commit */, true /* allow_large */, memid, stats); + } + *memid = _mi_memid_create_os(base, oversize, alignment, true, os_is_zero, os_is_pinned); + memid->memkind = MI_MEM_OS_REMAP; + memid->mem.os.prim_info = remap_info; + return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats); +} + +void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) { + mi_assert_internal(size > 0); + mi_assert_internal(newsize > 0); + mi_assert_internal(p != NULL && memid != NULL); + mi_assert_internal(mi_memkind_is_os(memid->memkind)); + if (p == NULL) return NULL; + if (!mi_memkind_is_os(memid->memkind)) return NULL; + + newsize = mi_os_alloc_size(newsize); + const size_t oversize = newsize + memid->mem.os.alignment - 1; + + if (memid->memkind == MI_MEM_OS_REMAP) { + bool extend_is_zero = false; + void* newp = NULL; + int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info); + if (err == 0 && newp != NULL) { + memid->initially_committed = true; + memid->mem.os.base = newp; + memid->mem.os.size = oversize; + return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats); + } + else { + _mi_warning_message("failed to remap OS re-allocation (error %d (0x%x) at %p of size %zu to size %zu)\n", err, err, p, size, newsize); + } + } + + // fall back to regular realloc + mi_memid_t newmemid = _mi_memid_none(); + void* newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, memid->initially_committed, true /* allow large? */, &newmemid, stats); + if (newp == NULL) return NULL; + + size_t csize = (size > newsize ? newsize : size); + _mi_memcpy_aligned(newp, p, csize); + _mi_os_free(p, size, *memid, stats); + *memid = newmemid; + return newp; +} + + /* ----------------------------------------------------------- OS memory API: reset, commit, decommit, protect, unprotect. ----------------------------------------------------------- */ @@ -640,16 +730,17 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse } } } - mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size); + const size_t alloc_size = page * MI_HUGE_OS_PAGE_SIZE; + mi_assert_internal(alloc_size <= size); if (pages_reserved != NULL) { *pages_reserved = page; } - if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; } + if (psize != NULL) { *psize = alloc_size; } if (page != 0) { mi_assert(start != NULL); - *memid = _mi_memid_create_os(true /* is committed */, all_zero, true /* is_large */); + *memid = _mi_memid_create_os(start, alloc_size, _mi_os_page_size(), true /* is committed */, all_zero, true /* is_large */); memid->memkind = MI_MEM_OS_HUGE; mi_assert(memid->is_pinned); #ifdef MI_TRACK_ASAN - if (all_zero) { mi_track_mem_defined(start,size); } + if (all_zero) { mi_track_mem_defined(start,alloc_size); } #endif } return (page == 0 ? NULL : start); diff --git a/src/page.c b/src/page.c index 5fefc3b5..b99b8d68 100644 --- a/src/page.c +++ b/src/page.c @@ -814,11 +814,10 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex General allocation ----------------------------------------------------------- */ -// A huge page is allocated directly without being in a queue. -// Because huge pages contain just one block, and the segment contains -// just that page, we always treat them as abandoned and any thread -// that frees the block can free the whole page and segment directly. -// Huge pages are also use if the requested alignment is very large (> MI_ALIGNMENT_MAX). +// A huge page always occupies a single segment. +// It is used for large allocations, (very) large alignments (> MI_ALIGNMENT_MAX), or remappable blocks. +// When a huge page is freed from another thread, it is immediately reset to reduce memory pressure. +// We use a page_alignment of 1 for remappable memory. static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) { size_t block_size = _mi_os_good_alloc_size(size); mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0); @@ -918,7 +917,9 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al // note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case. void* p = _mi_page_malloc(heap, page, size, false); mi_assert_internal(p != NULL); - _mi_memzero_aligned(p, mi_page_usable_block_size(page)); + if (!page->free_is_zero) { + _mi_memzero_aligned(p, mi_page_usable_block_size(page)); + } return p; } else { diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index b64f0173..76e006ed 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -11,6 +11,10 @@ terms of the MIT license. A copy of the license can be found in the file #define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined #endif +#ifndef _GNU_SOURCE +#define _GNU_SOURCE // ensure mremap is defined +#endif + #if defined(__sun) // illumos provides new mman.h api when any of these are defined // otherwise the old api based on caddr_t which predates the void pointers one. @@ -861,3 +865,41 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { } #endif + + + + +//---------------------------------------------------------------- +// Remappable memory +//---------------------------------------------------------------- + +int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { + #if !defined(MREMAP_MAYMOVE) + MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); + return EINVAL; + #else + MI_UNUSED(future_reserve); + *remap_info = NULL; + return _mi_prim_alloc(size,1,true /* commit? */, true /* allow_large */, is_pinned /* is_large? */, is_zero, addr); + #endif +} + +int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { + #if !defined(MREMAP_MAYMOVE) + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); + return EINVAL; + #else + mi_assert_internal(*remap_info == NULL); MI_UNUSED(remap_info); + void* p = mremap(addr,size,newsize,MREMAP_MAYMOVE); + if (p == MAP_FAILED) { return errno; } + *extend_is_zero = true; + *newaddr = p; + return 0; + #endif +} + +int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { + MI_UNUSED(remap_info); + mi_assert_internal(remap_info == NULL); + return _mi_prim_free(addr,size); +} diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 50511f0b..0960f683 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -273,3 +273,23 @@ void _mi_prim_thread_done_auto_done(void) { void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { MI_UNUSED(heap); } + + +//---------------------------------------------------------------- +// Remappable memory +//---------------------------------------------------------------- + +int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { + MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); + return EINVAL; +} + +int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); + return EINVAL; +} + +int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info); + return EINVAL; +} diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index e6b61079..76df6fd6 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -620,3 +620,24 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { } #endif + + + +//---------------------------------------------------------------- +// Remappable memory +//---------------------------------------------------------------- + +int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { + MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); + return EINVAL; +} + +int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); + return EINVAL; +} + +int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info); + return EINVAL; +} diff --git a/src/segment.c b/src/segment.c index 6798bb66..010ff1cd 100644 --- a/src/segment.c +++ b/src/segment.c @@ -517,13 +517,19 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme bool allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy size_t align_offset = 0; size_t alignment = MI_SEGMENT_SIZE; - if (page_alignment > 0) { + if (page_alignment > MI_PAGE_ALIGN_REMAPPABLE) { alignment = page_alignment; align_offset = _mi_align_up(pre_size, MI_SEGMENT_SIZE); segment_size = segment_size + (align_offset - pre_size); // adjust the segment size } - mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os); + mi_segment_t* segment = NULL; + if (page_alignment == MI_PAGE_ALIGN_REMAPPABLE) { + segment = (mi_segment_t*)_mi_os_alloc_remappable(segment_size, 0, alignment, &memid, tld_os->stats); + } + else { + segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os); + } if (segment == NULL) { return NULL; // failed to allocate } @@ -1224,7 +1230,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, page->xblock_size = (psize > MI_HUGE_BLOCK_SIZE ? MI_HUGE_BLOCK_SIZE : (uint32_t)psize); // reset the part of the page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE) - if (page_alignment > 0 && segment->allow_decommit && page->is_committed) { + if (page_alignment > MI_PAGE_ALIGN_REMAPPABLE && segment->allow_decommit && page->is_committed) { uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment); mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment)); mi_assert_internal(psize - (aligned_p - start) >= size); @@ -1283,17 +1289,52 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc } #endif +mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld) { + mi_assert_internal(segment == _mi_page_segment(page)); + mi_assert_internal(page->used == 1); + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + mi_assert_internal(mi_page_thread_free(page) == NULL); + mi_assert_internal(segment->next == NULL && segment->prev == NULL); + mi_assert_internal(page->next == NULL && page->prev == NULL); + const size_t bsize = mi_page_block_size(page); + const size_t newssize = _mi_align_up(_mi_align_up(newsize, _mi_os_page_size()) + (mi_segment_size(segment) - bsize), MI_SEGMENT_SIZE); + mi_memid_t memid = segment->memid; + const ptrdiff_t block_ofs = (uint8_t*)block - (uint8_t*)segment; + + mi_segment_protect(segment, false, tld->os); + if (segment->allow_decommit && !page->is_committed) { // TODO: always commit fully regardless of the page? + _mi_os_commit(segment, mi_segment_size(segment), NULL, tld->stats); + page->is_committed = true; + } + mi_segment_t* newsegment = (mi_segment_t*)_mi_os_realloc(segment, mi_segment_size(segment), newssize, &memid, tld->stats); + if (newsegment == NULL) { + mi_segment_protect(segment, true, tld->os); + return NULL; + } + else { + newsegment->memid = memid; + newsegment->segment_size = newssize; + newsegment->cookie = _mi_ptr_cookie(newsegment); + mi_segment_protect(newsegment, true, tld->os); + } + + mi_block_t* newblock = (mi_block_t*)((uint8_t*)newsegment + block_ofs); + mi_assert_internal(_mi_ptr_segment(newblock) == newsegment); + mi_page_t* newpage = _mi_ptr_page(newblock); + mi_assert_internal(mi_page_block_size(newpage) >= newsize); MI_UNUSED(newpage); + return newblock; +} + /* ----------------------------------------------------------- Page allocation ----------------------------------------------------------- */ mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; - if mi_unlikely(page_alignment > MI_ALIGNMENT_MAX) { - mi_assert_internal(_mi_is_power_of_two(page_alignment)); - mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE); - //mi_assert_internal((MI_SEGMENT_SIZE % page_alignment) == 0); - if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; } + if mi_unlikely(page_alignment > 0) { + mi_assert_internal(page_alignment == MI_PAGE_ALIGN_REMAPPABLE || (_mi_is_power_of_two(page_alignment) && page_alignment >= MI_SEGMENT_SIZE)); + if (page_alignment != MI_PAGE_ALIGN_REMAPPABLE && page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; } page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld); } else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) { diff --git a/src/static.c b/src/static.c index bc05dd72..34f0db42 100644 --- a/src/static.c +++ b/src/static.c @@ -5,8 +5,12 @@ terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #ifndef _DEFAULT_SOURCE -#define _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // ensure mmap is defined #endif +#ifndef _GNU_SOURCE +#define _GNU_SOURCE // ensure mremap is defined +#endif + #if defined(__sun) // same remarks as os.c for the static's context. #undef _XOPEN_SOURCE diff --git a/test/main-override-static.c b/test/main-override-static.c index bf1cc416..3bda56a6 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -18,11 +18,14 @@ static void test_reserved(void); static void negative_stat(void); static void alloc_huge(void); static void test_heap_walk(void); - +static void test_remap(void); int main() { mi_version(); mi_stats_reset(); + + test_remap(); + // detect double frees and heap corruption // double_free1(); // double_free2(); @@ -216,6 +219,20 @@ static void test_heap_walk(void) { mi_heap_visit_blocks(heap, true, &test_visit, NULL); } + +static void test_remap(void) { + size_t size = 64 * 1024 * 1024; + size_t inc = 1024 * 1024; + uint8_t* p = (uint8_t*)mi_malloc_remappable(size); + memset(p, 1, size); + for (int i = 2; i < 100; i++) { + p = mi_remap(p, size + inc); + memset(p + size, i, inc); + size += inc; + } + mi_free(p); +} + // ---------------------------- // bin size experiments // ------------------------------