diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 1754602d..b3dbff1f 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -116,7 +116,7 @@ size_t _mi_os_large_page_size(void); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); -void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats); +void* _mi_os_alloc_remappable(size_t size, size_t alignment, mi_memid_t* memid, mi_stats_t* stats); void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats); @@ -304,6 +304,11 @@ static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { } } +// Align upwards for a pointer +static inline void* _mi_align_up_ptr(void* p, size_t alignment) { + return (void*)_mi_align_up((uintptr_t)p, alignment); +} + // Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { mi_assert_internal(divider != 0); diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 71187294..0cd685d8 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -72,15 +72,16 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo // Allocate remappable memory that can be used with `_mi_prim_remap`. // Return `EINVAL` if this is not supported. -// The returned memory is always committed. +// The returned memory is always committed and aligned at `alignment`. // If `is_pinned` is `true` the memory cannot be decommitted or reset. // The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_remap` and `_mi_prim_free_remappable`. -int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ); +int _mi_prim_alloc_remappable(size_t size, size_t alignment, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ); // Remap remappable memory. Return `EINVAL` if this is not supported. +// If remapped, the alignment should be preserved. // pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`. // `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`. -int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ); +int _mi_prim_remap(void* addr, size_t size, size_t newsize, size_t alignment, bool* extend_is_zero, void** newaddr, void** remap_info ); // Free remappable memory. Return `EINVAL` if this is not supported. // pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`. diff --git a/src/os.c b/src/os.c index 3addcd50..8ac224e1 100644 --- a/src/os.c +++ b/src/os.c @@ -81,10 +81,6 @@ void _mi_os_init(void) { bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); -static void* mi_align_up_ptr(void* p, size_t alignment) { - return (void*)_mi_align_up((uintptr_t)p, alignment); -} - static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); uintptr_t mask = alignment - 1; @@ -255,10 +251,11 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo // aligns within an already allocated area; may modify `memid` with a new base and size. static void* mi_os_align_within(mi_memid_t* memid, size_t alignment, size_t size, mi_stats_t* stats) -{ - mi_assert_internal(alignment >= _mi_os_page_size()); +{ + mi_assert_internal(alignment <= 1 || (alignment >= _mi_os_page_size())); mi_assert_internal((size + alignment - 1) <= memid->mem.os.size); - void* p = mi_align_up_ptr(memid->mem.os.base, alignment); + memid->mem.os.alignment = alignment; + void* p = _mi_align_up_ptr(memid->mem.os.base, alignment); mi_assert_internal((uintptr_t)p + size <= (uintptr_t)memid->mem.os.base + memid->mem.os.size); if (!memid->is_pinned) { size_t pre_size = (uint8_t*)p - (uint8_t*)memid->mem.os.base; @@ -277,8 +274,7 @@ static void* mi_os_align_within(mi_memid_t* memid, size_t alignment, size_t size if (pre_size > 0) { mi_os_prim_free(memid->mem.os.base, pre_size, memid->initially_committed, stats); } if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, memid->initially_committed, stats); } memid->mem.os.base = p; - memid->mem.os.size = mid_size; - memid->mem.os.alignment = alignment; + memid->mem.os.size = mid_size; } } mi_assert_internal(_mi_is_aligned(p, alignment)); @@ -395,26 +391,26 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse Remappable memory ----------------------------------------------------------- */ -void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) { +void* _mi_os_alloc_remappable(size_t size, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) { mi_assert_internal(size > 0); mi_assert_internal(memid != NULL); *memid = _mi_memid_none(); if (alignment == 0) { alignment = 1; } - const size_t oversize = mi_os_get_alloc_size(size + alignment - 1); - if (future_reserve < oversize) { future_reserve = oversize; } + size = mi_os_get_alloc_size(size); bool os_is_pinned = true; bool os_is_zero = false; void* base = NULL; void* remap_info = NULL; - int err = _mi_prim_alloc_remappable(oversize, future_reserve, &os_is_pinned, &os_is_zero, &base, &remap_info); + int err = _mi_prim_alloc_remappable(size, alignment, &os_is_pinned, &os_is_zero, &base, &remap_info); if (err != 0 || base == NULL) { // fall back to regular allocation return _mi_os_alloc_aligned(size, alignment, true /* commit */, true /* allow_large */, memid, stats); } - *memid = _mi_memid_create_os(base, oversize, alignment, true, os_is_zero, os_is_pinned); + mi_assert_internal(_mi_is_aligned(base, alignment)); + *memid = _mi_memid_create_os(base, size, alignment, true, os_is_zero, os_is_pinned); memid->memkind = MI_MEM_OS_REMAP; memid->mem.os.prim_info = remap_info; - return mi_os_align_within(memid,alignment,size,stats); + return base; } void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) { @@ -426,8 +422,7 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s if (!mi_memkind_is_os(memid->memkind)) return NULL; newsize = mi_os_get_alloc_size(newsize); - const size_t oversize = mi_os_get_alloc_size(newsize + memid->mem.os.alignment - 1); - + if (memid->memkind == MI_MEM_OS_REMAP) { bool extend_is_zero = false; void* newp = NULL; @@ -435,12 +430,15 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s // if parts may have been decommitted, ensure it is committed now (or we get EFAULT from mremap) _mi_os_commit(memid->mem.os.base, memid->mem.os.size, NULL, stats); } - int err = _mi_prim_remap(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info); + const size_t alignment = memid->mem.os.alignment; + void* prim_info = memid->mem.os.prim_info; + int err = _mi_prim_remap(memid->mem.os.base, memid->mem.os.size, newsize, alignment, &extend_is_zero, &newp, &prim_info); if (err == 0 && newp != NULL) { - const size_t alignment = memid->mem.os.alignment; - *memid = _mi_memid_create_os(newp, oversize, 1, true /* committed */, false /* iszero */, false /* islarge */); + mi_assert_internal(_mi_is_aligned(newp, alignment)); + *memid = _mi_memid_create_os(newp, newsize, alignment, true /* committed */, false /* iszero */, memid->is_pinned /* is pinned */); + memid->mem.os.prim_info = prim_info; memid->memkind = MI_MEM_OS_REMAP; - return mi_os_align_within(memid, alignment, newsize, stats); + return newp; } else { _mi_warning_message("failed to remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, size, newsize); @@ -449,7 +447,7 @@ void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_s // fall back to copy (but in remappable memory if possible) mi_memid_t newmemid = _mi_memid_none(); - void* newp = _mi_os_alloc_remappable(newsize, 0, memid->mem.os.alignment, &newmemid, stats); + void* newp = _mi_os_alloc_remappable(newsize, memid->mem.os.alignment, &newmemid, stats); if (newp == NULL) { newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, true /* commit */, false /* allow_large */, &newmemid, stats); if (newp == NULL) return NULL; @@ -475,10 +473,10 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, if (size == 0 || addr == NULL) return NULL; // page align conservatively within the range - void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size()) + void* start = (conservative ? _mi_align_up_ptr(addr, _mi_os_page_size()) : mi_align_down_ptr(addr, _mi_os_page_size())); void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) - : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); + : _mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; if (diff <= 0) return NULL; diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 8660a54b..399ed2d1 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -873,25 +873,66 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { // Remappable memory //---------------------------------------------------------------- -int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { +static int mi_unix_alloc_aligned(size_t size, size_t alignment, bool commit, bool* is_pinned, bool* is_zero, void** addr) +{ + mi_assert_internal(alignment <= 1 || (alignment >= _mi_os_page_size())); + *addr = NULL; + void* base = NULL; + int err = _mi_prim_alloc(size,alignment,commit,false /*allow large*/, is_pinned, is_zero, &base); + if (err != 0) return err; + if (_mi_is_aligned(base,alignment)) { + *addr = base; + return 0; + } + _mi_prim_free(base,size); + const size_t oversize = _mi_align_up( _mi_align_up(size,alignment), _mi_os_page_size()); + err = _mi_prim_alloc(oversize,alignment,commit,false,is_pinned,is_zero,&base); + if (err != 0) return err; + mi_assert_internal(!(*is_pinned)); + if (!(*is_pinned)) { + void* p = _mi_align_up_ptr(base,alignment); + *addr = p; + size_t pre_size = (uint8_t*)p - (uint8_t*)base; + size_t mid_size = _mi_align_up(size,_mi_os_page_size()); + size_t post_size = oversize - pre_size - mid_size; + mi_assert_internal(pre_size < oversize && post_size < oversize && mid_size >= size); + if (pre_size > 0) { _mi_prim_free(base, pre_size); } + if (post_size > 0) { _mi_prim_free((uint8_t*)p + mid_size, post_size); } + return 0; + } + else { + _mi_prim_free(base,oversize); + return EINVAL; + } +} + +int _mi_prim_alloc_remappable(size_t size, size_t alignment, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { #if !defined(MREMAP_MAYMOVE) - MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); + MI_UNUSED(size); MI_UNUSED(alignment); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); return EINVAL; #else - MI_UNUSED(future_reserve); *remap_info = NULL; - return _mi_prim_alloc(size,1,true /* commit? */, true /* allow_large */, is_pinned /* is_large? */, is_zero, addr); + return mi_unix_alloc_aligned(size, alignment, true, is_pinned, is_zero, addr); #endif } -int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { - #if !defined(MREMAP_MAYMOVE) - MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); +int _mi_prim_remap(void* addr, size_t size, size_t newsize, size_t alignment, bool* extend_is_zero, void** newaddr, void** remap_info ) { + #if !defined(MREMAP_MAYMOVE) || !defined(MREMAP_FIXED) + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(alignment); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); return EINVAL; #else - mi_assert_internal(*remap_info == NULL); MI_UNUSED(remap_info); - void* p = mremap(addr,size,newsize,MREMAP_MAYMOVE); - if (p == MAP_FAILED) { return errno; } + mi_assert_internal(*remap_info == NULL); MI_UNUSED(remap_info); + void* p = NULL; + bool is_pinned = false; + // don't commit yet, mremap will take over the virtual rang completely due to MREMAP_FIXED + int err = mi_unix_alloc_aligned(size, alignment, false, &is_pinned, extend_is_zero, &p); + if (err != 0) return err; + void* res = mremap(addr, size, newsize, MREMAP_MAYMOVE | MREMAP_FIXED, p); + if (res == MAP_FAILED || res != p) { + err = errno; + _mi_prim_free(p,size); + return err; + } *extend_is_zero = true; *newaddr = p; return 0; diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 57afaef7..359ad1b3 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -67,7 +67,7 @@ static PGetNumaProcessorNode pGetNumaProcessorNode = NULL; // Enable large page support dynamically (if possible) //--------------------------------------------- -static bool win_enable_large_os_pages(size_t* large_page_size) +static bool mi_win_enable_large_os_pages(size_t* large_page_size) { static bool large_initialized = false; if (large_initialized) return (_mi_os_large_page_size() > 0); @@ -99,7 +99,7 @@ static bool win_enable_large_os_pages(size_t* large_page_size) } if (!ok) { if (err == 0) err = GetLastError(); - _mi_warning_message("cannot enable large OS page support, error %lu\n", err); + _mi_warning_message("cannot acquire the lock memory privilege (needed for large OS page or remap support), error %lu\n", err); } return (ok!=0); } @@ -144,7 +144,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) FreeLibrary(hDll); } if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { - win_enable_large_os_pages(&config->large_page_size); + mi_win_enable_large_os_pages(&config->large_page_size); } } @@ -308,7 +308,7 @@ static void* _mi_prim_alloc_huge_os_pagesx(void* hint_addr, size_t size, int num { const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; - win_enable_large_os_pages(NULL); + if (!mi_win_enable_large_os_pages(NULL)) return NULL; MI_MEM_EXTENDED_PARAMETER params[3] = { {{0,0},{0}},{{0,0},{0}},{{0,0},{0}} }; // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages @@ -627,19 +627,208 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { // Remappable memory //---------------------------------------------------------------- -int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { - MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); - // return EINVAL; - return _mi_prim_alloc(size, 1, true, true, is_pinned, is_zero, addr); +typedef struct mi_win_remap_info_s { + void* base; // base of the virtual address space + size_t base_pages; // total virtual alloc'd pages from `base` + size_t alignment; // alignment: _mi_align_up_ptr(mapped_base,alignment) maps to the first physical page (= `mapped_base`) + size_t page_count; // allocated physical pages (with info in page_info) + size_t page_reserved; // available entries in page_info + ULONG_PTR page_info[1]; +} mi_win_remap_info_t; + + +static void* mi_win_mapped_base(mi_win_remap_info_t* rinfo) { + return _mi_align_up_ptr(rinfo->base, rinfo->alignment); } -int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { +static size_t mi_win_mapped_pages(mi_win_remap_info_t* rinfo) { + const size_t presize = (uint8_t*)mi_win_mapped_base(rinfo) - (uint8_t*)rinfo->base; + return (rinfo->base_pages - _mi_divide_up(presize, _mi_os_page_size())); +} + + +// allocate remap info +static mi_win_remap_info_t* mi_win_alloc_remap_info(size_t page_count, size_t alignment) { + const size_t remap_info_size = _mi_align_up(sizeof(mi_win_remap_info_t) + (page_count * sizeof(ULONG_PTR)), _mi_os_page_size()); + mi_win_remap_info_t* rinfo = NULL; + bool os_is_zero = false; + bool os_is_large = false; + int err = _mi_prim_alloc(remap_info_size, 1, true, false, &os_is_large, &os_is_zero, (void**)&rinfo); + if (err != 0) return NULL; + if (!os_is_zero) { _mi_memzero_aligned(rinfo, remap_info_size); } + rinfo->base = NULL; + rinfo->base_pages = 0; + rinfo->alignment = alignment; + rinfo->page_count = 0; + rinfo->page_reserved = (remap_info_size - offsetof(mi_win_remap_info_t, page_info)) / sizeof(ULONG_PTR); + return rinfo; +} + +// reallocate remap info +static mi_win_remap_info_t* mi_win_realloc_remap_info(mi_win_remap_info_t* rinfo, size_t newpage_count, size_t alignment) { + if (rinfo == NULL) { + return mi_win_alloc_remap_info(newpage_count,alignment); + } + else if (rinfo->page_reserved >= newpage_count) { + mi_assert_internal(alignment <= rinfo->alignment); + return rinfo; // still fits + } + else { + mi_assert_internal(alignment <= rinfo->alignment); + mi_win_remap_info_t* newrinfo = mi_win_alloc_remap_info(newpage_count,rinfo->alignment); + if (newrinfo == NULL) return NULL; + newrinfo->base = rinfo->base; + newrinfo->base_pages = rinfo->base_pages; + newrinfo->page_count = rinfo->page_count; + _mi_memcpy(newrinfo->page_info, rinfo->page_info, rinfo->page_count * sizeof(ULONG_PTR)); + _mi_prim_free(rinfo, sizeof(mi_win_remap_info_t) + ((rinfo->page_reserved - 1) * sizeof(ULONG_PTR))); + return newrinfo; + } +} + +// ensure enough physical pages are allocated +static int mi_win_ensure_physical_pages(mi_win_remap_info_t** prinfo, size_t newpage_count, size_t alignment) { + // ensure meta data is large enough + mi_win_remap_info_t* rinfo = mi_win_realloc_remap_info(*prinfo, newpage_count, alignment); + if (rinfo == NULL) return ENOMEM; + *prinfo = rinfo; + + // allocate physical pages + if (newpage_count > rinfo->page_count) { + mi_assert_internal(rinfo->page_reserved >= newpage_count); + const size_t extra_pages = newpage_count - rinfo->page_count; + ULONG_PTR req_pages = extra_pages; + if (!AllocateUserPhysicalPages(GetCurrentProcess(), &req_pages, &rinfo->page_info[rinfo->page_count])) { + return (int)GetLastError(); + } + rinfo->page_count += req_pages; + if (req_pages < extra_pages) return ENOMEM; + } + return 0; +} + + + +static int mi_win_remap_virtual_pages(mi_win_remap_info_t* rinfo, size_t newpage_count) { + mi_assert_internal(rinfo != NULL && rinfo->page_count >= newpage_count); + if (mi_win_mapped_pages(rinfo) >= newpage_count) return 0; // still good? should we shrink the mapping? + + + if (rinfo->base != NULL) { + printf("remap existing remap\n"); + } + + // we can now free the original virtual range ? + if (rinfo->base != NULL) { + if (!VirtualFree(rinfo->base, 0, MEM_RELEASE)) { + return (int)GetLastError(); + } + } + + // unmap the old range + /*if (rinfo->mapped_base != NULL) { + if (!MapUserPhysicalPages(rinfo->mapped_base, rinfo->mapped_pages, NULL)) { + return (int)GetLastError(); + } + }*/ + + // allocate new virtual address range + const size_t newsize = _mi_align_up( newpage_count * _mi_os_page_size() + rinfo->alignment - 1, _mi_os_page_size()); + void* newbase = VirtualAlloc(NULL, newsize, MEM_RESERVE | MEM_PHYSICAL, PAGE_READWRITE); + if (newbase == NULL) { + // todo: remap old range? + return (int)GetLastError(); + } + if (rinfo->base != newbase) { + printf("different base %p vs previous %p\n", newbase, rinfo->base); + } + + // find the aligned point where we map to our physical pages + rinfo->base = newbase; + rinfo->base_pages = _mi_divide_up(newsize, _mi_os_page_size()); + mi_assert_internal(mi_win_mapped_pages(rinfo) >= newpage_count); + + // and remap it + if (!MapUserPhysicalPages(mi_win_mapped_base(rinfo), newpage_count, &rinfo->page_info[0])) { + // todo: remap old range? + int err = (int)GetLastError(); + VirtualFree(newbase, 0, MEM_RELEASE); + return err; + } + + return 0; +} + +static int mi_win_remap(mi_win_remap_info_t** prinfo, size_t newsize, size_t alignment) { + size_t newpage_count = _mi_divide_up(newsize, _mi_os_page_size()); + int err = mi_win_ensure_physical_pages(prinfo, newpage_count, alignment); + if (err != 0) return err; + err = mi_win_remap_virtual_pages(*prinfo, newpage_count); + if (err != 0) return err; + return 0; +} + +static int mi_win_free_remap_info(mi_win_remap_info_t* rinfo) { + int err = 0; + if (rinfo == NULL) return 0; + if (rinfo->base != NULL) { + if (!VirtualFree(rinfo->base, 0, MEM_RELEASE)) { + err = (int)GetLastError(); + } + rinfo->base = NULL; + rinfo->base_pages = 0; + } + if (rinfo->page_count > 0) { + size_t req_pages = rinfo->page_count; + if (!FreeUserPhysicalPages(GetCurrentProcess(), &req_pages, &rinfo->page_info[0])) { + err = (int)GetLastError(); + } + rinfo->page_count = 0; + } + VirtualFree(rinfo, 0, MEM_RELEASE); + return err; +} + +int _mi_prim_alloc_remappable(size_t size, size_t alignment, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { + // MI_UNUSED(size); MI_UNUSED(alignment); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); + // return EINVAL; + // return _mi_prim_alloc(size, 1, true, true, is_pinned, is_zero, addr); + + if (!mi_win_enable_large_os_pages(NULL)) return EINVAL; + + mi_win_remap_info_t* rinfo = NULL; + int err = mi_win_remap(&rinfo, size, alignment); + if (err != 0) { + if (rinfo != NULL) { mi_win_free_remap_info(rinfo); } + return err; + } + *is_pinned = true; + *is_zero = true; + *addr = mi_win_mapped_base(rinfo); + *remap_info = rinfo; + return 0; +} + +int _mi_prim_remap(void* addr, size_t size, size_t newsize, size_t alignment, bool* extend_is_zero, void** newaddr, void** remap_info ) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); - return EINVAL; + // return EINVAL; + mi_win_remap_info_t** prinfo = (mi_win_remap_info_t**)remap_info; + mi_assert_internal(*prinfo != NULL); + mi_assert_internal(mi_win_mapped_base(*prinfo) == addr); + mi_assert_internal((*prinfo)->alignment == alignment); + int err = mi_win_remap(prinfo, newsize, alignment); + if (err != 0) return err; + *extend_is_zero = false; + *newaddr = mi_win_mapped_base(*prinfo); + return 0; } int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info); - return _mi_prim_free(addr, size); + // return _mi_prim_free(addr, size); // return EINVAL; + mi_win_remap_info_t* rinfo = (mi_win_remap_info_t*)remap_info; + mi_assert_internal(rinfo != NULL); + mi_assert_internal(mi_win_mapped_base(rinfo) == addr); + return mi_win_free_remap_info(rinfo); } diff --git a/src/segment.c b/src/segment.c index 46509a42..81b2f51f 100644 --- a/src/segment.c +++ b/src/segment.c @@ -525,7 +525,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme mi_segment_t* segment = NULL; if (page_alignment == MI_PAGE_ALIGN_REMAPPABLE) { - segment = (mi_segment_t*)_mi_os_alloc_remappable(segment_size, 0, alignment, &memid, tld_os->stats); + segment = (mi_segment_t*)_mi_os_alloc_remappable(segment_size, alignment, &memid, tld_os->stats); } else { segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os); @@ -1303,14 +1303,14 @@ mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, const size_t newssize = _mi_align_up(_mi_align_up(newsize, _mi_os_page_size()) + (mi_segment_size(segment) - bsize), MI_SEGMENT_SIZE); mi_memid_t memid = segment->memid; const ptrdiff_t block_ofs = (uint8_t*)block - (uint8_t*)segment; - + const uintptr_t cookie = segment->cookie; mi_segment_protect(segment, false, tld->os); mi_segment_t* newsegment = (mi_segment_t*)_mi_os_remap(segment, mi_segment_size(segment), newssize, &memid, tld->stats); if (newsegment == NULL) { mi_segment_protect(segment, true, tld->os); return NULL; } - + mi_assert_internal(cookie == newsegment->cookie); newsegment->memid = memid; newsegment->segment_size = newssize; newsegment->cookie = _mi_ptr_cookie(newsegment); diff --git a/test/main-override-static.c b/test/main-override-static.c index 3e3257a2..19c2ecd1 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -61,7 +61,7 @@ int main() { //mi_free(p2); //mi_collect(true); - //mi_stats_print(NULL); + mi_stats_print(NULL); // test_process_info();