From 0c0a753aa06d43179f9209883fb3c37046bf1e23 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 26 Apr 2023 16:06:36 -0700 Subject: [PATCH] wip: adding mi_remap; first working version --- include/mimalloc/internal.h | 3 ++ src/alloc.c | 69 +++++++++++++++++++++++-------------- src/page.c | 21 +++++++++++ src/prim/windows/prim.c | 6 ++-- src/segment.c | 2 ++ test/main-override-static.c | 1 + 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 14544b60..151ae084 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -174,6 +174,9 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback fro size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats +void _mi_heap_huge_page_attach(mi_heap_t* heap, mi_page_t* page); +void _mi_heap_huge_page_detach(mi_heap_t* heap, mi_page_t* page); + // "heap.c" void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); diff --git a/src/alloc.c b/src/alloc.c index 900c2f71..dbc6e52d 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -24,6 +24,30 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if MI_PADDING +static void mi_padding_init(mi_page_t* page, mi_block_t* block, size_t size /* block size minus MI_PADDING_SIZE */) { + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); + ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - size); + #if (MI_DEBUG>=2) + mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size + delta)); + #endif + mi_track_mem_defined(padding, sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + padding->canary = (uint32_t)(mi_ptr_encode(page, block, page->keys)); + padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)padding - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + } + #endif +} +#else +static void mi_padding_init(mi_page_t* page, mi_block_t* block, size_t size) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(size); +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { @@ -81,24 +105,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } #endif -#if MI_PADDING // && !MI_TRACK_ENABLED - mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>=2) - mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); - #endif - mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess - padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); - padding->delta = (uint32_t)(delta); - #if MI_PADDING_CHECK - if (!mi_page_is_huge(page)) { - uint8_t* fill = (uint8_t*)padding - delta; - const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes - for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } - } - #endif -#endif - + mi_padding_init(page, block, size - MI_PADDING_SIZE); return block; } @@ -807,12 +814,14 @@ mi_decl_nodiscard void* mi_zalloc_remappable(size_t size) mi_attr_noexcept { mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept { if (p == NULL) return mi_malloc(newsize); + const size_t padsize = newsize + MI_PADDING_SIZE; mi_segment_t* segment = mi_checked_ptr_segment(p, "mi_remap"); mi_assert_internal(segment != NULL); - mi_page_t* const page = _mi_segment_page_of(segment, p); + mi_page_t* page = _mi_segment_page_of(segment, p); + mi_block_t* block = _mi_page_ptr_unalign(segment, page, p); const size_t bsize = mi_page_usable_block_size(page); - if (bsize >= newsize) { - // TODO: adjust padding + if (bsize >= padsize) { + mi_padding_init(page, block, newsize); return p; } @@ -820,14 +829,22 @@ mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept { if (segment->thread_id == heap->thread_id && segment->memid.memkind == MI_MEM_OS_REMAP) { - mi_heap_t* heap = mi_prim_get_default_heap(); - mi_block_t* block = _mi_segment_huge_page_remap(segment, page, (mi_block_t*)p, newsize, &heap->tld->segments); - if (block != NULL) { - // TODO: adjust padding? + mi_assert_internal((void*)block == p); + _mi_heap_huge_page_detach(heap, page); + block = _mi_segment_huge_page_remap(segment, page, block, padsize, &heap->tld->segments); + if (block != NULL) { + segment = mi_checked_ptr_segment(block, "mi_remap"); + page = _mi_segment_page_of(segment, block); + mi_padding_init(page, block, newsize); + _mi_heap_huge_page_attach(heap, page); return block; } + else { + _mi_heap_huge_page_attach(heap, page); + } } - _mi_warning_message("unable to remap block, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, 0, newsize); + _mi_warning_message("unable to remap block, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize); + return mi_realloc(p, newsize); } diff --git a/src/page.c b/src/page.c index b99b8d68..49de68b3 100644 --- a/src/page.c +++ b/src/page.c @@ -403,6 +403,27 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { _mi_segment_page_abandon(page,segments_tld); } +// Detach a huge page (used for remapping) +void _mi_heap_huge_page_detach(mi_heap_t* heap, mi_page_t* page) { + mi_assert_internal(mi_page_heap(page) == heap); + #if !MI_HUGE_PAGE_ABANDON + mi_page_queue_t* pq = mi_page_queue_of(page); + mi_assert_internal(mi_page_queue_is_huge(pq)); + mi_page_queue_remove(pq, page); + #endif +} + +// (re)attach a huge page +void _mi_heap_huge_page_attach(mi_heap_t* heap, mi_page_t* page) { + mi_assert_internal(mi_page_heap(page) == heap); +#if !MI_HUGE_PAGE_ABANDON + mi_page_queue_t* pq = mi_page_queue(heap, MI_HUGE_OBJ_SIZE_MAX); // not block_size as that can be low if the page_alignment > 0 + mi_assert_internal(mi_page_queue_is_huge(pq)); + mi_page_queue_push(heap, pq, page); +#endif +} + + // Free a page with no more free blocks void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 76df6fd6..f4d9c39b 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -629,7 +629,8 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); - return EINVAL; + // return EINVAL; + return _mi_prim_alloc(size, 1, true, true, is_pinned, is_zero, addr); } int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { @@ -639,5 +640,6 @@ int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* e int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info); - return EINVAL; + return _mi_prim_free(addr, size); + // return EINVAL; } diff --git a/src/segment.c b/src/segment.c index 010ff1cd..e1949258 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1317,6 +1317,8 @@ mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, newsegment->segment_size = newssize; newsegment->cookie = _mi_ptr_cookie(newsegment); mi_segment_protect(newsegment, true, tld->os); + _mi_segment_map_freed_at(segment); + _mi_segment_map_allocated_at(newsegment); } mi_block_t* newblock = (mi_block_t*)((uint8_t*)newsegment + block_ofs); diff --git a/test/main-override-static.c b/test/main-override-static.c index 3bda56a6..2a762331 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -229,6 +229,7 @@ static void test_remap(void) { p = mi_remap(p, size + inc); memset(p + size, i, inc); size += inc; + printf("%3d: increased to size %zu\n", i, size); } mi_free(p); }