kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip: enable mremap

Browse source
This commit is contained in:
daanx 2023-04-29 09:29:20 -07:00
parent 0c0a753aa0
commit 3e1c1f8e9f
5 changed files with 78 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/mimalloc/internal.h
View file

@ -117,7 +117,7 @@ size_t _mi_os_large_page_size(void);
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats); void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats);
void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats); void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats);
// arena.c // arena.c

25
src/alloc.c
View file

@ -814,25 +814,33 @@ mi_decl_nodiscard void* mi_zalloc_remappable(size_t size) mi_attr_noexcept {
mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept { mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept {
if (p == NULL) return mi_malloc(newsize); if (p == NULL) return mi_malloc(newsize);
const size_t padsize = newsize + MI_PADDING_SIZE;
mi_segment_t* segment = mi_checked_ptr_segment(p, "mi_remap"); mi_segment_t* segment = mi_checked_ptr_segment(p, "mi_remap");
const mi_threadid_t tid = _mi_prim_thread_id();
if (segment->thread_id != tid) {
_mi_warning_message("cannot remap memory from a different thread (address %p, newsize %zu bytes)\n", p, newsize);
return mi_realloc(p, newsize);
}
const size_t padsize = newsize + MI_PADDING_SIZE;
mi_assert_internal(segment != NULL); mi_assert_internal(segment != NULL);
mi_page_t* page = _mi_segment_page_of(segment, p); mi_page_t* page = _mi_segment_page_of(segment, p);
mi_block_t* block = _mi_page_ptr_unalign(segment, page, p); mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
const size_t bsize = mi_page_usable_block_size(page); const size_t bsize = mi_page_usable_block_size(page);
if (bsize >= padsize) { if (bsize >= padsize && 9*(bsize/10) <= padsize) { // if smaller and not more than 10% waste, keep it
_mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
mi_padding_init(page, block, newsize); mi_padding_init(page, block, newsize);
return p; return p;
} }
if (segment->memid.memkind == MI_MEM_OS_REMAP) {
// we can remap
mi_heap_t* heap = mi_prim_get_default_heap(); mi_heap_t* heap = mi_prim_get_default_heap();
if (segment->thread_id == heap->thread_id &&
segment->memid.memkind == MI_MEM_OS_REMAP)
{
mi_assert_internal((void*)block == p); mi_assert_internal((void*)block == p);
mi_assert_internal(heap->thread_id == tid);
_mi_heap_huge_page_detach(heap, page); _mi_heap_huge_page_detach(heap, page);
block = _mi_segment_huge_page_remap(segment, page, block, padsize, &heap->tld->segments); block = _mi_segment_huge_page_remap(segment, page, block, padsize, &heap->tld->segments);
if (block != NULL) { if (block != NULL) {
// succes! re-establish the pointers to the potentially relocated memory
segment = mi_checked_ptr_segment(block, "mi_remap"); segment = mi_checked_ptr_segment(block, "mi_remap");
page = _mi_segment_page_of(segment, block); page = _mi_segment_page_of(segment, block);
mi_padding_init(page, block, newsize); mi_padding_init(page, block, newsize);
@ -840,11 +848,14 @@ mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept {
return block; return block;
} }
else { else {
_mi_verbose_message("unable to remap memory, huge remap (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
_mi_heap_huge_page_attach(heap, page); _mi_heap_huge_page_attach(heap, page);
} }
} }
_mi_warning_message("unable to remap block, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize); else {
_mi_verbose_message("unable to remap memory, not remappable (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
}
_mi_warning_message("unable to remap memory, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
return mi_realloc(p, newsize); return mi_realloc(p, newsize);
} }

59
src/os.c
View file

@ -50,7 +50,7 @@ bool _mi_os_use_large_page(size_t size, size_t alignment) {
return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);
} }
static size_t mi_os_alloc_size(size_t size) { static size_t mi_os_get_alloc_size(size_t size) {
return _mi_align_up(size, mi_os_mem_config.alloc_granularity); return _mi_align_up(size, mi_os_mem_config.alloc_granularity);
} }
@ -164,7 +164,7 @@ static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_st
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free(addr, size); int err = _mi_prim_free(addr, size);
if (err != 0) { if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); _mi_warning_message("unable to free OS memory (error: %d (0x%02x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
} }
mi_stats_t* stats = &_mi_stats_main; mi_stats_t* stats = &_mi_stats_main;
if (still_committed) { _mi_stat_decrease(&stats->committed, size); } if (still_committed) { _mi_stat_decrease(&stats->committed, size); }
@ -177,7 +177,7 @@ static void mi_os_prim_free_remappable(void* addr, size_t size, bool still_commi
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free_remappable(addr, size, remap_info); int err = _mi_prim_free_remappable(addr, size, remap_info);
if (err != 0) { if (err != 0) {
_mi_warning_message("unable to free remappable OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); _mi_warning_message("unable to free remappable OS memory (error: %d (0x%02x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
} }
mi_stats_t* stats = &_mi_stats_main; mi_stats_t* stats = &_mi_stats_main;
if (still_committed) { _mi_stat_decrease(&stats->committed, size); } if (still_committed) { _mi_stat_decrease(&stats->committed, size); }
@ -186,7 +186,7 @@ static void mi_os_prim_free_remappable(void* addr, size_t size, bool still_commi
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) { void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) {
if (mi_memkind_is_os(memid.memkind)) { if (mi_memkind_is_os(memid.memkind)) {
size_t csize = mi_os_alloc_size(size); size_t csize = mi_os_get_alloc_size(size);
void* base = addr; void* base = addr;
// different base? (due to alignment) // different base? (due to alignment)
if (memid.mem.os.base != NULL) { if (memid.mem.os.base != NULL) {
@ -225,7 +225,7 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats)
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) { static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) {
mi_assert_internal(size > 0 && size == mi_os_alloc_size(size)); mi_assert_internal(size > 0 && size == mi_os_get_alloc_size(size));
mi_assert_internal(is_zero != NULL); mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL); mi_assert_internal(is_large != NULL);
if (size == 0) return NULL; if (size == 0) return NULL;
@ -236,7 +236,7 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
void* p = NULL; void* p = NULL;
int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, is_zero, &p); int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, is_zero, &p);
if (err != 0) { if (err != 0) {
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large); _mi_warning_message("unable to allocate OS memory (error: %d (0x%02x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large);
} }
mi_stat_counter_increase(stats->mmap_calls, 1); mi_stat_counter_increase(stats->mmap_calls, 1);
if (p != NULL) { if (p != NULL) {
@ -255,7 +255,8 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats) static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats)
{ {
//mi_assert_internal((size + alignment - 1) <= over_size); mi_assert_internal(alignment >= _mi_os_page_size());
mi_assert_internal((size + alignment - 1) <= over_size);
void* p = mi_align_up_ptr(base, alignment); void* p = mi_align_up_ptr(base, alignment);
mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size); mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size);
if (!is_pinned) { if (!is_pinned) {
@ -263,6 +264,12 @@ static void* mi_os_align_within(void* base, size_t over_size, size_t alignment,
size_t mid_size = _mi_align_up(size, _mi_os_page_size()); size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size; size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size); mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
// decommit the pre- and post part (if needed)
if (committed) {
if (pre_size > 0) { _mi_os_decommit(base, pre_size, stats); }
if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); }
}
/*
if (mi_os_mem_config.must_free_whole) { if (mi_os_mem_config.must_free_whole) {
// decommit the pre- and post part (if needed) // decommit the pre- and post part (if needed)
if (committed) { if (committed) {
@ -271,10 +278,12 @@ static void* mi_os_align_within(void* base, size_t over_size, size_t alignment,
} }
} }
else { else {
// TODO: we can only free parts if we also return an adjusted base and fullsize.
// free the pre- and post part // free the pre- and post part
if (pre_size > 0) { mi_os_prim_free(base, pre_size, committed, stats); } if (pre_size > 0) { mi_os_prim_free(base, pre_size, committed, stats); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); } if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); }
} }
*/
} }
mi_assert_internal(_mi_is_aligned(p, alignment)); mi_assert_internal(_mi_is_aligned(p, alignment));
return p; return p;
@ -284,13 +293,13 @@ static void* mi_os_align_within(void* base, size_t over_size, size_t alignment,
// This function guarantees the allocated memory is aligned. // This function guarantees the allocated memory is aligned.
static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) { static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) {
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && size == mi_os_alloc_size(size)); mi_assert_internal(size > 0 && size == mi_os_get_alloc_size(size));
mi_assert_internal(is_large != NULL); mi_assert_internal(is_large != NULL);
mi_assert_internal(is_zero != NULL); mi_assert_internal(is_zero != NULL);
mi_assert_internal(base != NULL); mi_assert_internal(base != NULL);
if (!commit) allow_large = false; if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = mi_os_alloc_size(size); size = mi_os_get_alloc_size(size);
// try first with a hint (this will be aligned directly on Win 10+ or BSD) // try first with a hint (this will be aligned directly on Win 10+ or BSD)
void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats); void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats);
@ -313,7 +322,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
if (p == NULL) return NULL; if (p == NULL) return NULL;
*base = p; *base = p;
*fullsize = oversize; *fullsize = oversize;
return mi_os_align_within(base, oversize, alignment, size, commit, *is_large, stats); return mi_os_align_within(p, oversize, alignment, size, commit, *is_large, stats);
} }
mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0)); mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0));
@ -381,7 +390,7 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse
else { else {
// overallocate to align at an offset // overallocate to align at an offset
const size_t extra = _mi_align_up(offset, alignment) - offset; const size_t extra = _mi_align_up(offset, alignment) - offset;
const size_t oversize = mi_os_alloc_size(size + extra); const size_t oversize = mi_os_get_alloc_size(size + extra);
void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, tld_stats); void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, tld_stats);
if (start == NULL) return NULL; if (start == NULL) return NULL;
@ -405,7 +414,7 @@ void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignme
mi_assert_internal(memid != NULL); mi_assert_internal(memid != NULL);
*memid = _mi_memid_none(); *memid = _mi_memid_none();
if (alignment == 0) { alignment = 1; } if (alignment == 0) { alignment = 1; }
const size_t oversize = mi_os_alloc_size(size + alignment - 1); const size_t oversize = mi_os_get_alloc_size(size + alignment - 1);
if (future_reserve < oversize) { future_reserve = oversize; } if (future_reserve < oversize) { future_reserve = oversize; }
bool os_is_pinned = true; bool os_is_pinned = true;
bool os_is_zero = false; bool os_is_zero = false;
@ -422,7 +431,7 @@ void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignme
return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats); return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats);
} }
void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) { void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) {
mi_assert_internal(size > 0); mi_assert_internal(size > 0);
mi_assert_internal(newsize > 0); mi_assert_internal(newsize > 0);
mi_assert_internal(p != NULL && memid != NULL); mi_assert_internal(p != NULL && memid != NULL);
@ -430,12 +439,16 @@ void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi
if (p == NULL) return NULL; if (p == NULL) return NULL;
if (!mi_memkind_is_os(memid->memkind)) return NULL; if (!mi_memkind_is_os(memid->memkind)) return NULL;
newsize = mi_os_alloc_size(newsize); newsize = mi_os_get_alloc_size(newsize);
const size_t oversize = newsize + memid->mem.os.alignment - 1; const size_t oversize = mi_os_get_alloc_size(newsize + memid->mem.os.alignment - 1);
if (memid->memkind == MI_MEM_OS_REMAP) { if (memid->memkind == MI_MEM_OS_REMAP) {
bool extend_is_zero = false; bool extend_is_zero = false;
void* newp = NULL; void* newp = NULL;
if (!memid->is_pinned || !memid->initially_committed) {
// if parts may have been decommitted, ensure it is committed now (or we get EFAULT from mremap)
_mi_os_commit(memid->mem.os.base, memid->mem.os.size, NULL, stats);
}
int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info); int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info);
if (err == 0 && newp != NULL) { if (err == 0 && newp != NULL) {
memid->initially_committed = true; memid->initially_committed = true;
@ -444,13 +457,13 @@ void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi
return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats); return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats);
} }
else { else {
_mi_warning_message("failed to remap OS re-allocation (error %d (0x%x) at %p of size %zu to size %zu)\n", err, err, p, size, newsize); _mi_warning_message("failed to remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, size, newsize);
} }
} }
// fall back to regular realloc // fall back to copy (but in remappable memory if possible)
mi_memid_t newmemid = _mi_memid_none(); mi_memid_t newmemid = _mi_memid_none();
void* newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, memid->initially_committed, true /* allow large? */, &newmemid, stats); void* newp = _mi_os_alloc_remappable(newsize, 0, memid->mem.os.alignment, &newmemid, stats);
if (newp == NULL) return NULL; if (newp == NULL) return NULL;
size_t csize = (size > newsize ? newsize : size); size_t csize = (size > newsize ? newsize : size);
@ -505,7 +518,7 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats
bool os_is_zero = false; bool os_is_zero = false;
int err = _mi_prim_commit(start, csize, &os_is_zero); int err = _mi_prim_commit(start, csize, &os_is_zero);
if (err != 0) { if (err != 0) {
_mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); _mi_warning_message("cannot commit OS memory (error: %d (0x%02x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
return false; return false;
} }
if (os_is_zero && is_zero != NULL) { if (os_is_zero && is_zero != NULL) {
@ -535,7 +548,7 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_
*needs_recommit = true; *needs_recommit = true;
int err = _mi_prim_decommit(start,csize,needs_recommit); int err = _mi_prim_decommit(start,csize,needs_recommit);
if (err != 0) { if (err != 0) {
_mi_warning_message("cannot decommit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); _mi_warning_message("cannot decommit OS memory (error: %d (0x%02x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
} }
mi_assert_internal(err == 0); mi_assert_internal(err == 0);
return (err == 0); return (err == 0);
@ -565,7 +578,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
int err = _mi_prim_reset(start, csize); int err = _mi_prim_reset(start, csize);
if (err != 0) { if (err != 0) {
_mi_warning_message("cannot reset OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); _mi_warning_message("cannot reset OS memory (error: %d (0x%02x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
} }
return (err == 0); return (err == 0);
} }
@ -614,7 +627,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
*/ */
int err = _mi_prim_protect(start,csize,protect); int err = _mi_prim_protect(start,csize,protect);
if (err != 0) { if (err != 0) {
_mi_warning_message("cannot %s OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize); _mi_warning_message("cannot %s OS memory (error: %d (0x%02x), address: %p, size: 0x%zx bytes)\n", (protect ? "protect" : "unprotect"), err, err, start, csize);
} }
return (err == 0); return (err == 0);
} }
@ -696,7 +709,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p); int err = _mi_prim_alloc_huge_os_pages(addr, MI_HUGE_OS_PAGE_SIZE, numa_node, &is_zero, &p);
if (!is_zero) { all_zero = false; } if (!is_zero) { all_zero = false; }
if (err != 0) { if (err != 0) {
_mi_warning_message("unable to allocate huge OS page (error: %d (0x%x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE); _mi_warning_message("unable to allocate huge OS page (error: %d (0x%02x), address: %p, size: %zx bytes)\n", err, err, addr, MI_HUGE_OS_PAGE_SIZE);
break; break;
} }

11
src/segment.c
View file

@ -1290,6 +1290,7 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc
#endif #endif
mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld) { mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld) {
// assert there are no pointers into the segment/page/block anymore
mi_assert_internal(segment == _mi_page_segment(page)); mi_assert_internal(segment == _mi_page_segment(page));
mi_assert_internal(page->used == 1); mi_assert_internal(page->used == 1);
mi_assert_internal(page->free == NULL); mi_assert_internal(page->free == NULL);
@ -1297,29 +1298,25 @@ mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page,
mi_assert_internal(mi_page_thread_free(page) == NULL); mi_assert_internal(mi_page_thread_free(page) == NULL);
mi_assert_internal(segment->next == NULL && segment->prev == NULL); mi_assert_internal(segment->next == NULL && segment->prev == NULL);
mi_assert_internal(page->next == NULL && page->prev == NULL); mi_assert_internal(page->next == NULL && page->prev == NULL);
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
const size_t newssize = _mi_align_up(_mi_align_up(newsize, _mi_os_page_size()) + (mi_segment_size(segment) - bsize), MI_SEGMENT_SIZE); const size_t newssize = _mi_align_up(_mi_align_up(newsize, _mi_os_page_size()) + (mi_segment_size(segment) - bsize), MI_SEGMENT_SIZE);
mi_memid_t memid = segment->memid; mi_memid_t memid = segment->memid;
const ptrdiff_t block_ofs = (uint8_t*)block - (uint8_t*)segment; const ptrdiff_t block_ofs = (uint8_t*)block - (uint8_t*)segment;
mi_segment_protect(segment, false, tld->os); mi_segment_protect(segment, false, tld->os);
if (segment->allow_decommit && !page->is_committed) { // TODO: always commit fully regardless of the page? mi_segment_t* newsegment = (mi_segment_t*)_mi_os_remap(segment, mi_segment_size(segment), newssize, &memid, tld->stats);
_mi_os_commit(segment, mi_segment_size(segment), NULL, tld->stats);
page->is_committed = true;
}
mi_segment_t* newsegment = (mi_segment_t*)_mi_os_realloc(segment, mi_segment_size(segment), newssize, &memid, tld->stats);
if (newsegment == NULL) { if (newsegment == NULL) {
mi_segment_protect(segment, true, tld->os); mi_segment_protect(segment, true, tld->os);
return NULL; return NULL;
} }
else {
newsegment->memid = memid; newsegment->memid = memid;
newsegment->segment_size = newssize; newsegment->segment_size = newssize;
newsegment->cookie = _mi_ptr_cookie(newsegment); newsegment->cookie = _mi_ptr_cookie(newsegment);
mi_segment_protect(newsegment, true, tld->os); mi_segment_protect(newsegment, true, tld->os);
_mi_segment_map_freed_at(segment); _mi_segment_map_freed_at(segment);
_mi_segment_map_allocated_at(newsegment); _mi_segment_map_allocated_at(newsegment);
}
mi_block_t* newblock = (mi_block_t*)((uint8_t*)newsegment + block_ofs); mi_block_t* newblock = (mi_block_t*)((uint8_t*)newsegment + block_ofs);
mi_assert_internal(_mi_ptr_segment(newblock) == newsegment); mi_assert_internal(_mi_ptr_segment(newblock) == newsegment);

12
test/main-override-static.c
View file

@ -221,8 +221,9 @@ static void test_heap_walk(void) {
static void test_remap(void) { static void test_remap(void) {
size_t size = 64 * 1024 * 1024; const size_t size0 = 64 * 1024 * 1024;
size_t inc = 1024 * 1024; const size_t inc = 1024 * 1024;
size_t size = size0;
uint8_t* p = (uint8_t*)mi_malloc_remappable(size); uint8_t* p = (uint8_t*)mi_malloc_remappable(size);
memset(p, 1, size); memset(p, 1, size);
for (int i = 2; i < 100; i++) { for (int i = 2; i < 100; i++) {
@ -231,6 +232,13 @@ static void test_remap(void) {
size += inc; size += inc;
printf("%3d: increased to size %zu\n", i, size); printf("%3d: increased to size %zu\n", i, size);
} }
for (int i = 1; i < 100; i++) {
size_t idx = size0 + ((i - 1) * inc) - 1;
uint8_t v = p[idx];
if (v != i) {
printf("error: corrupted memory in remap: i=%d, index=0x%zx, value=%u \n", i, idx,v);
};
}
mi_free(p); mi_free(p);
} }