kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-11 09:49:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve remap primitive interface

Browse source
This commit is contained in:
daanx 2023-04-30 13:01:17 -07:00
parent 1246f46625
commit 10fbe6cf0f
5 changed files with 205 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
include/mimalloc/prim.h
View file

@ -70,22 +70,23 @@ int _mi_prim_protect(void* addr, size_t size, bool protect);
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr); int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
// Allocate remappable memory that can be used with `_mi_prim_remap`. // Reserve a pure virtual address range that can be used with `_mi_prim_remap_to`.
// Return `EINVAL` if this is not supported. // Return `EINVAL` if this is not supported.
// The returned memory is always committed and aligned at `alignment`. // If `is_pinned` is set to `true`, the memory cannot be decommitted or reset.
// If `is_pinned` is `true` the memory cannot be decommitted or reset. // The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_remap_to` and `_mi_prim_remap_free`.
// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_remap` and `_mi_prim_free_remappable`. int _mi_prim_remap_reserve(size_t size, bool* is_pinned, void** addr, void** remap_info );
int _mi_prim_alloc_remappable(size_t size, size_t alignment, bool* is_pinned, bool* is_zero, void** addr, void** remap_info );
// Remap remappable memory. Return `EINVAL` if this is not supported. // Remap remappable memory from `addr` to `newaddr` with the new `newsize`. Return `EINVAL` if this is not supported.
// If remapped, the alignment should be preserved. // Both `addr` (if not NULL) and `newaddr` are inside ranges returned from `_mi_prim_remap_reserve`.
// pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`. // The `addr` can be NULL to allocate freshly. The `base` pointer is always `<= addr` and if `base != addr`,
// `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`. // then it was the pointer returned from `_mi_prim_remap_reserve`.
int _mi_prim_remap(void* addr, size_t size, size_t newsize, size_t alignment, bool* extend_is_zero, void** newaddr, void** remap_info ); // This is used to ensure we can remap _aligned_ addresses (`addr` and `newaddr`).
// pre: `newsize > 0`, `size > 0`, `newaddr != NULL`, `extend_zero != NULL`, `remap_info != NULL`.
int _mi_prim_remap_to(void* base, void* addr, size_t size, void* newaddr, size_t newsize, bool* extend_is_zero, void** remap_info, void** new_remap_info );
// Free remappable memory. Return `EINVAL` if this is not supported. // Free remappable memory. Return `EINVAL` if this is not supported.
// pre: `addr != NULL` and previously allocated using `_mi_prim_remap` or `_mi_prim_alloc_remappable`. // pre: `addr != NULL`
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ); int _mi_prim_remap_free(void* addr, size_t size, void* remap_info );
// Return the current NUMA node // Return the current NUMA node

114
src/os.c
View file

@ -171,7 +171,7 @@ static void mi_os_prim_free_remappable(void* addr, size_t size, bool still_commi
MI_UNUSED(tld_stats); MI_UNUSED(tld_stats);
mi_assert_internal((size % _mi_os_page_size()) == 0); mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free_remappable(addr, size, remap_info); int err = _mi_prim_remap_free(addr, size, remap_info);
if (err != 0) { if (err != 0) {
_mi_warning_message("unable to free remappable OS memory (error: %d (0x%02x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); _mi_warning_message("unable to free remappable OS memory (error: %d (0x%02x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
} }
@ -392,73 +392,73 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse
----------------------------------------------------------- */ ----------------------------------------------------------- */
void* _mi_os_alloc_remappable(size_t size, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) { void* _mi_os_alloc_remappable(size_t size, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) {
mi_assert_internal(size > 0); if (alignment < _mi_os_page_size()) {
mi_assert_internal(memid != NULL); alignment = _mi_os_page_size();
*memid = _mi_memid_none();
if (alignment == 0) { alignment = 1; }
size = mi_os_get_alloc_size(size);
bool os_is_pinned = true;
bool os_is_zero = false;
void* base = NULL;
void* remap_info = NULL;
int err = _mi_prim_alloc_remappable(size, alignment, &os_is_pinned, &os_is_zero, &base, &remap_info);
if (err != 0 || base == NULL) {
// fall back to regular allocation
return _mi_os_alloc_aligned(size, alignment, true /* commit */, true /* allow_large */, memid, stats);
} }
mi_assert_internal(_mi_is_aligned(base, alignment)); *memid = _mi_memid_none();
*memid = _mi_memid_create_os(base, size, alignment, true, os_is_zero, os_is_pinned); memid->mem.os.alignment = alignment;
memid->memkind = MI_MEM_OS_REMAP; return _mi_os_remap(NULL, 0, size, memid, stats);
memid->mem.os.prim_info = remap_info;
return base;
} }
void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) { void* _mi_os_remap(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) {
mi_assert_internal(size > 0); mi_assert_internal(memid != NULL);
mi_assert_internal(newsize > 0); mi_assert_internal((memid->memkind == MI_MEM_NONE && p == NULL && size == 0) || (memid->memkind == MI_MEM_OS_REMAP && p != NULL && size > 0));
mi_assert_internal(p != NULL && memid != NULL);
mi_assert_internal(mi_memkind_is_os(memid->memkind));
if (p == NULL) return NULL;
if (!mi_memkind_is_os(memid->memkind)) return NULL;
newsize = mi_os_get_alloc_size(newsize); newsize = mi_os_get_alloc_size(newsize);
if (memid->memkind == MI_MEM_OS_REMAP) { // reserve virtual range
bool extend_is_zero = false; const size_t alignment = memid->mem.os.alignment;
void* newp = NULL; mi_assert_internal(alignment >= _mi_os_page_size());
if (!memid->is_pinned || !memid->initially_committed) { const size_t oversize = mi_os_get_alloc_size(newsize + alignment - 1);
// if parts may have been decommitted, ensure it is committed now (or we get EFAULT from mremap) bool os_is_pinned = false;
_mi_os_commit(memid->mem.os.base, memid->mem.os.size, NULL, stats); void* base = NULL;
void* remap_info = NULL;
int err = _mi_prim_remap_reserve(oversize, &os_is_pinned, &base, &remap_info);
if (err != 0) {
// fall back to regular allocation
if (err != EINVAL) { // EINVAL means not supported
_mi_warning_message("failed to reserve remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, 0, size);
} }
const size_t alignment = memid->mem.os.alignment; return NULL;
void* prim_info = memid->mem.os.prim_info;
int err = _mi_prim_remap(memid->mem.os.base, memid->mem.os.size, newsize, alignment, &extend_is_zero, &newp, &prim_info);
if (err == 0 && newp != NULL) {
mi_assert_internal(_mi_is_aligned(newp, alignment));
*memid = _mi_memid_create_os(newp, newsize, alignment, true /* committed */, false /* iszero */, memid->is_pinned /* is pinned */);
memid->mem.os.prim_info = prim_info;
memid->memkind = MI_MEM_OS_REMAP;
return newp;
}
else {
_mi_warning_message("failed to remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, size, newsize);
}
}
// fall back to copy (but in remappable memory if possible)
mi_memid_t newmemid = _mi_memid_none();
void* newp = _mi_os_alloc_remappable(newsize, memid->mem.os.alignment, &newmemid, stats);
if (newp == NULL) {
newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, true /* commit */, false /* allow_large */, &newmemid, stats);
if (newp == NULL) return NULL;
} }
size_t csize = (size > newsize ? newsize : size); // create an aligned pointer within
_mi_memcpy_aligned(newp, p, csize); mi_memid_t newmemid = _mi_memid_create_os(base, oversize, 1, false /* commit */, false /* iszero */, os_is_pinned);
_mi_os_free(p, size, *memid, stats); newmemid.memkind = MI_MEM_OS_REMAP;
newmemid.mem.os.prim_info = remap_info;
void* newp = mi_os_align_within(&newmemid, alignment, newsize, stats);
// now map the new virtual adress range to physical memory
// this also releases the old virtual memory range (if there is no error)
bool extend_is_zero = false;
err = _mi_prim_remap_to(memid->mem.os.base, p, size, newp, newsize, &extend_is_zero, &memid->mem.os.prim_info, &newmemid.mem.os.prim_info);
if (err != 0) {
_mi_warning_message("failed to remap OS memory (error %d (0x%02x) at %p of %zu bytes to %zu bytes)\n", err, err, p, 0, size);
_mi_prim_remap_free(newmemid.mem.os.base, newmemid.mem.os.size, newmemid.mem.os.prim_info);
return NULL;
}
newmemid.initially_committed = true;
if (p == NULL && extend_is_zero) {
newmemid.initially_zero = true;
}
*memid = newmemid; *memid = newmemid;
return newp; return newp;
} }
// // fall back to copy (but in remappable memory if possible)
// mi_memid_t newmemid = _mi_memid_none();
// void* newp = _mi_os_alloc_remappable(newsize, memid->mem.os.alignment, &newmemid, stats);
// if (newp == NULL) {
// newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, true /* commit */, false /* allow_large */, &newmemid, stats);
// if (newp == NULL) return NULL;
// }
//
// size_t csize = (size > newsize ? newsize : size);
// _mi_memcpy_aligned(newp, p, csize);
// _mi_os_free(p, size, *memid, stats);
// *memid = newmemid;
// return newp;
//}
/* ----------------------------------------------------------- /* -----------------------------------------------------------

91
src/prim/unix/prim.c
View file

@ -873,74 +873,39 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
// Remappable memory // Remappable memory
//---------------------------------------------------------------- //----------------------------------------------------------------
static int mi_unix_alloc_aligned(size_t size, size_t alignment, bool commit, bool* is_pinned, bool* is_zero, void** addr) int _mi_prim_remap_reserve(size_t size, bool* is_pinned, void** base, void** remap_info) {
{ mi_assert_internal((size%_mi_os_page_size()) == 0);
mi_assert_internal(alignment <= 1 || (alignment >= _mi_os_page_size())); *remap_info = NULL;
*addr = NULL; bool is_zero = false;
void* base = NULL; int err = _mi_prim_alloc(size, 1, false /* commit */, false /*allow large*/, is_pinned, &is_zero, base);
int err = _mi_prim_alloc(size,alignment,commit,false /*allow large*/, is_pinned, is_zero, &base);
if (err != 0) return err; if (err != 0) return err;
if (_mi_is_aligned(base,alignment)) { return 0;
*addr = base; }
return 0;
} int _mi_prim_remap_to(void* base, void* addr, size_t size, void* newaddr, size_t newsize, bool* extend_is_zero, void** remap_info, void** new_remap_info)
_mi_prim_free(base,size); {
const size_t oversize = _mi_align_up( _mi_align_up(size,alignment), _mi_os_page_size()); mi_assert_internal(base <= addr);
err = _mi_prim_alloc(oversize,alignment,commit,false,is_pinned,is_zero,&base); mi_assert_internal((size % _mi_os_page_size()) == 0);
if (err != 0) return err; mi_assert_internal((newsize % _mi_os_page_size()) == 0);
mi_assert_internal(!(*is_pinned)); *new_remap_info = NULL;
if (!(*is_pinned)) { *remap_info = NULL;
void* p = _mi_align_up_ptr(base,alignment); *extend_is_zero = false; // todo: can we assume zero'd?
*addr = p; int err = 0;
size_t pre_size = (uint8_t*)p - (uint8_t*)base; if (addr == NULL) {
size_t mid_size = _mi_align_up(size,_mi_os_page_size()); err = _mi_prim_commit(newaddr, newsize, extend_is_zero);
size_t post_size = oversize - pre_size - mid_size;
mi_assert_internal(pre_size < oversize && post_size < oversize && mid_size >= size);
if (pre_size > 0) { _mi_prim_free(base, pre_size); }
if (post_size > 0) { _mi_prim_free((uint8_t*)p + mid_size, post_size); }
return 0;
} }
else { else {
_mi_prim_free(base,oversize); void* p = mremap(addr, size, newsize, (MREMAP_MAYMOVE | MREMAP_FIXED), newaddr);
return EINVAL; if (p == MAP_FAILED || newaddr != p) {
} p = NULL;
} err = errno;
int _mi_prim_alloc_remappable(size_t size, size_t alignment, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
#if !defined(MREMAP_MAYMOVE)
MI_UNUSED(size); MI_UNUSED(alignment); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
return EINVAL;
#else
*remap_info = NULL;
return mi_unix_alloc_aligned(size, alignment, true, is_pinned, is_zero, addr);
#endif
}
int _mi_prim_remap(void* addr, size_t size, size_t newsize, size_t alignment, bool* extend_is_zero, void** newaddr, void** remap_info ) {
#if !defined(MREMAP_MAYMOVE) || !defined(MREMAP_FIXED)
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(alignment); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
return EINVAL;
#else
mi_assert_internal(*remap_info == NULL); MI_UNUSED(remap_info);
void* p = NULL;
bool is_pinned = false;
// don't commit yet, mremap will take over the virtual rang completely due to MREMAP_FIXED
int err = mi_unix_alloc_aligned(size, alignment, false, &is_pinned, extend_is_zero, &p);
if (err != 0) return err;
void* res = mremap(addr, size, newsize, MREMAP_MAYMOVE | MREMAP_FIXED, p);
if (res == MAP_FAILED || res != p) {
err = errno;
_mi_prim_free(p,size);
return err;
} }
*extend_is_zero = true; }
*newaddr = p; return err;
return 0;
#endif
} }
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { int _mi_prim_remap_free(void* base, size_t size, void* remap_info) {
MI_UNUSED(remap_info); MI_UNUSED(remap_info);
mi_assert_internal(remap_info == NULL); mi_assert_internal((size % _mi_os_page_size()) == 0);
return _mi_prim_free(addr,size); return _mi_prim_free(base,size);
} }

12
src/prim/wasi/prim.c
View file

@ -279,17 +279,17 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
// Remappable memory // Remappable memory
//---------------------------------------------------------------- //----------------------------------------------------------------
int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) { int _mi_prim_remap_reserve(size_t size, bool* is_pinned, void** base, void** remap_info) {
MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info); MI_UNUSED(size); MI_UNUSED(is_pinned); MI_UNUSED(base); MI_UNUSED(remap_info);
return EINVAL; return EINVAL;
} }
int _mi_prim_remap(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) { int _mi_prim_remap_to(void* base, void* addr, size_t size, void* newaddr, size_t newsize, bool* extend_is_zero, void** remap_info, void** new_remap_info) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info); MI_UNUSED(base); MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newaddr); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(remap_info); MI_UNUSED(new_remap_info);
return EINVAL; return EINVAL;
} }
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) { int _mi_prim_remap_free(void* base, size_t size, void* remap_info) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info); MI_UNUSED(base); MI_UNUSED(size); MI_UNUSED(remap_info);
return EINVAL; return EINVAL;
} }

235
src/prim/windows/prim.c
View file

@ -178,7 +178,7 @@ int _mi_prim_free(void* addr, size_t size ) {
// VirtualAlloc // VirtualAlloc
//--------------------------------------------- //---------------------------------------------
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) { static void* mi_win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
#if (MI_INTPTR_SIZE >= 8) #if (MI_INTPTR_SIZE >= 8)
// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
if (addr == NULL) { if (addr == NULL) {
@ -207,7 +207,7 @@ static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignmen
return VirtualAlloc(addr, size, flags, PAGE_READWRITE); return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
} }
static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
mi_assert_internal(!(large_only && !allow_large)); mi_assert_internal(!(large_only && !allow_large));
static _Atomic(size_t) large_page_try_ok; // = 0; static _Atomic(size_t) large_page_try_ok; // = 0;
void* p = NULL; void* p = NULL;
@ -223,7 +223,7 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW
else { else {
// large OS pages must always reserve and commit. // large OS pages must always reserve and commit.
*is_large = true; *is_large = true;
p = win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES); p = mi_win_virtual_alloc_prim(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
if (large_only) return p; if (large_only) return p;
// fall back to non-large page allocation on error (`p == NULL`). // fall back to non-large page allocation on error (`p == NULL`).
if (p == NULL) { if (p == NULL) {
@ -234,7 +234,7 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW
// Fall back to regular page allocation // Fall back to regular page allocation
if (p == NULL) { if (p == NULL) {
*is_large = ((flags&MEM_LARGE_PAGES) != 0); *is_large = ((flags&MEM_LARGE_PAGES) != 0);
p = win_virtual_alloc_prim(addr, size, try_alignment, flags); p = mi_win_virtual_alloc_prim(addr, size, try_alignment, flags);
} }
//if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); } //if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x, large only: %d, allow large: %d)\n", size, GetLastError(), addr, try_alignment, flags, large_only, allow_large); }
return p; return p;
@ -247,7 +247,7 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la
*is_zero = true; *is_zero = true;
int flags = MEM_RESERVE; int flags = MEM_RESERVE;
if (commit) { flags |= MEM_COMMIT; } if (commit) { flags |= MEM_COMMIT; }
*addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); *addr = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
return (*addr != NULL ? 0 : (int)GetLastError()); return (*addr != NULL ? 0 : (int)GetLastError());
} }
@ -627,28 +627,16 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
// Remappable memory // Remappable memory
//---------------------------------------------------------------- //----------------------------------------------------------------
// tracks allocated physical memory backing a virtual address range
typedef struct mi_win_remap_info_s { typedef struct mi_win_remap_info_s {
void* base; // base of the virtual address space
size_t base_pages; // total virtual alloc'd pages from `base`
size_t alignment; // alignment: _mi_align_up_ptr(mapped_base,alignment) maps to the first physical page (= `mapped_base`)
size_t page_count; // allocated physical pages (with info in page_info) size_t page_count; // allocated physical pages (with info in page_info)
size_t page_reserved; // available entries in page_info size_t page_reserved; // available entries in page_info
ULONG_PTR page_info[1]; ULONG_PTR page_info[1];
} mi_win_remap_info_t; } mi_win_remap_info_t;
static void* mi_win_mapped_base(mi_win_remap_info_t* rinfo) {
return _mi_align_up_ptr(rinfo->base, rinfo->alignment);
}
static size_t mi_win_mapped_pages(mi_win_remap_info_t* rinfo) {
const size_t presize = (uint8_t*)mi_win_mapped_base(rinfo) - (uint8_t*)rinfo->base;
return (rinfo->base_pages - _mi_divide_up(presize, _mi_os_page_size()));
}
// allocate remap info // allocate remap info
static mi_win_remap_info_t* mi_win_alloc_remap_info(size_t page_count, size_t alignment) { static mi_win_remap_info_t* mi_win_alloc_remap_info(size_t page_count) {
const size_t remap_info_size = _mi_align_up(sizeof(mi_win_remap_info_t) + (page_count * sizeof(ULONG_PTR)), _mi_os_page_size()); const size_t remap_info_size = _mi_align_up(sizeof(mi_win_remap_info_t) + (page_count * sizeof(ULONG_PTR)), _mi_os_page_size());
mi_win_remap_info_t* rinfo = NULL; mi_win_remap_info_t* rinfo = NULL;
bool os_is_zero = false; bool os_is_zero = false;
@ -656,29 +644,22 @@ static mi_win_remap_info_t* mi_win_alloc_remap_info(size_t page_count, size_t al
int err = _mi_prim_alloc(remap_info_size, 1, true, false, &os_is_large, &os_is_zero, (void**)&rinfo); int err = _mi_prim_alloc(remap_info_size, 1, true, false, &os_is_large, &os_is_zero, (void**)&rinfo);
if (err != 0) return NULL; if (err != 0) return NULL;
if (!os_is_zero) { _mi_memzero_aligned(rinfo, remap_info_size); } if (!os_is_zero) { _mi_memzero_aligned(rinfo, remap_info_size); }
rinfo->base = NULL;
rinfo->base_pages = 0;
rinfo->alignment = alignment;
rinfo->page_count = 0; rinfo->page_count = 0;
rinfo->page_reserved = (remap_info_size - offsetof(mi_win_remap_info_t, page_info)) / sizeof(ULONG_PTR); rinfo->page_reserved = (remap_info_size - offsetof(mi_win_remap_info_t, page_info)) / sizeof(ULONG_PTR);
return rinfo; return rinfo;
} }
// reallocate remap info // reallocate remap info
static mi_win_remap_info_t* mi_win_realloc_remap_info(mi_win_remap_info_t* rinfo, size_t newpage_count, size_t alignment) { static mi_win_remap_info_t* mi_win_realloc_remap_info(mi_win_remap_info_t* rinfo, size_t newpage_count) {
if (rinfo == NULL) { if (rinfo == NULL) {
return mi_win_alloc_remap_info(newpage_count,alignment); return mi_win_alloc_remap_info(newpage_count);
} }
else if (rinfo->page_reserved >= newpage_count) { else if (rinfo->page_reserved >= newpage_count) {
mi_assert_internal(alignment <= rinfo->alignment);
return rinfo; // still fits return rinfo; // still fits
} }
else { else {
mi_assert_internal(alignment <= rinfo->alignment); mi_win_remap_info_t* newrinfo = mi_win_alloc_remap_info(newpage_count);
mi_win_remap_info_t* newrinfo = mi_win_alloc_remap_info(newpage_count,rinfo->alignment);
if (newrinfo == NULL) return NULL; if (newrinfo == NULL) return NULL;
newrinfo->base = rinfo->base;
newrinfo->base_pages = rinfo->base_pages;
newrinfo->page_count = rinfo->page_count; newrinfo->page_count = rinfo->page_count;
_mi_memcpy(newrinfo->page_info, rinfo->page_info, rinfo->page_count * sizeof(ULONG_PTR)); _mi_memcpy(newrinfo->page_info, rinfo->page_info, rinfo->page_count * sizeof(ULONG_PTR));
_mi_prim_free(rinfo, sizeof(mi_win_remap_info_t) + ((rinfo->page_reserved - 1) * sizeof(ULONG_PTR))); _mi_prim_free(rinfo, sizeof(mi_win_remap_info_t) + ((rinfo->page_reserved - 1) * sizeof(ULONG_PTR)));
@ -686,14 +667,32 @@ static mi_win_remap_info_t* mi_win_realloc_remap_info(mi_win_remap_info_t* rinfo
} }
} }
// free meta info and the managed physical pages
static int mi_win_free_remap_info(mi_win_remap_info_t* rinfo) {
int err = 0;
if (rinfo == NULL) return 0;
if (rinfo->page_count > 0) {
size_t req_pages = rinfo->page_count;
if (!FreeUserPhysicalPages(GetCurrentProcess(), &req_pages, &rinfo->page_info[0])) {
err = (int)GetLastError();
}
rinfo->page_count = 0;
}
if (!VirtualFree(rinfo, 0, MEM_RELEASE)) {
err = (int)GetLastError();
}
return err;
}
// ensure enough physical pages are allocated // ensure enough physical pages are allocated
static int mi_win_ensure_physical_pages(mi_win_remap_info_t** prinfo, size_t newpage_count, size_t alignment) { static int mi_win_ensure_physical_pages(mi_win_remap_info_t** prinfo, size_t newpage_count)
{
// ensure meta data is large enough // ensure meta data is large enough
mi_win_remap_info_t* rinfo = mi_win_realloc_remap_info(*prinfo, newpage_count, alignment); mi_win_remap_info_t* rinfo = mi_win_realloc_remap_info(*prinfo, newpage_count);
if (rinfo == NULL) return ENOMEM; if (rinfo == NULL) return ENOMEM;
*prinfo = rinfo; *prinfo = rinfo;
// allocate physical pages // allocate physical pages; todo: allow shrinking?
if (newpage_count > rinfo->page_count) { if (newpage_count > rinfo->page_count) {
mi_assert_internal(rinfo->page_reserved >= newpage_count); mi_assert_internal(rinfo->page_reserved >= newpage_count);
const size_t extra_pages = newpage_count - rinfo->page_count; const size_t extra_pages = newpage_count - rinfo->page_count;
@ -702,133 +701,101 @@ static int mi_win_ensure_physical_pages(mi_win_remap_info_t** prinfo, size_t new
return (int)GetLastError(); return (int)GetLastError();
} }
rinfo->page_count += req_pages; rinfo->page_count += req_pages;
if (req_pages < extra_pages) return ENOMEM; if (req_pages < extra_pages) {
return ENOMEM;
}
} }
return 0; return 0;
} }
// Remap physical memory to another virtual address range
static int mi_win_remap_virtual_pages(mi_win_remap_info_t* rinfo, void* oldaddr, size_t oldpage_count, void* newaddr, size_t newpage_count) {
static int mi_win_remap_virtual_pages(mi_win_remap_info_t* rinfo, size_t newpage_count) {
mi_assert_internal(rinfo != NULL && rinfo->page_count >= newpage_count); mi_assert_internal(rinfo != NULL && rinfo->page_count >= newpage_count);
if (mi_win_mapped_pages(rinfo) >= newpage_count) return 0; // still good? should we shrink the mapping?
if (rinfo->base != NULL) {
printf("remap existing remap\n");
}
// we can now free the original virtual range ?
if (rinfo->base != NULL) {
if (!VirtualFree(rinfo->base, 0, MEM_RELEASE)) {
return (int)GetLastError();
}
}
// unmap the old range // unmap the old range
/*if (rinfo->mapped_base != NULL) { if (oldaddr != NULL) {
if (!MapUserPhysicalPages(rinfo->mapped_base, rinfo->mapped_pages, NULL)) { if (!MapUserPhysicalPages(oldaddr, oldpage_count, NULL)) {
return (int)GetLastError(); return (int)GetLastError();
} }
}*/
// allocate new virtual address range
const size_t newsize = _mi_align_up( newpage_count * _mi_os_page_size() + rinfo->alignment - 1, _mi_os_page_size());
void* newbase = VirtualAlloc(NULL, newsize, MEM_RESERVE | MEM_PHYSICAL, PAGE_READWRITE);
if (newbase == NULL) {
// todo: remap old range?
return (int)GetLastError();
}
if (rinfo->base != newbase) {
printf("different base %p vs previous %p\n", newbase, rinfo->base);
} }
// find the aligned point where we map to our physical pages // and remap the virtual addresses from newbase
rinfo->base = newbase; if (!MapUserPhysicalPages(newaddr, newpage_count, &rinfo->page_info[0])) {
rinfo->base_pages = _mi_divide_up(newsize, _mi_os_page_size()); // if this fails that would be very bad as we already unmapped the old range..
mi_assert_internal(mi_win_mapped_pages(rinfo) >= newpage_count); // todo: try to remap old range?
// and remap it
if (!MapUserPhysicalPages(mi_win_mapped_base(rinfo), newpage_count, &rinfo->page_info[0])) {
// todo: remap old range?
int err = (int)GetLastError(); int err = (int)GetLastError();
VirtualFree(newbase, 0, MEM_RELEASE); VirtualFree(newaddr, 0, MEM_RELEASE);
return err; return err;
} }
return 0; return 0;
} }
static int mi_win_remap(mi_win_remap_info_t** prinfo, size_t newsize, size_t alignment) { // Reserve a virtual address range to be mapped to physical memory later
size_t newpage_count = _mi_divide_up(newsize, _mi_os_page_size()); int _mi_prim_remap_reserve(size_t size, bool* is_pinned, void** base, void** remap_info) {
int err = mi_win_ensure_physical_pages(prinfo, newpage_count, alignment); if (!mi_win_enable_large_os_pages(NULL)) return EINVAL;
if (err != 0) return err; mi_assert_internal((size % _mi_os_page_size()) == 0);
err = mi_win_remap_virtual_pages(*prinfo, newpage_count); size = _mi_align_up(size, _mi_os_page_size());
if (err != 0) return err; mi_win_remap_info_t** prinfo = (mi_win_remap_info_t**)remap_info;
*prinfo = NULL;
*is_pinned = true;
*base = NULL;
void* p = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_PHYSICAL, PAGE_READWRITE);
if (p == NULL) { return (int)GetLastError(); }
*base = p;
return 0; return 0;
} }
static int mi_win_free_remap_info(mi_win_remap_info_t* rinfo) { // Remap to a new virtual address range
int _mi_prim_remap_to(void* base, void* addr, size_t size, void* newaddr, size_t newsize, bool* extend_is_zero, void** remap_info, void** new_remap_info)
{
*extend_is_zero = false; // todo: can we assume zero's ?
mi_win_remap_info_t**prinfo = (mi_win_remap_info_t**)remap_info;
mi_win_remap_info_t** pnewrinfo = (mi_win_remap_info_t**)new_remap_info;
mi_assert_internal(base <= addr);
mi_assert_internal(*pnewrinfo == NULL);
mi_assert_internal((size % _mi_os_page_size()) == 0);
mi_assert_internal((newsize % _mi_os_page_size()) == 0);
size = _mi_align_up(size, _mi_os_page_size());
newsize = _mi_align_up(newsize, _mi_os_page_size());
size_t oldpage_count = _mi_divide_up(size, _mi_os_page_size());
size_t newpage_count = _mi_divide_up(newsize, _mi_os_page_size());
int err = mi_win_ensure_physical_pages(prinfo, newpage_count);
if (err != 0) { return err; }
err = mi_win_remap_virtual_pages(*prinfo, addr, oldpage_count, newaddr, newpage_count);
if (err != 0) { return err; }
// release old virtual range
if (base != NULL) {
if (!VirtualFree(base, 0, MEM_RELEASE)) {
err = (int)GetLastError();
_mi_warning_message("unable to release virtual address range on remap (error %d (0x%02x), address: %p)\n", err, err, base);
}
}
*pnewrinfo = *prinfo;
*prinfo = NULL;
return 0;
}
// Free remappable memory
int _mi_prim_remap_free(void* base, size_t size, void* remap_info) {
MI_UNUSED(size);
int err = 0; int err = 0;
if (rinfo == NULL) return 0; // release virtual address range
if (rinfo->base != NULL) { if (base != NULL) {
if (!VirtualFree(rinfo->base, 0, MEM_RELEASE)) { if (!VirtualFree(base, 0, MEM_RELEASE)) {
err = (int)GetLastError(); err = (int)GetLastError();
} }
rinfo->base = NULL;
rinfo->base_pages = 0;
} }
if (rinfo->page_count > 0) { // release backing physical pages
size_t req_pages = rinfo->page_count; mi_win_remap_info_t* rinfo = (mi_win_remap_info_t*)remap_info;
if (!FreeUserPhysicalPages(GetCurrentProcess(), &req_pages, &rinfo->page_info[0])) { if (rinfo != NULL) {
err = (int)GetLastError(); err = mi_win_free_remap_info(rinfo);
}
rinfo->page_count = 0;
} }
VirtualFree(rinfo, 0, MEM_RELEASE);
return err; return err;
} }
int _mi_prim_alloc_remappable(size_t size, size_t alignment, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
// MI_UNUSED(size); MI_UNUSED(alignment); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
// return EINVAL;
// return _mi_prim_alloc(size, 1, true, true, is_pinned, is_zero, addr);
if (!mi_win_enable_large_os_pages(NULL)) return EINVAL;
mi_win_remap_info_t* rinfo = NULL;
int err = mi_win_remap(&rinfo, size, alignment);
if (err != 0) {
if (rinfo != NULL) { mi_win_free_remap_info(rinfo); }
return err;
}
*is_pinned = true;
*is_zero = true;
*addr = mi_win_mapped_base(rinfo);
*remap_info = rinfo;
return 0;
}
int _mi_prim_remap(void* addr, size_t size, size_t newsize, size_t alignment, bool* extend_is_zero, void** newaddr, void** remap_info ) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
// return EINVAL;
mi_win_remap_info_t** prinfo = (mi_win_remap_info_t**)remap_info;
mi_assert_internal(*prinfo != NULL);
mi_assert_internal(mi_win_mapped_base(*prinfo) == addr);
mi_assert_internal((*prinfo)->alignment == alignment);
int err = mi_win_remap(prinfo, newsize, alignment);
if (err != 0) return err;
*extend_is_zero = false;
*newaddr = mi_win_mapped_base(*prinfo);
return 0;
}
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info);
// return _mi_prim_free(addr, size);
// return EINVAL;
mi_win_remap_info_t* rinfo = (mi_win_remap_info_t*)remap_info;
mi_assert_internal(rinfo != NULL);
mi_assert_internal(mi_win_mapped_base(rinfo) == addr);
return mi_win_free_remap_info(rinfo);
}