kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-04 22:49:32 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

rewrite of low-level OS (aligned) allocation to better handle large pages and aligned allocation

Browse source
This commit is contained in:
daan 2019-07-03 14:52:32 -07:00
parent 7b4f3591f0
commit c3a5f84ad3
2 changed files with 225 additions and 232 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/init.c
View file

@ -103,7 +103,11 @@ mi_heap_t _mi_heap_main = {
NULL, NULL,
0, 0,
0, 0,
0xCDCDCDCDCDCDCDL, #if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!)
0xCDCDCDCDCDCDCDCDUL,
#else
0xCDCDCDCDUL,
#endif
0, 0,
false // can reclaim false // can reclaim
}; };

451
src/os.c
View file

@ -14,18 +14,38 @@ terms of the MIT license. A copy of the license can be found in the file
#include <string.h> // memset #include <string.h> // memset
#include <errno.h> #include <errno.h>
#if defined(_WIN32)
#include <windows.h>
#else
#include <sys/mman.h> // mmap
#include <unistd.h> // sysconf
#endif
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Initialization. Initialization.
On windows initializes support for aligned allocation and On windows initializes support for aligned allocation and
large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
----------------------------------------------------------- */ ----------------------------------------------------------- */
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
#if defined(_WIN32) uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
#include <windows.h> uintptr_t x = (sz / alignment) * alignment;
#else if (x < sz) x += alignment;
#include <sys/mman.h> // mmap if (x < sz) return 0; // overflow
#include <unistd.h> // sysconf return x;
#endif }
static void* mi_align_up_ptr(void* p, size_t alignment) {
return (void*)_mi_align_up((uintptr_t)p, alignment);
}
static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
return (sz / alignment) * alignment;
}
static void* mi_align_down_ptr(void* p, size_t alignment) {
return (void*)_mi_align_down((uintptr_t)p, alignment);
}
// page size (initialized properly in `os_init`) // page size (initialized properly in `os_init`)
static size_t os_page_size = 4096; static size_t os_page_size = 4096;
@ -61,8 +81,8 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
#if defined(_WIN32) #if defined(_WIN32)
// We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016.
// So, we need to look it up dynamically to run on older systems. // So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility)
typedef PVOID (__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG ); typedef PVOID(__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
static VirtualAlloc2Ptr pVirtualAlloc2 = NULL; static VirtualAlloc2Ptr pVirtualAlloc2 = NULL;
void _mi_os_init(void) { void _mi_os_init(void) {
@ -74,7 +94,7 @@ void _mi_os_init(void) {
// get the VirtualAlloc2 function // get the VirtualAlloc2 function
HINSTANCE hDll; HINSTANCE hDll;
hDll = LoadLibrary("kernelbase.dll"); hDll = LoadLibrary("kernelbase.dll");
if (hDll!=NULL) { if (hDll != NULL) {
// use VirtualAlloc2FromApp as it is available to Windows store apps // use VirtualAlloc2FromApp as it is available to Windows store apps
pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp"); pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp");
FreeLibrary(hDll); FreeLibrary(hDll);
@ -106,7 +126,7 @@ void _mi_os_init(void) {
CloseHandle(token); CloseHandle(token);
} }
if (!ok) { if (!ok) {
if (err==0) err = GetLastError(); if (err == 0) err = GetLastError();
_mi_warning_message("cannot enable large OS page support, error %lu\n", err); _mi_warning_message("cannot enable large OS page support, error %lu\n", err);
} }
} }
@ -120,37 +140,16 @@ void _mi_os_init() {
os_alloc_granularity = os_page_size; os_alloc_granularity = os_page_size;
} }
if (mi_option_is_enabled(mi_option_large_os_pages)) { if (mi_option_is_enabled(mi_option_large_os_pages)) {
large_os_page_size = (1UL<<21); // 2MiB large_os_page_size = (1UL << 21); // 2MiB
} }
} }
#endif #endif
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Raw allocation on Windows (VirtualAlloc) and Unix's (mmap). Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).
Defines a portable `mmap`, `munmap` and `mmap_trim`.
----------------------------------------------------------- */ ----------------------------------------------------------- */
uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
uintptr_t x = (sz / alignment) * alignment;
if (x < sz) x += alignment;
if (x < sz) return 0; // overflow
return x;
}
static void* mi_align_up_ptr(void* p, size_t alignment) {
return (void*)_mi_align_up((uintptr_t)p, alignment);
}
static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
return (sz / alignment) * alignment;
}
static void* mi_align_down_ptr(void* p, size_t alignment) {
return (void*)_mi_align_down((uintptr_t)p, alignment);
}
static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats) static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
{ {
if (addr == NULL || size == 0) return true; if (addr == NULL || size == 0) return true;
@ -163,7 +162,7 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
_mi_stat_decrease(&stats->committed, size); // TODO: what if never committed? _mi_stat_decrease(&stats->committed, size); // TODO: what if never committed?
_mi_stat_decrease(&stats->reserved, size); _mi_stat_decrease(&stats->reserved, size);
if (err) { if (err) {
#pragma warning(suppress:4996) #pragma warning(suppress:4996)
_mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size); _mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size);
return false; return false;
} }
@ -172,38 +171,53 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
} }
} }
static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_flags, mi_stats_t* stats) { #ifdef _WIN32
if (size == 0) return NULL; static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
// on modern Windows try use VirtualAlloc2
MEM_ADDRESS_REQUIREMENTS reqs = { 0 };
reqs.Alignment = try_alignment;
MEM_EXTENDED_PARAMETER param = { 0 };
param.Type = MemExtendedParameterAddressRequirements;
param.Pointer = &reqs;
return (*pVirtualAlloc2)(addr, NULL, size, flags, PAGE_READWRITE, &param, 1);
}
#endif
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) {
void* p = NULL; void* p = NULL;
#if defined(_WIN32) if (use_large_os_page(size, try_alignment)) {
int flags = MEM_RESERVE | extra_flags; p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | flags);
if (commit) flags |= MEM_COMMIT; // fall back to non-large page allocation on error (`p == NULL`).
if (use_large_os_page(size, 0)) {
p = VirtualAlloc(addr, size, MEM_LARGE_PAGES | flags, PAGE_READWRITE);
} }
if (p == NULL) { if (p == NULL) {
p = VirtualAlloc(addr, size, flags, PAGE_READWRITE); p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
} }
return p;
}
#else #else
static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) {
void* p = NULL;
#if !defined(MAP_ANONYMOUS) #if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON #define MAP_ANONYMOUS MAP_ANON
#endif #endif
int flags = MAP_PRIVATE | MAP_ANONYMOUS | extra_flags; int flags = MAP_PRIVATE | MAP_ANONYMOUS;
if (addr != NULL) { #if defined(MAP_ALIGNED) // BSD
#if defined(MAP_EXCL) if (try_alignment > 0) {
flags |= MAP_FIXED | MAP_EXCL; // BSD size_t n = _mi_bsr(try_alignment);
#elif defined(MAP_FIXED_NOREPLACE) if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
flags |= MAP_FIXED_NOREPLACE; // Linux flags |= MAP_ALIGNED(n);
#elif defined(MAP_FIXED) }
flags |= MAP_FIXED;
#endif
} }
int pflags = (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE);
#if defined(PROT_MAX)
pflags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
#endif #endif
#if defined(PROT_MAX)
if (large_os_page_size > 0 && use_large_os_page(size, 0) && ((uintptr_t)addr % large_os_page_size) == 0) { protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
#endif
if (large_os_page_size > 0 && use_large_os_page(size, try_alignment)) {
int lflags = flags; int lflags = flags;
#ifdef MAP_ALIGNED_SUPER #ifdef MAP_ALIGNED_SUPER
lflags |= MAP_ALIGNED_SUPER; lflags |= MAP_ALIGNED_SUPER;
@ -215,66 +229,145 @@ static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_fla
lflags |= MAP_HUGE_2MB; lflags |= MAP_HUGE_2MB;
#endif #endif
if (lflags != flags) { if (lflags != flags) {
// try large page allocation // try large page allocation
p = mmap(addr, size, pflags, lflags, -1, 0); // TODO: if always failing due to permissions or no huge pages, try to avoid repeatedly trying?
// Should we check this in _mi_os_init? (as on Windows)
p = mmap(NULL, size, protect_flags, lflags, -1, 0);
if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission
} }
} }
if (p == NULL) { if (p == NULL) {
p = mmap(addr, size, pflags, flags, -1, 0); p = mmap(NULL, size, protect_flags, flags, -1, 0);
if (p == MAP_FAILED) p = NULL; if (p == MAP_FAILED) p = NULL;
} }
if (addr != NULL && p != addr) { return p;
mi_os_mem_free(p, size, stats); }
p = NULL;
}
#endif #endif
mi_assert(p == NULL || (addr == NULL && p != addr) || (addr != NULL && p == addr));
if (p != NULL) {
_mi_stat_increase(&stats->mmap_calls, 1);
_mi_stat_increase(&stats->reserved, size);
if (commit) _mi_stat_increase(&stats->committed, size);
}
return p;
}
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) { // Primitive allocation from the OS.
if (alignment < _mi_os_page_size() || ((alignment & (~alignment + 1)) != alignment)) return NULL; // Note: the `alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
if (size == 0) return NULL;
void* p = NULL; void* p = NULL;
#if defined(_WIN32) && defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) #if defined(_WIN32)
if (pVirtualAlloc2 != NULL) { int flags = MEM_RESERVE;
// on modern Windows try use VirtualAlloc2 if (commit) flags |= MEM_COMMIT;
MEM_ADDRESS_REQUIREMENTS reqs = {0}; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags);
reqs.Alignment = alignment; #else
MEM_EXTENDED_PARAMETER param = { 0 }; int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
param.Type = MemExtendedParameterAddressRequirements; p = mi_unix_mmap(size, try_alignment, protect_flags);
param.Pointer = &reqs; #endif
DWORD flags = MEM_RESERVE; _mi_stat_increase(&stats->mmap_calls, 1);
if (commit) flags |= MEM_COMMIT;
if (use_large_os_page(size, alignment)) flags |= MEM_LARGE_PAGES;
p = (*pVirtualAlloc2)(NULL, NULL, size, flags, PAGE_READWRITE, &param, 1);
}
#elif defined(MAP_ALIGNED)
// on BSD, use the aligned mmap api
size_t n = _mi_bsr(alignment);
if (((size_t)1 << n) == alignment && n >= 12) { // alignment is a power of 2 and >= 4096
p = mi_os_mem_alloc(suggest, size, commit, MAP_ALIGNED(n), tld->stats); // use the NetBSD/freeBSD aligned flags
}
#else
UNUSED(size);
UNUSED(alignment);
#endif
mi_assert(p == NULL || (uintptr_t)p % alignment == 0);
if (p != NULL) { if (p != NULL) {
_mi_stat_increase(&stats->mmap_calls, 1);
_mi_stat_increase(&stats->reserved, size); _mi_stat_increase(&stats->reserved, size);
if (commit) _mi_stat_increase(&stats->committed, size); if (commit) _mi_stat_increase(&stats->committed, size);
} }
return p; return p;
} }
// OS page align within a given area,
// either conservative (pages inside the area only), // Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned.
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) {
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = _mi_align_up(size, _mi_os_page_size());
// try first with a hint (this will be aligned directly on Win 10+ or BSD)
void* p = mi_os_mem_alloc(size, alignment, commit, stats);
if (p == NULL) return NULL;
// if not aligned, free it, overallocate, and unmap around it
if (((uintptr_t)p % alignment != 0)) {
mi_os_mem_free(p, size, stats);
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
size_t over_size = size + alignment;
#if _WIN32
// over-allocate and than re-allocate exactly at an aligned address in there.
// this may fail due to threads allocating at the same time so we
// retry this at most 3 times before giving up.
// (we can not decommit around the overallocation on Windows, because we can only
// free the original pointer, not one pointing inside the area)
int flags = MEM_RESERVE;
if (commit) flags |= MEM_COMMIT;
for (int tries = 0; tries < 3; tries++) {
// over-allocate to determine a virtual memory range
p = mi_os_mem_alloc(over_size, alignment, commit, stats);
if (p == NULL) return NULL; // error
if (((uintptr_t)p % alignment) == 0) {
// if p happens to be aligned, just decommit the left-over area
_mi_os_decommit((uint8_t*)p + size, over_size - size, stats);
break;
}
else {
// otherwise free and allocate at an aligned address in there
mi_os_mem_free(p, over_size, stats);
void* aligned_p = mi_align_up_ptr(p, alignment);
p = mi_win_virtual_alloc(aligned_p, size, alignment, flags);
if (p == aligned_p) break; // success!
if (p != NULL) { // should not happen?
mi_os_mem_free(p, size, stats);
p = NULL;
}
}
}
#else
// overallocate...
p = mi_os_mem_alloc(over_size, alignment, commit, stats);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area.
void* aligned_p = mi_align_up_ptr(p, alignment);
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
if (pre_size > 0) mi_os_mem_free(p, pre_size, stats);
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
#endif
}
mi_assert_internal(p == NULL || (p != NULL && ((uintptr_t)p % alignment) == 0));
return p;
}
/* -----------------------------------------------------------
OS API: alloc, free, alloc_aligned
----------------------------------------------------------- */
void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, 0);
return mi_os_mem_alloc(size, 0, true, stats);
}
void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
if (size == 0 || p == NULL) return;
size = mi_os_good_alloc_size(size, 0);
mi_os_mem_free(p, size, stats);
}
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
{
if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, alignment);
alignment = _mi_align_up(alignment, _mi_os_page_size());
return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats);
}
/* -----------------------------------------------------------
OS memory API: reset, commit, decommit, protect, unprotect.
----------------------------------------------------------- */
// OS page align within a given area, either conservative (pages inside the area only),
// or not (straddling pages outside the area is possible) // or not (straddling pages outside the area is possible)
static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) { static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) {
mi_assert(addr != NULL && size > 0); mi_assert(addr != NULL && size > 0);
@ -283,9 +376,9 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size,
// page align conservatively within the range // page align conservatively within the range
void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size()) void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size())
: mi_align_down_ptr(addr, _mi_os_page_size())); : mi_align_down_ptr(addr, _mi_os_page_size()));
void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size())
: mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size()));
ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start;
if (diff <= 0) return NULL; if (diff <= 0) return NULL;
@ -295,7 +388,7 @@ static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size,
} }
static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) { static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) {
return mi_os_page_align_areax(true,addr,size,newsize); return mi_os_page_align_areax(true, addr, size, newsize);
} }
@ -307,8 +400,8 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
// page align conservatively within the range // page align conservatively within the range
size_t csize; size_t csize;
void* start = mi_os_page_align_area_conservative(addr,size,&csize); void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize==0) return true; if (csize == 0) return true;
_mi_stat_increase(&stats->reset, csize); _mi_stat_increase(&stats->reset, csize);
#if defined(_WIN32) #if defined(_WIN32)
@ -329,17 +422,17 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
*/ */
return true; return true;
#else #else
#if defined(MADV_FREE) #if defined(MADV_FREE)
static int advice = MADV_FREE; static int advice = MADV_FREE;
int err = madvise(start, csize, advice); int err = madvise(start, csize, advice);
if (err!=0 && errno==EINVAL && advice==MADV_FREE) { if (err != 0 && errno == EINVAL && advice == MADV_FREE) {
// if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on
advice = MADV_DONTNEED; advice = MADV_DONTNEED;
err = madvise(start, csize, advice); err = madvise(start, csize, advice);
} }
#else #else
int err = madvise(start, csize, MADV_DONTNEED); int err = madvise(start, csize, MADV_DONTNEED);
#endif #endif
if (err != 0) { if (err != 0) {
_mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno); _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno);
} }
@ -353,32 +446,31 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
// page align conservatively within the range // page align conservatively within the range
size_t csize = 0; size_t csize = 0;
void* start = mi_os_page_align_area_conservative(addr, size, &csize); void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize==0) return false; if (csize == 0) return false;
int err = 0; int err = 0;
#ifdef _WIN32 #ifdef _WIN32
DWORD oldprotect = 0; DWORD oldprotect = 0;
BOOL ok = VirtualProtect(start,csize,protect ? PAGE_NOACCESS : PAGE_READWRITE,&oldprotect); BOOL ok = VirtualProtect(start, csize, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);
err = (ok ? 0 : GetLastError()); err = (ok ? 0 : GetLastError());
#else #else
err = mprotect(start,csize,protect ? PROT_NONE : (PROT_READ|PROT_WRITE)); err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
#endif #endif
if (err != 0) { if (err != 0) {
_mi_warning_message("mprotect error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err); _mi_warning_message("mprotect error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err);
} }
return (err==0); return (err == 0);
} }
bool _mi_os_protect(void* addr, size_t size) { bool _mi_os_protect(void* addr, size_t size) {
return mi_os_protectx(addr,size,true); return mi_os_protectx(addr, size, true);
} }
bool _mi_os_unprotect(void* addr, size_t size) { bool _mi_os_unprotect(void* addr, size_t size) {
return mi_os_protectx(addr, size, false); return mi_os_protectx(addr, size, false);
} }
// Commit/Decommit memory. // Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative.
// We page align to a conservative area inside the range to reset.
static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) { static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) {
// page align in the range, commit liberally, decommit conservative // page align in the range, commit liberally, decommit conservative
size_t csize; size_t csize;
@ -387,7 +479,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stat
int err = 0; int err = 0;
if (commit) { if (commit) {
_mi_stat_increase(&stats->committed, csize); _mi_stat_increase(&stats->committed, csize);
_mi_stat_increase(&stats->commit_calls,1); _mi_stat_increase(&stats->commit_calls, 1);
} }
else { else {
_mi_stat_decrease(&stats->committed, csize); _mi_stat_decrease(&stats->committed, csize);
@ -423,123 +515,20 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
// page align conservatively within the range // page align conservatively within the range
mi_assert_internal(oldsize > newsize && p != NULL); mi_assert_internal(oldsize > newsize && p != NULL);
if (oldsize < newsize || p==NULL) return false; if (oldsize < newsize || p == NULL) return false;
if (oldsize == newsize) return true; if (oldsize == newsize) return true;
// oldsize and newsize should be page aligned or we cannot shrink precisely // oldsize and newsize should be page aligned or we cannot shrink precisely
void* addr = (uint8_t*)p + newsize; void* addr = (uint8_t*)p + newsize;
size_t size = 0; size_t size = 0;
void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size); void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size);
if (size==0 || start != addr) return false; if (size == 0 || start != addr) return false;
#ifdef _WIN32 #ifdef _WIN32
// we cannot shrink on windows, but we can decommit // we cannot shrink on windows, but we can decommit
return _mi_os_decommit(start, size, stats); return _mi_os_decommit(start, size, stats);
#else #else
return mi_os_mem_free(start, size, stats); return mi_os_mem_free(start, size, stats);
#endif
}
/* -----------------------------------------------------------
OS allocation using mmap/munmap
----------------------------------------------------------- */
void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, 0);
void* p = mi_os_mem_alloc(NULL, size, true, 0, stats);
mi_assert(p!=NULL);
return p;
}
void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
if (size==0) return;
size = mi_os_good_alloc_size(size, 0);
mi_os_mem_free(p, size, stats);
}
// Slow but guaranteed way to allocated aligned memory
// by over-allocating and then reallocating at a fixed aligned
// address that should be available then.
static void* mi_os_alloc_aligned_ensured(size_t size, size_t alignment, bool commit, size_t trie, mi_stats_t* stats)
{
if (trie >= 3) return NULL; // stop recursion (only on Windows)
if (size > SIZE_MAX - alignment) return NULL; // overflow
size_t alloc_size = size + alignment; // no need for -1 as we need to be page aligned anyways
// allocate a chunk that includes the alignment
void* p = mi_os_mem_alloc(NULL, alloc_size, commit, 0, stats);
if (p == NULL) return NULL;
// create an aligned pointer in the allocated area
void* aligned_p = mi_align_up_ptr(p, alignment);
mi_assert(aligned_p != NULL);
#if _WIN32
// free it and try to allocate `size` at exactly `aligned_p`
// note: this may fail in case another thread happens to allocate
// concurrently at that spot. We try up to 3 times to mitigate this.
mi_os_mem_free(p, alloc_size, stats);
p = mi_os_mem_alloc(aligned_p, size, commit, 0, stats);
if (p != aligned_p) {
if (p != NULL) mi_os_mem_free(p, size, stats);
return mi_os_alloc_aligned_ensured(size, alignment, commit, trie+1, stats);
}
#else
// we selectively unmap parts around the over-allocated area.
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = alloc_size - pre_size - mid_size;
if (pre_size > 0) mi_os_mem_free(p, pre_size, stats);
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
#endif #endif
mi_assert(((uintptr_t)aligned_p) % alignment == 0);
return aligned_p;
} }
// Allocate an aligned block.
// Since `mi_mmap` is relatively slow we try to allocate directly at first and
// hope to get an aligned address; only when that fails we fall back
// to a guaranteed method by overallocating at first and adjusting.
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
{
if (size == 0) return NULL;
size = mi_os_good_alloc_size(size,alignment);
if (alignment < 1024) return mi_os_mem_alloc(NULL, size, commit, 0, tld->stats);
// try direct OS aligned allocation; only supported on BSD and Windows 10+
void* suggest = NULL;
void* p = mi_os_mem_alloc_aligned(size,alignment,commit,tld->stats);
// Fall back
if (p==NULL && (tld->mmap_next_probable % alignment) == 0) {
// if the next probable address is aligned,
// then try to just allocate `size` and hope it is aligned...
p = mi_os_mem_alloc(suggest, size, commit, 0, tld->stats);
if (p == NULL) return NULL;
if (((uintptr_t)p % alignment) == 0) _mi_stat_increase(&tld->stats->mmap_right_align, 1);
}
//fprintf(stderr, "segment address guess: %s, p=%lxu, guess:%lxu\n", (p != NULL && (uintptr_t)p % alignment ==0 ? "correct" : "incorrect"), (uintptr_t)p, next_probable);
if (p==NULL || ((uintptr_t)p % alignment) != 0) {
// if `p` is not yet aligned after all, free the block and use a slower
// but guaranteed way to allocate an aligned block
if (p != NULL) mi_os_mem_free(p, size, tld->stats);
_mi_stat_increase( &tld->stats->mmap_ensure_aligned, 1);
//fprintf(stderr, "mimalloc: slow mmap 0x%lx\n", _mi_thread_id());
p = mi_os_alloc_aligned_ensured(size, alignment,commit,0,tld->stats);
}
if (p != NULL) {
// next probable address is the page-aligned address just after the newly allocated area.
size_t probable_size = MI_SEGMENT_SIZE;
if (tld->mmap_previous > p) {
// Linux tends to allocate downward
tld->mmap_next_probable = _mi_align_down((uintptr_t)p - probable_size, os_alloc_granularity); // ((uintptr_t)previous - (uintptr_t)p);
}
else {
// Otherwise, guess the next address is page aligned `size` from current pointer
tld->mmap_next_probable = _mi_align_up((uintptr_t)p + probable_size, os_alloc_granularity);
}
tld->mmap_previous = p;
}
return p;
}