diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index e0868f04..b25250f7 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -89,7 +89,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE) #define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) -#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1MiB on 64-bit +#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/8) // 512KiB on 64-bit #define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT) diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c index 7158e920..cd1953c8 100644 --- a/src/alloc-override-win.c +++ b/src/alloc-override-win.c @@ -377,6 +377,7 @@ typedef enum patch_apply_e { typedef struct mi_patch_s { const char* name; // name of the function to patch + int priority; // priority to patch this one (used to prioritize over multiple entries in various dll's) void* original; // the resolved address of the function (or NULL) void* target; // the address of the new target (never NULL) void* target_term;// the address of the target during termination (or NULL) @@ -384,8 +385,8 @@ typedef struct mi_patch_s { mi_jump_t save; // the saved instructions in case it was applied } mi_patch_t; -#define MI_PATCH_NAME3(name,target,term) { name, NULL, &target, &term, false } -#define MI_PATCH_NAME2(name,target) { name, NULL, &target, NULL, false } +#define MI_PATCH_NAME3(name,target,term) { name, 0, NULL, &target, &term, PATCH_NONE } +#define MI_PATCH_NAME2(name,target) { name, 0, NULL, &target, NULL, PATCH_NONE } #define MI_PATCH3(name,target,term) MI_PATCH_NAME3(#name, target, term) #define MI_PATCH2(name,target) MI_PATCH_NAME2(#name, target) #define MI_PATCH1(name) MI_PATCH2(name,mi_##name) @@ -447,7 +448,7 @@ static mi_patch_t patches[] = { MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free, mi_free_term), #endif - { NULL, NULL, NULL, false } + { NULL, 0, NULL, NULL, NULL, PATCH_NONE } }; @@ -522,15 +523,16 @@ static int __cdecl mi_setmaxstdio(int newmax) { // ------------------------------------------------------ // Try to resolve patches for a given module (DLL) -static void mi_module_resolve(HMODULE mod) { +static void mi_module_resolve(HMODULE mod, int priority) { // see if any patches apply for (size_t i = 0; patches[i].name != NULL; i++) { mi_patch_t* patch = &patches[i]; - if (!patch->applied && patch->original==NULL) { + if (!patch->applied && patch->priority < priority) { void* addr = GetProcAddress(mod, patch->name); if (addr != NULL) { // found it! set the address patch->original = addr; + patch->priority = priority; } } } @@ -554,9 +556,11 @@ static bool mi_patches_resolve(void) { size_t count = needed / sizeof(HMODULE); size_t ucrtbase_index = 0; size_t mimalloc_index = 0; - // iterate through the loaded modules - for (size_t i = 0; i < count; i++) { - HMODULE mod = modules[i]; + // iterate through the loaded modules; do this from the end so we prefer the + // first loaded DLL as sometimes both "msvcr" and "ucrt" are both loaded and we should + // override "ucrt" in that situation. + for (size_t i = count; i > 0; i--) { + HMODULE mod = modules[i-1]; char filename[MAX_PATH] = { 0 }; DWORD slen = GetModuleFileName(mod, filename, MAX_PATH); if (slen > 0 && slen < MAX_PATH) { @@ -564,16 +568,19 @@ static bool mi_patches_resolve(void) { filename[slen] = 0; const char* lastsep = strrchr(filename, '\\'); const char* basename = (lastsep==NULL ? filename : lastsep+1); - if (i==0 // main module to allow static crt linking - || _strnicmp(basename, "ucrt", 4) == 0 // new ucrtbase.dll in windows 10 - || _strnicmp(basename, "msvcr", 5) == 0) // older runtimes - { + + int priority = 0; + if (i == 0) priority = 2; // main module to allow static crt linking + else if (_strnicmp(basename, "ucrt", 4) == 0) priority = 3; // new ucrtbase.dll in windows 10 + else if (_strnicmp(basename, "msvcr", 5) == 0) priority = 1; // older runtimes + + if (priority > 0) { // remember indices so we can check load order (in debug mode) if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i; if (_stricmp(basename, UCRTBASE_NAME) == 0) ucrtbase_index = i; // probably found a crt module, try to patch it - mi_module_resolve(mod); + mi_module_resolve(mod,priority); // try to find the atexit functions for the main process (in `ucrtbase.dll`) if (crt_atexit==NULL) crt_atexit = (atexit_fun_t*)GetProcAddress(mod, "_crt_atexit"); diff --git a/src/alloc-override.c b/src/alloc-override.c index 42c9f3d6..7c052690 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -52,7 +52,7 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(realloc), MI_INTERPOSE_MI(free), MI_INTERPOSE_MI(strdup), - MI_INTERPOSE_MI(strndup) + MI_INTERPOSE_MI(strndup) }; #else // On all other systems forward to our API @@ -60,8 +60,8 @@ terms of the MIT license. A copy of the license can be found in the file void* calloc(size_t size, size_t n) mi_attr_noexcept MI_FORWARD2(mi_calloc, size, n); void* realloc(void* p, size_t newsize) mi_attr_noexcept MI_FORWARD2(mi_realloc, p, newsize); void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p); - char* strdup(const char* s) MI_FORWARD1(mi_strdup, s); - char* strndup(const char* s, size_t n) MI_FORWARD2(mi_strndup, s, n); + //char* strdup(const char* s) MI_FORWARD1(mi_strdup, s); + //char* strndup(const char* s, size_t n) MI_FORWARD2(mi_strndup, s, n); #endif #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) diff --git a/src/init.c b/src/init.c index 9f64c439..9b7e60ce 100644 --- a/src/init.c +++ b/src/init.c @@ -103,7 +103,11 @@ mi_heap_t _mi_heap_main = { NULL, 0, 0, - 0xCDCDCDCDCDCDCDL, +#if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!) + 0xCDCDCDCDCDCDCDCDUL, +#else + 0xCDCDCDCDUL, +#endif 0, false // can reclaim }; diff --git a/src/os.c b/src/os.c index dbc2ad01..d5b7cb65 100644 --- a/src/os.c +++ b/src/os.c @@ -14,18 +14,38 @@ terms of the MIT license. A copy of the license can be found in the file #include // memset #include +#if defined(_WIN32) +#include +#else +#include // mmap +#include // sysconf +#endif + /* ----------------------------------------------------------- Initialization. - On windows initializes support for aligned allocation and + On windows initializes support for aligned allocation and large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). ----------------------------------------------------------- */ +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); -#if defined(_WIN32) - #include -#else - #include // mmap - #include // sysconf -#endif +uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { + uintptr_t x = (sz / alignment) * alignment; + if (x < sz) x += alignment; + if (x < sz) return 0; // overflow + return x; +} + +static void* mi_align_up_ptr(void* p, size_t alignment) { + return (void*)_mi_align_up((uintptr_t)p, alignment); +} + +static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { + return (sz / alignment) * alignment; +} + +static void* mi_align_down_ptr(void* p, size_t alignment) { + return (void*)_mi_align_down((uintptr_t)p, alignment); +} // page size (initialized properly in `os_init`) static size_t os_page_size = 4096; @@ -56,13 +76,13 @@ static bool use_large_os_page(size_t size, size_t alignment) { static size_t mi_os_good_alloc_size(size_t size, size_t alignment) { UNUSED(alignment); if (size >= (SIZE_MAX - os_alloc_granularity)) return size; // possible overflow? - return _mi_align_up(size, os_alloc_granularity); + return _mi_align_up(size, os_alloc_granularity); } #if defined(_WIN32) // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. -// So, we need to look it up dynamically to run on older systems. -typedef PVOID (*VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG ); +// So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility) +typedef PVOID(__stdcall *VirtualAlloc2Ptr)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG); static VirtualAlloc2Ptr pVirtualAlloc2 = NULL; void _mi_os_init(void) { @@ -74,9 +94,9 @@ void _mi_os_init(void) { // get the VirtualAlloc2 function HINSTANCE hDll; hDll = LoadLibrary("kernelbase.dll"); - if (hDll!=NULL) { + if (hDll != NULL) { // use VirtualAlloc2FromApp as it is available to Windows store apps - pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp"); + pVirtualAlloc2 = (VirtualAlloc2Ptr)GetProcAddress(hDll, "VirtualAlloc2FromApp"); FreeLibrary(hDll); } // Try to see if large OS pages are supported @@ -106,10 +126,10 @@ void _mi_os_init(void) { CloseHandle(token); } if (!ok) { - if (err==0) err = GetLastError(); + if (err == 0) err = GetLastError(); _mi_warning_message("cannot enable large OS page support, error %lu\n", err); } - } + } } #else void _mi_os_init() { @@ -120,7 +140,7 @@ void _mi_os_init() { os_alloc_granularity = os_page_size; } if (mi_option_is_enabled(mi_option_large_os_pages)) { - large_os_page_size = (1UL<<21); // 2MiB + large_os_page_size = (1UL << 21); // 2MiB } } #endif @@ -128,29 +148,8 @@ void _mi_os_init() { /* ----------------------------------------------------------- Raw allocation on Windows (VirtualAlloc) and Unix's (mmap). - Defines a portable `mmap`, `munmap` and `mmap_trim`. ----------------------------------------------------------- */ -uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { - uintptr_t x = (sz / alignment) * alignment; - if (x < sz) x += alignment; - if (x < sz) return 0; // overflow - return x; -} - -static void* mi_align_up_ptr(void* p, size_t alignment) { - return (void*)_mi_align_up((uintptr_t)p, alignment); -} - -static uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { - return (sz / alignment) * alignment; -} - -static void* mi_align_down_ptr(void* p, size_t alignment) { - return (void*)_mi_align_down((uintptr_t)p, alignment); -} - - static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats) { if (addr == NULL || size == 0) return true; @@ -163,7 +162,7 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats) _mi_stat_decrease(&stats->committed, size); // TODO: what if never committed? _mi_stat_decrease(&stats->reserved, size); if (err) { - #pragma warning(suppress:4996) +#pragma warning(suppress:4996) _mi_warning_message("munmap failed: %s, addr 0x%8li, size %lu\n", strerror(errno), (size_t)addr, size); return false; } @@ -172,39 +171,53 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats) } } -static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_flags, mi_stats_t* stats) { - UNUSED(stats); - if (size == 0) return NULL; +#ifdef _WIN32 +static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { +#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) + if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + // on modern Windows try use VirtualAlloc2 + MEM_ADDRESS_REQUIREMENTS reqs = { 0 }; + reqs.Alignment = try_alignment; + MEM_EXTENDED_PARAMETER param = { 0 }; + param.Type = MemExtendedParameterAddressRequirements; + param.Pointer = &reqs; + return (*pVirtualAlloc2)(addr, NULL, size, flags, PAGE_READWRITE, ¶m, 1); + } +#endif + return VirtualAlloc(addr, size, flags, PAGE_READWRITE); +} + +static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) { void* p = NULL; -#if defined(_WIN32) - int flags = MEM_RESERVE | extra_flags; - if (commit) flags |= MEM_COMMIT; - if (use_large_os_page(size, 0)) { - p = VirtualAlloc(addr, size, MEM_LARGE_PAGES | flags, PAGE_READWRITE); + if (use_large_os_page(size, try_alignment)) { + p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | flags); + // fall back to non-large page allocation on error (`p == NULL`). } if (p == NULL) { - p = VirtualAlloc(addr, size, flags, PAGE_READWRITE); + p = mi_win_virtual_allocx(addr, size, try_alignment, flags); } + return p; +} + #else +static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) { + void* p = NULL; #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON #endif - int flags = MAP_PRIVATE | MAP_ANONYMOUS | extra_flags; - if (addr != NULL) { - #if defined(MAP_EXCL) - flags |= MAP_FIXED | MAP_EXCL; // BSD - #elif defined(MAP_FIXED_NOREPLACE) - flags |= MAP_FIXED_NOREPLACE; // Linux - #elif defined(MAP_FIXED) - flags |= MAP_FIXED; - #endif + int flags = MAP_PRIVATE | MAP_ANONYMOUS; + #if defined(MAP_ALIGNED) // BSD + if (try_alignment > 0) { + size_t n = _mi_bsr(try_alignment); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + flags |= MAP_ALIGNED(n); + } } - int pflags = (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE); - #if defined(PROT_MAX) - pflags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif - - if (large_os_page_size > 0 && use_large_os_page(size, 0) && ((uintptr_t)addr % large_os_page_size) == 0) { + #if defined(PROT_MAX) + protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD + #endif + if (large_os_page_size > 0 && use_large_os_page(size, try_alignment)) { int lflags = flags; #ifdef MAP_ALIGNED_SUPER lflags |= MAP_ALIGNED_SUPER; @@ -217,74 +230,155 @@ static void* mi_os_mem_alloc(void* addr, size_t size, bool commit, int extra_fla #endif if (lflags != flags) { // try large page allocation - p = mmap(addr, size, pflags, lflags, -1, 0); + // TODO: if always failing due to permissions or no huge pages, try to avoid repeatedly trying? + // Should we check this in _mi_os_init? (as on Windows) + p = mmap(NULL, size, protect_flags, lflags, -1, 0); if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission } } if (p == NULL) { - p = mmap(addr, size, pflags, flags, -1, 0); + p = mmap(NULL, size, protect_flags, flags, -1, 0); if (p == MAP_FAILED) p = NULL; } - if (addr != NULL && p != addr) { - mi_os_mem_free(p, size, stats); - p = NULL; - } + return p; +} #endif - UNUSED(stats); - mi_assert(p == NULL || (addr == NULL && p != addr) || (addr != NULL && p == addr)); - if (p != NULL) { - mi_stat_increase(stats->mmap_calls, 1); - mi_stat_increase(stats->reserved, size); - if (commit) mi_stat_increase(stats->committed, size); - } - return p; -} -static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) { - if (alignment < _mi_os_page_size() || ((alignment & (~alignment + 1)) != alignment)) return NULL; +// Primitive allocation from the OS. +// Note: the `alignment` is just a hint and the returned pointer is not guaranteed to be aligned. +static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) { + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + if (size == 0) return NULL; + void* p = NULL; - #if defined(_WIN32) && defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) - if (pVirtualAlloc2 != NULL) { - // on modern Windows try use VirtualAlloc2 - MEM_ADDRESS_REQUIREMENTS reqs = {0}; - reqs.Alignment = alignment; - MEM_EXTENDED_PARAMETER param = { 0 }; - param.Type = MemExtendedParameterAddressRequirements; - param.Pointer = &reqs; - DWORD flags = MEM_RESERVE; - if (commit) flags |= MEM_COMMIT; - if (use_large_os_page(size, alignment)) flags |= MEM_LARGE_PAGES; - p = (*pVirtualAlloc2)(NULL, NULL, size, flags, PAGE_READWRITE, ¶m, 1); - } - #elif defined(MAP_ALIGNED) - // on BSD, use the aligned mmap api - size_t n = _mi_bsr(alignment); - if (((size_t)1 << n) == alignment && n >= 12) { // alignment is a power of 2 and >= 4096 - p = mi_os_mem_alloc(suggest, size, commit, MAP_ALIGNED(n), tld->stats); // use the NetBSD/freeBSD aligned flags - } - #else - UNUSED(size); - UNUSED(alignment); - #endif - UNUSED(stats); // if !STATS - mi_assert(p == NULL || (uintptr_t)p % alignment == 0); +#if defined(_WIN32) + int flags = MEM_RESERVE; + if (commit) flags |= MEM_COMMIT; + p = mi_win_virtual_alloc(NULL, size, try_alignment, flags); +#else + int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); + p = mi_unix_mmap(size, try_alignment, protect_flags); +#endif + _mi_stat_increase(&stats->mmap_calls, 1); if (p != NULL) { - mi_stat_increase(stats->mmap_calls, 1); - mi_stat_increase(stats->reserved, size); - if (commit) mi_stat_increase(stats->committed, size); + _mi_stat_increase(&stats->reserved, size); + if (commit) _mi_stat_increase(&stats->committed, size); } return p; } -// Conservatively OS page align within a given area -static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) { + +// Primitive aligned allocation from the OS. +// This function guarantees the allocated memory is aligned. +static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) { + mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); + mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; + size = _mi_align_up(size, _mi_os_page_size()); + + // try first with a hint (this will be aligned directly on Win 10+ or BSD) + void* p = mi_os_mem_alloc(size, alignment, commit, stats); + if (p == NULL) return NULL; + + // if not aligned, free it, overallocate, and unmap around it + if (((uintptr_t)p % alignment != 0)) { + mi_os_mem_free(p, size, stats); + if (size >= (SIZE_MAX - alignment)) return NULL; // overflow + size_t over_size = size + alignment; + +#if _WIN32 + // over-allocate and than re-allocate exactly at an aligned address in there. + // this may fail due to threads allocating at the same time so we + // retry this at most 3 times before giving up. + // (we can not decommit around the overallocation on Windows, because we can only + // free the original pointer, not one pointing inside the area) + int flags = MEM_RESERVE; + if (commit) flags |= MEM_COMMIT; + for (int tries = 0; tries < 3; tries++) { + // over-allocate to determine a virtual memory range + p = mi_os_mem_alloc(over_size, alignment, commit, stats); + if (p == NULL) return NULL; // error + if (((uintptr_t)p % alignment) == 0) { + // if p happens to be aligned, just decommit the left-over area + _mi_os_decommit((uint8_t*)p + size, over_size - size, stats); + break; + } + else { + // otherwise free and allocate at an aligned address in there + mi_os_mem_free(p, over_size, stats); + void* aligned_p = mi_align_up_ptr(p, alignment); + p = mi_win_virtual_alloc(aligned_p, size, alignment, flags); + if (p == aligned_p) break; // success! + if (p != NULL) { // should not happen? + mi_os_mem_free(p, size, stats); + p = NULL; + } + } + } +#else + // overallocate... + p = mi_os_mem_alloc(over_size, alignment, commit, stats); + if (p == NULL) return NULL; + // and selectively unmap parts around the over-allocated area. + void* aligned_p = mi_align_up_ptr(p, alignment); + size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; + size_t mid_size = _mi_align_up(size, _mi_os_page_size()); + size_t post_size = over_size - pre_size - mid_size; + mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size); + if (pre_size > 0) mi_os_mem_free(p, pre_size, stats); + if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats); + // we can return the aligned pointer on `mmap` systems + p = aligned_p; +#endif + } + + mi_assert_internal(p == NULL || (p != NULL && ((uintptr_t)p % alignment) == 0)); + return p; +} + +/* ----------------------------------------------------------- + OS API: alloc, free, alloc_aligned +----------------------------------------------------------- */ + +void* _mi_os_alloc(size_t size, mi_stats_t* stats) { + if (size == 0) return NULL; + size = mi_os_good_alloc_size(size, 0); + return mi_os_mem_alloc(size, 0, true, stats); +} + +void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { + if (size == 0 || p == NULL) return; + size = mi_os_good_alloc_size(size, 0); + mi_os_mem_free(p, size, stats); +} + +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld) +{ + if (size == 0) return NULL; + size = mi_os_good_alloc_size(size, alignment); + alignment = _mi_align_up(alignment, _mi_os_page_size()); + return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats); +} + + + +/* ----------------------------------------------------------- + OS memory API: reset, commit, decommit, protect, unprotect. +----------------------------------------------------------- */ + + +// OS page align within a given area, either conservative (pages inside the area only), +// or not (straddling pages outside the area is possible) +static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) { mi_assert(addr != NULL && size > 0); if (newsize != NULL) *newsize = 0; if (size == 0 || addr == NULL) return NULL; // page align conservatively within the range - void* start = mi_align_up_ptr(addr, _mi_os_page_size()); - void* end = mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()); + void* start = (conservative ? mi_align_up_ptr(addr, _mi_os_page_size()) + : mi_align_down_ptr(addr, _mi_os_page_size())); + void* end = (conservative ? mi_align_down_ptr((uint8_t*)addr + size, _mi_os_page_size()) + : mi_align_up_ptr((uint8_t*)addr + size, _mi_os_page_size())); ptrdiff_t diff = (uint8_t*)end - (uint8_t*)start; if (diff <= 0) return NULL; @@ -293,6 +387,12 @@ static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) { return start; } +static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) { + return mi_os_page_align_areax(true, addr, size, newsize); +} + + + // Signal to the OS that the address range is no longer in use // but may be used later again. This will release physical memory // pages and reduce swapping while keeping the memory committed. @@ -300,10 +400,9 @@ static void* mi_os_page_align_area(void* addr, size_t size, size_t* newsize) { bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { // page align conservatively within the range size_t csize; - void* start = mi_os_page_align_area(addr,size,&csize); - if (csize==0) return true; - UNUSED(stats); // if !STATS - mi_stat_increase(stats->reset, csize); + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return true; + _mi_stat_increase(&stats->reset, csize); #if defined(_WIN32) // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory @@ -318,22 +417,22 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { /* // VirtualUnlock removes the memory eagerly from the current working set (which MEM_RESET does lazily on demand) // TODO: put this behind an option? - DWORD ok = VirtualUnlock(start, csize); + DWORD ok = VirtualUnlock(start, csize); if (ok != 0) return false; */ - return true; + return true; #else - #if defined(MADV_FREE) - static int advice = MADV_FREE; - int err = madvise(start, csize, advice); - if (err!=0 && errno==EINVAL && advice==MADV_FREE) { - // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on - advice = MADV_DONTNEED; - err = madvise(start, csize, advice); - } - #else - int err = madvise(start, csize, MADV_DONTNEED); - #endif +#if defined(MADV_FREE) + static int advice = MADV_FREE; + int err = madvise(start, csize, advice); + if (err != 0 && errno == EINVAL && advice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on + advice = MADV_DONTNEED; + err = madvise(start, csize, advice); + } +#else + int err = madvise(start, csize, MADV_DONTNEED); +#endif if (err != 0) { _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno); } @@ -346,46 +445,44 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { static bool mi_os_protectx(void* addr, size_t size, bool protect) { // page align conservatively within the range size_t csize = 0; - void* start = mi_os_page_align_area(addr, size, &csize); - if (csize==0) return false; + void* start = mi_os_page_align_area_conservative(addr, size, &csize); + if (csize == 0) return false; int err = 0; #ifdef _WIN32 DWORD oldprotect = 0; - BOOL ok = VirtualProtect(start,csize,protect ? PAGE_NOACCESS : PAGE_READWRITE,&oldprotect); + BOOL ok = VirtualProtect(start, csize, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); err = (ok ? 0 : GetLastError()); #else - err = mprotect(start,csize,protect ? PROT_NONE : (PROT_READ|PROT_WRITE)); + err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); #endif if (err != 0) { _mi_warning_message("mprotect error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err); } - return (err==0); + return (err == 0); } bool _mi_os_protect(void* addr, size_t size) { - return mi_os_protectx(addr,size,true); + return mi_os_protectx(addr, size, true); } bool _mi_os_unprotect(void* addr, size_t size) { return mi_os_protectx(addr, size, false); } -// Commit/Decommit memory. -// We page align to a conservative area inside the range to reset. +// Commit/Decommit memory. Commit is aligned liberal, while decommit is aligned conservative. static bool mi_os_commitx(void* addr, size_t size, bool commit, mi_stats_t* stats) { - // page align conservatively within the range + // page align in the range, commit liberally, decommit conservative size_t csize; - void* start = mi_os_page_align_area(addr, size, &csize); + void* start = mi_os_page_align_areax(!commit, addr, size, &csize); if (csize == 0) return true; int err = 0; - UNUSED(stats); // if !STATS if (commit) { - mi_stat_increase(stats->committed, csize); - mi_stat_increase(stats->commit_calls,1); + _mi_stat_increase(&stats->committed, csize); + _mi_stat_increase(&stats->commit_calls, 1); } else { - mi_stat_decrease(stats->committed, csize); + _mi_stat_decrease(&stats->committed, csize); } #if defined(_WIN32) @@ -415,107 +512,22 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) { return mi_os_commitx(addr, size, false, stats); } +bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { + // page align conservatively within the range + mi_assert_internal(oldsize > newsize && p != NULL); + if (oldsize < newsize || p == NULL) return false; + if (oldsize == newsize) return true; -/* ----------------------------------------------------------- - OS allocation using mmap/munmap ------------------------------------------------------------ */ + // oldsize and newsize should be page aligned or we cannot shrink precisely + void* addr = (uint8_t*)p + newsize; + size_t size = 0; + void* start = mi_os_page_align_area_conservative(addr, oldsize - newsize, &size); + if (size == 0 || start != addr) return false; -void* _mi_os_alloc(size_t size, mi_stats_t* stats) { - if (size == 0) return NULL; - size = mi_os_good_alloc_size(size, 0); - void* p = mi_os_mem_alloc(NULL, size, true, 0, stats); - mi_assert(p!=NULL); - return p; -} - -void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { - UNUSED(stats); - mi_os_mem_free(p, size, stats); -} - -// Slow but guaranteed way to allocated aligned memory -// by over-allocating and then reallocating at a fixed aligned -// address that should be available then. -static void* mi_os_alloc_aligned_ensured(size_t size, size_t alignment, bool commit, size_t trie, mi_stats_t* stats) -{ - if (trie >= 3) return NULL; // stop recursion (only on Windows) - size_t alloc_size = size + alignment; - mi_assert(alloc_size >= size); // overflow? - if (alloc_size < size) return NULL; - - // allocate a chunk that includes the alignment - void* p = mi_os_mem_alloc(NULL, alloc_size, commit, 0, stats); - if (p == NULL) return NULL; - // create an aligned pointer in the allocated area - void* aligned_p = mi_align_up_ptr(p, alignment); - mi_assert(aligned_p != NULL); - - // free it and try to allocate `size` at exactly `aligned_p` - // note: this may fail in case another thread happens to allocate - // concurrently at that spot. We try up to 3 times to mitigate this. - mi_os_mem_free(p, alloc_size, stats); - p = mi_os_mem_alloc(aligned_p, size, commit, 0, stats); - if (p != aligned_p) { - if (p != NULL) mi_os_mem_free(p, size, stats); - return mi_os_alloc_aligned_ensured(size, alignment, commit, trie++, stats); - } -#if 0 // could use this on mmap systems - // we selectively unmap parts around the over-allocated area. - size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; - size_t mid_size = _mi_align_up(size, _mi_os_page_size()); - size_t post_size = alloc_size - pre_size - mid_size; - if (pre_size > 0) mi_os_mem_free(p, pre_size, stats); - if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats); +#ifdef _WIN32 + // we cannot shrink on windows, but we can decommit + return _mi_os_decommit(start, size, stats); +#else + return mi_os_mem_free(start, size, stats); #endif - - mi_assert(((uintptr_t)aligned_p) % alignment == 0); - return aligned_p; -} - -// Allocate an aligned block. -// Since `mi_mmap` is relatively slow we try to allocate directly at first and -// hope to get an aligned address; only when that fails we fall back -// to a guaranteed method by overallocating at first and adjusting. -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld) -{ - if (size == 0) return NULL; - size = mi_os_good_alloc_size(size,alignment); - if (alignment < 1024) return mi_os_mem_alloc(NULL, size, commit, 0, tld->stats); - - // try direct OS aligned allocation; only supported on BSD and Windows 10+ - void* suggest = NULL; - void* p = mi_os_mem_alloc_aligned(size,alignment,commit,tld->stats); - - // Fall back - if (p==NULL && (tld->mmap_next_probable % alignment) == 0) { - // if the next probable address is aligned, - // then try to just allocate `size` and hope it is aligned... - p = mi_os_mem_alloc(suggest, size, commit, 0, tld->stats); - if (p == NULL) return NULL; - if (((uintptr_t)p % alignment) == 0) mi_stat_increase(tld->stats->mmap_right_align, 1); - } - //fprintf(stderr, "segment address guess: %s, p=%lxu, guess:%lxu\n", (p != NULL && (uintptr_t)p % alignment ==0 ? "correct" : "incorrect"), (uintptr_t)p, next_probable); - - if (p==NULL || ((uintptr_t)p % alignment) != 0) { - // if `p` is not yet aligned after all, free the block and use a slower - // but guaranteed way to allocate an aligned block - if (p != NULL) mi_os_mem_free(p, size, tld->stats); - mi_stat_increase( tld->stats->mmap_ensure_aligned, 1); - //fprintf(stderr, "mimalloc: slow mmap 0x%lx\n", _mi_thread_id()); - p = mi_os_alloc_aligned_ensured(size, alignment,commit,0,tld->stats); - } - if (p != NULL) { - // next probable address is the page-aligned address just after the newly allocated area. - size_t probable_size = MI_SEGMENT_SIZE; - if (tld->mmap_previous > p) { - // Linux tends to allocate downward - tld->mmap_next_probable = _mi_align_down((uintptr_t)p - probable_size, os_alloc_granularity); // ((uintptr_t)previous - (uintptr_t)p); - } - else { - // Otherwise, guess the next address is page aligned `size` from current pointer - tld->mmap_next_probable = _mi_align_up((uintptr_t)p + probable_size, os_alloc_granularity); - } - tld->mmap_previous = p; - } - return p; } diff --git a/src/page.c b/src/page.c index fae67ef2..ea86c138 100644 --- a/src/page.c +++ b/src/page.c @@ -391,6 +391,9 @@ void _mi_page_retire(mi_page_t* page) { static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats) { UNUSED(stats); + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + mi_assert_internal(page->capacity + extend <= page->reserved); void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL ); size_t bsize = page->block_size; mi_block_t* start = mi_page_block_at(page, page_area, page->capacity); diff --git a/src/segment.c b/src/segment.c index 13cab17a..f249b02c 100644 --- a/src/segment.c +++ b/src/segment.c @@ -134,7 +134,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa // secure > 1: every page has an os guard page psize -= _mi_os_page_size(); } - + if (page_size != NULL) *page_size = psize; mi_assert_internal(_mi_ptr_page(p) == page); mi_assert_internal(_mi_ptr_segment(p) == segment); diff --git a/test/main-override.cpp b/test/main-override.cpp index 406d3bea..e24ed6d3 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -13,7 +13,7 @@ void free_p() { } int main() { - mi_stats_reset(); + mi_stats_reset(); atexit(free_p); void* p1 = malloc(78); void* p2 = malloc(24);