From 8c9ccea2f576394b69d0489b6f8883b95c4ddc1a Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 10:46:06 -0800 Subject: [PATCH 1/4] fix huge page madvise in case mmap failed --- src/os.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/os.c b/src/os.c index 26b8bcd2..77e8216e 100644 --- a/src/os.c +++ b/src/os.c @@ -510,7 +510,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (uintptr_t)8); // on error, don't try again for the next N allocations } } } @@ -518,29 +518,30 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro if (p == NULL) { *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); - #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (allow_large && use_large_os_page(size, try_alignment)) { - if (madvise(p, size, MADV_HUGEPAGE) == 0) { - *is_large = true; // possibly - }; - } - #endif - #if defined(__sun) - if (allow_large && use_large_os_page(size, try_alignment)) { - struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = large_os_page_size; - cmd.mha_cmd = MHA_MAPSIZE_VA; - if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - *is_large = true; + if (p != NULL) { + #if defined(MADV_HUGEPAGE) + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (allow_large && use_large_os_page(size, try_alignment)) { + if (madvise(p, size, MADV_HUGEPAGE) == 0) { + *is_large = true; // possibly + }; } + #elif defined(__sun) + if (allow_large && use_large_os_page(size, try_alignment)) { + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = large_os_page_size; + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + *is_large = true; + } + } + #endif } - #endif } if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); From 89abbe75d81ed4ed4858a3319a3ed938bc751ca1 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:23:11 -0800 Subject: [PATCH 2/4] improve aligned support on BSD and MAP_ALIGN systems --- src/os.c | 56 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/os.c b/src/os.c index 77e8216e..8334771c 100644 --- a/src/os.c +++ b/src/os.c @@ -413,23 +413,41 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + UNUSED(try_alignment); void* p = NULL; + #if defined(MAP_ALIGNED) // BSD + if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + size_t n = mi_bsr(try_alignment); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + flags |= MAP_ALIGNED(n); + p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #elif defined(MAP_ALIGN) // Solaris + if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + #endif #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations - void* hint; - if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { - p = mmap(hint,size,protect_flags,flags,fd,0); - if (p==MAP_FAILED) p = NULL; // fall back to regular mmap + if (addr == NULL) { + void* hint = mi_os_get_aligned_hint(try_alignment, size); + if (hint != NULL) { + p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } } - #else - UNUSED(try_alignment); - UNUSED(mi_os_get_aligned_hint); #endif - if (p==NULL) { - p = mmap(addr,size,protect_flags,flags,fd,0); - if (p==MAP_FAILED) p = NULL; - } - return p; + // regular mmap + p = mmap(addr, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // failed to allocate + return NULL; } static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { @@ -444,24 +462,17 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro int fd = -1; if (os_overcommit) { flags |= MAP_NORESERVE; - } - #if defined(MAP_ALIGNED) // BSD - if (try_alignment > 0) { - size_t n = mi_bsr(try_alignment); - if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB - flags |= MAP_ALIGNED(n); - } - } - #endif + } #if defined(PROT_MAX) protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif #if defined(VM_MAKE_TAG) // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) int os_tag = (int)mi_option_get(mi_option_os_tag); - if (os_tag < 100 || os_tag > 255) os_tag = 100; + if (os_tag < 100 || os_tag > 255) { os_tag = 100; } fd = VM_MAKE_TAG(os_tag); #endif + // huge page allocation if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { static _Atomic(uintptr_t) large_page_try_ok; // = 0; uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); @@ -515,6 +526,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro } } } + // regular allocation if (p == NULL) { *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); From fd61997cef6af8eab86fc40da5636f95c6b6fb47 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:26:36 -0800 Subject: [PATCH 3/4] improve aligned support on BSD and MAP_ALIGN systems --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 8334771c..d43f9afd 100644 --- a/src/os.c +++ b/src/os.c @@ -564,7 +564,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // On 64-bit systems, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate them. -#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) +#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || defined(MI_OS_USE_MMAP)) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // Return a 4MiB aligned address that is probably available. From f72e5688f5b6bbbe6a61289f0ed73541518de08e Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:58:04 -0800 Subject: [PATCH 4/4] remove assign in while condition --- src/os.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/os.c b/src/os.c index d43f9afd..426ead4a 100644 --- a/src/os.c +++ b/src/os.c @@ -303,20 +303,21 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations - void* hint; - if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { - void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); - if (p != NULL) return p; - // for robustness always fall through in case of an error - /* - DWORD err = GetLastError(); - if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) - err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) - return NULL; + if (addr == NULL) { + void* hint = mi_os_get_aligned_hint(try_alignment,size); + if (hint != NULL) { + void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); + if (p != NULL) return p; + // for robustness always fall through in case of an error + /* + DWORD err = GetLastError(); + if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) + err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) + return NULL; + } + */ + _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); } - */ - _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); - // fall through on error } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) @@ -414,20 +415,19 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { UNUSED(try_alignment); - void* p = NULL; #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); - p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { - p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); + void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } @@ -437,14 +437,14 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr if (addr == NULL) { void* hint = mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { - p = mmap(hint, size, protect_flags, flags, fd, 0); + void* p = mmap(hint, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } } #endif // regular mmap - p = mmap(addr, size, protect_flags, flags, fd, 0); + void* p = mmap(addr, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // failed to allocate return NULL;