From 657135de36edad2082323426aea3e2fa1a9cf19a Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 23 Dec 2024 09:53:52 -0800 Subject: [PATCH] commit 2level page-map on over-commit systems --- CMakeLists.txt | 18 +++++++++++------- include/mimalloc/internal.h | 26 ++++++++++++-------------- src/options.c | 2 +- src/page-map.c | 3 ++- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 07a292e0..c184a0b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,10 +10,9 @@ option(MI_PADDING "Enable padding to detect heap block overflow (alway option(MI_OVERRIDE "Override the standard malloc interface (i.e. define entry points for 'malloc', 'free', etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) -option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) -option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) -option(MI_TRACK_ETW "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF) +option(MI_GUARDED "Build with guard pages behind certain object allocations (implies MI_NO_PADDING=ON)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) + option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" ON) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) @@ -21,14 +20,19 @@ option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_WIN_REDIRECT "Use redirection module ('mimalloc-redirect') on Windows if compiling mimalloc as a DLL" ON) option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix)" OFF) -option(MI_LIBC_MUSL "Set this when linking with musl libc" OFF) +option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) + +option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) +option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) +option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) +option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) +option(MI_TRACK_ETW "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF) + option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) -option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) -option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) -option(MI_GUARDED "Build with guard pages behind certain object allocations (implies MI_NO_PADDING=ON)" OFF) + option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index e98a37f5..4cb54d6f 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -435,13 +435,14 @@ static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t si /* ----------------------------------------------------------- - Pages + The page map maps addresses to `mi_page_t` pointers ----------------------------------------------------------- */ #if MI_PAGE_MAP_FLAT -// flat page-map committed on demand +// flat page-map committed on demand, using one byte per slice (64 KiB). // single indirection and low commit, but large initial virtual reserve (4 GiB with 48 bit virtual addresses) +// used by default on <= 40 bit virtual address spaces. extern uint8_t* _mi_page_map; static inline size_t _mi_page_map_index(const void* p) { @@ -468,26 +469,23 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { #else // 2-level page map: -// double indirection but low commit and low virtual reserve. -// -// The page-map is usually 4 MiB and points to sub maps of 64 KiB. -// The page-map is committed on-demand (in 64 KiB) parts (and sub-maps are committed on-demand as well) -// One sub page-map = 64 KiB => covers 2^13 * 2^16 = 2^32 = 512 MiB address space -// The page-map needs 48-16-13 = 19 bits => 2^19 sub map pointers = 4 MiB size. -// (Choosing a MI_PAGE_MAP_SUB_SHIFT of 16 gives slightly better code but will commit the initial sub-map at 512 KiB) - +// double indirection, but low commit and low virtual reserve. +// +// the page-map is usually 4 MiB and points to sub maps of 64 KiB. +// the page-map is committed on-demand (in 64 KiB parts) (and sub-maps are committed on-demand as well) +// one sub page-map = 64 KiB => covers 2^(16-3) * 2^16 = 2^29 = 512 MiB address space +// the page-map needs 48-(16+13) = 19 bits => 2^19 sub map pointers = 4 MiB size. #define MI_PAGE_MAP_SUB_SHIFT (13) #define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) - #define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) #define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT) extern mi_page_t*** _mi_page_map; static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { - const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE; - if (sub_idx != NULL) { *sub_idx = (uint32_t)u % MI_PAGE_MAP_SUB_COUNT; } - return (size_t)(u / MI_PAGE_MAP_SUB_COUNT); + const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); + if (sub_idx != NULL) { *sub_idx = u % MI_PAGE_MAP_SUB_COUNT; } + return (u / MI_PAGE_MAP_SUB_COUNT); } static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { diff --git a/src/options.c b/src/options.c index fc3a2838..7562cd46 100644 --- a/src/options.c +++ b/src/options.c @@ -103,7 +103,7 @@ typedef struct mi_option_desc_s { #endif #ifndef MI_DEFAULT_PAGEMAP_COMMIT -#if defined(__APPLE__) +#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access #define MI_DEFAULT_PAGEMAP_COMMIT 1 #else #define MI_DEFAULT_PAGEMAP_COMMIT 0 diff --git a/src/page-map.c b/src/page-map.c index 37ce3082..db14265b 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -187,7 +187,8 @@ bool _mi_page_map_init(void) { const size_t os_page_size = _mi_os_page_size(); const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size); const size_t reserve_size = page_map_size + os_page_size; - const bool commit = page_map_size <= 64*MI_KiB || mi_option_is_enabled(mi_option_pagemap_commit); // _mi_os_has_overcommit(); // commit on-access on Linux systems? + const bool commit = page_map_size <= 64*MI_KiB || + mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); _mi_page_map = (mi_page_t***)_mi_os_alloc_aligned(reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); if (_mi_page_map==NULL) { _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);