From f38816d4ed3fb4af588c6addf8ec11ec47b79c55 Mon Sep 17 00:00:00 2001 From: Philip Brown <122590765+asdf-bro@users.noreply.github.com> Date: Sun, 6 Oct 2024 15:42:46 -0500 Subject: [PATCH 01/38] Musl needs __libc* functions too --- src/alloc-override.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index 12837cdd..82a98b80 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -289,8 +289,8 @@ mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_r void __libc_free(void* p) MI_FORWARD0(mi_free, p) void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -#elif defined(__GLIBC__) && defined(__linux__) - // forward __libc interface (needed for glibc-based Linux distributions) +#elif defined(__linux__) + // forward __libc interface (needed for glibc-based and musl-based Linux distributions) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) From bf251b27b187d36bec964e9d6e830dcf0e5b75a4 Mon Sep 17 00:00:00 2001 From: ArtSin Date: Tue, 15 Oct 2024 13:39:28 +0400 Subject: [PATCH 02/38] Fix int and long handling and the use of (u)intptr_t in _mi_vsnprintf 'va_arg' with type 'long' was used for both 'int' and 'long' arguments, but x86_64 Linux has 64-bit 'long', so passing an 'int' argument to '_mi_vsnprintf` results in undefined behaviour. 'intptr_t' was used as the largest integer type, but on 32-bit systems it is 32-bit wide, while 'long long' is 64-bit. 'intmax_t' can be used instead. --- src/libc.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/libc.c b/src/libc.c index dd6b4007..ce541f1b 100644 --- a/src/libc.c +++ b/src/libc.c @@ -130,7 +130,7 @@ static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, } -static void mi_out_num(uintptr_t x, size_t base, char prefix, char** out, char* end) +static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end) { if (x == 0 || base == 0 || base > 16) { if (prefix != 0) { mi_outc(prefix, out, end); } @@ -206,12 +206,13 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { } else if (c == 'p' || c == 'x' || c == 'u') { // unsigned - uintptr_t x = 0; + uintmax_t x = 0; if (c == 'x' || c == 'u') { if (numtype == 'z') x = va_arg(args, size_t); else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t - else if (numtype == 'L') x = (uintptr_t)va_arg(args, unsigned long long); - else x = va_arg(args, unsigned long); + else if (numtype == 'L') x = va_arg(args, unsigned long long); + else if (numtype == 'l') x = va_arg(args, unsigned long); + else x = va_arg(args, unsigned int); } else if (c == 'p') { x = va_arg(args, uintptr_t); @@ -228,20 +229,21 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { } else if (c == 'i' || c == 'd') { // signed - intptr_t x = 0; + intmax_t x = 0; if (numtype == 'z') x = va_arg(args, intptr_t ); else if (numtype == 't') x = va_arg(args, ptrdiff_t); - else if (numtype == 'L') x = (intptr_t)va_arg(args, long long); - else x = va_arg(args, long); + else if (numtype == 'L') x = va_arg(args, long long); + else if (numtype == 'l') x = va_arg(args, long); + else x = va_arg(args, int); char pre = 0; if (x < 0) { pre = '-'; - if (x > INTPTR_MIN) { x = -x; } + if (x > INTMAX_MIN) { x = -x; } } else if (numplus != 0) { pre = numplus; } - mi_out_num((uintptr_t)x, 10, pre, &out, end); + mi_out_num((uintmax_t)x, 10, pre, &out, end); } else if (c >= ' ' && c <= '~') { // unknown format From 394e8c27d8b924e8b0248adc2835f65451c4f2ab Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:02:24 -0700 Subject: [PATCH 03/38] add cmake option to add C pre processor definitions more easily --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e78ba9fd..b32542ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) option(MI_NO_THP "Disable transparent huge pages support on Linux/Android for the mimalloc process only" OFF) +option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") # deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) @@ -62,8 +63,8 @@ set(mi_sources set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build set(mi_cflags_dynamic "") # extra flags for a shared-object library build -set(mi_defines "") set(mi_libraries "") +set(mi_defines ${MI_EXTRA_CPPDEFS}) # ----------------------------------------------------------------------------- # Convenience: set default build type depending on the build directory From 638ea539de088390b36f646e6c342dd230d0c93a Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:04:01 -0700 Subject: [PATCH 04/38] allow certain options to have defaults set via the pre-processor at build time -- see issue #945 --- src/options.c | 62 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/src/options.c b/src/options.c index 27122c78..76d2cf82 100644 --- a/src/options.c +++ b/src/options.c @@ -47,6 +47,47 @@ typedef struct mi_option_desc_s { #define MI_OPTION(opt) mi_option_##opt, #opt, NULL #define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy +// Some options can be set at build time for statically linked libraries (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) +// This is useful if we cannot pass them as environment variables +// (and setting them programmatically would be too late) + +#ifndef MI_DEFAULT_VERBOSE +#define MI_DEFAULT_VERBOSE 0 +#endif + +#ifndef MI_DEFAULT_EAGER_COMMIT +#define MI_DEFAULT_EAGER_COMMIT 1 +#endif + +#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT +#define MI_DEFAULT_ARENA_EAGER_COMMIT 2 +#endif + +#ifndef MI_DEFAULT_ARENA_RESERVE + #if (MI_INTPTR_SIZE>4) + #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L + #else + #define MI_DEFAULT_ARENA_RESERVE 128L*1024L + #endif +#endif + +#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC +#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0 +#endif + +#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES +#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES +#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_OS_MEMORY +#define MI_DEFAULT_RESERVE_OS_MEMORY 0 +#endif + + static mi_option_desc_t options[_mi_option_last] = { // stable options @@ -56,16 +97,16 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(show_errors) }, #endif { 0, UNINIT, MI_OPTION(show_stats) }, - { 0, UNINIT, MI_OPTION(verbose) }, + { MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) }, // the following options are experimental and not all combinations make sense. - { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) - { 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + { MI_DEFAULT_EAGER_COMMIT, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) + { MI_DEFAULT_ARENA_EAGER_COMMIT, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) - { 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) + { MI_DEFAULT_RESERVE_OS_MEMORY, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION(abandoned_page_purge) }, // purge free page memory when a thread terminates @@ -83,16 +124,11 @@ static mi_option_desc_t options[_mi_option_last] = { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! - #if (MI_INTPTR_SIZE>4) - { 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - #else - { 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit - #endif - + { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free - { 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. #if defined(MI_VISIT_ABANDONED) { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandonded segments; requires taking locks during reclaim. From 50d3525a8c948a5943c89590629e869fc7cdbe0d Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:04:27 -0700 Subject: [PATCH 05/38] add test for issue #944 --- test/main-override.cpp | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index fc7f70f0..50eb0267 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -37,6 +37,7 @@ static void tsan_numa_test(); // issue #414 static void strdup_test(); // issue #445 static void heap_thread_free_huge(); static void test_std_string(); // issue #697 +static void test_thread_local(); // issue #944 static void test_stl_allocators(); @@ -44,7 +45,8 @@ static void test_stl_allocators(); int main() { // mi_stats_reset(); // ignore earlier allocations - test_std_string(); + //test_std_string(); + test_thread_local(); // heap_thread_free_huge(); /* heap_thread_free_large(); @@ -312,3 +314,31 @@ static void tsan_numa_test() { dummy_worker(); t1.join(); } + + +class MTest +{ + char *data; +public: + MTest() { data = (char*)malloc(1024); } + ~MTest() { free(data); }; +}; + +thread_local MTest tlVariable; + +void threadFun( int i ) +{ + printf( "Thread %d\n", i ); + std::this_thread::sleep_for( std::chrono::milliseconds(100) ); +} + +void test_thread_local() +{ + for( int i=1; i < 100; ++i ) + { + std::thread t( threadFun, i ); + t.join(); + mi_stats_print(NULL); + } + return; +} \ No newline at end of file From 34e66778ecc40c58164c14ada56c15e8f38cf1f7 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:10:09 -0700 Subject: [PATCH 06/38] fix MI_EXTRA_CPPDEFS setting --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b32542ef..adf45bbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,7 +64,12 @@ set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build set(mi_cflags_dynamic "") # extra flags for a shared-object library build set(mi_libraries "") -set(mi_defines ${MI_EXTRA_CPPDEFS}) + +if(MI_EXTRA_CPPDEFS) + set(mi_defines ${MI_EXTRA_CPPDEFS}) +else() + set(mi_defines "") +endif() # ----------------------------------------------------------------------------- # Convenience: set default build type depending on the build directory From aa881733d7830e44b39bd16080fab803e468828c Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 22:56:59 -0700 Subject: [PATCH 07/38] reorganize primitives for process initialization; use special data segment on Windows for thread termination by default on Windows now (issue #869) --- include/mimalloc/internal.h | 5 + src/init.c | 114 ++------------------- src/prim/prim.c | 44 ++++++++ src/prim/windows/prim.c | 197 ++++++++++++++++++++++++++++-------- 4 files changed, 209 insertions(+), 151 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 2f21d88e..63a1e6f0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -77,6 +77,11 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; +void _mi_process_load(void); +void mi_cdecl _mi_process_done(void); +bool _mi_is_redirected(void); +bool _mi_allocator_init(const char** message); +void _mi_allocator_done(void); bool _mi_is_main_thread(void); size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not initialized yet diff --git a/src/init.c b/src/init.c index ead5a147..eaa7f5be 100644 --- a/src/init.c +++ b/src/init.c @@ -459,10 +459,6 @@ void mi_thread_init(void) mi_attr_noexcept //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); } -void mi_thread_done(void) mi_attr_noexcept { - _mi_thread_done(NULL); -} - void _mi_thread_done(mi_heap_t* heap) { // calling with NULL implies using the default heap @@ -508,54 +504,15 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) { // -------------------------------------------------------- // Run functions on process init/done, and thread init/done // -------------------------------------------------------- -static void mi_cdecl mi_process_done(void); - static bool os_preloading = true; // true until this module is initialized -static bool mi_redirected = false; // true if malloc redirects to mi_malloc // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. bool mi_decl_noinline _mi_preloading(void) { return os_preloading; } -mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { - return mi_redirected; -} - -// Communicate with the redirection module on Windows -#if defined(_WIN32) && defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) -#ifdef __cplusplus -extern "C" { -#endif -mi_decl_export void _mi_redirect_entry(DWORD reason) { - // called on redirection; careful as this may be called before DllMain - if (reason == DLL_PROCESS_ATTACH) { - mi_redirected = true; - } - else if (reason == DLL_PROCESS_DETACH) { - mi_redirected = false; - } - else if (reason == DLL_THREAD_DETACH) { - mi_thread_done(); - } -} -__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); -__declspec(dllimport) void mi_cdecl mi_allocator_done(void); -#ifdef __cplusplus -} -#endif -#else -static bool mi_allocator_init(const char** message) { - if (message != NULL) *message = NULL; - return true; -} -static void mi_allocator_done(void) { - // nothing to do -} -#endif - -// Called once by the process loader -static void mi_process_load(void) { +// Called once by the process loader from `src/prim/prim.c` +void _mi_process_load(void) { mi_heap_main_init(); #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; @@ -563,17 +520,14 @@ static void mi_process_load(void) { #endif os_preloading = false; mi_assert_internal(_mi_is_main_thread()); - #if !(defined(_WIN32) && defined(MI_SHARED_LIB)) // use Dll process detach (see below) instead of atexit (issue #521) - atexit(&mi_process_done); - #endif _mi_options_init(); mi_process_setup_auto_thread_done(); mi_process_init(); - if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); + if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); // show message from the redirector (if present) const char* msg = NULL; - mi_allocator_init(&msg); + _mi_allocator_init(&msg); if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { _mi_fputs(NULL,NULL,NULL,msg); } @@ -651,7 +605,7 @@ void mi_process_init(void) mi_attr_noexcept { } // Called when the process is done (through `at_exit`) -static void mi_cdecl mi_process_done(void) { +void mi_cdecl _mi_process_done(void) { // only shutdown if we were initialized if (!_mi_process_is_initialized) return; // ensure we are called once @@ -683,64 +637,8 @@ static void mi_cdecl mi_process_done(void) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { mi_stats_print(NULL); } - mi_allocator_done(); + _mi_allocator_done(); _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); os_preloading = true; // don't call the C runtime anymore } - - -#if defined(_WIN32) && defined(MI_SHARED_LIB) - // Windows DLL: easy to hook into process_init and thread_done - __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { - MI_UNUSED(reserved); - MI_UNUSED(inst); - if (reason==DLL_PROCESS_ATTACH) { - mi_process_load(); - } - else if (reason==DLL_PROCESS_DETACH) { - mi_process_done(); - } - else if (reason==DLL_THREAD_DETACH) { - if (!mi_is_redirected()) { - mi_thread_done(); - } - } - return TRUE; - } - -#elif defined(_MSC_VER) - // MSVC: use data section magic for static libraries - // See - static int _mi_process_init(void) { - mi_process_load(); - return 0; - } - typedef int(*_mi_crt_callback_t)(void); - #if defined(_M_X64) || defined(_M_ARM64) - __pragma(comment(linker, "/include:" "_mi_msvc_initu")) - #pragma section(".CRT$XIU", long, read) - #else - __pragma(comment(linker, "/include:" "__mi_msvc_initu")) - #endif - #pragma data_seg(".CRT$XIU") - mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init }; - #pragma data_seg() - -#elif defined(__cplusplus) - // C++: use static initialization to detect process start - static bool _mi_process_init(void) { - mi_process_load(); - return (_mi_heap_main.thread_id != 0); - } - static bool mi_initialized = _mi_process_init(); - -#elif defined(__GNUC__) || defined(__clang__) - // GCC,Clang: use the constructor attribute - static void __attribute__((constructor)) _mi_process_init(void) { - mi_process_load(); - } - -#else -#pragma message("define a way to call mi_process_load on your platform") -#endif diff --git a/src/prim/prim.c b/src/prim/prim.c index 3b7d3736..242cd618 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -25,3 +25,47 @@ terms of the MIT license. A copy of the license can be found in the file #include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.) #endif + +// Generic process initialization +#ifndef MI_PRIM_HAS_PROCESS_ATTACH +#if defined(__GNUC__) || defined(__clang__) + // GCC,Clang: use the constructor attribute + #if defined(__clang__) + #define mi_attr_constructor __attribute__((constructor(101))) + #define mi_attr_destructor __attribute__((destructor(101))) + #else + #define mi_attr_constructor __attribute__((constructor)) + #define mi_attr_destructor __attribute__((destructor)) + #endif + static void mi_attr_constructor mi_process_attach(void) { + _mi_process_load(); + } + static void mi_attr_destructor mi_process_detach(void) { + _mi_process_done(); + } +#elif defined(__cplusplus) + // C++: use static initialization to detect process start + static bool mi_process_attach(void) { + _mi_process_load(); + atexit(&_mi_process_done); + return (_mi_heap_main.thread_id != 0); + } + static bool mi_initialized = mi_process_attach(); +#else + #pragma message("define a way to call _mi_process_load/done on your platform") +#endif +#endif + +// Generic allocator init/done callback +#ifndef MI_PRIM_HAS_ALLOCATOR_INIT +bool _mi_is_redirected(void) { + return false; +} +bool _mi_allocator_init(const char** message) { + if (message != NULL) *message = NULL; + return true; +} +void _mi_allocator_done(void) { + // nothing to do +} +#endif diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index bd874f9b..c62ea497 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -499,8 +499,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) } // get process info - PROCESS_MEMORY_COUNTERS info; - memset(&info, 0, sizeof(info)); + PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); if (pGetProcessMemoryInfo != NULL) { pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); } @@ -602,60 +601,172 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #endif // MI_USE_RTLGENRANDOM + + //---------------------------------------------------------------- -// Thread init/done +// Process & Thread Init/Done //---------------------------------------------------------------- -#if !defined(MI_SHARED_LIB) - -// use thread local storage keys to detect thread ending -// note: another design could be to use special linker sections (see issue #869) -#include -#if (_WIN32_WINNT < 0x600) // before Windows Vista -WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); -WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); -WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); -WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); -#endif - -static DWORD mi_fls_key = (DWORD)(-1); - -static void NTAPI mi_fls_done(PVOID value) { - mi_heap_t* heap = (mi_heap_t*)value; - if (heap != NULL) { - _mi_thread_done(heap); - FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 +static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { + MI_UNUSED(reserved); + MI_UNUSED(module); + if (reason==DLL_PROCESS_ATTACH) { + _mi_process_load(); } + else if (reason==DLL_PROCESS_DETACH) { + _mi_process_done(); + } + else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) { + _mi_thread_done(NULL); + } } -void _mi_prim_thread_init_auto_done(void) { - mi_fls_key = FlsAlloc(&mi_fls_done); -} -void _mi_prim_thread_done_auto_done(void) { - // call thread-done on all threads (except the main thread) to prevent - // dangling callback pointer if statically linked with a DLL; Issue #208 - FlsFree(mi_fls_key); -} +#if defined(MI_SHARED_LIB) + #define MI_PRIM_HAS_PROCESS_INIT 1 -void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { - mi_assert_internal(mi_fls_key != (DWORD)(-1)); - FlsSetValue(mi_fls_key, heap); -} + // Windows DLL: easy to hook into process_init and thread_done + __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { + win_main((PVOID)inst,reason,reserved); + return TRUE; + } -#else + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) { } + void _mi_prim_thread_done_auto_done(void) { } + void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); + } -// Dll; nothing to do as in that case thread_done is handled through the DLL_THREAD_DETACH event. +#elif !defined(MI_WIN_USE_FLS) + #define MI_PRIM_HAS_PROCESS_INIT 1 -void _mi_prim_thread_init_auto_done(void) { -} + // Set up TLS callbacks in a statically linked library by using special data sections. + // See + // We may use ".CRT$XLY" instead of "B" -- see also issue #869. + #if defined(__cplusplus) + extern "C" { + #endif -void _mi_prim_thread_done_auto_done(void) { -} + #if defined(_WIN64) + #pragma comment(linker, "/INCLUDE:_tls_used") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback") + #pragma const_seg(".CRT$XLB") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback[] = { &mi_win_main }; + #pragma const_seg() + #else + #pragma comment(linker, "/INCLUDE:__tls_used") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback") + #pragma data_seg(".CRT$XLB") + const PIMAGE_TLS_CALLBACK _mi_tls_callback[] = { &mi_win_main }; + #pragma data_seg() + #endif -void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { - MI_UNUSED(heap); -} + #if defined(__cplusplus) + } + #endif + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. + void _mi_prim_thread_init_auto_done(void) { } + void _mi_prim_thread_done_auto_done(void) { } + void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + MI_UNUSED(heap); + } + +#else // statically linked, use fiber api + + #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`) + // MSVC: use data section magic for static libraries + // See + #define MI_PRIM_HAS_PROCESS_INIT 1 + + static int mi_process_attach(void) { + mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); + atexit(&_mi_process_done); + return 0; + } + typedef int(*_mi_crt_callback_t)(void); + #if defined(_M_X64) || defined(_M_ARM64) + __pragma(comment(linker, "/include:" "_mi_msvc_initu")) + #pragma section(".CRT$XIU", long, read) + #else + __pragma(comment(linker, "/include:" "__mi_msvc_initu")) + #endif + #pragma data_seg(".CRT$XIU") + mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &mi_process_attach }; + #pragma data_seg() + #endif + + // use the fiber api for calling `_mi_thread_done`. + #include + #if (_WIN32_WINNT < 0x600) // before Windows Vista + WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); + WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); + WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); + WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); + #endif + + static DWORD mi_fls_key = (DWORD)(-1); + + static void NTAPI mi_fls_done(PVOID value) { + mi_heap_t* heap = (mi_heap_t*)value; + if (heap != NULL) { + _mi_thread_done(heap); + FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672 + } + } + + void _mi_prim_thread_init_auto_done(void) { + mi_fls_key = FlsAlloc(&mi_fls_done); + } + + void _mi_prim_thread_done_auto_done(void) { + // call thread-done on all threads (except the main thread) to prevent + // dangling callback pointer if statically linked with a DLL; Issue #208 + FlsFree(mi_fls_key); + } + + void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) { + mi_assert_internal(mi_fls_key != (DWORD)(-1)); + FlsSetValue(mi_fls_key, heap); + } #endif +// ---------------------------------------------------- +// Communicate with the redirection module on Windows +// ---------------------------------------------------- +#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) + static bool mi_redirected = false; // true if malloc redirects to mi_malloc + + bool _mi_is_redirected(void) { + return mi_redirected; + } + + #ifdef __cplusplus + extern "C" { + #endif + mi_decl_export void _mi_redirect_entry(DWORD reason) { + // called on redirection; careful as this may be called before DllMain + if (reason == DLL_PROCESS_ATTACH) { + mi_redirected = true; + } + else if (reason == DLL_PROCESS_DETACH) { + mi_redirected = false; + } + else if (reason == DLL_THREAD_DETACH) { + _mi_thread_done(NULL); + } + } + __declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message); + __declspec(dllimport) void mi_cdecl mi_allocator_done(void); + #ifdef __cplusplus + } + #endif + bool _mi_allocator_init(const char** message) { + return mi_allocator_init(message); + } + void _mi_allocator_done(void) { + mi_allocator_done(); + } +#endif \ No newline at end of file From 4377abe017d59796845ec0266531d62e5873a917 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 22:59:41 -0700 Subject: [PATCH 08/38] add cmake option to fall back on the fiber api do detect thread termination on windows --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index adf45bbf..e0aa59b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ option(MI_NO_THP "Disable transparent huge pages support on Linux/And option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") # deprecated options +option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) @@ -313,6 +314,11 @@ if(MI_LIBC_MUSL) list(APPEND mi_defines MI_LIBC_MUSL=1) endif() +if(MI_WIN_USE_FLS) + message(STATUS "Use the Fiber API to detect thread termination") + list(APPEND mi_defines MI_WIN_USE_FLS=1) +endif() + # On Haiku use `-DCMAKE_INSTALL_PREFIX` instead, issue #788 # if(CMAKE_SYSTEM_NAME MATCHES "Haiku") # SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) From f971bd6d749ccdea5b3622ef6272896b2d45ab28 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 23:05:51 -0700 Subject: [PATCH 09/38] fix build on windows --- src/prim/prim.c | 1 + src/prim/windows/prim.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/prim/prim.c b/src/prim/prim.c index 242cd618..8a2400c8 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -45,6 +45,7 @@ terms of the MIT license. A copy of the license can be found in the file } #elif defined(__cplusplus) // C++: use static initialization to detect process start + extern mi_heap_t _mi_heap_main; static bool mi_process_attach(void) { _mi_process_load(); atexit(&_mi_process_done); diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index c62ea497..ed160541 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -623,11 +623,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #if defined(MI_SHARED_LIB) - #define MI_PRIM_HAS_PROCESS_INIT 1 + #define MI_PRIM_HAS_PROCESS_ATTACH 1 // Windows DLL: easy to hook into process_init and thread_done __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { - win_main((PVOID)inst,reason,reserved); + mi_win_main((PVOID)inst,reason,reserved); return TRUE; } @@ -639,7 +639,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { } #elif !defined(MI_WIN_USE_FLS) - #define MI_PRIM_HAS_PROCESS_INIT 1 + #define MI_PRIM_HAS_PROCESS_ATTACH 1 // Set up TLS callbacks in a statically linked library by using special data sections. // See @@ -679,7 +679,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`) // MSVC: use data section magic for static libraries // See - #define MI_PRIM_HAS_PROCESS_INIT 1 + #define MI_PRIM_HAS_PROCESS_ATTACH 1 static int mi_process_attach(void) { mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); From e55ae0aeb76f5205edce57b97031be6aa80f962a Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 23:09:14 -0700 Subject: [PATCH 10/38] fix duplicate definition on windows --- src/prim/windows/prim.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index ed160541..90fab080 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -737,6 +737,8 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { // Communicate with the redirection module on Windows // ---------------------------------------------------- #if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) + #define MI_PRIM_HAS_ALLOCATOR_INIT 1 + static bool mi_redirected = false; // true if malloc redirects to mi_malloc bool _mi_is_redirected(void) { From 46e9e7fdd08f017eded0c88538647c82752c668f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 22 Oct 2024 06:06:15 -0700 Subject: [PATCH 11/38] fix win32 compilation --- src/prim/windows/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 90fab080..7ba1f1de 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -659,7 +659,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback") #pragma data_seg(".CRT$XLB") - const PIMAGE_TLS_CALLBACK _mi_tls_callback[] = { &mi_win_main }; + PIMAGE_TLS_CALLBACK _mi_tls_callback[] = { &mi_win_main }; #pragma data_seg() #endif @@ -738,7 +738,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { // ---------------------------------------------------- #if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) #define MI_PRIM_HAS_ALLOCATOR_INIT 1 - + static bool mi_redirected = false; // true if malloc redirects to mi_malloc bool _mi_is_redirected(void) { From 104e82170912bffdc7228202abc897204ca747be Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 22 Oct 2024 06:08:56 -0700 Subject: [PATCH 12/38] fix fast divisor for 32-bit platforms --- src/heap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/heap.c b/src/heap.c index a3aaaf6b..206d3a36 100644 --- a/src/heap.c +++ b/src/heap.c @@ -543,13 +543,14 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) { mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX); - *shift = 64 - mi_clz(divisor - 1); + *shift = MI_INTPTR_BITS - mi_clz(divisor - 1); *magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1); } static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) { mi_assert_internal(n <= UINT32_MAX); - return ((((uint64_t)n * magic) >> 32) + n) >> shift; + const uint64_t hi = ((uint64_t)n * magic) >> 32; + return (size_t)((hi + n) >> shift); } bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) { From dfdb9cb8772f8fabf63ce81aaa7e2aac9621a1e3 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 22 Oct 2024 06:52:34 -0700 Subject: [PATCH 13/38] cleanup process init/done --- src/prim/prim.c | 20 ++++++++++++++++---- src/prim/windows/prim.c | 12 ++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/prim/prim.c b/src/prim/prim.c index 8a2400c8..a4d3814f 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -29,7 +29,8 @@ terms of the MIT license. A copy of the license can be found in the file // Generic process initialization #ifndef MI_PRIM_HAS_PROCESS_ATTACH #if defined(__GNUC__) || defined(__clang__) - // GCC,Clang: use the constructor attribute + // gcc,clang: use the constructor/destructor attribute + // which for both seem to run before regular constructors/destructors #if defined(__clang__) #define mi_attr_constructor __attribute__((constructor(101))) #define mi_attr_destructor __attribute__((destructor(101))) @@ -44,7 +45,17 @@ terms of the MIT license. A copy of the license can be found in the file _mi_process_done(); } #elif defined(__cplusplus) - // C++: use static initialization to detect process start + // C++: use static initialization to detect process start/end + struct mi_init_done_t { + mi_init_done_t() { + _mi_process_load(); + } + ~mi_init_done_t() { + _mi_process_done(); + } + }; + static mi_init_done_t mi_init_done; + /* extern mi_heap_t _mi_heap_main; static bool mi_process_attach(void) { _mi_process_load(); @@ -52,7 +63,8 @@ terms of the MIT license. A copy of the license can be found in the file return (_mi_heap_main.thread_id != 0); } static bool mi_initialized = mi_process_attach(); -#else + */ + #else #pragma message("define a way to call _mi_process_load/done on your platform") #endif #endif @@ -63,7 +75,7 @@ bool _mi_is_redirected(void) { return false; } bool _mi_allocator_init(const char** message) { - if (message != NULL) *message = NULL; + if (message != NULL) { *message = NULL; } return true; } void _mi_allocator_done(void) { diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 7ba1f1de..b0631b81 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -674,7 +674,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { MI_UNUSED(heap); } -#else // statically linked, use fiber api +#else // deprecated: statically linked, use fiber api #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`) // MSVC: use data section magic for static libraries @@ -686,15 +686,15 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { atexit(&_mi_process_done); return 0; } - typedef int(*_mi_crt_callback_t)(void); - #if defined(_M_X64) || defined(_M_ARM64) - __pragma(comment(linker, "/include:" "_mi_msvc_initu")) + typedef int(*mi_crt_callback_t)(void); + #if defined(_WIN64) + #pragma comment(linker, "/INCLUDE:_mi_tls_callback") #pragma section(".CRT$XIU", long, read) #else - __pragma(comment(linker, "/include:" "__mi_msvc_initu")) + #pragma comment(linker, "/INCLUDE:__mi_tls_callback") #endif #pragma data_seg(".CRT$XIU") - mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &mi_process_attach }; + mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach }; #pragma data_seg() #endif From 6e9b38ac126c301f6e6c3e47a642c2c8388c333d Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 22 Oct 2024 18:58:55 -0700 Subject: [PATCH 14/38] fix issue where searching for abandoned blocks would skip the first one --- src/arena-abandon.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/arena-abandon.c b/src/arena-abandon.c index eaa8c7c9..c40d3ce3 100644 --- a/src/arena-abandon.c +++ b/src/arena-abandon.c @@ -237,7 +237,7 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_at(mi_arena_t* arena, mi_s static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_cursor_t* previous) { const size_t max_arena = mi_arena_get_count(); size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx); - size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx) + 1; + size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx); // visit arena's (from the previous cursor) for (; previous->start < previous->end; previous->start++, field_idx = 0, bit_idx = 0) { // index wraps around @@ -266,11 +266,12 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_ // pre-check if the bit is set size_t mask = ((size_t)1 << bit_idx); if mi_unlikely((field & mask) == mask) { - previous->bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); - mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, previous->bitmap_idx); + mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx); + mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, bitmap_idx); if (segment != NULL) { //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); } + previous->bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx + 1); // start at next one for the next iteration return segment; } } From d951b4dd234c165552521419ef2365a55fdcd454 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 23 Oct 2024 00:53:17 -0700 Subject: [PATCH 15/38] add missing mi_thread_done definition --- src/init.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/init.c b/src/init.c index eaa7f5be..75458a1f 100644 --- a/src/init.c +++ b/src/init.c @@ -459,6 +459,10 @@ void mi_thread_init(void) mi_attr_noexcept //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); } +void mi_thread_done(void) mi_attr_noexcept { + _mi_thread_done(NULL); +} + void _mi_thread_done(mi_heap_t* heap) { // calling with NULL implies using the default heap From 925efaeac93f15408f001484e9ea4fa0fe3b8aec Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 23 Oct 2024 01:10:00 -0700 Subject: [PATCH 16/38] improve windows static library initialization to account for thread local destructors (issue #944) --- src/prim/windows/prim.c | 45 ++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index b0631b81..385354fc 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -641,26 +641,47 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #elif !defined(MI_WIN_USE_FLS) #define MI_PRIM_HAS_PROCESS_ATTACH 1 + static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) { + mi_win_main(module, reason, reserved); + } + } + static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) { + if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) { + mi_win_main(module, reason, reserved); + } + } + // Set up TLS callbacks in a statically linked library by using special data sections. // See - // We may use ".CRT$XLY" instead of "B" -- see also issue #869. + // We use 2 entries to ensure we call attach events before constructors + // are called, and detach events after destructors are called. #if defined(__cplusplus) extern "C" { #endif #if defined(_WIN64) - #pragma comment(linker, "/INCLUDE:_tls_used") - #pragma comment(linker, "/INCLUDE:_mi_tls_callback") - #pragma const_seg(".CRT$XLB") - extern const PIMAGE_TLS_CALLBACK _mi_tls_callback[]; - const PIMAGE_TLS_CALLBACK _mi_tls_callback[] = { &mi_win_main }; - #pragma const_seg() + #pragma comment(linker, "/INCLUDE:_tls_used") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma const_seg(".CRT$XLB") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; + #pragma const_seg() + #pragma const_seg(".CRT$XLY") + extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; + const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; + #pragma const_seg() #else - #pragma comment(linker, "/INCLUDE:__tls_used") - #pragma comment(linker, "/INCLUDE:__mi_tls_callback") - #pragma data_seg(".CRT$XLB") - PIMAGE_TLS_CALLBACK _mi_tls_callback[] = { &mi_win_main }; - #pragma data_seg() + #pragma comment(linker, "/INCLUDE:__tls_used") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") + #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma data_seg(".CRT$XLB") + PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; + #pragma data_seg() + #pragma data_seg(".CRT$XLY") + PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; + #pragma data_seg() #endif #if defined(__cplusplus) From 2b0d039cf3cf6df8b5269814344626e10565eb04 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 23 Oct 2024 01:21:41 -0700 Subject: [PATCH 17/38] fix assertion check --- src/arena-abandon.c | 2 +- src/bitmap.h | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/arena-abandon.c b/src/arena-abandon.c index c40d3ce3..9c3356fc 100644 --- a/src/arena-abandon.c +++ b/src/arena-abandon.c @@ -271,7 +271,7 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_ if (segment != NULL) { //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); } - previous->bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx + 1); // start at next one for the next iteration + previous->bitmap_idx = mi_bitmap_index_create_ex(field_idx, bit_idx + 1); // start at next one for the next iteration return segment; } } diff --git a/src/bitmap.h b/src/bitmap.h index a1e7686a..f8898935 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -35,9 +35,13 @@ typedef mi_bitmap_field_t* mi_bitmap_t; typedef size_t mi_bitmap_index_t; // Create a bit index. +static inline mi_bitmap_index_t mi_bitmap_index_create_ex(size_t idx, size_t bitidx) { + mi_assert_internal(bitidx <= MI_BITMAP_FIELD_BITS); + return (idx*MI_BITMAP_FIELD_BITS) + bitidx; +} static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) { mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS); - return (idx*MI_BITMAP_FIELD_BITS) + bitidx; + return mi_bitmap_index_create_ex(idx,bitidx); } // Get the field index from a bit index. From ee92b337b9e0f68f28a5eb374e5852c5e60baee3 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 24 Oct 2024 00:13:07 -0700 Subject: [PATCH 18/38] do not reclaim segments if free-ing from a thread with an already abandoned heap (issue #944) --- src/free.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/free.c b/src/free.c index f2e5f8e3..a85baa55 100644 --- a/src/free.c +++ b/src/free.c @@ -236,11 +236,12 @@ static void mi_decl_noinline mi_free_block_delayed_mt( mi_page_t* page, mi_block static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block) { // first see if the segment was abandoned and if we can reclaim it into our thread - if (mi_option_is_enabled(mi_option_abandoned_reclaim_on_free) && + if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 && #if MI_HUGE_PAGE_ABANDON segment->page_kind != MI_PAGE_HUGE && #endif - mi_atomic_load_relaxed(&segment->thread_id) == 0) + mi_atomic_load_relaxed(&segment->thread_id) == 0 && // segment is abandoned? + mi_prim_get_default_heap() != (mi_heap_t*)&_mi_heap_empty) // and we did not already exit this thread (without this check, a fresh heap will be initalized (issue #944)) { // the segment is abandoned, try to reclaim it into our heap if (_mi_segment_attempt_reclaim(mi_heap_get_default(), segment)) { From 532904c85c23f777f29650eb48bd61f1115e2bab Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 24 Oct 2024 01:01:53 -0700 Subject: [PATCH 19/38] update mimalloc redirect to v1.2 to handle static destructors that free memory (issue #944) --- bin/mimalloc-redirect.dll | Bin 68096 -> 70144 bytes bin/mimalloc-redirect.lib | Bin 2874 -> 2874 bytes bin/mimalloc-redirect32.dll | Bin 41984 -> 47616 bytes bin/mimalloc-redirect32.lib | Bin 2928 -> 2928 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index a3a3591ff9b07edada16ad83ca1391963a126b2d..dec318976ca1b881bb5d165eef8d762533ed9b6f 100644 GIT binary patch delta 10362 zcmb_ie_T{m+CO)k(E<4}@7S^a}RJM(Y3M<*DLrbMGiWY`! zgU5ZBEw@Gx925ncQc~B(vcz)t9o^otx3;LYx`bQUx`n0je$TyU2AB8!^X2n-?)N#* zuk)PeoO{l>qmI35#{sp|A%6DhOWiNN(R^j2#dP$;wxGHQsDT5$KkfqCMknnJjtD2D85OmOu zLS$^F#_s{_5{QqPIta<8!jLqzS0EjA32_mV?Q1IR%y2zO2P&gK31g@KF&L}^6aun= z?)WnkVxC;Q%DxJ+8?lJ$1mgg0q4%gpXI&i%wjE+E@D}KfKQnVJReaA3fdXj|HBc?k zmsKf(mcFA(6~a8w(R6Ooa#^VOM z?EX*Y@tg^dXe8h5V~^p??uQ6G^2XuOIt-6>^YM6-xjkgWqtk##26F0tnnk`}n2zTS zX7k-dJZ7@qqgm+Qg?RkTB7A_vyRT2eBZr0lnx%D_9#g+Ted?jqrkNs?QKw9drSwOL|pGBVr}S++k}LlQ42f0S^g$83mjpli%3nsp~sK!@Nt9kM!-55Mp^}tP97rdM5hYy>7p+bqd zN!2q0V>;f)JVQ@F8h*UcpskM5B5gOC%HUsOT&?DL#mtEoD?Uw_5Jers_r`7L>+ffo zqhmZ+3Zht3U@%KR|LB<#@}(ebpwVGd1d+}U(+h8T)`q>U5{}Wd5qiN*7mu*a4HIj2 z8!#(GTJuAHzf-I_ZNTeCNV%q4jTJ4PN10eV!&}uSgzxwiD~r{IRo6ktCQbK6`ss+w zxZgw0ro&X#mu%OuVqqSTnzOrt>CykHhz-I=)E6;N_#KUkoF^=yMUnRjH)(t1lCXn_ z#=SR}Pov>c6NmZ!$(&bHb5w%53H?K8pu3{NhAw>!xjst&7`1xrN3zX6Xt{H#J(zRF zyanlBvt&2#T|7ERjbv^`c|%e&IbRAK39Rlj-$Tl+icy3CO?Y4 zN`C6^KNc(rG-Oo9>Z)tmsE8i=5?qAv39S)2MfH(6kDluS5JTcPA`3MPCVT+YP4Y! zlLg7L@?aDK8yX=!TE?-?!jFTYzEP8A=P|>uZ!iGVfmxcEUbe)RCKt~{J*YiL5Dy7MTf~NkX5G9X&(yj6}%b5=OAFQwPFY z0%0xwuohOX%Fjz-I9(bTLSdkvLf%iIGK8W)h$4T8qUwBuVQ|+kvxvV%L_VUG0l$@g zzm@RAYQ=+oG|Eu9mU+;x2Xe3ma4PUr}3J zvg)(1=l5AbHY(tM3KJCx>`5*iVo z8vZD9V!LTC5|SSLii+{^mD`ymWYw^ge)$qT@6s%&#ExE-I~zdkqu2x(@RGya@ZTZ9)?&wUvlDv|b4i5DI6o7L;bx;J=?muij^<;W%R*0T)s`ZIT~!_l z?NdzqHni+jCq2xegNa^Y4%l1{!vhh+A@?%*bS9V0K4;Au#SFgUwsUO9VB4g}HpVJY zOJUTE+#j~Eu|%^o$c!Th2P?L%{^%}<*>0CTq&ylU?v$< z?$gkT=QW8e8d&MMFW3~YgC1#N2a3ReFFXQ8Hx$n9TGcy&TQqBtc9E-n~#qXJl8^Me0UQ-*r;}>LtQs|}0 zP$|bigmN9K4SkHc+{av~D>#pu^=4Im9o?>g5cYlR9x^MFt@SwV1zxTGS)sa>j-J z=<&=Vn(j%psD4bRS5rSzZU2bAZM?P+Cz15PRCLU$qtr&e6He~Xvg)q)3i9|-EPU=gdj1a>?{`dh!Sm=dxa#6xhn4^>oYV(B{ z<|5yaE#W7`q0*x_`uo>HWT0~rN8gVcd5y9K8@N9FM1jFD>+IG!EDpvMBE`ZS)<(38 zFh1jN61hq%{el}BE8URJnzK+9Z5}48delAj8D0!;8g=DqD#d@BM{F}zIW?)Dy-A3C z=Nem#USxIAz-W!~=xY{5Ne{99ysT*)P-A2$PR*haX?8$u;-3ap=ft;YQ*atJPY73a z{gp16us35=l|Wdp@=L!}(>)ARfw$d)6I%MmWhEt1tSvMYvB0?9eYM>aGjWBGv~~q7 zp1P!1WV06o8-=DD*E@OHWlOG0v@R_vB@K3LT;`q-98eSdEw)FP(b0GoX38b{-)XNK zw~uGOs9cS?B$<7($_GlVrgXUy!YYSYznq?$6gTWw5K3u2`nO3JRgJy$@Z@o-`;4B8 zljjT(R?(33rv(G8Ngt<4d5>*BY=izwdV2bc>{UPyX0fUwn>?=RevS!{yG8f7KCeS3 ztJ$)w4P9bE{HgEJ^o$bC^mkd1OPN!!%SX_zjHEPWL9o41-tCm5Li#8Cojpl;^45@X znN`dry+kKxro%CuS~Jr|t-2szBclD|*WQb6+Aq+?Ort9C0{wI53gIj@Ov}+x6(J|X zE}kxOmUrmEs@({WiTbYE5)eR#T--k!ElsG#$w*9mLshtn4bFVnGEEAW*vHY*i2 z1*y00M)2TwkbL)bM8d!=n2F4j8aXWcMywIFNH8pWlHi~(WyjLx*`hEO+??PP<{R}k zeD|bT@h#z|y)z~VU(mRj7S)2cXxowm`rDaFs?xWpXXf*&>NjcrtntD9i{P~LO;)YP zXPJZ?dT`c#37^SJA|Ic0RTV}OzVjpW9)5g>!42H$4c~czj?0OUFb!8Gxr9we2?^iv zA9Qif^I30*h!}mA4dDl#HIK97D>;tli3ixBUyjeUK*$~M(TLpSh#|L?)mg+s79r$b zx+u3c}1uLH|U46XHFWAPV)JXlGFr{l%OF1{NJ{`X&h9$n2{XSY2#e{t zya^){d3B6fEG08UUHPzfp*xhS=OznZ(+P8P%+H>ZR}A(;Y&QipW_i1_iZwb!Z^%RR zcVUq3?cR=Z$fYX}Wmy}gBi~_&Jqu~m-0359EFXE58l|hfY<_y?Ca69;NyFyZ_Jl{$ z4Vl_?>?_&G4mEZ*;%#7;6XquGIA$s1q@0I3-P?oL2d@mVBPu(A(3@GT2@-e7se#2t zfrcE0ZT0#)qWt4^tntazF>n5`uje574_~Jj=dFwW!%~6BBgZcg$~x6Nvy_VYONDg$ zV!lmv_9q%T|EpPfX1LsY0@H)AR(Y8C{iRb%Vj?VUQ%^HX4Ob#oor%o)DX4IG+(RcX z7?pP7$NqjP1d_l<16EEvJH1Nyy-~w_=D@;9Z42(D-UVY-&mN~;3*sxg@EOK>v`B5o z>T~`&7h`@{t2} zwV}l<_8C|@>2)zRqTmi6efRAC{(io{j`Y|T#R&r2v!gt&yIiU<@hrAU+KIPWdAWuw zf+z@>StNBW)_EL@KUBP8JhpoesAJ!lix(qV0EKq-o9e5_}S6pG8u9cQ(jjT3fS*yYl@m&Y2Cn6)^O-4hDf$r<>0 zu%O19aap>4kY)bt@|1*oaGmIx(-U!L^k62N*fSpA4UKh*68hz>DQKSDAwD{ zHtW}Hux~0|vl(IWcZWc~YSSA1nn!K+Cmog4Hwi3=&<*6(tNkQ#Y;` zLhW4S2#AuMTvX4|%@GmhKbuwYUM3`;=BiGPzCos@0n;w7k~sDaGIb4@>i((Zujgpu z=uf~gV4BZW7LL|IrtJaKP#ae{7%GFW=OXuj$je2(0Z|th^>R^}V7Q?qsN<;T=uhyD z3T1PZ$k8&$^o|Nyxyr_|nxpq&UXE2t;;6fetJ*nsa_r@%mI2c)zltOImlDmzO-aCX zY7AEyIr@jwwN`QQx3jtaif7v^3kBTJ6Lyb^wXFMhqY`e^B^ynx;7#Y?*n`~MTrbIP zK^?r=N%z-gd(0cN1Wgwc(c5)#nyxkAFq0^V=?A5EGe?o5Kjk~hT*Os2j@5%q2^*~v z)y-8M9DROVt@PF&NFYe^NAaxs!(&2n%#VtNiKEETpO1c^6B}1KIkpcnH3v+`T;VEd zkcpGe;e*1MKCaUCDGB&HB#$VgdQhOQ#u$xvz{AWfErUGF0gq%WSJ^l^IJPqr^k8N6 zP)b(MRc?-6jy7(pA27YbRT9TOj=kK}6)>f?R&BKL=AaQ3`$v>)=Bgr&6&%gXgyk0~ zVRAiJwR7y?=&#cq3#F5*x;XZ6^s&5PKQQ~rx}Ox0k)z1b!4q%<5}@YSw9(aE?eEk# zFcddeb#jzA>ey0)y(8Xyv9}Zu*|eF0k6l^=@cjjd@rzLByLIIb@)B-Nze`4 zV_d+hM&d{hg(3wQ%m2wpcI_Migr zdi-Qi4cZI79hicj9LjcJ{w-i~@w3bpD9i=8$bxo*cLL9WPJ{0P&R>kI!CQfIEYN`` zOYpx+mLh5JKH$-1s2uoSAp3o7ACm(gT!AWqcK~mKGQsQbBV=|F=D!sUXAvRW?#HSI z?*;ZffO!IMc@R^w5)}e(1MUG`1@8i$2VDd21O5oQ0iHaBDFWRBZw9UdsdvH`SO*FL z?*hIFiURMe1oJ&89u4vQCG z6=Vx&FL<0ewjty_{SI8Nhd~C1mD01O|^=4LJ$QWO86%38vr~ z9Cd$DEvi2&AOnn)<3*0wIR3&h=1s*eo#Q z|K3mZ7WGnl*`f`l_Ph<7OPApbaKk$C4XwRBP8C0t9=&Z6eDt&1u|fg8d3&0WPpABn zH71$4EL>xsSH5WzJ{m1+Flig?#bsraK`6(1;k7W~2Ti3Osk;uj4|N@iY1OwDwK|W~ zAMqX0w&~i$HsZoEh2G(+c6nWW`$;o zH<_BMo1M+|&F*I6*1C0Wy}P2t(Nf(~-_m=i?@&prt+nfjbfl=w+D5#DU`~n8b;Z@? z>UD_?mWHB+k_K;MXQQw2N@G`Jc9XeDY_c>JHMKW)G<%ynn|;kjx5;gGTiiu%SBtx) zqouRO*AjDBf4I_g*nC(#>};)X)gILyjX7#O>S(KObGEtKh)?-tqR&P4Yxl?Ow>4BW zI2x)OoDEWAZ=l6^~ku zy4%{@I@-K#ova#NIOYKQd-ognoAzh#H}9`+a5c0y=nljj&>t`!Fde9FayHdBxtqvA z?Lpnan1lL*6>f*S+U<1LyL(&uTC|5s4%-e_9CjS8KHSwRwf465wUQ&4S(EC20PE@5 A>i_@% delta 8766 zcmbVReOS}iy+6M|ARtH((IB7!0z^TL;0r2hsHi^`YgE*%VjEkn*cZhT9QCOs6y5Ma z?xLr=(%EeRK_eifPgWP(+D5O{baiX=t{q!nM!S0ZW7tZ)YN?&$e$Fo^!Lk1?&+|Q> z@Au;X@NbcYCd;;9m^!KxV8NWIEM;YTi z0lXpi*Y0?tocSJ3(;V(qrDebQmnR7S8zAZ0QHs%NWGe)Pf-E)BDB-avQ#~Q~A#l>& zLPC-&Jm>-J704&d)J8}F6~c1Ey#jF_PD`fhT2=IQ7F!M4O7961XRnG57?K!ZIp~2u zBO%6VTen!ZKvtN9h;}eGkd;2Hm@ua_0;~&S74SCbfj=X2-4XDu4}$_F5LV+q2>rDp zOCa=;B3p=Xf1}U{H4$lq^nDwJ=B+F=kqR{F$!N}{qVY{YbDVX*Nk`L%Y<(9|b>HeK zXrfVf-@B|)Gn0?Up!p~TO>-ogvJ5oKGSNKFe5az4zIqmK9m?tZ=R!1RGpD1S$#TZA zsP|BM-yP=cN4b4(W~0esQEMlnxq>m%^Wkq%n{v8vfV!0VbDzIYh}aOjcb`Bmg{z2I zw_qn&V%2!i67gv-5gTJmVHEdvxNyK*6`t?yj}#k~MedXcQBm@by)d&8QaKy4{(CG| zj81GQ)oI1Bt760N#D>IAVG?&Hsy;?7HYok*h(liSpz;#9{#W7ky6hEuW-oQAcIhU? zyu(+5 zzDIT`))iokEzcu&j!$ej&uk6MwL+W=t};ukU0{P&9!U~QTrL&T+fm}^oME5F*l~14 z^rZQB2U!)RI(aD#N^&kNZ}68iDnErH?o7B)R9~zfV$tB7Tw{b;T52lt-Q?<$0N zS{SbtJalussVG)#IHW@{MA~*2k?P*k;loKtm9lRLJ5Ice*<<~DZ{45}xBt&fR=eOu zwiq^yK-qVh-i)6);Rxhx@(p!^nbtTqhgdaK)@NmEp_o>V*(rQTzZkPvI83t=77MHB zQwbY|JM==rn$gD*j5}9W&7^5#b0V(&FMKU@-PpA7R%C}vM_(N~I%3-{6k0)DW1mR< zgqi#TO76M?n}SCP`OctkcGwL4%J>!ARqQXJCBi1}#t0(~^|*h3~L__zhIX_B|5jvV{FA z#(^PqF955U!oa$+!oW;e1_PZjA#GFz3$X+tkY6()WnD12HRFLIOPSG*{^2|Yp*->m zBTqpfPhlvJd}^~ih4jjVG@X)#Xh#T)ixp%H85)BH8EJfS*2Bf2(8bIIp^F2d#gK;> z^bqTnSNi^aPVG-4SE`(gu1ualA4*29i@~amtjMFQ0E6pzn|F%OMEar!Z z$!BhB$a8DZb1UC9R7}ZQx1HI4DGk?99;%_7*H9j)0ZYO+#WTT3&mdB|*qE4yAlRMr zS*$-w^vTpr#rqO%OilmM4?7^*&%!q<Ax?6eKN^=)#F9HH2A0#ts+h?_VMFLFFME4z{`l`vJr{j%~9H zd<5hJNp`JdW{rub_YiWHjZN}HfTOMg`>BFhqnJlJJSGO27>u@?4ebLMUQ%Wtu+Tr5 zS=T74#zFbNOzDOao7aknq@TG=MQzeAA&onu{I*6Q2ss+!*-V@bv20{!z3PC`m4MNB z6z>B|tg#pwJ0FG;&!Ufm*ecduWar0!T?zcT=T7>uK>gw4>A?8(gvYOAU^t`=;z0p9 zKgOL)Uo$6Dqb4#}iMsONlX93z-9>#cz?Cav-Higw#`VhJY^bi&MKPPHUlSQq#e{eU z5UJ@)S!ho;Dqg1aTKZyzeHI;`v4fg)akM)_8};cwhlUV8a+bR%BUy<4b{kt4pEUS6 zy|4R0aXydU%bcb-nn$N(EmrKwqr0+RRP@ZCH?!VTeEu1IBU`8V`ZITL_KSkz;YMni zyh0H-opw&n87EC6q@)oSUP&Xaq5Nz4?+qosatIcQR58V**glPxPWh`s`k2njxk0by z&QjFoQq@#5wHe|Rcd+!)pXk-RjI2Lyf&f8`Y(bUR zf<1Pf8M!V-Z?VqYU@XTIOvU$vGM+nG?$cchvgnNkxwN}bMOPIr6YA;D z3l9r>X!e4oLO0#HV1r;rn+=ISSSk9!7YmVD-L(b@;=6H%(GzZ?bK+TBHW@+8%>J;d{3U#&x{$0?)PX| z(a#h;KKgReq4rZyTSp}2CkAmt=(|Ig{K9{SnriiZ(!%ViAU4zHPq*?(OQqfEdhfa6{LK*asO zqgxkej#1tZ>`gO^XhuXzk1cLYII#*5w_ifUUowN*i}i#~pLF@il1xSUyEJ>rys2-N zz#{iuC>s;s0Uub)=@78X0>U>!!dn;Ve=OM=`3Unkc9DL+WP!fvB0JK&ILt2&_#Oe}ZKM2ynTjjd3SZD; zD=QScf1t4?|1-atxvY2_GZiQY8S!PrCrxA`ET*CV1e<&VsbYmTFzek=;rq-gI&IbX zob7n5N|BK8TUdU(C9{jAoL>jqn9nR&(A}$6(|1-)QtUfNdsn5@h>HUYW0Km=$#av9 zg;~diqGW^+f;?H^lP=r}6r}Bc0n);d?~Y;L{E%-r_kDx3zIMtlYf>1`-4 zDYMq&Qe~5@#r0Hx>xtfb_^@z{9$KBfaCbo7V8W(g;hNq=1>m&e!6FU!sn(DxoIT8+ z*)_&qWU+q(OL}W{_L#*FxM=$p4M7|@@W#6q?aT!BrGQj-*mZx?KrVNTerT9-iyI~D8->B_RCjC5u# z=HMd(d)!~4ugE10-u!FQS6vuqeOXr8>f88`Eb1TgVDM2)meW5OuZ_o+(GSWdD2kWS z{<8IS(dPL4%>o(9?3UZtugmRk<@T)HzA3lo(7F$Aemwrju1R^pA8QmC*mh8H zEeJ0p_7~K}fFBF$gsyXkd(EEFLgw0a#p`A~|F`H<8}2RpX-nT6=cuXnFxV`fSGK-# z+jEs=Teoc4Ngv(4)O~#SH6bEVK@x=p^jPIudZ$uDXIVAw_12Z)%dxgY@FMMa6JqiS zy9yXpqkjnG0A=7RtOO;2R)X{(FXY!iUxEffqugItCxlUTnCZw1Hmcbd?~d9RDU4dW z?QyMkhBnWTmrs*l{Pal8t*{wSOs02g*2*Y^b>TN9C-v4^BOZIUa>w3nH1DNb?%XC^JCWiPaYG(KbFqnaSA=b%t9T9* z>P{e~h9OcS>~KykR~kY}V@N3&Y4hvjX-t@2=?Dp9%ejY*d&~^q7%I}vb*@k}|FENv zD|>{%)Xdk)d3-V_0e7N?S50sHHT{SbLButX$dK# zV!jLH)VRNWXO|GA<6VQ?%{20^$kEIZ>nZgB)EtAgx}TwqcQ- ziyR?Qri+Wb9Q_=F$sU*(iK_;KCf^6@3z|MKF>0>TaxCEJH3wX@j|9rZbep)!!m*m; z05>%ao7%a`!O<1$b5jyBJ@VhG$vs>fEU)*me=S$ydae8F>&t{0gS@M{_pfFq%fI$D zT(1wf>3P>U!mW3No0;nkvKt-vP^GM9B6+RS>|E>Q=;auy-F^7ya|9e|X}Vto%29LF za12@wFG8B0tBf4YBTR{V_c1xBHm#bg>Nz?%`pW{@T*3U>SWi&JP_8~v;v#Y^V69>3 z{`!^0tYU4Nk&7)HZ5)Fo8$)%ZI=RX-!o>baAlW7%)yGvmBTUGqkjX$$^=hD=U?T5u z6g5|AN0=BxQPK=tW#nk$=wc=aX>ngY$fvKItExHLIaY8}Y%y+{>f|a9M<2%lIq~lg z&KIbqDJ_Oe2N}9Q{E0l1`c(BV1F7{KMUKYJ%yh{3=k3Z8OKvDkc?hm6>BOuiyRoPvjZXr&V)R6Gsn6^`n7kjzBb) zmutwc0~$R?3%B&jmTt?QCc*9h*b?D>{`sFO-1e_t8Bz_27j23~?c@O_X3o=hNQA4mlo z2e5AfsUbZ~k&IUi{)saP-UK`a>ILrw-UAuopvM1(o&xE?+ko$+5n=}KPbVZb1ONE~ zZvbwZ3_W;~L&y|RHFzU%Bgg{Y0_*~ngZJmu5Hc~BkP2o5`~lPp-Y}IAH>d}EFYqJ$ zQq#ob_}|e91O5vJUOfXdl!uXk_X9`eW0c^vz+<3#@LpiiEX)Xa7chM`Y{46W?I3Rr zI({(90zy3CEx>W}2=Rfp18;+z;MManZG{-+e#{WC6{G?01x{T6J(B|`<6m5Q@D`vO zqyp~;9$$nR0pAN;X2kL_8(_^xXtd~%BK-bQj1ht#Sc0JV=4J$M1X@8A;O)RWpnC9X zd~Ln49McQlwE`JVbVsha3paBdu(2RTZ^@a z!pm?S)&>ryQbMNVcN`mdJQT?hP!kIZybN-H$1{+82XcZZ8wkk-xxnL*N4A4J;PDzH zc90i5-g)E#$Oj%zH*#wO*58i~*@(3P^?=8-jcf$RD;(7*MRE5TY!f^cJL11Tc9TJIQYmdkOMr~jAS4ucpR~0H^>DZ zM;B1^K|^c$z~T==jlb0k462z~dQ0=2?K?O~5xm5|abJ+k%1* zxNE;!8x~c4o?S91@~99ljL-OtcLzC+xe!oK<5+TmJ+!wa_`Lx+bA6KOyu#tM#Q$fs z#K*(eTAx|IbBDEPXXTD{*5`KY+)n;{^t4i#q`0Fz`Uj<85H1}ZQVL1J%%e#W!Ym=} z=;{bz4z<@Ba3WU=za6a-?ka2WC5@Q6&E58HFb@1M4DdPp4pp<~N59j;DKr?=DJ zsqZp$6?BPRj&3Il=*7wbG|k#(drMP`s#V=;YpZXww>jDdPmq(UlbVwi&T6O4S?{zv zrBef^22YW8RlB95yrZI{y2IAd)7jfO&`D0KPMf;SUFBWXUAC?oU$?)zw|k&_u$%Y^ z!H1t1o5f~Ni?_ws($ivSEoe2inp&N0t~O7buT6VWf6{QW;H2?nlhff;w`kzV_7x@#9p7gjvcOD8!EYZBS$9_0B67-?*IS* delta 100 zcmdlbwo7b-1PjaM`aORp%dqTVQDo?=o}9&@Joz_^15;o1W?NQWMwavsuD>Sdu}wh8 lc5?6lWhdLRN=^1)7GiwV{%mH*$0`0RTALBn$um diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index 522723e5017b71b458afb4152bd6c7eec55e9e62..049c298189737481a44fbf45c1ed71ce86eb1d40 100644 GIT binary patch delta 8480 zcmb7JeSA~py+0?UZ76{R2v8uj0SXl8Ag@hd@=}p_D>j-|icHun*s_2XYDryj?HUZ7 z4QaRHBi?SJ*2!F_y<1&7K*Sli9qqK+idN@F=dIqlotn-Z2VqCFJ|LM|iaMIs6CtEp(pSPad4j>_d>1sSF)v~^t zy~{W*RbGDvb6oMJsy)6vz;;Za@qpxH~Y6=`TX z6=-gz&38>`g0s;KWTN>S75TwZG@nonodwNRYT%YNXs#I1Xjh?mgf{O}wf5y`;?&V$ zYIzeiaE&WKJE}!MjEHoAbIO(TYWGp5~CXZ6i8G z^}m@YrORC>i>zny9Cs^PnTBX4aET8xdyFYf0>-zPglFHE@rP^E z(}4lG7*$!QLRpS+dW2WV&FOlX>I@lQsv}|f9HN_-+V^<6h3D6k;W=xwn&VVW5p*R2 zO$pAI6>`NrA-56+66M19zxw7Y^23@8tz3v-EQ-c9@qDO(;FYWI!9eP;FBZ#iVeNG`!I1 zIqr^I7h$GF5G|l}~b@q)hTu8=nf!YW(U0^YsOB!U_$KB^z=i0Dm#V}6NahH1s zsI6;LH*B5$q*gx*!`0vF!me{qxBS{?IZq+7VDXaXF{*c%=}F7nn~b5C8VEG?vKV@j z)_O#1J*}?Ky;D&3#;Ju>qJ=@%X(~79>ZXr&cXwHP#p!Y(SL6m=ogr5@lGSM>tIl$- z{9&$HEeH`Bw0f|mae8_>`(aK!=<=J{<5($sv{tf5TO)gPbg;)`)B&>G=brX}!URaCmn(g`0~3t3tHLa#0Wo?YU(5?o}i zw}wI+L}=p|nN8(wrHrc|(&>;xu9tRaiE=ET(>$uq`0Jyo@*=yLCBcOums(2xit_12 z6clse;$)hf8*(Wa5q{pYashm@{5O98TU`KsCe9G2XN){(8t3zApt(`#opD>>h6 zOw=G90Y~ozcPvy93sy{pTvI3+W|i22?kMj6qnF|k2v)>R;eB*8i}oR=eSwSoK6b6f z1DA5ZgvUh49aWtzOD($8RCFN4+k1@ezPLv?|3UJEuEL$g`R3rB3RNtPhqmA@!koeD ztTJV=LI}Bph&=KPS*gsz-8r9hEx$#kUrU7LYh=^de;~c`|d0%(|Mqm-!vp z_unJ?v)YOjLIuh~sjOi2Sk5^)MtgA_Gm;2x=j9k$pefEVo{|j*mgyypN<)X+UB2=F zW)UUKGYn3aC`>+;+PFyx$>k)Pl_P6hNtWqKn|EAxP4mzRjPpL+*^0n>*Mn{aK#rY6 zx*)KR9W~?lLDkuM>O559D--_0Y9V=1w2sz~yHsZ@7^`%op-gyP^L?e3R|j2FfuGyV$H<&jnM(%%(VAmwm>!{usdDca6-pT+TUIsa z<9j6J9uJk7kG{`w3LIioAyG+VEI~C~W4Is&XlXzf)vKft+^$!J$%Kfh7uU(IU4h0vC1AMK8l682`YIFDv2wYjm5n+=H^nzCe>Y z?ZIUS(n1yLa&HOs@Z390ZqTL1R~2$1@|`+Zq9wmula-kPo1ro--C&`T3~D-XNmnj( zh$={FW>UPix>+4)q630dph3>nr%Ya@mZ&PuVFJl&qbnhVw5pbe8s8mY=DWX6`>ZXd_ zYY_{)A|u0dxv)VrnJjPM>ask262~kcq%ggU@8Luzf^wWPC1y1SwM4nsPbcf_+cX?C zD)gDOJv>Y{ueOuux=h`>pHXfy1*lrK;W&D`phrKLgweOirkt&tf^V@Ap`u&>yMasK zd}Bs$l9rg>ZmI&`!rr$?S5E0h)eS26lnT%D2!H)F=_<0v+fU{35g|lI$&&T?nLVRl zxWJrVoLwdypOL30oPMm`NygVNNb@pL`c1I}r+$UZGfFmZFw<|c7|zK0M^jVwYO$jc z71!>X!fe3HBX3gkNG%g|lZRXr45oAIjeQ4`^gYJc47y@e4fiXq;V_R3vfmw}#Ul1z zz?P0`rxo+F-xa$+dN(Y^jriJz<;}MMqWj>8hEuhC55h=;nwB?IEdi>CyrsZuNME0F z{3?LGfzugi5;)&I_qpRan9{+m={RdBU0vwXFGwM{o!8CuCeX{7=}n=Ri?zoPXqGUs zLyoOW*E}Tic(WhDfKI+a2b?kaJdHsL?%0#)pdzwWk;5BvWN&{?p4_-jO&tzy<+T_E zcV9(3hm3CAsPa%7f4)L(XeW1FCt0iaVE)$ z8etQSV%XbHF`D9^86lC}ZA2@}`h9ny>Y{h_ zpes%v?XHC9fSeAA9m?y&NwaceIATPa;(R!N7&D9qF&3mHWl+)1@wN<*1Oo$b;md3+!NGJG0E;FLMoPzVU%UnC3tw;qHEuHfH;0c@4`z#I8Iy&$OgIud=&I&(5Ileb$w5jo#7XM`1-W=qk@}m zdg9#6nw7a%|3z(XEhP1KZsSw>_~31_M|ae^>ksbPTT`v`9n$UFf3QmDt2wNzKIp64 zUyZ9oWu5Qi#T%EpQ%|i{j zx`TWA{(Scie)Ii@s_JX1!9FZ<3U&7$s;br<_U)W$ zCV~1SC;&)EU~Ljq{(@8MKs1uxB&Y{aDuEqIunIt<1db%Z4glQ}7)^o~0K6lC>TS~K zrT{3;h>rA0PzS&!fu1C|9e_^)Ym;CHfKCu*s$X#$FFjI6a}FgHp_$7v-5kMP=Mb(9u-ehM0 zFQduMI9{SsN0Y#d@*Jnm0DURmTmbo?Vu>Ne+pX~cs08^yy_k^kWL#|kPEimv4u-l^ z8u}1`QP2da1B|ewp-li#yd*l--yuFT(5IOUgXq#`qobDuIwXN&^l3U{0;T9wN&=im zw9tS)O=eU;aSWZuKy*$t-FUeG8UfM4_TDZ<18*z{(^z0$@x6yChIE1t1PmT)@HrEJnNtuUn%7pa*RQg~8Az8M^^^K@A`dV5P)R zw*ly)ApW{Uut#$azzApx6b2Ygj#BvwY=Uw?RAXjZ^8u8CDnWi|Bxg_40H6cZ0~!Z| z!=lqr)x!V;5Hlm%@6jlJ$*HqJTS0n&YDrpC1E3Am3#tS|2S(q;JFECU@3%V_^^M$_ zBU?9K%X2Z1<^i5FgKhy;f*t}r4(bQ}_5mK=Eenr9F8TL9aITg=v2xM>#{^$Rt~|Do z|A>6>SQdYcq?|M?f1AJE=czm7tG?H_V}G5meou|N`u;<8xIgkp*-83?*&`>d3l_@D zeKr49?Yp_Yt`7O%Q@ywE!pTiCGOx2J)dtSy`@ z{G+Yke%}79Bh{&P7C9ex_B(&?|gCm>uSZX7h3L33I3UY4daD|1kf`{NLu7`BU>^i^EcCx!ZETrP0!2dCc-jUb=OO zHQQQZEwgr8&scwBU68*re{KGj{2%9ktDvONYJb7b>G7{$I^m{;rd(6H`Brned6)SU z^L6tA%bV7p6qpKsRH!YwVtd^_Wq-(VhjX{H*7=sR84AU?@$uN8zuWk^=`M4PxySOh z<_PUrp3%g(D#Wg9Zo^&!=V#`*?t-3zp@OPHZ{e{*T~T&XanV@Ocu}ILv)OjacFwll zzSHirs~sAL-ZAVLaZEW{oo&u;CoFex+$lV^>Pz%hdd?s>=nNMOVZ)fA!Psc*G@4EM zrtPMvDP~fed(6FNucgM4otKlBn`h1&N8mzUH1CwP%X-?{W8Im*JHImDo3AOzD9{#U z7mO5)7BsgNb`+Krl@`fu3R~DVU>mZH*c$DA`!Rc)J>Ow-6g##$N*pmq+>vl_PPwxe zD}2s*!5OB(b#dG|JZki{`UbsUe@vfa$TgS^Hbb#t!XOx;hPWYN=rW!*?lx7Ls!TPe z471j(Gv}B`&F`4U%oFC0W~^?f<&>q%Qkv(%BrC0IYld~eI%FNTj#~Zs$MW0qZ3V>z zB?a+Bb5 S8HbG{#!=%tMpSiUvi|{u)nG*c delta 6930 zcmb7JeOy#!zCULiWTa6JItnWCq9~HA&O0;1yr4418guVJVs^8F#CCkSN;tZ-!45XC z3l7|iKJG3R6}5Di%r?5PMjN%G7nSWp8?BUV;f6(Z5Hf11rkb4lea@U?c0c!zd-;66 z&pFTc_xij)bEut>)egv7Y734&R6F_B2UmL~oNoB^rMHFpr@f83Q@GfJR z*+TXECsRwbZ05o;(2#C%v&?F$QOYt_`O)+R$_kd5fq=R(k{m&Y)9Ys<^LG|=X^QB(kSU@Ni9mdqQYG2oh^E{>D? zWT|tIg5ao?{2LVZPyvz=sK>ic$=s~(*9#eD;>md^3SmC+yBHK3DEA3kIpI;DSU`(? z*(mOqgQA~G-=lhQYf+q~X6jTFHzlJuM2pi@uPF(|$0`)Xu_)$JGad;{Tv&|a?qw); zExj3K1~QuPQS!C9D88VsYZswdvjoNLL=;6(A*bapklGkkOa;U7a`JFYuDULp<<8aC zo<#roZM4F87rX>jX3v{U{HqU9L4EL`To;^->7^BNIw83`ZFUW75tM5g=D>=8Q!&dK z4$P&@5w77V)q9dw{6Y4&^f~!*RCw;N5Qwm`S;rK1SzR{CQ|w$sHP$$%8=W%&*G#S} z=|IL|WI|q6*i2DT=tzSPE*o6e0t*GEBouH?&vJsN{u+)@X`p0!R!QiH^I8LvCh9pm z4OEpuf|at#EFqmEE=7Z25q0A~!!`>31fkdoXx}0-HfK-L*I!2>{(gBhtD-=OvKY;D z^OwkuI5m5OB&d?RgK@bmdj|>Mv?{edM70%-&al5G%y?4+&X7CcQo=4=%yXgc^~w-C z>xeUqrmILWZuZSFRmd4VE1S%2b}1WOVX8?Zc;s|&axen^e&y~sei^Bqx1N2OoSBzN zJ8W{EjrDeC#qVR;4@oPxQ5ed{$Qb8lM~NokCz;+aX(p=4x$&n;BE|d{UkS~a$owu6 z`9V9^XWc`FhRO)8xoaa_s}an^JjSIw#;Lu(W?aZ)j8g*Yk+4|M{%{(JAB9Y59%0 zj7sLCQF9E`+C|H^k4UjbsN7#OBE%Y&l!pan=kQZ>qC{KUOuN0v6QU75`t*j@4oOBs zf>Fs3Ik_lvZ_@qHD;4y_<_$_A2csbeg^&Z1^+42mprIuEgYI2T9o=?gz<$Z3Uodg? z6GvjEFOTZIew8LB1`J98gVBINAz+_m-50g)YnZFv#EOml6>Vg($4z~`BKT59=T@&| z)hk#zdmEe>Qdgw@&tAhHDMda^O!RrEUCxa;^hi1MM04m7a_Ex0bw#~(H8{WXRKeI& zM?HM>iO`Dk&Q7X#-g%Nfnq4Q0noBy1`RS;%(b*bsp2Rxu#5!*+_O#s3=sEtRAmq;1 zS4Sd|E;u{ytPKi}x)I^gFeN;i6qQugl3YoiPg>;bs-k5r2KoNa**?>eJoy6<;xWl61qLFQenULj)()(M+)Dp{veM0pv% z^hVy7oW;7yGsy;a1$i&Ioa|VZG^cbGx{BfX<)m(zS+svH<;SwL?yXnihsfbGlj9kxIPjD^-3R<;|{X%#~2_Ze}+V zS*UcA=h8RGGE>QC>A#RYGfo~@-b9{Wu|lRyA+N0{j1`WSW#goImG?((mTPoQ`D><_ znwW_f=-@Yb(T~+1&~e0h#5b3W;iTZ*k%eh)vNm%xHjT>b-zN#HlVra4$(q$Ei?cxq zJEtBmfs(0WPd-&z^gg+Fwa>JaVFIqPK#~6N`wXLC=ng!(O|%#-rxwmpcXM+;-EIOU zGsPY+HNNLES)BE_A_hmDn?Jmy`%hUD58G#u-9C23&ljYi4(hkfq$skKu^K&gBoqRR4dDB3p9 zr(>vdG%ggqIBY~`q7NqLBcLlp^DpMtMLA(Z+cUsG_nbU#BH)~XAK?;q`Z6UO<#AZG z5xQ!{t62LjIZ%6JNg>JDrE8ql?Q7 zKmIPUugzck@Vi1<*kb472I4;h$?ezeCg{=RnHDAlhOzQp(zUj5P3jFMxbFn#;a2pG z6l$`}6Q=rD+Wfcgkj!?PtPghe|B9zfSXh*E+SjV^t$=6y98Bw$xI~f zm9xnzb)u~39pX^y$%R$RovQQB8DzvvVr*w{xy0pcmusepj&2F(x5i<&EJq`EIR$Q( z6ALLUUP{mf@|wkaT%lx=u373^4z&Q1c|gS++>eWa3U`X11aOH}U}VIr-bMwE<$k5O z7`+>*G5Y9yejG07xF9=GrT4a}#w~0bMgX%u94Mm3Q*;5qQ8*xTd#r~rv)RAVK1iB4 zL34G5<7wddy2nt%{;A$Xu4-1x)?Xt@+Dwi*J)h4e({pK03G)4C?V5NGby7S+og`2A zt`SF889FV2aJmJuM=z3a8&ufQ1-eUoDg zJtuG^&Fn^ciKI&(%O-dC>QyY8OkOwSupEgoTGM_#Og+-;Zwl?C4RYFFfrMg@{2=_c z50f&ZN_NW^Disrdqtn{G?g<6OvfDh|eWu@ga| zZxxMm^_$2()3Ov{UbvKgE6aE@=z6<8FVLvt_dV8Sm+&y3J7^lk{iSEKF^(J}VEu)8sb=oOCb6QgXaWe3nGtUNoI< zm;BKoArH99@Kl5JG*Ai6c2+c&o8LSms9*_99`YQgVi#A$Z@a;;of>vh3EdE0i%Q&l z!uN(YPnY1AbMvVX-zQw`5Hd(D2dIc9Cnn?ONB(g=ByEa~{uVl@_Y4Uku;0m}A>vO3 zkvQf#Cv1bBG01tFAXA=;!D@Igw<00Bsxf+TjJJB1Gt@-Fxhb-z-XO`==f z4(kfpzr4|1Zk?H(XvI`uyo;nRqoWvEXx&8iZ@fJ*?|+l!Z6cR8J}SF%j{Gb?Z)v`u zvLxVCp~q=r(On?2Fn<$yD}S2~$8uyPK0BaP2&Gae@w*F*N+^?rGFd3oP~sv}oItK@ zt<*}NN$BT=J%D=$a2sCmC&`rb9q5Cs@3!vKcc!qLaf6IdofBmYL-zyx0uxY$dK1tN zlyeNT7}y8I06D-J@Wa4Iz&F6$d%GXM;}n~|=9}*H;MLnU{GmFq|CJMaKB4(+v}`Xq zu(R^O_VTi_ePs8Yo4dR3TrWFzcSZO3-3s=m&1L)2bec4+TB{>p-E)=%ORumR@@!e5 zOp}E#F;9@*dk4DPcWhz1Gj@7q3QsG;RRj1wB%fZ7?**T2^(Y!Zw2EYlM0SAa5y>uz z41yRE$svgx1u-R(>J6gL84&V^qk%Yy@QP%GL^grw5Xmly z>;*9-l0y0^qD>^-64?tP2%yXA6oYsf7AvYT zyzsybfDvQV`GQuJO7SSeaX=D41=CSqRjSn>)PPkKxFriWi0yz!6y!;QY7lk6aZykz z3KSh6dH^A<0lbVzl_|W0rHY~rX#iABm4O#MkO%xA;VmFa0gouSo^Tb22A~aSK`Vk% zUOgcCfe|1G!L%fp1`!67k1<>Y1nr_AIRk_mpnxa0GI;zJp8-Z+q3a&TS3c?MP zi-LB^q83C8&;gWgLc2tPq7TF{AOxJki~I?OO9rUlF3CI(#CD)c6a+;9*9M|fB-5ON zn`*8X#DGYaO5`XAUL^A*G6X{29t~D2k}4I5G(ZgmL8c*Jq*tXX0I?0&3$#PPNP;>L zEtCYtAjlI1Tn~r=fHtN!N>;1JK!gC*ldu3eB(+ixq5yCM)Z%)#mV)pC4L~U@q~24t zgXjf@fgl89QoqcA;GPoQNP*QL3W2=<2hC|wJK;Ep9$*B>1Idu}yBC+`w+&7eGf5i|)}ZsD_U8_bPaNZ$6s-e>P_e$>kFZ*h}P#6RGS7 zS@^7G=^*=0UU%hw@6H|Gt-C6{2g~-lcK+-B$_E%Gjg&oWV3p)I&*pT$|7^DG*zFl) z-tTi`%2=j`#R;>V?XGyfRqi{W>CyaAb6RsjGp4E0{k!frx;9;h?xgNNbt(FK{loeZ z{bl_X{nz@4exBi=;UPo7aMbXK;R(ajhD=MI&1inCA)3Ev-qU1h@6+zlKCOLG`>OV=c2Ijk`;Jfhxt7zV>oRp4b$fIV z=zght48i}P>(^b-UD93Ac@1@j-x*?!zcl9NWLh4zj9LF-RobrDvg}%WuKg|h-|e5% zPxxwl+sC6@vqJwXz0%;$xj!e8^C#1wY0Ts_KVp6@_mVZoR%mOopR&Je|JHuU5pc9P zo_F*+e1CTEj_(}Q8a^1AdX4&4^&RRYjmhLP{fFsg(9X}|yusmiI&smozZa+AWOG;yXTQ;W%W+|+7nGbNdm&1vQg^J;U4xzpTb?lt$B z)fPQknr|tv^jU(I0n3nO*iw*Nm^++1k~^9^o?B`yw|cA<)+y_>b;cUDR@tg;wYEB2 zgH3K%*jwz!?XC89d$J?Vk>SX8bUIEtdg$C}!8*r7uQsXk)WQAge)XWbP~+BY(~M|F zF$u>tyrx|1(eBk&XuaBL?Tj|04Qm-)wXRmzpi}6TdQPv>C+UytTlH=F4t<&-!?4qpDfR6$K>6*IiA&!k<&tj_194b1_q|d z_AJtqx3Cq#l<5PNOpa%jnq0~r%Xuc!`Nc!1DznLg9Q80&(v#P7a6naVzRWR~2>=MW BA`Som delta 118 zcmew$_Caie84F7UkK4D&RxG<%6dC%eCu^~*PnKu3W9qBk9M5XV$Qe;K{r^!01_q|d z_AJtqx3Cq#l<5PNOpa%jnq0~r%ejdy_Sr+IDznLg9Q80&(v#P7a6naVzRWR~2>^ga BBijG~ From 3cba10e51060e2dd5399ea44b499d741ee58355b Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 27 Oct 2024 01:02:13 -0700 Subject: [PATCH 20/38] update mimalloc-redirect --- bin/mimalloc-redirect.dll | Bin 70144 -> 70656 bytes bin/mimalloc-redirect32.dll | Bin 47616 -> 47616 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index dec318976ca1b881bb5d165eef8d762533ed9b6f..ed001d64f7f70516577b3f3724b9a1279b73d51c 100644 GIT binary patch delta 6805 zcmZ`;3wTpiw%$8^rBG-<0s#x85FluQ!$V0yG)Tb%6bV`-4lsmAAt;Yh3e-Db(&1sb zp<{6Cg;D3;goY;cMY)Fy{T#%Aj8=zFgV(Fo>!XZZ=hQMz1=Na+z5hOE2S&!}_pSZ^ zd+oK>-h1tJ_BjE6B-I~F4XT5pmj|yV;;)kov-ULf9!=IlaW6B_XEWcUpngIejHcwV zH|Z!ko_$T{_c%5P13`@an$0oXNq=Np45Mf|`**ZG*z0Ib%2x&p^?fm%kapSwP2LzZ z0}N<}^+U7Wil*f@G}XM@+m2?>2sBrXXx`x-FH305S!fcx>B;TB?TcpLP&D-!X!g%U zbD6u1;Q_jAXeOqhd5TAVjOL@Y%|bIY56u)lyP1c!%4mMz5#Glh+H=RExrc{dn2YA! zbVAq#^BLw(TQTLs9|@5Iy*_%1lGjs>MBXv&ZBSBXY38YspNJE=s@E|XWuHo=yW*Qu zC&oK67|1@J zN~c#An7Vm5yBsLAIph>=j;daRuCYiDRexh9L^GS;Fhp)6?*OfQTCu!E7 zcT=(}UUnB!{yEz>AkT1OH+yx!^3=J|v!m|YnbVSCFmGkEEko!x%xl?T*tJFZ&{AbE z%xYJ>x6h*VBjrerpBlz&WaEcSGp75olpWJht3#~xb+&&>c3}uPbRA$Z1$STr? zZ-e2S?QCLhk?AV0fU>mqle}KuNc$7ppZl+dMcdf$VV7wM>py(&)IuAQR$w!}c8>qa zJ1#j?6{X7$)+%FSE78j8wq`sVyyPffm=!M6$d2aS#zL-MtYP>tZf741 z|D9B5^QiAZ!J}w9aj4H{>OmzQR3e)*CdS+2JR~NUSj0BT2U`>;cAb4i@ z&QibrTT-2@Elld`J`TH!q%uV#5z4V|k zmE=7*7oJDk4|-JRs+UniKIoikI2A6poX}S8Yu(`>Wl!j=Jd2chG#Zdi>ZxnVtvkhI z1%Epqw#(dUq)xS|^NpJ`v~I^RVGuX+>Fgcv;F{11Bi>J&zazlz8?gZO*Mm7n5244| z($RYj-|u0b(Pn12W2&!G+x~O5101V(lCDkQ>Dp`K7r1KrUf-KE@m;=INqq*_q&5>z zh@F)6)aKt&jw<8@GXT}BX#%oURl1Tpl5XzEDZhb|Lu}H4Wcqmd@e55WL#+rb>NedE z_mv}i$63BGY{{vePavyOUGj5bWqxg;*{khDE&Y(!UG}aD;f><5skVP~WK->#UTn;f zeIiR%vth6bORK~Su~9s_Y71X9KTSQy+T)$6Er~Vc4P%MCyXk3`F@~)& z@NC2I?~Aos>{tp^?gUl8`lh>^$Kq>tPX#H>(P#lLtUvZ?6G;N|BDHkbGjAvbSLvmJs;=il1!L2`vjRb?-#hv2Jyx6t`>hC zRO;XUM~i>IHX^TZ-oxkkheEG#xMsp%ToWcN&cmJKaAxa95683?);V?@oyM%=X5_SA z(YJ=l+$a|NR_$Ss`1SHO^NkD9G0Z9rp}pCCl8ZL6p!EENzy6vMzFsZw+tb=Jcm}I_ zwYV|MEAQ&3BFj}pHaB0N+-*LkEF8a(4*T>`7|cDd8#ws6;QB~w-^JZ|`KTo&?>F4a)#GH)xp;NEF!v;?}M?bIr za#CvAw{LZK>u-?W`AgC-xG+xH#{4yD>DoZ<)N#m3UuKgob14S82h1| z5bv_skf(O>@CFTM}J`V-TT)b$LE4yJj-WtPVgXLj5(dZGW36q z9dYNfPo2YQJ~K=YQa5AM??``U4y+$hUYq_nwauRm{(o_Jwtp+?PL_9++e!A$FVN3E zo|d_YEtv5^#!eZAU%tx56?|*>%PBT$=9Ih_AAsuUDWtT@{2$<2(W%FvsN_#qmHCAm z%B`$wW^wuy?qFqIGw&TW3|864SJ;9%W$9a;7>u$%&heT@ zKTL7KHPLo_st~xl_N4WHSdr(}(MjjI)zA~zhaR?&U)otM>XqYqwMZQ`yk77|q<_$M|e4G`|yFLARuE}QW=53@4*=O?xuPm9SC&Q)s5CU({QAozGgi=-q@wipy zUucRJweMwp8f(WkXjk6wopIB5#SPz2|4Z|oGmUv3>}Qv67-!rxo_oXipl&>28hhzM z>o}{>JJn+7wd8Le%s+-)$%5+*3-tg7Ur$>Z|CCKw1&OC_;y2JI_p@xa?18c zDd`^J)7lEH-J!L+wDxJO-J`Vut*z49XV5D9mTc&Qde3}z!+#G={@B3_jx2^0R6sU@ z;16TU@RAYKbnnyOeg4B^i{xG9gD%I;%&J%18y_;5`ViWilHLaip%zLk2ieP;W|&hc zNnH+2e*@`H#}JxHN#=5;Z&^wYI$7!e)WEH1UK zTef=53hVmK>z1RlW|MWz`Yp@Wu32fdOXVZVN9&6s4;CfOH?3K{ep%V((&Aff*WNIO z&&oBW#Vg7-ls?I=Hf%1lZdhe4UABHTymNkN}s?6n%O0~ zMs|TsM*73g!racz|C>X(uzNDil)l!@oqSx0??nWcP$&}Wj(Z&o&N1)8dUZ@kNF%z>DH79U0EWOK}~5=HogeUm{7K8N|&x| z6H3B1#&}H4GUtjukzLss!|Eo9u2bvwb&0N9p!bUM!ogKEcbn)}3JeP5iROPEWqf!c z%?$~4OkndZW(m!#PeQ2M1a>NW4y~p%m7T92O4~G>UeZ;OYmUl-El<#1ZlUttRQZJ} zctgb;M@>r67BB4^7h?&5ZCCZZbcw$48+`_oKz%34z4WyTwIopfMwxRIsqlz?iNH#M zR*xReuUt6drb+K6F&Y<`SgKn|uv$p(Y87f#U~&zn)w)@--zCNm)KV=QGw5B$Z*>!k zz@(}F7FW9%58ZTi2$ky=SCjXrvwFpN?1n2XXY>_|vSiCrN<(bpsR8&=rvI6giqoio z&Sc2%^iW2DNi)4rnCG+=kE7<{6&B?JgGrB-dW1G*->HSDaG{PXU%$G6GS^w-z)qnu zcKvkOE|sy{5H1KWhU^OoUG?Iia} znU?SMWtapew_E~$548%dB+x0aZMp6hR2IC2{JDkNE6^`6B+OheLv;ylOklG>>k2(s zLJKx9A=IkC%&1| z)~V1ByN4;BEl&EM*PYatieDv7;|Tc^38@D+;b-XXiG-Zx zHo%;_Fa}-%tb^nsP8>M(ZbDvy-UIw>G9eLgCw^Z(4Y>p!0`|qfms|vw@`;b!F@+Et zoJ)XLAX(tnse~jT{lE!6S9U^jz?*=B@LBU74+`9TFCpi^6Tsb0LOup3)A447oCNm+ z%kfFM6TWd^zZn<;6EZ*tE0LX6(@BxH@1ki?`JVoG6U_X4t7J|Eg&5-B7+klVa z+jlLvAGi*m>)XIf@Cn}l*$W;AUWJ?lH_gLBA@6|)fgA9v;VihcfRGn`4`J2lkcSER z6cPcq7GXh;AdOs)pC|!H0eBN|^g===fxCd~-8dTXIPmCWNECP{(2gIF$H4tS4?dmF zfro%+AQ!0|EQRa^_XF!8&wSGNS3(o(d4oXvQn{|e2z2yS^XDI#;Vcq`bRO7y)IA>ocGpeK48>9sVUr_Ng!s%x!p ztG6_`8r_Y-#+Dw=A;A|*w^fxv)b{^^Cv6^t;04d}RmxS96st4T>PsAH3k0gAD zS`W1y>O3SzT~T+`8;!?WVu@H=%u(yCm22I#O?Az6_6DgT)EH|-gt&G-#1gWGg5d+< zP`D{g&x(<;VV~wfCQd1HwkNTsP(O^`Kb;ihH(_zbDPp!AM zytcA7Sle3HR_E)iBlX65SA)Bur7_Xi+Nd^?1R;2Ci8CaJTp>@WIUEnSgj>T>wWHcu zEmym$W09tawZ>i})i`SoL_^V7v^mN{+Yj@wp;{i%QeWQSZwNMYHg+|VBgP}9Bcx4V zyf;)5Di8TXm7%tiV(*KK?}DE(lF&w z47~;pv*dF}J~PwEM1J*`?_d{o&FdFzPy>m>!IA(~;6$dfS4Vj8+HGfF`{gxDBOD`IEp zL`v9YdT&-=9vnXIV~aGC={2@VGm&c9@1XVUeduAz*BT?O-Bd`(C7K0wtqAHC4OGDh zs3|6>$jwkKy!iD*sQYh)>eEAg%xk=8h04u_igGoC$9;DbRK<9xU`B0~Mwbd4= zh#6{n8q`+a^8K_7x{t?vGZ|_o?>(0{-Fp|*kGzF{#~dzQo&q(CH~l3a)+gzNunW1T znMe2N%)Flnk-bAlZKdRWt)9qrvxk9_YGXG~jr>HI$PGjL{z*vAJ6gIUyiq$n+>;?U z=q~(I8Jm8Wrs|c=i1QHg>>V(Af8>p2n`Q4Zvq?@nD|f9*E ziLks|w-&7D71X*;19M+s;%gp-v5qeA_fMRkI zzvHYy_fj&FmI`GR=Q%WuJ^I)^u)F(#D^rw(qv^ zmCdcfVPl?3zGw}m#qWM*8 zrP(>Ks#eYKxg-U8V8|yo?c4nQP*k&L!d{xRhfabX z$)?>_UfZbVWWPDUv0;zd1B`z^jVElo*^h$SWU`0_HH9##p9o0Qju=n%BiVTV#k??dXe9r%96TKF!M^Pt{|3fi`kNl z$>N<@;zmrFEh#rL_oPho+BY)^H%sv|0}FJ$)l5DN6Wdx;z_gWr#hJSBdY7pfLjSAHFza(J5y7H2Zc+ zk#;Xm7jsvba7E=alRj%6?_S=mMV__hoa1En;*~h9yrJ8q{i~AQ9%kN67Lo>Xx^9Wqtf77Ap4B)y!@kN9VBhRy&Qc zu=Ur|ZmFXLRZMDLUeI02;CDd7P`d-8|NAG&J3^KlmYLN}BqwtJrtF%!f)=b@j)2W) zlL4vgL3ars1vTZixPRYt^MoQq@q;1tWaIO5TdKtOorGQ{@kWZzvPIL+<~)91GSsPt zKEZA)8I$oKZ1JKfTU_!HZD|=bL#vzcr$i!o(a$Mcoxa2l|5Vmg_PFkj54mGWc`?#^ z6SL1eG4|mz>UPdeb*8@;!Cf#;3-?&qgNtrqQuz=~!5L+7`RKHaYj?w&^*&oa=bNkv zoR4^qH^{Gps5Zdd(q_$qKe8_z`D|E4A$^=ptMJi!_Q#6b(i002y+ip&#V@J(qzs;W z3N;-F2yEt6D!^u2)Gh4B^P?LR}ZCN}zUB@kR*{h2;&|NI&p0R6Uvy#cM z$3w@|kxxW2dW2DqFM<(<4^=}Xe$$m?H>RE~uzosF_u1=p|81b|uw>mgXEEPBBP`tm z!KK%O-x>)19=2`5cdrZYXVVB~+hFwkm#5j=udP~0 zN3x$*jn@1qvC*s7%sjlDlEkecRUcKgqUzUGeO%RVsQN8czpd(%(8{UR?%~+)8OoAP zS~@=Us{{{rvI znGkBE#CV#$`Ru&hks30RP9ZdtlFUco;aH&DRGpSZOO%mY&(pMNlL-5^cBy9CB>d9( zKzDofV>Q+6?QNZk=T|Z9(C9bBgG7-yM+B58nR`lUX1_4&*@gO2>Q?r?2&1t#8Ea84 z)UT(RHc^y$jN*IIMl&74+2?Dql{H!KpOuN3li}8+kDGD-Sb_vr8DGsxg){#)VPO_8usu zCZ*yxh%{YEwkD}YXVSPz7+u_`I5%NT)xzW%Fy;G%sZn4+pt_P!Zsl^*3YvdHm?HwC zgTkz880-D5J}WL<{a2J<`q$EY!#|RPGYPZ`O!eSVs}8ReX1hSAa<+9H@-B)VWy7mT zRHG;clEttnMw7*uDE6vF{0J#)ACFa6XJG3~4nr1b7nmAGat8TMVRj3w7Z`p>%tC4E zT0xmHXwXFZlzm;8U`&Mc3MAhqL;a!(QLD@_3bRFEs^i$&WO%A<6aI4@)I)LF=Fu6l zFgXTQwXgemRhRI`u2-!QCf}f{u|ZV>!XF=~I<#AuA{;T0C+OIiF!c*Ge0ReDl*K0> zp!rr|k_5^(hG34#JZCzD+0B-`VN@zkV~+)dsas$)g^wmX=~9-QTtPGA!c3H-;q{bt zzir~jfH&a&J{jl}*d;KPSxF6JrqlZ2@j))1x<-4g`n-PO(f8jlHQs`<_l%RKtX3uO zz4bIt5<&Js!MGpZe~9Kgh1o68Be3g{WZO<<_@A(u4DT6+hf|@#Wl~-F=Y%;fuwP)W z2=l99d4{XW1}y?pGbYN0k1-BOm>mLZ1WIdC z`m;64H~b*RBTyFT5gmHemB@1mbG^V+XE@G$*a2aV2#gCfJeF+TpImmskIAA{phKWj zgoTw;pC=0b@RLMquhkLaFD9fA|GDY|w*wd8XGM_Pfg`6A@-Das_+KT2yaq1KAY?N{ zi-)}rIOGmOUV+^TJX%UfE4ax<$WxH7cwOKzND<gV}Sn+`4p86vk93$hmeoJ{lMiCA?LxPz!E&nPJr8i@|}ci zM_nKAB4iCXnTNqaR&B3^1LW~aJSpJt1N$J$zzz7}a1l}kZo>Pp9cD$} z{DJ=pw*w!*!@m;T1H1|;0XMB6WR3&N1nvWFyC47l1P=rIS7MpK?GIo{@D%w7+zs3Z zxd`qDegOFjyfzB*J){o>vKngyxdtu)*Fm&9P#L%vk`3+$o`#G7j{>hi#( z!L7g*kRos=a1W#e+z&hpnFSsLW6Yx&R9&kHw3uHgI@8MePOcY*0Aq@Nq(h5%S z;wpj!!6o2o$ZOzkU=!p7IIb*m9`Y_YE+3NpC{hh>0xp4k3ho5%ft&~T15ZFMg5z=^ zS0G>2qCnQ5Ge{pe?jN!Nat$0;133oK?nDJ3eGHv|imD5jT!Xi z0}J$Y*@0<#I^y`tdir!mt^ZKqQ0$N~XbL)lzQc`&qlfh&Lr4x0KUNogc>MMLu)n{F zG~1fpEj2A&E%DagR!f_+-PP`IKX)*8&=|0HI68bCkwei#`c5e*2R*^=!{NjIhpi!7 z$Q?o>T{zW%fWOOcYPK{tw)k5DE!{1KHe+p@sm@;;+JEhKAxziVH4C;>yUvs>rx5e4!YIC=F+RnAd+T-p0?YwDM zfHxcKkUHf~M`w319E=97M{Gx=Bk~dZkwB;`)Ex?kB0L*0LT~{PQ&%b+FIY{Yin!^w2_1QgNB2~gQkNu0Z*Vl;0rVcdOP|%^oLxX?#`M{ ZPiK8+EEo^=2K$5L_}SsKSfe52e*od*3grL* diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index 049c298189737481a44fbf45c1ed71ce86eb1d40..ec4ff1d521292a6fd1403d35c825e1cd1b060b3b 100644 GIT binary patch delta 5053 zcmZu#4^&fEntv}OBx-1aMuCbDOR!L}=Kudc)RI%TlKPdpYAbfq4m0Oj*t2PyZJBLG_jg~yv}boY z=ezHIzwgie?)QE7zDK$eqOOFfkI(4;X4#SHU(YMaU>K%_VXWlq4Q1pV_Ce9VKS{o1 zpAaqi1JR52uxBH0i4L%jHX9k{PVYPnCkrwBrvk%Qc^Ljnf`Kt(xFy1{Vg-it1`Npu zFl;Nr@Fg`{mQP*~=a?IoVZL62;RN+#VlgNwHl2lGNhSuCx*eg|EB9mA3>9)d?>*8b zsnJ%hW0hCB)?R0nOwdIcCa~xoC@_Eh0i$^L7gTT{+9ozcrzOLbA(ylBBkIf!)-EV_ zG0g6wkXM%BO@^{4a!}cGgzCLUnP8NCoj#{8rG!`Z3kwleT6$FG7FDhz<+8>lRAZ$# z-s??-dT1YtS}*x!;!Y~IhsqVntHsr&j6 zWMuljYoZ0z+nmlLrO%UbaVzOtxM#sk1FJmnVZa!4#Yy&}huAoAE_zx_QR*tQHZ7wWGh}cvf5&TD9T=F2xCL{llQ^Zo~)!bFmI1H()A99ycC^yf@&JbVT zYIY{lk=M@J6Of}sA+M4P`Q%lhT3PkI#T4udcE~f@mcUlf$O?Jo+`YwuyHWA8@5vdm zD>GN=O4%0c^Sq;Scd6Da*vNrDV5-N7cv1NoOkrc&~|{BAvp{OoUT zr2nw1D*o*fhUxVtgB?jmXb{D)7{-@lOvslERj^`|c_~q6N=W0fWuoUwNcXZ$ovIB8 zlL^d2bXTFwKv!19XWzSbFE}dh^~O=fYQ9>^Fanzhb|jcKsUdpjC{^_)B(r$-dS?yS zw)Xmx(~Gx;YLi=0$F04!$&hz87%hQNG1QhsCn<)iGOPJF?h39ILs3V(!&kXO;;%e_ zX7SBd@$3FTMRK&aQ3y&ELSL?-s!1I^wYA7l z(9Q(bg@Cpr^=oK#s+86I{FHfycLq75$+I$n`;nMXwxc%DCIK>?E94Y)nVtR|X2HuU zn)S1!N?wTmFDB~cIjn|Qm)k{R4GAnSW`9pkEMM1IMtkqDZ=#BS8MhT}QGVjT)3GLY z&^mH_2JSW?*4bdkEKQPcHn>-v%Et#nK2Qra$G?;^%V~u4pknhDo}d%rG62JLi!X zf0S-8G?`8Tf4DY=Gsfdu)R0vV-j>!-@v&PZyChfi%Ufh+NkLvcplRtE{)099=MjO3O6J^PQ5;nM4@wECuyTlyj( z{VNSp+226r$|vNrmFuK~pQL18sEveMDU0~a9$}^BRN<44$>6Hsf_lt_PEAY@_rn#U zwQ|6+m+K3pkUJwfh1TrfXCVHA~LKvt}_iEeyEHm!b5+Vc@zguaP# z2v?j*R%mlXEgun&_U8}0`5_hIvOuTLAT3ke-223T;blZ|?n81@`|Sf$bpG}SR5$m| zDdAuxgL_qsqMJ6W;^N{Y`48RUOl1$$GD(;IbG#Q`!J%&u)WnL5V+XDUO3^;m{yr++l``MKrf1?E z4BseVL~Y}6C*?#2jYn9u_P2By`sGqUTXKyUo0EmItNr_Ef<3<_o6I{!uiqf!<{(=@ zYAhDfAFqQkkf_Bay+C!s*NNErWDe8FDji-~n{<%96BTwZ>&;>sBh6Mft5fwtapIb= zBYJLp%8+V*H6IfY9LdD=kZ^}*`^%{2Z?2KswgOY!gZY(|74o3C=)H$Hmnts$RM@P- z&r#iQ%EvUl9WroX;^dX%driDx?#0hPagB7^b2_JJ-SO_=kBkX%z?H_I0O)5>IiXP& zrq<$k`?rD$Zl38qemNEUlvRA)PYiPb`qflIugFhR5{_qoG<5k(1iwm-FN8QZb%kpD zR%%&JMYIX~$H*#2!RobwCtUycr93GvMiE~=U6#HT{B@A?m4ZxuMUo^hI{HQDuabOc zzOyB_^nj{N_V$s5V-M_&~s-X$+q zlrL2ZD!Bht=mr{r9BnVdZJ=z7d|a`3A=*bwzS>|e#E1Mg)SAI7im3;2yFt%jf8l>pKG?ivHooCTaP#7d3Qz{o;fULEs<}YA7(RjTT02c`ZG;r?Us76gn64$s>laT_-fk~mZ-sL1#VzxSmFd%32sPIhnZi%8=AwC0Jv^} zJKhqO90C_6mv;?G1~3~D@R}FGk{jS+lwICwcZ-@242Tf-8~MMmkQ96KmT zIq-ggAE!LKkDP5S#D_JtJOkO7U>OgDNh}~{lY*v}d>wdIq8eh9MslQmDKQ4Z5)ND) zxFJacW=(XK-Huro=n#n3Q%f%QoX+kCHvk$;qj7jUogD=?2AW8toS@5tOHgDu<)NIS z^Ht#TL1n|_?#uryb7EQv!vBC5Wc8S|%rV`V9GheMF&RuV*&~?T0Nn)r7}^ZDS&F=q zLQ|nB!Kpz;P~SR685KgS1h)~?0*V5SQ&ial?ilDKl>jBto-_F)V8%dnAtjf3-ULt2 z7=rH4p8^w4E3~8)n14ap<`m>$)FA31mR7KWr^k$XkW_%FAywT&DnDlXKz$&p-nU-J z%s}=T03)E$v_ROLY62V&N~Y2Nw7M$y5~C~uIYISLU^i)jtOh_6h^AZBjY$}E0Yq0D zoHM@(AOTX2qy*8lAj=4#JPoSJ`Ovv64qSa2sTYs{xUMu(E+Bp2!fB+7BH3ra4THu& zQ6JJnUJIVij)9AV#Q(}D`++h7nh&l7WCg{5mQysV5?noq#@$7cBU-Vl9Y7!G45%O5 z9#6+jxW5%=#Q^dkYLQCcEI9eQSRM4!EUByjI8YNP3O}*A{T~7_0805tuQ~=`3MBqF zWDJ_|w6>}YKqaUJR1T0q@gtX?X<%P3Kbmv&RQ`frzQ8i4Kw~ekD2Rz{JXP_3u-K^+ z3m1v215J-L1|Dr|ZEb80aE+Tu`{@mlFHiqb)_L7FWWVZixEU?O;Fl7H(S4|6^fAM? z2D$MoW2s4Ra+p3geQx^36tVW&7dZ|%l8*DP)9xGYr#(zL!#H8GTI1)_DTomkU8bfQRkSm%hm1barL>J?s9j9yV5=DPP!S7*dy~?@C`l{TPl@6_(oc4^CWTAfu_uB*`Tx*1(UmqZCp>Id|L`eFTuzTVJa$TzBu z8e^I9rg733GxEk`rsJlt>7*O=?f8_o4*xkYIiv)r(ZTW(sqtv%L5)??OkTZOIC zmb5W;nSIzkVjr~!9PN%Sht_FyTAfa3xpVW3Gww_{XPrssfa{EF&~?Ez>}qiP-7Rjl zN8>5+lzFtCNl(l(<(cs?gQ-#+*M_zI+LPJ=ZH=x@w^3KGYtSk6Dt*5GhJIW>p}(o0 V)b|(;8Tt&z495+TFT$UR{s)yhnqmL| delta 4685 zcmZu!4^&&%d4KOAAvuU5*|L#?3<6{uJ4*c|J)wV3e{fJo9Fz!#5^DP-N{T(jAsM!X zLsK}GCp;rR$YkHFhbD9fH(o}A*N}=D{3osK!?B6uW^#jj+(pUM9JW1IoK9<%MO!tq z?>@;%cQ(%X?!CX?_uYHH@4Mf&@9wK^KVtqyuJ~d&&AMaOwfExLGz8x z(A+XWqi%ra1a3aV)5dDhK~Z({k{ZT;-U!VDbUhPUtq=YM8C+G$cX1LKvxv% zxTbuFvc3SGBMt32% zbl(PS@sKAQ_2f8j&hAwm)*b^-&?TwEQdH$nj+}rcN4)pA^#bG1bDnIO2Y9^R$Vsf_ z{Mj;p{-o#L2sn-CHTz81lmG`0QJHOmJ14!;5n)DgnaEW#Dy%0cOG$wEHpJ6wUKUZ| z8|a%~i-D8z%-7zhD)4IeZR)o_gOZ{{D7N-Ug|3H`AOA&&i8yGaSht;|5xZ`;2(x&p zazD2coQk6J>*^`!ezopt=;BtTi-PszuWZ;#l5G43)r}<9UROOVpJO?KOsO_9 zqz-vE?jUvXp^XD1&43<`i1WyOoLAPt`K6t!>oFON49Uv+H^5xMqLTB-e5>new7I6S zsyJzF#DNE72=zYjDAJUdD~bdW65o9tPW-x-)X#GsJ|oJUK^?L>I1&YN*L?$y;6&XPvI_mE?jcfw z{;_T^If~4hr;lnPLrNh4<^2F;9sKjlcqdWRvl1CvA^Ii8#N8PjoL}AvLHYQDf?gW+ zWFte_qCN(CVJRu+RYJ5;1>AsQ`9{4whn>om?PQ~#WzM@CxuS}C(-)-#v4^K&$CJgHHVp)K-ID``$gF`=!I#RP z6SO&do9b`?IL@;Ng3g!8Vo`63^QR!OG9UlFU*p6};fU=}1cDFRsQ(V+j`QBx&na8> z3rQ$$qmFu)s^)M~mhP{VJLDHOMUzqfreB4YOQj-MvDwAYY?q%A_vhowe6Jb zN%WSs)`2TZD5I~|KTDkHh>S-FXCw11^Dn8HTl6xz8BX|pA5s(`hgly7Wo;oj;hyk@!1pbT&J_{Zo! zxBU(Id3=2P2c*RCE4;M~Kd30IGfLX9$y;~OMcreP*qx%TemfI)Kl~^u31j8lFH!94 zk%~zebAAYT={Bm}(Mm;cqn;i06;qhTHTc|Z6x-2GRozCP?x=lW7Sr&k@)=Z4Td8wP zsD(Zw2`=Fc@M~UuTEB_%EulL7%MB+!!YtIsGL8=JiSPb%9L)VCIPg~=p?Ur98?+~2 z{MWz0wyL{tLoVS2BHvcRCHg*qs`Kkv^mD^lxeuHD3kn$Dq$+M9F|%Daff>auVUyq# z0HIhvNeaY>XEivBId8s$Ka)gznB9^Dp1>v1WyWr370g6c=NFH^AKC`B(iu!*^%KSE z2guS$bb#PJ0(aWxgXgn49y6UHHS!PfF8Fu9mV+y|Qv|U|IYnpiGEQyXhp5H;A{F^L znlVSndx)`^$!q8xGlM2A4oME14BkY`mOrg_!>^6qBkh;WkarV8?9ynZyC?p*)k*5R zM!<0Ph7b?FBwjhEJl4r41P0!>TxKpZlq32T!Cq|n<_+|ot=1%W!T7D>EXYFDdCw}i zSyOf1tAtRif-!76SX?7hyc(XIaFu-aO6u!3&@=Yxqgps1{J!%C-xs!r6|_MH$efHk<#F5I6;el^h%3<~SYef;X;>Z+>v z>S==mcxD)@aM6X|N0n^tjuBxcX(#_%Fa^u1^9k@zjkm==_Q7$`^EzP{*)=hXcC(Yz zsYO&^)siqi<7XC8ouiR@bTQuQ$dy;O310&n{`dDnTMv4o^{MLi|4-N6gWhX>gIf9- z+U;@I_ym&&Igb)*0DB;VT9?Aj_w~eo;#t2IefU%}@^vrQ!N>2P!4VVi$w~x3e1}2y zzySV07jNv=m&t$g!K(VV)|Qqx<}WGhbsv8b@7XgZBANK_o|z}h4BH6w_s{O44BJpk z_Z({f)_7dn6A($=vxK}02RL#@wGh=%(UcqCo`Zb=xWVU)!qMFB>#c|xuN`N}IDAPb1Q z#1_5s0SbVGKxTFk^0cu3D4+?H;9?h2l>16UANy z7}q$|z@%CWfG!!tN(OGgxX=ZIb^w8r0pTmIFaQ{rIW~|E19%Mtz^q~l2A4sSAUL`i z)N*oKu>z<7BDz>)i_s|oHG!}tHX+z{0}6r+f`q*g3udLS0XhdFY_b4@6i5yPZ<53& z$TuviWG;bYAec>KwlV}LTw>ffQONzgSO&yoi40@%6fG*B1tM_?efsi`0aYderAj7~ zB@@L85P1;kWypLxoEAD3omOZ7(IBlL31Bq>yBAOpWB`PKI0Bfh90fFq8Tgw7<{ia6 zpd`o&NCND#LKbB2!8#yKAlSB;F$`g(tqgCzL!G2eb2?y)4cTPcdTtTvt!yp(D1)K4#T+4 zs53smJj!%3dzjCed(2wXN9G?|jkdqFsqMGe4;(9wSDa6{_PY99w_RZGCI}X~?euQL ze;c1-dYN;kUz+}4s%idi^K|n}^FK5%H{WfRnm3sn%#WF0Fn`-@v2?p$bX{|O?1EZ7 zN@!s6MfxLJr#I*=dZ)fszf14aKdIlVe@>78&vk=-%6iVaXzj5DZNoN=U2AvSQ}(pI zU>|2E*m<_g(d`I1I~OB}3FunsHoAu<^dh}Re_5ZL@&$Ed^NWSwqjtFz6SbLO1|C*cyg zW?ZwbdDmrE0y{TJ5c5Z&@1^_b0Xj?%(@lDvp3$>$Cd2zM!8poHgt<28=z% zUZaXpGa9CeSz_)mDQ208nO-xEn5tdC45M3|mGmtkrF8 zv*xV@tH?HMo3|xwefE%jz&>aX+jVS!W*OGPvMkT8uvs?8=Gn83DaVW>;Oub*oxRRJ zr`n})X Date: Mon, 28 Oct 2024 02:06:58 +0000 Subject: [PATCH 21/38] fix build error on linux --- src/arena-abandon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arena-abandon.c b/src/arena-abandon.c index 9c3356fc..48e37794 100644 --- a/src/arena-abandon.c +++ b/src/arena-abandon.c @@ -148,7 +148,7 @@ static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) { void _mi_arena_segment_mark_abandoned(mi_segment_t* segment) { mi_assert_internal(segment->used == segment->abandoned); - mi_atomic_store_release(&segment->thread_id, 0); // mark as abandoned for multi-thread free's + mi_atomic_store_release(&segment->thread_id, (uintptr_t)0); // mark as abandoned for multi-thread free's if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) { mi_arena_segment_os_mark_abandoned(segment); return; From f126b503822addadbdfcc82f940c5a086d3f75dc Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 27 Oct 2024 21:10:46 -0700 Subject: [PATCH 22/38] update comments, set constructor priority to 101 on macOS --- src/prim/osx/alloc-override-zone.c | 4 ++-- src/prim/prim.c | 10 +--------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index 1515b886..d3af170d 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -418,9 +418,9 @@ static inline malloc_zone_t* mi_get_default_zone(void) } #if defined(__clang__) -__attribute__((constructor(0))) +__attribute__((constructor(101))) // highest priority #else -__attribute__((constructor)) // seems not supported by g++-11 on the M1 +__attribute__((constructor)) // priority level is not supported by gcc #endif __attribute__((used)) static void _mi_macos_override_malloc(void) { diff --git a/src/prim/prim.c b/src/prim/prim.c index a4d3814f..2002853f 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -46,6 +46,7 @@ terms of the MIT license. A copy of the license can be found in the file } #elif defined(__cplusplus) // C++: use static initialization to detect process start/end + // This is not guaranteed to be first/last but the best we can generally do? struct mi_init_done_t { mi_init_done_t() { _mi_process_load(); @@ -55,15 +56,6 @@ terms of the MIT license. A copy of the license can be found in the file } }; static mi_init_done_t mi_init_done; - /* - extern mi_heap_t _mi_heap_main; - static bool mi_process_attach(void) { - _mi_process_load(); - atexit(&_mi_process_done); - return (_mi_heap_main.thread_id != 0); - } - static bool mi_initialized = mi_process_attach(); - */ #else #pragma message("define a way to call _mi_process_load/done on your platform") #endif From 5f3593333107066434e77177e0e79d56aba5189d Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 27 Oct 2024 21:39:07 -0700 Subject: [PATCH 23/38] add 0 byte to canary to prevent spurious read overflow to read the canary (issue #951, pr #953) --- include/mimalloc/internal.h | 10 ++++++++++ src/alloc.c | 2 +- src/free.c | 2 +- test/main-override-static.c | 13 ++++++++++++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 63a1e6f0..82e8c766 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -667,6 +667,16 @@ static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; } +static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { + const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys)); + // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951) + #ifdef MI_BIG_ENDIAN + return (x & 0x00FFFFFF); + #else + return (x & 0xFFFFFF00); + #endif +} + static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; diff --git a/src/alloc.c b/src/alloc.c index ca60c11a..119dfe75 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -99,7 +99,7 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); #endif mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess - padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); + padding->canary = mi_ptr_encode_canary(page,block,page->keys); padding->delta = (uint32_t)(delta); #if MI_PADDING_CHECK if (!mi_page_is_huge(page)) { diff --git a/src/free.c b/src/free.c index a85baa55..046a34e2 100644 --- a/src/free.c +++ b/src/free.c @@ -414,7 +414,7 @@ static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* bloc uintptr_t keys[2]; keys[0] = page->keys[0]; keys[1] = page->keys[1]; - bool ok = ((uint32_t)mi_ptr_encode(page,block,keys) == canary && *delta <= *bsize); + bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize); mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; } diff --git a/test/main-override-static.c b/test/main-override-static.c index 1ac9bf87..4ad76d6a 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -19,6 +19,7 @@ static void test_reserved(void); static void negative_stat(void); static void alloc_huge(void); static void test_heap_walk(void); +static void test_canary_leak(void); // static void test_large_pages(void); @@ -31,7 +32,8 @@ int main() { // double_free2(); // corrupt_free(); // block_overflow1(); - block_overflow2(); + // block_overflow2(); + test_canary_leak(); // test_aslr(); // invalid_free(); // test_reserved(); @@ -226,6 +228,15 @@ static void test_heap_walk(void) { mi_heap_visit_blocks(heap, true, &test_visit, NULL); } +static void test_canary_leak(void) { + char* p = mi_mallocn_tp(char,23); + for(int i = 0; i < 23; i++) { + p[i] = '0'+i; + } + puts(p); + free(p); +} + // Experiment with huge OS pages #if 0 From b3828bba9e983e4a61cc82b1acc9965f7bb15add Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 27 Oct 2024 21:58:20 -0700 Subject: [PATCH 24/38] disable aligned hinting or SV39 mmu's, issue #939, and pr #949 --- CMakeLists.txt | 11 +++++++++++ src/os.c | 7 ++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e0aa59b0..4729e5b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -319,6 +319,17 @@ if(MI_WIN_USE_FLS) list(APPEND mi_defines MI_WIN_USE_FLS=1) endif() + + # Check /proc/cpuinfo for an SV39 MMU and define a constant if one is + # found. We will want to skip the aligned hinting in that case. Issue #939, #949 + if (EXISTS /proc/cpuinfo) + file(STRINGS /proc/cpuinfo mi_sv39_mmu REGEX "^mmu[ \t]+:[ \t]+sv39$") + if (mi_sv39_mmu) + MESSAGE( STATUS "Disable aligned hints (SV39 MMU detected)" ) + list(APPEND mi_defines MI_NO_ALIGNED_HINT=1) + endif() + endif() + # On Haiku use `-DCMAKE_INSTALL_PREFIX` instead, issue #788 # if(CMAKE_SYSTEM_NAME MATCHES "Haiku") # SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) diff --git a/src/os.c b/src/os.c index 4babd8da..4b9d6125 100644 --- a/src/os.c +++ b/src/os.c @@ -92,8 +92,9 @@ static void* mi_align_down_ptr(void* p, size_t alignment) { -------------------------------------------------------------- */ // On 64-bit systems, we can do efficient aligned allocation by using -// the 2TiB to 30TiB area to allocate those. -#if (MI_INTPTR_SIZE >= 8) +// the 2TiB to 30TiB area to allocate those. We assume we have +// at least 48 bits of virtual address space on 64-bit systems (but see issue #939) +#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; // Return a MI_SEGMENT_SIZE aligned address that is probably available. @@ -239,7 +240,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); - // try first with a hint (this will be aligned directly on Win 10+ or BSD) + // try first with a requested alignment hint (this will usually be aligned directly on Win 10+ or BSD) void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats); if (p == NULL) return NULL; From b5ae6fc555749e141d066987307d261f4d31a03a Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 29 Oct 2024 20:08:36 -0700 Subject: [PATCH 25/38] remove wrong assertion --- src/options.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/options.c b/src/options.c index 76d2cf82..cc39dd6d 100644 --- a/src/options.c +++ b/src/options.c @@ -196,7 +196,6 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma } mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { - mi_assert_internal(mi_option_has_size_in_kib(option)); const long x = mi_option_get(option); size_t size = (x < 0 ? 0 : (size_t)x); if (mi_option_has_size_in_kib(option)) { From e2f4fe647e8aff4603a7d5119b8639fd1a47c8a6 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 29 Oct 2024 22:23:21 -0700 Subject: [PATCH 26/38] update test file --- test/main-override.cpp | 91 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index 50eb0267..9c47d3a1 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -11,7 +11,7 @@ #include #include -#include +//#include #include #ifdef _WIN32 @@ -38,15 +38,17 @@ static void strdup_test(); // issue #445 static void heap_thread_free_huge(); static void test_std_string(); // issue #697 static void test_thread_local(); // issue #944 - +// static void test_mixed0(); // issue #942 +static void test_mixed1(); // issue #942 static void test_stl_allocators(); int main() { // mi_stats_reset(); // ignore earlier allocations + test_mixed1(); //test_std_string(); - test_thread_local(); + //test_thread_local(); // heap_thread_free_huge(); /* heap_thread_free_large(); @@ -179,6 +181,89 @@ static void test_stl_allocators() { #endif } +#if 0 +#include +#include +#include +#include +#include +#include + +static void test_mixed0() { + std::vector> numbers(1024 * 1024 * 100); + std::vector threads(1); + + std::atomic index{}; + + auto start = std::chrono::system_clock::now(); + + for (auto& thread : threads) { + thread = std::thread{[&index, &numbers]() { + while (true) { + auto i = index.fetch_add(1, std::memory_order_relaxed); + if (i >= numbers.size()) return; + + numbers[i] = std::make_unique(i); + } + }}; + } + + for (auto& thread : threads) thread.join(); + + auto end = std::chrono::system_clock::now(); + + auto duration = + std::chrono::duration_cast(end - start); + std::cout << "Running on " << threads.size() << " threads took " << duration + << std::endl; +} +#endif + +void asd() { + void* p = malloc(128); + free(p); +} +static void test_mixed1() { + std::thread thread(asd); + thread.join(); +} + +#if 0 +// issue #691 +static char* cptr; + +static void* thread1_allocate() +{ + cptr = mi_calloc_tp(char,22085632); + return NULL; +} + +static void* thread2_free() +{ + assert(cptr); + mi_free(cptr); + cptr = NULL; + return NULL; +} + +static void test_large_migrate(void) { + auto t1 = std::thread(thread1_allocate); + t1.join(); + auto t2 = std::thread(thread2_free); + t2.join(); + /* + pthread_t thread1, thread2; + + pthread_create(&thread1, NULL, &thread1_allocate, NULL); + pthread_join(thread1, NULL); + + pthread_create(&thread2, NULL, &thread2_free, NULL); + pthread_join(thread2, NULL); + */ + return; +} +#endif + // issue 445 static void strdup_test() { #ifdef _MSC_VER From 54940a6a65d28c0d6499490899f6d420b3bb7e24 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 5 Nov 2024 02:07:45 -0800 Subject: [PATCH 27/38] update mimalloc-redirect to potentially fix issue #957 --- bin/mimalloc-redirect.dll | Bin 70656 -> 70656 bytes bin/mimalloc-redirect32.dll | Bin 47616 -> 48128 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index ed001d64f7f70516577b3f3724b9a1279b73d51c..4702fec0175e66168151f1d995a6148a0e69a1eb 100644 GIT binary patch delta 5842 zcmZu#3s_TEw%$7=ydP1BhL;35LKKw%K2Sh|MLjBNtf=vUQj3bC_cm61%v4Jdbfhsh z&UQN3Y6q&<(#Ooi9*d8Q8m&cqT`8kgr`BS}cEX??+B(&?PILc#PQ-q5FWb2JP%lr1Mz_T}<6<1&ya>wuP>wpR=!NI{gESRn4G1(hAjH zTD49fq^mF#>f%tSbQRQ+7^ny}l-B?iz>BxIddmnElLnO;0u>qtwKNOL5)Ks>0X32P zWko|hN`m@Z5Y+gIP);7!#v7;^2sJhU>S**>=yD2mnfn|Y0yTlBVBr~E5TQl}L)|7& zEyJMJ_k}v03>AT<*=^k=X4hm!EPas>Z!oer80u|gEt>Sv=YAqYbVQtgk&;$*FcE7C zPJoh1{Sjy)uJjPm8L=NuQ76=Ni)XDm$I~4oIyF0`gup;mYKfo~>xzY7vF5I^*ij_p z=5&aTV!`Ac_Q>aRi8W`8A240;jN*iiXwFW^nvsYNd&K(;7eq&iU=Ra75*_D7hwfi5 zP!iP~vMpRIIy5~fh(3>4r}+}Bdk>g)nXcHZ$#JVEE zBq9TwI2vqCnpj=1m)!}mMv6~-oDn%}N@!AE^bfqL5&0U zSxpPK6={9EH6tHQt{5P9;f(mp_iTSCE1LXEWi-#Lx<@0kk%+<^j)FC+xkL!w788si z7*SM)Fn}|sCQ5PBqJ*htrX(G6h3$&VdFb=;?B)4GeAE;{CmCDzfXW=$Bp*llY(GwXvwuC5~_=bZP4 zZf4Q<4}Tw}qkKPjuXVF=;iary9~3@;8?gzAce!FT$0i5PT8VNLTOOMil+Ee?{GGiO zJ0*z9yo>!1YYlww7qAat*0_;@uW*{fcE!yLoWyAj`!;S(d|($L%LhHX)I(|w}qs;%tI^w4WF6Mj|OHG&>n5NLmgfVfiR@48}!siw6{3C0F=UWPm zO&k|d35w=7aZ3|hnwUoSv+aqg)&g#@eu)WkYG!hux>})LPDS&yzu2G@7x;@!O0nEu zbSuTR{^Aj(_#z7$V2r$Hj%s!Ji_)_{x*WaApqQ7ddfvhpo-OwAbrEZ&YzpO4*5Ny#$uGLG~Z zzRb9viXCR$5&oIOd56iTRk93`OKeN3Rywe?sve&Uv8F|60+Pz`sKd2_8x-RS%JFo~M_x~^b?i3o){1OJ z$1b6PhayRvcjhQUj`67%o!bRF8uFSxMaQ0H7_j*WH*hvybk+*Iz{f03$nl8I66Q(^ z?x*A7!MGUYa5Urn>LzAkPg{+{7k@S`Ww6y}>H}D2(kj-Io~^3>gbBuE z)o(vxlZ;t0>K~P@WnOlS&jMefgKWDoo4(FY8ZXl{_VS>q(I<0y2gKL3@die7YtW#6 zb+9Y@!r8ko=GQUTARWsdJd7}J48PV)NGPKd3J@EPXj2Qft3D^d%j8xTb#icD{ z6CK4S4sp?s*tH>t=?V7o&}2ix0(3j`W4?IZ6;X#Vck)4XY(KS)C$x#R4;84D{d4Hs zftTjNu!Q~Buu&l%P6uAaDEAIZ#o^F1Lni+KSu&qWIQp&6hb)Z9(2DP@aeWHU$K()$ zy|1%BWj3h(^$nY8N>ttYhS^LrsE&C|zs;Vp`Dce^lcp<(ucR}=XhlCM9m)!b*gi^n zqE9zK`gPV^S~YeK9Ng`SLoPr4xdaFEcRZY5D&DL9y2m$zrVEb+e1oy;f%0YP)Tq%4 z-#6y&a9rl9$0fcA&B%iCPRh~H&%(mxL9x;tYn6YGV;kD#l_|!Pp!dOH^&fTr0`I>3 z_?X!6^-^~5^CFH3Uw)p#2Hvh!tntX_#nG;uw@riicu?UmR}3U1_;KPxW)m`)hn2~h z4?|Hd6=UpW{4UynAs8?O0X5zkta$h^)lWaMjl(zpW<)IAw=ZslS~KxHzvTF>J9czw zP?iObbI!9jMptO&oZ~e!^D?o*eVJuUOXAixz6Eh}`)9^G9o|=A%=UB}*`g`^nK>_v zPLhiA5&{BmPDN!V`_HjGp@E!^J|o|z*=1<1iP_9AsFs~(pIMVxWMQgm$7z;R__FE` zr`VaoK|!BQhX32Ar2B;{sPKCcg)trkcYP;MFfJ3ZW`mJL)qaUGzDV=tv+Ad=1zwp1 z$8l^?(E~c3&6$ulv>Q)hZ{P{+4yW!P;PTe?hQP3m9}}nU@4d#0$JvPq^~e2Rh00VuUK6JADPFw+_Jh>B4uBi#x#QA~iB?tWyT|0I!HX4pUb(xmETTh; zu+4=C`^PaBGIwPW@eq1g2&(s z(-MNd;N6rrv)8BDRPVmew9^yHJ^9L{SiD!>m(x$jq+IYDMelPk!tl0iX!G{{M(N1V zy%E+!y|MLAVx`{L93}SEeCC=SW32CWUiifMwO;23hvben=d+8`4VgB--RwOEyYk6P zTyTx{V}}+_R()q?zJ+nBPs}X7w0z8^*_8MOUY7Movi`HI zugdzGtZ&Hr6Ip*M>(8L2Q>C^(*!&@k2Bak6e@nc=2*GENEQAzQLsoMOA*hp5Z3C%x z<<)!54rZM2YL~g=z=7ExOV-uLsV0(=$nmW2nj>_LbYaa6^-fdpWFP)`XlNZF7%$QI zkNhwQok2p3Zb1+ zAw6B<{St{I(*loY#0i6D3x0!mt&@qv|ZbV(rYut@fsba4-Kx%MXDQj;uP_5MH zo=cgy&%hJo)U)3}%M_Y>z(8Fx-RFKYPzehDQ*V&ntq*A^BX&M*L~3kB@WX?8O*=P9 zds`P!W|Qda5#2NWW@c$BSPdexTyno<_L2rMK}T#|;>>Y&D(K zhq%j?eq6FiYVy=m4Oz}Rnz?QkW%VBzG=?Wd?T315Lq>}+E3JCI2(6fl2F0@BoE~%B z=fYfy^=Ccj^&zfjmB!7|>dVNY;-Wsp;x`e+)OkrC((X6e6_fQcZ+P|P^)$k;)F1yO z1F@rbB%VM2d-^Q*sy?F3A5``v$o;V%i#zkAl9}-n&+zz|~ zSqR<(JcDm~2{^&)p#rj}7VGZ<3B|Jwf*aTY5x_fvs>N7Ka4m2YWH`7PxC)XFZU-KO6oGqycOk{#q>PX( zND21`u7)h+{=j-jId}u`G^A=33hf||AeG?SXRzBL8^K+`vyfWwPT)hxc5t$UkQB%+ za1(GLWDmFv_zGk{cmwbv?8pus>GjJ8;4!9k-2XYr2S3T)~+z0OjhA+i{z;VlxnGp3>ObYN7NI19~cpDP4 z6`KziCP`a{B|rskNn(cx;4a{MkRjln<%GmlUD}a8WIuJ-B F{}q3uKPUqdCqg5 z^PK1Wyoa$jtL@Edmp9&BjA9ZW-N0QNMHIKWhqDAl z{8$uIxa;}xC_V{f$JFmJmnJ1--6lf3Uk)VX&;ajhc1n{t@!VZPL`TS1n<(j02NAJ4 z^J7p_7S)cDh)=tT=nQFvQCO#%*1FfJ)7?E<(Wz;Yk^=)(Y;n-k+`}dunJ@`!)71s*MTf?Mis*BT^_r1jJujh`LGBeYq=NMY?V@&8 ztFy=XHo`3Fgy=XX)@KVA(UC2biW9-sjuba%{*?&>3PS7d+c-nkvh4%nCeQha2aqqA za~+yIH0LHApXu=AIyIBf5Z0!qS#{ZZANOV!9l1h=7$6Vf42k}UT^Yc#>wl}vW{s+6 zA|hLVi@OWi0Bu-Xz7SLw5rip}psGyaG0vQtJjKk08q&y0;|$C*a98xoAALR^J%W+P z%NImGsMUL0+7x`1(v|GV&=e}N-JwSMuGAL#zKX796@!elne7|2mHt%{!n;)TQK=_l z2o1XZogB&?NjGF3RgGA|o`@_C{`*!+YTaeyv2d5!naE^%gx!tIAN))&2F6Ghv7Uyt zX>wr28Ws#=_9(5+uz`^Dliu;Y%oP+m=qzm(=b_)SNzny?*Pa4j!1hE>2t3Q_TJ~A=g1~K@ zcCz$gYlqF~CS++=yPAs5U4n=2FlR_OZ%*xI$A-<)UR(}7!OOy9asprG{5iG+bSbBd zwZ=?}{%SM*4-W~vxtj&W<^*<^fPTjo#6A?#3JUi}x#lwaL+nWU4LcQUDyZZNb`=S7 zY8tpreT}e34n?!mUu{&Xd;QgBrMll=ZBeR+{nd7*+Q}XsZVnx@nlI80p${qbE@YL% zV;|na?VzfIo%inF_>Ry7+Och_Gw)WxtSmCDEn5(0j^YsERrkhhV~&CVp<(4Sq+IX}UXjuKuFAgenxoyZw@KKrA9$TgbX_#CCEq z8oJr7_yw8UIo$;tS5GRA@IrobI1M#-mCbGRjzv>gn;Z91xN_k*Lss!F^LZC$oN7cs z@ywGja)L$tOSX`KAxW+iG4t73(mzQ*=`&ZL-RzSEA6#HTJS8i}jhttGp9gqEHgF0`cO&Rk9a^7P^~> zBc2_wsF*uD;Z6I7eKewLPO}=->JFhDl9UC99jOuApqQ67-Q7JA;oQHDv6Hx2mm0Zn z>=IgdR|Hb(9fZ$Ax_c?+Vr>snu^&e!sQ*5J5N1hCNp#^b=Q`Z{)Y&lvqB(^om}c-L zB^A-ikgPh<6vjv9UKaCIw5!?S#Dq-VpJ$|CdODosW7-(9$ZLj^d%^lnhbF=X}0ecUbLaSjsEr5CyLwn}#Io2Z!O-<_R$R+hV-`V3uT) z>f}v!ENL(+Nv@>J*wthgJ;pYSIYPfOAr`g$G=a?U6Q8T>S6kae2wTHc)>a&M`>?cPb%bT3C+ zSX&MDG~cPXyquc-@WQu=wI>+0SMJsLX|W=TCO(MI!_RDgw_tfv?C11?D*l1T3{m9b zCb5%5{Gdhf8i@vZP^r>{o8+WGiPBYbWs@ z#!hx_My5FdcJgglX5#b3?HO_%Z&t4f`a{$k4}t3^x>I`a;R2Ou{@ZeD@Lue{S+j*A z(V<7Tw5jO!;@j-anYHu{_Ry@?RRM3Y-dWl7C`-*L({8Y$`M{g(V9qkbs5z9Z)3(rV zyu%1w+3Gbz=1At;*Xj5(+$`=0W&=Gu^S$K=KHXgzDmcN)CL$@%`jw(LntOH{ej_VRc*CzL z0~vcNpe^q2JMDhoCH;NBJ}md0mC3Am5oWR9SbN_%r{DM$#W*dKRpc2{jDEe<`y}+r z2Pu|+=VIuL#bfqSud(M;8AS?rBP(JDt4`*2b5q{?}e_(oL8 zOt82T6Sqw0@DW^io?Hb%dpM_A0<)R~C;g9$D2^Tab zLo|MzK>a+>2blDPR02Z*8(;?X^@Jn<;{Xv@DE(X>5P-n+f3DE00`r!X8Pgt+hHTod zqZX-Y+Xkv>EF^X+J+eNXGTU~m#&f@V!ePY(o3=ebnbm2fjS6jl&Pq!aYH?YqN1@FP zRyf@3gsqYBZcnfclolx({qL;QrO>(;co%NB^Y>Qi%Uv0imA`1EHtCVyp=;k>Zpf)r zvQHm?AyZ$1S|4C!o+cyQ^;gALjquZ7b?-OQmiyI=*NpT+fA!Ut1A)JE74*ihT-jTq zbbXH_2}rR``o3vCWj`K@(iw{sl^v=A%2rE}I*;GVA~i^>DAh~!@CK@~FOxR5t>@M6 zkK9y2Jn{})SUY|-Nc&ME@*}rc-iZn!#ftL_Pb$fuL#1Z-9?J8;JMVedh#bpQ|9%v8 z%TYC0f>>Qw6koS4y z8KtxGDw6AwE`EwkHl2&qx%{d=xsz11zMM*~^QG}s_CbVv1PIlHSnw;s0YDAzfvm<4 z4H4iikUk(7+=yQ)-UgWT+B@wa%yahk0sPLms8MqPh zDWC$}4w;W1RBFIokU1&HRt@YSF9CO;?}J=xK_0;2FOk+?qkiHlP>05i)!t(hF{Z96bq>MPL3u$#ozY z+&CGt10ul56a);!fVV(KXCk}YCuG&S2MIZaiU)G573%~}vXBv=1KbW-J{1no*$o*n z4Q=2S$bCQ|*F*k38`%Q)Knl~5C2%Wb1itpU;5Nv1U>EoW$a#1qJX3@9w}Y&})20Fq zrFiZ95!eClhP(lEfb0K~!CN3l&BuNMw?VEf!1^CV#a)13!4@D>;C+y0yx&{F?T|%<$PhU0ZE_sA0**_Y z+yr{T$s$4$fg9i!$U@)_cq!xyfVviaA&&vU;2y|pKm<6cdJLZrAO;l+l3WGUTVeksA*nzxI4(%C9*EdF1p|P1fEYAffK)BT5`gRRs(1=W2Dd|g Y3ykCX66}xV2zINK<^5RoS|APm7ZjQ+JOBUy diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index ec4ff1d521292a6fd1403d35c825e1cd1b060b3b..17c0555027cd73317e893549e9831235c35d4d83 100644 GIT binary patch delta 3611 zcmY)x3s6&6_T6VNk;d{+zA6wtf+%W!k35J9vZj?Tu}Ma(R$J+4HAS>kq1sUj8d1VC z7ChQzMt6r*XUYt=rOj6AryZYwZwqaB6hQic(r`=tyxM$$sJEkp{3 zYZ3I3Y2;;yV$(|MbIz_K{B=c zH)?rF*c^lkg5Xvhf<7w9Q`5PWCQ@3|vk2x=`vtKGUW!L>hn~3>Mc|XzZ=fn9CdP

xmre(Uhf(;I)85s}9yaEqM|7)j zTl}KcZx;*I4$-2rNqf92LK_JCm`UP?UK99~P^MQ?-Ykt@-`_XPWOv7up zG|`90+GO_>9(G7%;Hj4lqNNG_JBnx$9;XXNgVMimW zeD$TWFlP;=w*=`Z)k zezfD%q%oYF+rT~0O^Q;gWwr_ur|O|ccqhxFFUnefY*u=p|TfW4S0 zW`-lxrG#@^Bv*yRY;oenx(kP#wAwt?Jx_**ZBocRIbDWxtK8OsFX3|BPI3^I#=nlYeky)%H0@beg_>o^(N$E&iB zDLw!5(SN)OgG3*H^;A=?+cH#)mwy#~^gEI*{=;^INd2}!!77mGdlWfx7KbY`^wOWzklWlK?^~vRg2PwIfT!$qowPXzRq$Ezb zd6QaK<0UM`G^*(d1znr$(#406Fgb}dL*C?lL=9IbPiJr50A*@Ej(U5l8OOdSHI_UF zAEdq$*ZDvWY1AMVwP;s6q!U5Hv_bLgt2ifQC%bVcFD7ka@~bju=6ZFS6d2JPnTU9I)ZNpveNIfkSi`XpkmG72^5DVgtwyKJ)~jLFd0?NNgO!jB zRa2vh#oI9TJW+n|ot$hq6&?ju83LII$1)0(Ay27^Q;pu-=?e(SfcIv2NIt~Pw6ISu z!^)Xc+3zkx-OK`Z`(?N}GmR~}3`(s)5+PT6Px}iZA zPZZFqdqhIv6a7c*(k^(-aFDIK2qDIqsXH#pd2%{yFiNak$q^*fScaWtl!o0aMZwF) zBAhKJjI;E|E>HzU*wK#ab7kpRa1Li3%1LKrx(dg95K~~!AD3x+vTc}wXm1@~Pc+q? zvYS}O@i=|OJO$1Nl^E`DZrOoG_IAQt(^_`uJhYfx?1uBe3z@8$?hMc<Fp6d zB-7H*$zIbkwQ^#_R)@sP6#9$FQqAV81!8@WXcbAapmy8MBkY*9HRm2A5{YcJMV$ zevw76`}$`)r%<+qvq@!tP!yF@2} znAJo^!|$__*ehotVRkb6-{#QYB-H*Jy}O z#v)2I_^m=Ux)5Wad;NacwU+8qZ2me8e$P-$@g}v>z5W24dNTp7P}-JK4W33Z;Vo)~ zVt&MYGS;%5+NBuH`re&0pew;BWJN{7?J?erI-L zcC0yccF*j-*_Y>J=H%wA$vK!d7-kbbl-kwWYuZTNa$Sq=uFkA)(jU{0G87oz zH*_2B7&44+7}r%An~cYer;HuOi^gw_H;fOAelNU^;JeN9x>MP z=VZ5QGV3!BX0FJ}GM_f{vyRMqZO+o1Ejg7aoh$z{q1s#80WG6@U3Xm9XYla%_y9l4 zw8|u!noX&KS;!TZ3uVGPLWA(3@Tu^F@VhW3t0N2bZ45I7ACmU5c9Cw4?gjk{{Ym{- t`VRdC{T2N+y-$BvKcN3v&s5+eMjV<8A`h3x@CK2X*JXw`y4@C&(P0)&l$F@3O@>Mdt8TJ@gfd>o zz)y=)c9u;`ib`x0^3le&269A0V^5aWj#hiF!tRbWwWCeTe*YagN6z^_zW;mu-+OM0 zk8Sa?ol;cK_wi3LcmLcBn>2^qm&H_d{cp$Y%pfG!GmII!Y8Ju?k`a30w@BgeRKZP> zO==*D%^*)e3BtEwEyDL;55i-v0oF>Iht&*oR1FL!3dtDwD=GN-AV+%reK2+ zfnzp;Lx~9fLq%@JBKVPNFnI{BPy^}d2#y*M_>&Plm4e_pRf~^9;G++1qL%Nc25!rG ztK$(}o{HcVYMLhqI7*ugLl89!0iiFQrnHymBX|;q6rVt6a7;`jnr=IzS{Ewa#6+~6 zQH%(FnkNVk(YV~f2Z`%o$YnO>i`&!F$aDx^(c-NXt5r5pP+FxAT@cz$*eoC9sL~Re zeN;&45d3XTex@N&6ns|EqC#=MO>(>HRS$)*%3E-HZZ6pXQ*)EaQcy>(B$Kc)vYsU# zxE7gDhT&K9mXI=7GtWx0VPGClPQcW>D$)asqMF#`f5T8zIXM8cqsz&QV2fVD&i)s? z7X1*p4N~+HvI+DtiwcvcWIslPLo|{Rp_l%1T6!y;)rV}-!T{GHbc;eS`gaKZ>$cey z+Zj%h;Jp|t*$UG!@#G_jk4*}}NTzIo@ zMDPYerzP{*)-mgt>3OPp!Z`#_Ta$;`_OK|Zuv~`GwGNAPQh|-L)=_=a+5YYSs0UN) z7}4UxtVuSB&te!+@QdXNH08AgL@;}PF?{+GHjsK?sC}q01PQ~7l#KY@;~%S_Hhv9z zJQGIZtCTI7m_9~&0#Xuo5*zd-tij@kC+>?jE3oUDR7`_XJt%P0gkk8$R}7p^oZsAn zqhJ_Xi>WxF#L++>e{$|0{{>yE_N`-_!ey%yn^9VcVt2dNiJfMrVL02*{yaTxRS#Oc z7jIMsv@m|@ag?12K7ziOFZegjZf){YrRNlep~>ZPGTdv$>IXSY#&tPE-(p-BugOUY zctVpzUyzgJv`CmFw@td5#xT?E(yjRd?j)@&%8&trL9`EgrFYN>uE!O8Eg+ zkl=L)l8BMe;g;_kqOl<1t0fIL>kk3A(Wj}bihu!)$98OwQGsE$Wk25S6FM-bUJBDl2ULh zQb3y>P2O;oXP+Y>w##xN;Z}GQXmiZCJ)h302Hg^K{XfxP36ELVXV~sr8*F!o!ll?)X(a^7pXX`(MPj#Ew!~T15%uUygQ$3~K-hpe0_UA93%7hMK+TK8ExJ#tJ zLMUHL`rsM9NWWUUQ|&Q38P2XjAkW$t0ghE7 zKgc3DS(C-~Ov)BGtCV<;qM0g|Q4apBND*y$X>V zG=!6}am1`LwtJ(7bjnzHy@u4wnBrLt?hL9igxI)@ZF^3GpBj{A-bAh3+VCoP5!1?8 ze*?9LG*?|?2SMV|&3(|bnL`yS*nZQ)Ra(qp!A_@zRz3B+nCcChmgL8#T$bm|@r z-yc=Mol6;Rh)RT4oM5=e@YLfOLaIl$P~W0b)Z-2MH@^c5TUGc_e(Nc=-U>gpo(ywa z3+~L z`qvCc41D| zzQJ$EHIwPykm^wOZW=@QGPwYjc@1o@jvE;nxajq zrgBrIsn;}MI&X?FCz;dDi_M44=kf;&Mv4nd7%jsXX%=;tFci1p2ZPFZ)0oBU`9l8h z{MY>Vyvyt;m|NIh=r254+*dMI@@OejwhE^ab#k>uTC4WBHdI$&_{wO)$9Lxq Date: Sat, 16 Nov 2024 15:37:08 -0800 Subject: [PATCH 28/38] add virtual address bits and physical memory to the mem config --- CMakeLists.txt | 8 ++++---- include/mimalloc/prim.h | 2 ++ src/options.c | 25 ++++++++++++++++--------- src/os.c | 40 +++++++++++++++++++++++++++++----------- src/prim/unix/prim.c | 6 ++++++ src/prim/windows/prim.c | 12 ++++++++++++ 6 files changed, 69 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4729e5b5..87837026 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,13 +320,13 @@ if(MI_WIN_USE_FLS) endif() - # Check /proc/cpuinfo for an SV39 MMU and define a constant if one is - # found. We will want to skip the aligned hinting in that case. Issue #939, #949 + # Check /proc/cpuinfo for an SV39 MMU and limit the virtual address bits. + # (this will skip the aligned hinting in that case. Issue #939, #949) if (EXISTS /proc/cpuinfo) file(STRINGS /proc/cpuinfo mi_sv39_mmu REGEX "^mmu[ \t]+:[ \t]+sv39$") if (mi_sv39_mmu) - MESSAGE( STATUS "Disable aligned hints (SV39 MMU detected)" ) - list(APPEND mi_defines MI_NO_ALIGNED_HINT=1) + MESSAGE( STATUS "Set virtual address bits to 39 (SV39 MMU detected)" ) + list(APPEND mi_defines MI_DEFAULT_VIRTUAL_ADDRESS_BITS=39) endif() endif() diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 640c966f..fb2a9434 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -25,6 +25,8 @@ typedef struct mi_os_mem_config_s { size_t page_size; // default to 4KiB size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows) size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) + size_t physical_memory; // physical memory size + size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization) bool has_overcommit; // can we reserve more memory than can be actually committed? bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc) bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory) diff --git a/src/options.c b/src/options.c index cc39dd6d..9ddf86ba 100644 --- a/src/options.c +++ b/src/options.c @@ -47,7 +47,9 @@ typedef struct mi_option_desc_s { #define MI_OPTION(opt) mi_option_##opt, #opt, NULL #define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy -// Some options can be set at build time for statically linked libraries (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) +// Some options can be set at build time for statically linked libraries +// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) +// // This is useful if we cannot pass them as environment variables // (and setting them programmatically would be too late) @@ -99,14 +101,19 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(show_stats) }, { MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) }, - // the following options are experimental and not all combinations make sense. - { MI_DEFAULT_EAGER_COMMIT, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) - { MI_DEFAULT_ARENA_EAGER_COMMIT, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + // some of the following options are experimental and not all combinations are allowed. + { MI_DEFAULT_EAGER_COMMIT, + UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) + { MI_DEFAULT_ARENA_EAGER_COMMIT, + UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) - { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, + UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, + UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { MI_DEFAULT_RESERVE_OS_MEMORY, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) + { MI_DEFAULT_RESERVE_OS_MEMORY, + UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION(abandoned_page_purge) }, // purge free page memory when a thread terminates @@ -124,11 +131,11 @@ static mi_option_desc_t options[_mi_option_last] = { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! - { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) + { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free - { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. #if defined(MI_VISIT_ABANDONED) { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandonded segments; requires taking locks during reclaim. diff --git a/src/os.c b/src/os.c index 4b9d6125..827fc6cf 100644 --- a/src/os.c +++ b/src/os.c @@ -11,16 +11,33 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- - Initialization. + Initialization. ----------------------------------------------------------- */ +#ifndef MI_DEFAULT_VIRTUAL_ADDRESS_BITS +#if MI_INTPTR_SIZE < 8 +#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 32 +#else +#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 48 +#endif +#endif + +#ifndef MI_DEFAULT_PHYSICAL_MEMORY +#if MI_INTPTR_SIZE < 8 +#define MI_DEFAULT_PHYSICAL_MEMORY 4*MI_GiB +#else +#define MI_DEFAULT_PHYSICAL_MEMORY 32*MI_GiB +#endif +#endif static mi_os_mem_config_t mi_os_mem_config = { - 4096, // page size - 0, // large page size (usually 2MiB) - 4096, // allocation granularity - true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) - false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) - true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) + 4096, // page size + 0, // large page size (usually 2MiB) + 4096, // allocation granularity + MI_DEFAULT_PHYSICAL_MEMORY, + MI_DEFAULT_VIRTUAL_ADDRESS_BITS, + true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems) + false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span) + true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory) }; bool _mi_os_has_overcommit(void) { @@ -91,9 +108,9 @@ static void* mi_align_down_ptr(void* p, size_t alignment) { aligned hinting -------------------------------------------------------------- */ -// On 64-bit systems, we can do efficient aligned allocation by using -// the 2TiB to 30TiB area to allocate those. We assume we have -// at least 48 bits of virtual address space on 64-bit systems (but see issue #939) +// On systems with enough virtual address bits, we can do efficient aligned allocation by using +// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address +// space (64TiB) we use this technique. (but see issue #939) #if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; @@ -111,6 +128,7 @@ static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; + if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space size = _mi_align_up(size, MI_SEGMENT_SIZE); if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) @@ -276,7 +294,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats); if (p == NULL) return NULL; - // and selectively unmap parts around the over-allocated area. + // and selectively unmap parts around the over-allocated area. void* aligned_p = mi_align_up_ptr(p, alignment); size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; size_t mid_size = _mi_align_up(size, _mi_os_page_size()); diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 0ea8189c..9075f9bd 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -139,6 +139,12 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) if (psize > 0) { config->page_size = (size_t)psize; config->alloc_granularity = (size_t)psize; + #if defined(_SC_PHYS_PAGES) + long pphys = sysconf(_SC_PHYS_PAGES); + if (pphys > 0 && (size_t)pphys < (SIZE_MAX/(size_t)psize)) { + config->physical_memory = (size_t)pphys * (size_t)psize; + } + #endif } config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this? config->has_overcommit = unix_detect_overcommit(); diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 385354fc..494fac78 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -118,6 +118,18 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) GetSystemInfo(&si); if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; } if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; } + // get virtual address bits + if ((uintptr_t)si.lpMaximumApplicationAddress > 0) { + const size_t vbits = MI_INTPTR_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress); + config->virtual_address_bits = vbits; + } + // get physical memory + ULONGLONG memInKiB = 0; + if (GetPhysicallyInstalledSystemMemory(&memInKiB)) { + if (memInKiB > 0 && memInKiB < (SIZE_MAX / MI_KiB)) { + config->physical_memory = memInKiB * MI_KiB; + } + } // get the VirtualAlloc2 function HINSTANCE hDll; hDll = LoadLibrary(TEXT("kernelbase.dll")); From 0e76fe3798ee975db1db4289f054a34175859c37 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 16 Nov 2024 16:57:37 -0800 Subject: [PATCH 29/38] add address hint to primitive allocation API --- ide/vs2022/mimalloc.vcxproj | 2 +- include/mimalloc/prim.h | 3 ++- src/arena.c | 2 +- src/os.c | 11 ++++++++--- src/prim/emscripten/prim.c | 4 ++-- src/prim/unix/prim.c | 4 ++-- src/prim/wasi/prim.c | 4 ++-- src/prim/windows/prim.c | 22 +++++++++++----------- 8 files changed, 29 insertions(+), 23 deletions(-) diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 5efc8fd0..fd7d99d5 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -116,7 +116,7 @@ true Default ../../include - MI_DEBUG=4;%(PreprocessorDefinitions); + MI_DEBUG=1;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index fb2a9434..f8bf948e 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -43,9 +43,10 @@ int _mi_prim_free(void* addr, size_t size ); // If `commit` is false, the virtual memory range only needs to be reserved (with no access) // which will later be committed explicitly using `_mi_prim_commit`. // `is_zero` is set to true if the memory was zero initialized (as on most OS's) +// The `hint_addr` address is either `NULL` or a preferred allocation address but can be ignored. // pre: !commit => !allow_large // try_alignment >= _mi_os_page_size() and a power of 2 -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr); // Commit memory. Returns error code or 0 on success. // For example, on Linux this would make the memory PROT_READ|PROT_WRITE. diff --git a/src/arena.c b/src/arena.c index 3bb8f502..8ca5aaf3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -289,7 +289,7 @@ static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_no bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld ) { MI_UNUSED_RELEASE(alignment); - mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); + mi_assert(alignment <= MI_SEGMENT_ALIGN); const size_t bcount = mi_block_count_of_size(size); const size_t arena_index = mi_arena_id_index(arena_id); mi_assert_internal(arena_index < mi_atomic_load_relaxed(&mi_arena_count)); diff --git a/src/os.c b/src/os.c index 827fc6cf..62c8c934 100644 --- a/src/os.c +++ b/src/os.c @@ -214,7 +214,8 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats) { -------------------------------------------------------------- */ // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) { +// Also `hint_addr` is a hint and may be ignored. +static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(is_zero != NULL); mi_assert_internal(is_large != NULL); @@ -223,9 +224,9 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning *is_zero = false; void* p = NULL; - int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, is_zero, &p); + int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p); if (err != 0) { - _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large); + _mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large); } MI_UNUSED(tld_stats); @@ -245,6 +246,10 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo return p; } +static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) { + return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero, tld_stats); +} + // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. diff --git a/src/prim/emscripten/prim.c b/src/prim/emscripten/prim.c index 944c0cb4..82147de7 100644 --- a/src/prim/emscripten/prim.c +++ b/src/prim/emscripten/prim.c @@ -71,8 +71,8 @@ int _mi_prim_free(void* addr, size_t size) { extern void* emmalloc_memalign(size_t alignment, size_t size); // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { - MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; // TODO: Track the highest address ever seen; first uses of it are zeroes. // That assumes no one else uses sbrk but us (they could go up, diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 9075f9bd..59421e52 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -357,14 +357,14 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); *is_zero = true; int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); - *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); + *addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large); return (*addr != NULL ? 0 : errno); } diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 5d7a8132..e1e7de5e 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -119,8 +119,8 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) { } // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { - MI_UNUSED(allow_large); MI_UNUSED(commit); +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { + MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr); *is_large = false; *is_zero = false; *addr = mi_prim_mem_grow(size, try_alignment); diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 494fac78..1d3d6f41 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -123,7 +123,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) const size_t vbits = MI_INTPTR_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress); config->virtual_address_bits = vbits; } - // get physical memory + // get physical memory ULONGLONG memInKiB = 0; if (GetPhysicallyInstalledSystemMemory(&memInKiB)) { if (memInKiB > 0 && memInKiB < (SIZE_MAX / MI_KiB)) { @@ -203,7 +203,7 @@ static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_ali } #endif // on modern Windows try use VirtualAlloc2 for aligned allocation - if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; reqs.Alignment = try_alignment; MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; @@ -291,14 +291,14 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW return p; } -int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { +int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(commit || !allow_large); mi_assert_internal(try_alignment > 0); *is_zero = true; int flags = MEM_RESERVE; if (commit) { flags |= MEM_COMMIT; } - *addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + *addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large); return (*addr != NULL ? 0 : (int)GetLastError()); } @@ -629,8 +629,8 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { _mi_process_done(); } else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) { - _mi_thread_done(NULL); - } + _mi_thread_done(NULL); + } } @@ -693,7 +693,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #pragma data_seg() #pragma data_seg(".CRT$XLY") PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; - #pragma data_seg() + #pragma data_seg() #endif #if defined(__cplusplus) @@ -707,13 +707,13 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { MI_UNUSED(heap); } -#else // deprecated: statically linked, use fiber api +#else // deprecated: statically linked, use fiber api #if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`) // MSVC: use data section magic for static libraries // See #define MI_PRIM_HAS_PROCESS_ATTACH 1 - + static int mi_process_attach(void) { mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); atexit(&_mi_process_done); @@ -766,9 +766,9 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { } #endif -// ---------------------------------------------------- +// ---------------------------------------------------- // Communicate with the redirection module on Windows -// ---------------------------------------------------- +// ---------------------------------------------------- #if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT) #define MI_PRIM_HAS_ALLOCATOR_INIT 1 From c57e9b855cc97ea53112ad755ce1d0290cef23af Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sat, 16 Nov 2024 19:43:13 -0800 Subject: [PATCH 30/38] fix std malloc compile of the stress test --- test/test-stress.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index 30ad0e77..9d40a6d9 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -47,10 +47,6 @@ static int ITER = 50; // N full iterations destructing and re-creating a #define STRESS // undefine for leak test -#ifndef NDEBUG -#define HEAP_WALK // walk the heap objects? -#endif - static bool allow_large_objects = true; // allow very large objects? (set to `true` if SCALE>100) static size_t use_one_size = 0; // use single object size of `N * sizeof(uintptr_t)`? @@ -66,6 +62,9 @@ static bool main_participates = false; // main thread participates as a #define custom_calloc(n,s) mi_calloc(n,s) #define custom_realloc(p,s) mi_realloc(p,s) #define custom_free(p) mi_free(p) +#ifndef NDEBUG +#define HEAP_WALK // walk the heap objects? +#endif #endif // transfer pointer between threads @@ -220,7 +219,7 @@ static void test_stress(void) { uintptr_t r = rand(); for (int n = 0; n < ITER; n++) { run_os_threads(THREADS, &stress); - #ifndef NDEBUG + #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) // switch between arena and OS allocation for testing mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1); #endif @@ -270,7 +269,7 @@ int main(int argc, char** argv) { #ifdef HEAP_WALK mi_option_enable(mi_option_visit_abandoned); #endif - #ifndef NDEBUG + #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */); #endif #ifndef USE_STD_MALLOC From 498c92e34894977b0e599fb276b1fcf9faebacb4 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 16 Nov 2024 21:15:50 -0800 Subject: [PATCH 31/38] update guarded implementation to use block tags --- ide/vs2022/mimalloc.vcxproj | 2 +- include/mimalloc.h | 2 + include/mimalloc/internal.h | 17 ++++-- include/mimalloc/types.h | 7 +++ src/alloc-aligned.c | 44 ++++++++++---- src/alloc.c | 116 +++++++++++++++++++----------------- src/free.c | 63 +++++++++++--------- src/heap.c | 2 +- src/options.c | 8 ++- src/page.c | 6 -- test/test-stress.c | 2 + 11 files changed, 161 insertions(+), 108 deletions(-) diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index fd7d99d5..5a614289 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -116,7 +116,7 @@ true Default ../../include - MI_DEBUG=1;%(PreprocessorDefinitions); + MI_DEBUG=4;MI_DEBUG_GUARDED=1;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 diff --git a/include/mimalloc.h b/include/mimalloc.h index e5133c96..940284b6 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -368,6 +368,8 @@ typedef enum mi_option_e { mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) mi_option_debug_guarded_min, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) mi_option_debug_guarded_max, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) + mi_option_debug_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + mi_option_debug_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 82e8c766..d73532e0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -600,16 +600,25 @@ static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { page->flags.x.has_aligned = has_aligned; } +/* ------------------------------------------------------------------- + Guarded objects +------------------------------------------------------------------- */ #if MI_DEBUG_GUARDED -static inline bool mi_page_has_guarded(const mi_page_t* page) { - return page->flags.x.has_guarded; +static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED); } -static inline void mi_page_set_has_guarded(mi_page_t* page, bool has_guarded) { - page->flags.x.has_guarded = has_guarded; +static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) { + MI_UNUSED(heap); + return (size <= (size_t)_mi_option_get_fast(mi_option_debug_guarded_max) + && size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min)); } + +mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; #endif + /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d1e6e5d8..e01754e2 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -232,6 +232,13 @@ typedef struct mi_block_s { mi_encoded_t next; } mi_block_t; +#if MI_DEBUG_GUARDED +// we always align guarded pointers in a block at an offset +// the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones +#define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) +#endif + // The delayed flags are used for efficient multi-threaded free-ing typedef enum mi_delayed_e { diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 248c932d..12815689 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -20,14 +20,24 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0)); if (alignment > size) return false; if (alignment <= MI_MAX_ALIGN_SIZE) return true; - #if MI_DEBUG_GUARDED - return false; - #else const size_t bsize = mi_good_size(size); - return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0); - #endif + return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0); } +#if MI_DEBUG_GUARDED +static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, size_t size, size_t alignment, bool zero) mi_attr_noexcept { + // use over allocation for guarded blocksl + mi_assert_internal(alignment > 0 && alignment < MI_BLOCK_ALIGNMENT_MAX); + const size_t oversize = size + alignment - 1; + void* base = _mi_heap_malloc_guarded(heap, oversize, zero); + void* p = mi_align_up_ptr(base, alignment); + mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size); + mi_assert_internal(mi_usable_size(p) >= size); + mi_assert_internal(_mi_is_aligned(p, alignment)); + return p; +} +#endif + // Fallback aligned allocation that over-allocates -- split out for better codegen static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept { @@ -68,6 +78,13 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t void* aligned_p = (void*)((uintptr_t)p + adjust); if (aligned_p != p) { mi_page_set_has_aligned(page, true); + #if MI_DEBUG_GUARDED + // set tag to aligned so mi_usable_size works with guard pages + if (adjust > sizeof(mi_block_t)) { + mi_block_t* const block = (mi_block_t*)p; + block->next = MI_BLOCK_TAG_ALIGNED; + } + #endif _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); } // todo: expand padding if overallocated ? @@ -76,10 +93,8 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); - #if !MI_DEBUG_GUARDED mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_page(aligned_p), aligned_p)); - #endif - + // now zero the block if needed if (alignment > MI_BLOCK_ALIGNMENT_MAX) { // for the tracker, on huge aligned allocations only from the start of the large block is defined @@ -128,6 +143,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* return mi_heap_malloc_zero_aligned_at_overalloc(heap,size,alignment,offset,zero); } + // Primitive aligned allocation static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept { @@ -138,8 +154,13 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t #endif return NULL; } + + #if MI_DEBUG_GUARDED + if (offset==0 && alignment < MI_BLOCK_ALIGNMENT_MAX && mi_heap_malloc_use_guarded(heap,size)) { + return mi_heap_malloc_guarded_aligned(heap, size, alignment, zero); + } + #endif - #if !MI_DEBUG_GUARDED // try first if there happens to be a small block available with just the right alignment if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` @@ -160,8 +181,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t } } } - #endif - + // fallback to generic aligned allocation return mi_heap_malloc_zero_aligned_at_generic(heap, size, alignment, offset, zero); } @@ -313,3 +333,5 @@ mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment); } + + diff --git a/src/alloc.c b/src/alloc.c index 119dfe75..b4713ff1 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -31,22 +31,22 @@ terms of the MIT license. A copy of the license can be found in the file extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size); - + // check the free list mi_block_t* const block = page->free; if mi_unlikely(block == NULL) { return _mi_malloc_generic(heap, size, zero, 0); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); - + // pop from the free list page->free = mi_block_next(page, block); page->used++; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); - + #if MI_DEBUG>3 - if (page->free_is_zero && size > sizeof(*block)) { + if (page->free_is_zero && size > sizeof(*block)) { mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); } #endif @@ -122,9 +122,7 @@ extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t siz } #if MI_DEBUG_GUARDED -static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; -static inline bool mi_heap_malloc_use_guarded(size_t size, bool has_huge_alignment); -static inline bool mi_heap_malloc_small_use_guarded(size_t size); +mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; #endif static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { @@ -138,7 +136,9 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, if (size == 0) { size = sizeof(void*); } #endif #if MI_DEBUG_GUARDED - if (mi_heap_malloc_small_use_guarded(size)) { return mi_heap_malloc_guarded(heap, size, zero); } + if (mi_heap_malloc_use_guarded(heap,size)) { + return _mi_heap_malloc_guarded(heap, size, zero); + } #endif // get page in constant time, and allocate from it @@ -171,13 +171,15 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t si // The main allocation function extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept { - // fast path for small objects + // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); return mi_heap_malloc_small_zero(heap, size, zero); } #if MI_DEBUG_GUARDED - else if (mi_heap_malloc_use_guarded(size,huge_alignment>0)) { return mi_heap_malloc_guarded(heap, size, zero); } + else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) { + return _mi_heap_malloc_guarded(heap, size, zero); + } #endif else { // regular allocation @@ -185,7 +187,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p,size,zero); - + #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } @@ -602,61 +604,65 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } #if MI_DEBUG_GUARDED -static inline bool mi_heap_malloc_small_use_guarded(size_t size) { - return (size <= (size_t)_mi_option_get_fast(mi_option_debug_guarded_max) - && size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min)); +// We always allocate a guarded allocation at an offset (`mi_page_has_aligned` will be true). +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` + +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) { + // TODO: we can still make padding work by moving it out of the guard page area + mi_page_t* const page = _mi_ptr_page(block); + mi_page_set_has_aligned(page, true); + block->next = MI_BLOCK_TAG_GUARDED; + + // set guard page at the end of the block + mi_segment_t* const segment = _mi_page_segment(page); + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); + if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { + // should never happen + mi_free(block); + return NULL; + } + uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); + if (segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) { + _mi_os_protect(guard_page, os_page_size); + } + else { + _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); + } + + // align pointer just in front of the guard page + size_t offset = block_size - os_page_size - obj_size; + mi_assert_internal(offset > sizeof(mi_block_t)); + if (offset > MI_BLOCK_ALIGNMENT_MAX) { + // give up to place it right in front of the guard page if the offset is too large for unalignment + offset = MI_BLOCK_ALIGNMENT_MAX; + } + void* p = (uint8_t*)block + offset; + mi_track_align(block, p, offset, obj_size); + return p; } -static inline bool mi_heap_malloc_use_guarded(size_t size, bool has_huge_alignment) { - return (!has_huge_alignment // guarded pages do not work with huge aligments at the moment - && _mi_option_get_fast(mi_option_debug_guarded_max) > 0 // guarded must be enabled - && (mi_heap_malloc_small_use_guarded(size) - || ((mi_good_size(size) & (_mi_os_page_size() - 1)) == 0)) // page-size multiple are always guarded so we can have a correct `mi_usable_size`. - ); -} - -static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept +mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { #if defined(MI_PADDING_SIZE) mi_assert(MI_PADDING_SIZE==0); #endif // allocate multiple of page size ending in a guard page - const size_t obj_size = _mi_align_up(size, MI_MAX_ALIGN_SIZE); // ensure minimal alignment requirement + // ensure minimal alignment requirement? const size_t os_page_size = _mi_os_page_size(); - const size_t req_size = _mi_align_up(obj_size + os_page_size, os_page_size); - void* const block = _mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */); + const size_t obj_size = (mi_option_is_enabled(mi_option_debug_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */); if (block==NULL) return NULL; - mi_page_t* page = _mi_ptr_page(block); - mi_segment_t* segment = _mi_page_segment(page); - - const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` - void* const guard_page = (uint8_t*)block + (block_size - os_page_size); - mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); - - // place block in front of the guard page - size_t offset = block_size - os_page_size - obj_size; - if (offset > MI_BLOCK_ALIGNMENT_MAX) { - // give up to place it right in front of the guard page if the offset is too large for unalignment - offset = MI_BLOCK_ALIGNMENT_MAX; - } - void* const p = (uint8_t*)block + offset; - mi_assert_internal(p>=block); - - // set page flags - if (offset > 0) { - mi_page_set_has_aligned(page, true); - } - - // set guard page - if (segment->allow_decommit) { - mi_page_set_has_guarded(page, true); - _mi_os_protect(guard_page, os_page_size); - } - else { - _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", p, size); - } + void* const p = mi_block_ptr_set_guarded(block, obj_size); // stats + const size_t usize = mi_usable_size(p); + mi_assert_internal(usize >= size); mi_track_malloc(p, size, zero); #if MI_STAT>1 if (p != NULL) { diff --git a/src/free.c b/src/free.c index 046a34e2..73c05c87 100644 --- a/src/free.c +++ b/src/free.c @@ -70,20 +70,29 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { } // forward declaration for a MI_DEBUG_GUARDED build -static void mi_block_unguard(mi_page_t* page, mi_block_t* block); +#if MI_DEBUG_GUARDED +static void mi_block_unguard_prim(mi_page_t* page, mi_block_t* block, void* p); // forward declaration +static inline void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard_prim(page, block, p); } +} +#else +static inline void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); +} +#endif // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { MI_UNUSED(segment); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p); - mi_block_unguard(page,block); + mi_block_unguard(page, block, p); mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865) - mi_block_unguard(page, block); + mi_block_unguard(page, block, p); mi_free_block_mt(page, segment, block); } @@ -297,20 +306,19 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p const size_t size = mi_page_usable_size_of(page, block); const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block; mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); - return (size - adjust); + const size_t aligned_size = (size - adjust); + #if MI_DEBUG_GUARDED + if (mi_block_ptr_is_guarded(block, p)) { + return aligned_size - _mi_os_page_size(); + } + #endif + return aligned_size; } static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); if mi_unlikely(segment==NULL) return 0; - const mi_page_t* const page = _mi_segment_page_of(segment, p); - #if MI_DEBUG_GUARDED - if (mi_page_has_guarded(page)) { - const size_t bsize = mi_page_usable_aligned_size_of(page, p); - mi_assert_internal(bsize > _mi_os_page_size()); - return (bsize > _mi_os_page_size() ? bsize - _mi_os_page_size() : bsize); - } else - #endif + const mi_page_t* const page = _mi_segment_page_of(segment, p); if mi_likely(!mi_page_has_aligned(page)) { const mi_block_t* block = (const mi_block_t*)p; return mi_page_usable_size_of(page, block); @@ -534,22 +542,19 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { // Remove guard page when building with MI_DEBUG_GUARDED -#if !MI_DEBUG_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block) { - MI_UNUSED(page); - MI_UNUSED(block); - // do nothing -} -#else -static void mi_block_unguard(mi_page_t* page, mi_block_t* block) { - if (mi_page_has_guarded(page)) { - const size_t bsize = mi_page_block_size(page); - const size_t psize = _mi_os_page_size(); - mi_assert_internal(bsize > psize); - mi_assert_internal(_mi_page_segment(page)->allow_decommit); - void* gpage = (uint8_t*)block + (bsize - psize); - mi_assert_internal(_mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); - } +#if MI_DEBUG_GUARDED +static void mi_block_unguard_prim(mi_page_t* page, mi_block_t* block, void* p) { + mi_assert_internal(mi_block_ptr_is_guarded(block, p)); + mi_assert_internal(mi_page_has_aligned(page)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= sizeof(mi_block_t)); + mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); + + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(_mi_page_segment(page)->allow_decommit); + void* gpage = (uint8_t*)block + bsize - psize; + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); } #endif diff --git a/src/heap.c b/src/heap.c index 206d3a36..eb0ab991 100644 --- a/src/heap.c +++ b/src/heap.c @@ -370,7 +370,7 @@ void mi_heap_destroy(mi_heap_t* heap) { mi_assert_expensive(mi_heap_is_valid(heap)); if (heap==NULL || !mi_heap_is_initialized(heap)) return; #if MI_DEBUG_GUARDED - _mi_warning_message("'mi_heap_destroy' called but ignored as MI_DEBUG_GUARDED is enabled (heap at %p)\n", heap); + // _mi_warning_message("'mi_heap_destroy' called but MI_DEBUG_GUARDED is enabled -- using `mi_heap_delete` instead (heap at %p)\n", heap); mi_heap_delete(heap); return; #else diff --git a/src/options.c b/src/options.c index 9ddf86ba..3d9017f1 100644 --- a/src/options.c +++ b/src/options.c @@ -143,7 +143,13 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(visit_abandoned) }, #endif { 0, UNINIT, MI_OPTION(debug_guarded_min) }, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects - { 0, UNINIT, MI_OPTION(debug_guarded_max) }, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects + { MI_GiB, UNINIT, MI_OPTION(debug_guarded_max) }, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects + { 0, UNINIT, MI_OPTION(debug_guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) +#if MI_DEBUG_GUARDED + { 1000,UNINIT, MI_OPTION(debug_guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded(= 1000) +#else + { 0, UNINIT, MI_OPTION(debug_guarded_sample_rate)}, +#endif }; static void mi_option_init(mi_option_desc_t* desc); diff --git a/src/page.c b/src/page.c index 49f9ed52..d6dcfb15 100644 --- a/src/page.c +++ b/src/page.c @@ -414,9 +414,6 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // no more aligned blocks in here mi_page_set_has_aligned(page, false); - #if MI_DEBUG_GUARDED - mi_page_set_has_guarded(page, false); - #endif // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) @@ -443,9 +440,6 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_assert_internal(mi_page_all_free(page)); mi_page_set_has_aligned(page, false); - #if MI_DEBUG_GUARDED - mi_page_set_has_guarded(page, false); - #endif // don't retire too often.. // (or we end up retiring and re-allocating most of the time) diff --git a/test/test-stress.c b/test/test-stress.c index 30ad0e77..b062f2ce 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -22,6 +22,8 @@ terms of the MIT license. #include #include +#define MI_DEBUG_GUARDED + // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // // argument defaults From 8b6017d976ed98b0ec71bf34f0c8089b00b52e67 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 16 Nov 2024 22:42:32 -0800 Subject: [PATCH 32/38] rename mi_debug_guarded_ to mi_guarded_ --- CMakeLists.txt | 8 +++--- azure-pipelines.yml | 6 ++--- ide/vs2022/mimalloc.vcxproj | 2 +- include/mimalloc.h | 18 ++++++------- include/mimalloc/internal.h | 6 ++--- include/mimalloc/types.h | 8 +++--- src/alloc-aligned.c | 26 +++++++++---------- src/alloc.c | 20 +++++++------- src/free.c | 48 +++++++++++++++++----------------- src/heap.c | 4 +-- src/options.c | 52 ++++++++++++++++++------------------- src/segment.c | 2 +- test/test-api-fill.c | 2 +- test/test-stress.c | 12 ++++----- 14 files changed, 106 insertions(+), 108 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 87837026..5fc1808e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) -option(MI_DEBUG_GUARDED "Build with guard pages behind certain object allocations (implies MI_NO_PADDING=ON)" OFF) +option(MI_GUARDED "Build with guard pages behind certain object allocations (implies MI_NO_PADDING=ON)" OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) @@ -207,9 +207,9 @@ if(MI_TRACK_ETW) endif() endif() -if(MI_DEBUG_GUARDED) - message(STATUS "Compile guard pages behind certain object allocations (MI_DEBUG_GUARDED=ON)") - list(APPEND mi_defines MI_DEBUG_GUARDED=1) +if(MI_GUARDED) + message(STATUS "Compile guard pages behind certain object allocations (MI_GUARDED=ON)") + list(APPEND mi_defines MI_GUARDED=1) if(NOT MI_NO_PADDING) message(STATUS " Disabling padding due to guard pages (MI_NO_PADDING=ON)") set(MI_NO_PADDING ON) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4455dfeb..e4361f98 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -117,8 +117,8 @@ jobs: CC: clang CXX: clang BuildType: debug-guarded-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_DEBUG_GUARDED=ON - + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON + steps: - task: CMake@1 inputs: @@ -129,7 +129,7 @@ jobs: - script: ctest --verbose --timeout 180 workingDirectory: $(BuildType) displayName: CTest - env: + env: MIMALLOC_DEBUG_GUARDED_MAX: 1024 # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj index 5a614289..160f1436 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc.vcxproj @@ -116,7 +116,7 @@ true Default ../../include - MI_DEBUG=4;MI_DEBUG_GUARDED=1;%(PreprocessorDefinitions); + MI_DEBUG=4;MI_GUARDED=1;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 diff --git a/include/mimalloc.h b/include/mimalloc.h index 940284b6..4ecb8be0 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -290,7 +290,7 @@ mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t a #endif -// Experimental: allow sub-processes whose memory segments stay separated (and no reclamation between them) +// Experimental: allow sub-processes whose memory segments stay separated (and no reclamation between them) // Used for example for separate interpreter's in one process. typedef void* mi_subproc_id_t; mi_decl_export mi_subproc_id_t mi_subproc_main(void); @@ -349,7 +349,7 @@ typedef enum mi_option_e { mi_option_deprecated_segment_cache, mi_option_deprecated_page_reset, mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination - mi_option_deprecated_segment_reset, + mi_option_deprecated_segment_reset, mi_option_eager_commit_delay, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10) mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. @@ -366,10 +366,10 @@ typedef enum mi_option_e { mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) - mi_option_debug_guarded_min, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) - mi_option_debug_guarded_max, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) - mi_option_debug_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) - mi_option_debug_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) + mi_option_guarded_min, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0) + mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) + mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) + mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, @@ -539,7 +539,7 @@ template struct _mi_heap_stl_allocator_common : publi protected: std::shared_ptr heap; template friend struct _mi_heap_stl_allocator_common; - + _mi_heap_stl_allocator_common() { mi_heap_t* hp = mi_heap_new(); this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ @@ -556,7 +556,7 @@ private: template struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common { using typename _mi_heap_stl_allocator_common::size_type; mi_heap_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is deleted when the destructor is called - mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap + mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap template mi_heap_stl_allocator(const mi_heap_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { } mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } @@ -573,7 +573,7 @@ template bool operator!=(const mi_heap_stl_allocator& x, template struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common { using typename _mi_heap_stl_allocator_common::size_type; mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common() { } // creates fresh heap that is destroyed when the destructor is called - mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap + mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common(hp) { } // no delete nor destroy on the passed in heap template mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator& x) mi_attr_noexcept : _mi_heap_stl_allocator_common(x) { } mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index d73532e0..ae3a3358 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -603,7 +603,7 @@ static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { /* ------------------------------------------------------------------- Guarded objects ------------------------------------------------------------------- */ -#if MI_DEBUG_GUARDED +#if MI_GUARDED static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED); @@ -611,8 +611,8 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) { MI_UNUSED(heap); - return (size <= (size_t)_mi_option_get_fast(mi_option_debug_guarded_max) - && size >= (size_t)_mi_option_get_fast(mi_option_debug_guarded_min)); + return (size <= (size_t)_mi_option_get_fast(mi_option_guarded_max) + && size >= (size_t)_mi_option_get_fast(mi_option_guarded_min)); } mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index e01754e2..29ba8564 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -74,8 +74,8 @@ terms of the MIT license. A copy of the license can be found in the file // Use guard pages behind objects of a certain size (set by the MIMALLOC_DEBUG_GUARDED_MIN/MAX options) // Padding should be disabled when using guard pages -// #define MI_DEBUG_GUARDED 1 -#if defined(MI_DEBUG_GUARDED) +// #define MI_GUARDED 1 +#if defined(MI_GUARDED) #define MI_PADDING 0 #endif @@ -232,7 +232,7 @@ typedef struct mi_block_s { mi_encoded_t next; } mi_block_t; -#if MI_DEBUG_GUARDED +#if MI_GUARDED // we always align guarded pointers in a block at an offset // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones #define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) @@ -257,7 +257,6 @@ typedef union mi_page_flags_s { struct { uint8_t in_full : 1; uint8_t has_aligned : 1; - uint8_t has_guarded : 1; // only used with MI_DEBUG_GUARDED } x; } mi_page_flags_t; #else @@ -267,7 +266,6 @@ typedef union mi_page_flags_s { struct { uint8_t in_full; uint8_t has_aligned; - uint8_t has_guarded; // only used with MI_DEBUG_GUARDED } x; } mi_page_flags_t; #endif diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 12815689..86b13dea 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -21,10 +21,10 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { if (alignment > size) return false; if (alignment <= MI_MAX_ALIGN_SIZE) return true; const size_t bsize = mi_good_size(size); - return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0); + return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0); } -#if MI_DEBUG_GUARDED +#if MI_GUARDED static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, size_t size, size_t alignment, bool zero) mi_attr_noexcept { // use over allocation for guarded blocksl mi_assert_internal(alignment > 0 && alignment < MI_BLOCK_ALIGNMENT_MAX); @@ -69,7 +69,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t if (p == NULL) return NULL; } mi_page_t* page = _mi_ptr_page(p); - + // .. and align within the allocation const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` const uintptr_t poffset = ((uintptr_t)p + offset) & align_mask; @@ -78,7 +78,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t void* aligned_p = (void*)((uintptr_t)p + adjust); if (aligned_p != p) { mi_page_set_has_aligned(page, true); - #if MI_DEBUG_GUARDED + #if MI_GUARDED // set tag to aligned so mi_usable_size works with guard pages if (adjust > sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; @@ -94,7 +94,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_page(aligned_p), aligned_p)); - + // now zero the block if needed if (alignment > MI_BLOCK_ALIGNMENT_MAX) { // for the tracker, on huge aligned allocations only from the start of the large block is defined @@ -115,27 +115,27 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* { mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) - if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { + if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } - + // use regular allocation if it is guaranteed to fit the alignment constraints. // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { void* p = _mi_heap_malloc_zero(heap, size, zero); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); - const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { return p; } else { // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. mi_assert(false); - mi_free(p); + mi_free(p); } } @@ -155,16 +155,16 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t return NULL; } - #if MI_DEBUG_GUARDED + #if MI_GUARDED if (offset==0 && alignment < MI_BLOCK_ALIGNMENT_MAX && mi_heap_malloc_use_guarded(heap,size)) { return mi_heap_malloc_guarded_aligned(heap, size, alignment, zero); } #endif - + // try first if there happens to be a small block available with just the right alignment if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - const size_t padsize = size + MI_PADDING_SIZE; + const size_t padsize = size + MI_PADDING_SIZE; mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize); if mi_likely(page->free != NULL) { const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; @@ -181,7 +181,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t } } } - + // fallback to generic aligned allocation return mi_heap_malloc_zero_aligned_at_generic(heap, size, alignment, offset, zero); } diff --git a/src/alloc.c b/src/alloc.c index b4713ff1..561b0026 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -121,7 +121,7 @@ extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t siz return _mi_page_malloc_zero(heap,page,size,true); } -#if MI_DEBUG_GUARDED +#if MI_GUARDED mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; #endif @@ -132,12 +132,12 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, const uintptr_t tid = _mi_thread_id(); mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local #endif - #if (MI_PADDING || MI_DEBUG_GUARDED) + #if (MI_PADDING || MI_GUARDED) if (size == 0) { size = sizeof(void*); } #endif - #if MI_DEBUG_GUARDED - if (mi_heap_malloc_use_guarded(heap,size)) { - return _mi_heap_malloc_guarded(heap, size, zero); + #if MI_GUARDED + if (mi_heap_malloc_use_guarded(heap,size)) { + return _mi_heap_malloc_guarded(heap, size, zero); } #endif @@ -176,9 +176,9 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z mi_assert_internal(huge_alignment == 0); return mi_heap_malloc_small_zero(heap, size, zero); } - #if MI_DEBUG_GUARDED - else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) { - return _mi_heap_malloc_guarded(heap, size, zero); + #if MI_GUARDED + else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) { + return _mi_heap_malloc_guarded(heap, size, zero); } #endif else { @@ -603,7 +603,7 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } } -#if MI_DEBUG_GUARDED +#if MI_GUARDED // We always allocate a guarded allocation at an offset (`mi_page_has_aligned` will be true). // We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) // and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` @@ -653,7 +653,7 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo // allocate multiple of page size ending in a guard page // ensure minimal alignment requirement? const size_t os_page_size = _mi_os_page_size(); - const size_t obj_size = (mi_option_is_enabled(mi_option_debug_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */); diff --git a/src/free.c b/src/free.c index 73c05c87..afbafae6 100644 --- a/src/free.c +++ b/src/free.c @@ -33,8 +33,8 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool // checks if mi_unlikely(mi_check_is_double_free(page, block)) return; mi_check_padding(page, block); - if (track_stats) { mi_stat_free(page, block); } - #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN && !MI_DEBUG_GUARDED + if (track_stats) { mi_stat_free(page, block); } + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN && !MI_GUARDED memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); #endif if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block)); } // faster then mi_usable_size as we already know the page and that p is unaligned @@ -69,14 +69,14 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { return (mi_block_t*)((uintptr_t)p - adjust); } -// forward declaration for a MI_DEBUG_GUARDED build -#if MI_DEBUG_GUARDED -static void mi_block_unguard_prim(mi_page_t* page, mi_block_t* block, void* p); // forward declaration -static inline void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { - if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard_prim(page, block, p); } +// forward declaration for a MI_GUARDED build +#if MI_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration +static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { + if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard(page, block, p); } } #else -static inline void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { +static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); } #endif @@ -85,14 +85,14 @@ static inline void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { MI_UNUSED(segment); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p); - mi_block_unguard(page, block, p); + mi_block_check_unguard(page, block, p); mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865) - mi_block_unguard(page, block, p); + mi_block_check_unguard(page, block, p); mi_free_block_mt(page, segment, block); } @@ -109,17 +109,17 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms { MI_UNUSED(msg); -#if (MI_DEBUG>0) - if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) { + #if (MI_DEBUG>0) + if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return NULL; } -#endif + #endif mi_segment_t* const segment = _mi_ptr_segment(p); if mi_unlikely(segment==NULL) return segment; -#if (MI_DEBUG>0) + #if (MI_DEBUG>0) if mi_unlikely(!mi_is_in_heap_region(p)) { _mi_warning_message("%s: pointer might not point to a valid heap region: %p\n" "(this may still be a valid very large allocation (over 64MiB))\n", msg, p); @@ -127,13 +127,13 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); } } -#endif -#if (MI_DEBUG>0 || MI_SECURE>=4) + #endif + #if (MI_DEBUG>0 || MI_SECURE>=4) if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) { _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); return NULL; } -#endif + #endif return segment; } @@ -307,7 +307,7 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block; mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); const size_t aligned_size = (size - adjust); - #if MI_DEBUG_GUARDED + #if MI_GUARDED if (mi_block_ptr_is_guarded(block, p)) { return aligned_size - _mi_os_page_size(); } @@ -318,7 +318,7 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); if mi_unlikely(segment==NULL) return 0; - const mi_page_t* const page = _mi_segment_page_of(segment, p); + const mi_page_t* const page = _mi_segment_page_of(segment, p); if mi_likely(!mi_page_has_aligned(page)) { const mi_block_t* block = (const mi_block_t*)p; return mi_page_usable_size_of(page, block); @@ -541,12 +541,12 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { #endif -// Remove guard page when building with MI_DEBUG_GUARDED -#if MI_DEBUG_GUARDED -static void mi_block_unguard_prim(mi_page_t* page, mi_block_t* block, void* p) { +// Remove guard page when building with MI_GUARDED +#if MI_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { mi_assert_internal(mi_block_ptr_is_guarded(block, p)); mi_assert_internal(mi_page_has_aligned(page)); - mi_assert_internal((uint8_t*)p - (uint8_t*)block >= sizeof(mi_block_t)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); const size_t bsize = mi_page_block_size(page); @@ -555,6 +555,6 @@ static void mi_block_unguard_prim(mi_page_t* page, mi_block_t* block, void* p) { mi_assert_internal(_mi_page_segment(page)->allow_decommit); void* gpage = (uint8_t*)block + bsize - psize; mi_assert_internal(_mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); + _mi_os_unprotect(gpage, psize); } #endif diff --git a/src/heap.c b/src/heap.c index eb0ab991..78ebcd1e 100644 --- a/src/heap.c +++ b/src/heap.c @@ -369,8 +369,8 @@ void mi_heap_destroy(mi_heap_t* heap) { mi_assert(heap->no_reclaim); mi_assert_expensive(mi_heap_is_valid(heap)); if (heap==NULL || !mi_heap_is_initialized(heap)) return; - #if MI_DEBUG_GUARDED - // _mi_warning_message("'mi_heap_destroy' called but MI_DEBUG_GUARDED is enabled -- using `mi_heap_delete` instead (heap at %p)\n", heap); + #if MI_GUARDED + // _mi_warning_message("'mi_heap_destroy' called but MI_GUARDED is enabled -- using `mi_heap_delete` instead (heap at %p)\n", heap); mi_heap_delete(heap); return; #else diff --git a/src/options.c b/src/options.c index 3d9017f1..c5f1e2a1 100644 --- a/src/options.c +++ b/src/options.c @@ -47,9 +47,9 @@ typedef struct mi_option_desc_s { #define MI_OPTION(opt) mi_option_##opt, #opt, NULL #define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy -// Some options can be set at build time for statically linked libraries +// Some options can be set at build time for statically linked libraries // (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) -// +// // This is useful if we cannot pass them as environment variables // (and setting them programmatically would be too late) @@ -102,17 +102,17 @@ static mi_option_desc_t options[_mi_option_last] = { MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) }, // some of the following options are experimental and not all combinations are allowed. - { MI_DEFAULT_EAGER_COMMIT, + { MI_DEFAULT_EAGER_COMMIT, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) - { MI_DEFAULT_ARENA_EAGER_COMMIT, + { MI_DEFAULT_ARENA_EAGER_COMMIT, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) - { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { MI_DEFAULT_RESERVE_OS_MEMORY, + { MI_DEFAULT_RESERVE_OS_MEMORY, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free @@ -137,18 +137,18 @@ static mi_option_desc_t options[_mi_option_last] = { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. -#if defined(MI_VISIT_ABANDONED) +#if defined(MI_VISIT_ABANDONED) { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandonded segments; requires taking locks during reclaim. #else - { 0, UNINIT, MI_OPTION(visit_abandoned) }, + { 0, UNINIT, MI_OPTION(visit_abandoned) }, #endif - { 0, UNINIT, MI_OPTION(debug_guarded_min) }, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects - { MI_GiB, UNINIT, MI_OPTION(debug_guarded_max) }, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects - { 0, UNINIT, MI_OPTION(debug_guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) -#if MI_DEBUG_GUARDED - { 1000,UNINIT, MI_OPTION(debug_guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded(= 1000) + { 0, UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects + { MI_GiB, UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects + { 0, UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) +#if MI_GUARDED + { 1000,UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded(= 1000) #else - { 0, UNINIT, MI_OPTION(debug_guarded_sample_rate)}, + { 0, UNINIT, MI_OPTION(guarded_sample_rate)}, #endif }; @@ -172,25 +172,25 @@ void _mi_options_init(void) { } mi_max_error_count = mi_option_get(mi_option_max_errors); mi_max_warning_count = mi_option_get(mi_option_max_warnings); - #if MI_DEBUG_GUARDED - if (mi_option_get(mi_option_debug_guarded_max) > 0) { + #if MI_GUARDED + if (mi_option_get(mi_option_guarded_max) > 0) { if (mi_option_is_enabled(mi_option_allow_large_os_pages)) { mi_option_disable(mi_option_allow_large_os_pages); _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n"); } } - _mi_verbose_message("guarded build: %s\n", mi_option_get(mi_option_debug_guarded_max) > 0 ? "enabled" : "disabled"); + _mi_verbose_message("guarded build: %s\n", mi_option_get(mi_option_guarded_max) > 0 ? "enabled" : "disabled"); #endif } long _mi_option_get_fast(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); - mi_option_desc_t* desc = &options[option]; + mi_option_desc_t* desc = &options[option]; mi_assert(desc->option == option); // index should match the option //mi_assert(desc->init != UNINIT); return desc->value; } - + mi_decl_nodiscard long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); @@ -225,11 +225,11 @@ void mi_option_set(mi_option_t option, long value) { desc->value = value; desc->init = INITIALIZED; // ensure min/max range; be careful to not recurse. - if (desc->option == mi_option_debug_guarded_min && _mi_option_get_fast(mi_option_debug_guarded_max) < value) { - mi_option_set(mi_option_debug_guarded_max, value); + if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) { + mi_option_set(mi_option_guarded_max, value); } - else if (desc->option == mi_option_debug_guarded_max && _mi_option_get_fast(mi_option_debug_guarded_min) > value) { - mi_option_set(mi_option_debug_guarded_min, value); + else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) { + mi_option_set(mi_option_guarded_min, value); } } @@ -565,7 +565,7 @@ static void mi_option_init(mi_option_desc_t* desc) { char* end = buf; long value = strtol(buf, &end, 10); if (mi_option_has_size_in_kib(desc->option)) { - // this option is interpreted in KiB to prevent overflow of `long` for large allocations + // this option is interpreted in KiB to prevent overflow of `long` for large allocations // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) size_t size = (value < 0 ? 0 : (size_t)value); bool overflow = false; @@ -580,7 +580,7 @@ static void mi_option_init(mi_option_desc_t* desc) { value = (size > LONG_MAX ? LONG_MAX : (long)size); } if (*end == 0) { - mi_option_set(desc->option, value); + mi_option_set(desc->option, value); } else { // set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose. diff --git a/src/segment.c b/src/segment.c index 837a65e9..18736818 100644 --- a/src/segment.c +++ b/src/segment.c @@ -455,7 +455,7 @@ static size_t mi_segment_calculate_sizes(size_t capacity, size_t required, size_ if (MI_SECURE == 0) { // normally no guard pages - #if MI_DEBUG_GUARDED + #if MI_GUARDED isize = _mi_align_up(minsize, _mi_os_page_size()); #else isize = _mi_align_up(minsize, 16 * MI_MAX_ALIGN_SIZE); diff --git a/test/test-api-fill.c b/test/test-api-fill.c index 3baee83d..eebbd394 100644 --- a/test/test-api-fill.c +++ b/test/test-api-fill.c @@ -271,7 +271,7 @@ int main(void) { mi_free(p); }; - #if !(MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_DEBUG_GUARDED) + #if !(MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_GUARDED) CHECK_BODY("fill-freed-small") { size_t malloc_size = MI_SMALL_SIZE_MAX / 2; uint8_t* p = (uint8_t*)mi_malloc(malloc_size); diff --git a/test/test-stress.c b/test/test-stress.c index b062f2ce..cb769dbf 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -22,21 +22,22 @@ terms of the MIT license. #include #include -#define MI_DEBUG_GUARDED +// #define MI_GUARDED +// #define USE_STD_MALLOC // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // // argument defaults #if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits -static int THREADS = 8; +static int THREADS = 8; static int SCALE = 25; static int ITER = 400; #elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits -static int THREADS = 8; +static int THREADS = 8; static int SCALE = 25; static int ITER = 20; -#elif defined(MI_DEBUG_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits -static int THREADS = 8; +#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +static int THREADS = 8; static int SCALE = 10; static int ITER = 10; #else @@ -58,7 +59,6 @@ static size_t use_one_size = 0; // use single object size of `N * static bool main_participates = false; // main thread participates as a worker too -// #define USE_STD_MALLOC #ifdef USE_STD_MALLOC #define custom_calloc(n,s) calloc(n,s) #define custom_realloc(p,s) realloc(p,s) From 8ba1879073785ebbc0a27e2aeace3a0d3e6ff076 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 17 Nov 2024 00:06:16 -0800 Subject: [PATCH 33/38] add sampling for guarded objects --- include/mimalloc.h | 1 + include/mimalloc/internal.h | 16 +++++++++++++--- include/mimalloc/types.h | 8 ++++++++ src/alloc-aligned.c | 25 +++++++++++++++++++++---- src/alloc.c | 1 + src/heap.c | 1 + src/init.c | 26 +++++++++++++++++++++++++- src/options.c | 3 ++- src/stats.c | 2 ++ test/test-stress.c | 3 +-- 10 files changed, 75 insertions(+), 11 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 4ecb8be0..8cbe265f 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -370,6 +370,7 @@ typedef enum mi_option_e { mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) + mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index ae3a3358..7809503b 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -91,6 +91,7 @@ void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_heap_guarded_init(mi_heap_t* heap); // os.c void _mi_os_init(void); // called from process init @@ -610,12 +611,21 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* } static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) { - MI_UNUSED(heap); - return (size <= (size_t)_mi_option_get_fast(mi_option_guarded_max) - && size >= (size_t)_mi_option_get_fast(mi_option_guarded_min)); + MI_UNUSED(heap); + if (heap->guarded_sample_rate==0 || + size > heap->guarded_size_max || + size < heap->guarded_size_min) { + return false; + } + if (++heap->guarded_sample_count < heap->guarded_sample_rate) { + return false; + } + heap->guarded_sample_count = 0; // reset + return true; } mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; + #endif diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 29ba8564..ec181e61 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -502,6 +502,13 @@ struct mi_heap_s { mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom tag, can be used for separating heaps based on the object types + #if MI_GUARDED + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) + size_t guarded_sample_seed; // starting sample count + size_t guarded_sample_count; // current sample count (wraps at `sample_rate`) + #endif mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") }; @@ -594,6 +601,7 @@ typedef struct mi_stats_s { mi_stat_counter_t arena_count; mi_stat_counter_t arena_crossover_count; mi_stat_counter_t arena_rollback_count; + mi_stat_counter_t guarded_alloc_count; #if MI_STAT>1 mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; #endif diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 86b13dea..6aee38c3 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -36,6 +36,18 @@ static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, si mi_assert_internal(_mi_is_aligned(p, alignment)); return p; } + +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { + const size_t rate = heap->guarded_sample_rate; + heap->guarded_sample_rate = 0; + void* p = _mi_heap_malloc_zero(heap, size, zero); + heap->guarded_sample_rate = rate; + return p; +} +#else +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { + return _mi_heap_malloc_zero(heap, size, zero); +} #endif // Fallback aligned allocation that over-allocates -- split out for better codegen @@ -58,6 +70,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t return NULL; } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); + // note: no guarded as alignment > 0 p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block // zero afterwards as only the area from the aligned_p may be committed! if (p == NULL) return NULL; @@ -65,7 +78,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t else { // otherwise over-allocate oversize = size + alignment - 1; - p = _mi_heap_malloc_zero(heap, oversize, zero); + p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero); if (p == NULL) return NULL; } mi_page_t* page = _mi_ptr_page(p); @@ -80,7 +93,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_page_set_has_aligned(page, true); #if MI_GUARDED // set tag to aligned so mi_usable_size works with guard pages - if (adjust > sizeof(mi_block_t)) { + if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; block->next = MI_BLOCK_TAG_ALIGNED; } @@ -93,7 +106,11 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); - mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_page(aligned_p), aligned_p)); + #if MI_DEBUG > 1 + mi_page_t* const apage = _mi_ptr_page(aligned_p); + void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == unalign_p); + #endif // now zero the block if needed if (alignment > MI_BLOCK_ALIGNMENT_MAX) { @@ -126,7 +143,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = _mi_heap_malloc_zero(heap, size, zero); + void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { diff --git a/src/alloc.c b/src/alloc.c index 561b0026..25b05526 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -668,6 +668,7 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + mi_heap_stat_counter_increase(heap, guarded_alloc_count, 1); } #endif #if MI_DEBUG>3 diff --git a/src/heap.c b/src/heap.c index 78ebcd1e..581b3f71 100644 --- a/src/heap.c +++ b/src/heap.c @@ -221,6 +221,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); + _mi_heap_guarded_init(heap); // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; diff --git a/src/init.c b/src/init.c index 75458a1f..822d5a18 100644 --- a/src/init.c +++ b/src/init.c @@ -86,7 +86,8 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ - { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ + { 0, 0 } \ MI_STAT_COUNT_END_NULL() // -------------------------------------------------------- @@ -111,6 +112,9 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { NULL, // next false, // can reclaim 0, // tag + #if MI_GUARDED + 0, 0, 0, 0, 0, + #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -151,6 +155,9 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = { NULL, // next heap false, // can reclaim 0, // tag + #if MI_GUARDED + 0, 0, 0, 0, 0, + #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -159,6 +166,22 @@ bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. mi_stats_t _mi_stats_main = { MI_STATS_NULL }; +#if MI_GUARDED +void _mi_heap_guarded_init(mi_heap_t* heap) { + heap->guarded_sample_rate = mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX); + heap->guarded_size_max = mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX); + heap->guarded_size_min = mi_option_get_clamp(mi_option_guarded_min, 0, (long)heap->guarded_size_max); + heap->guarded_sample_seed = (size_t)mi_option_get(mi_option_guarded_sample_seed); + if (heap->guarded_sample_seed == 0) { heap->guarded_sample_seed = _mi_heap_random_next(heap); } + heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; + heap->guarded_sample_count = heap->guarded_sample_seed; +} +#else +void _mi_heap_guarded_init(mi_heap_t* heap) { + MI_UNUSED(heap); +} +#endif + static void mi_heap_main_init(void) { if (_mi_heap_main.cookie == 0) { @@ -174,6 +197,7 @@ static void mi_heap_main_init(void) { _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); mi_lock_init(&mi_subproc_default.abandoned_os_lock); mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock); + _mi_heap_guarded_init(&_mi_heap_main); } } diff --git a/src/options.c b/src/options.c index c5f1e2a1..2c6814c8 100644 --- a/src/options.c +++ b/src/options.c @@ -150,6 +150,7 @@ static mi_option_desc_t options[_mi_option_last] = #else { 0, UNINIT, MI_OPTION(guarded_sample_rate)}, #endif + { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, }; static void mi_option_init(mi_option_desc_t* desc); @@ -173,7 +174,7 @@ void _mi_options_init(void) { mi_max_error_count = mi_option_get(mi_option_max_errors); mi_max_warning_count = mi_option_get(mi_option_max_warnings); #if MI_GUARDED - if (mi_option_get(mi_option_guarded_max) > 0) { + if (mi_option_get(mi_option_guarded_sample_rate) > 0) { if (mi_option_is_enabled(mi_option_allow_large_os_pages)) { mi_option_disable(mi_option_allow_large_os_pages); _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n"); diff --git a/src/stats.c b/src/stats.c index 99cf89c5..29376ace 100644 --- a/src/stats.c +++ b/src/stats.c @@ -118,6 +118,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_counter_add(&stats->searches, &src->searches, 1); mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); + mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { @@ -342,6 +343,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); + mi_stat_counter_print(&stats->guarded_alloc_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); diff --git a/test/test-stress.c b/test/test-stress.c index cb769dbf..88c39f23 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -22,7 +22,6 @@ terms of the MIT license. #include #include -// #define MI_GUARDED // #define USE_STD_MALLOC // > mimalloc-test-stress [THREADS] [SCALE] [ITER] @@ -36,7 +35,7 @@ static int ITER = 400; static int THREADS = 8; static int SCALE = 25; static int ITER = 20; -#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +#elif defined(MI_XGUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = 8; static int SCALE = 10; static int ITER = 10; From d57cb0765d4673ee10a577d741455ead9396b944 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 17 Nov 2024 22:45:09 -0800 Subject: [PATCH 34/38] add guarded objects that are sampled (and fit a size range). guarded sample rate etc can be set per heap as well as defaulted with options --- include/mimalloc.h | 6 ++++++ include/mimalloc/internal.h | 22 ++++++++++++-------- include/mimalloc/types.h | 2 +- src/alloc.c | 8 +++----- src/free.c | 1 + src/init.c | 41 +++++++++++++++++++++++++++++-------- src/options.c | 2 +- test/main-override.cpp | 2 +- test/test-stress.c | 3 ++- 9 files changed, 61 insertions(+), 26 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 8cbe265f..e3fecdf1 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -309,6 +309,12 @@ mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_ex(int heap_tag, bool al // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; +// Experimental: objects followed by a guard page. +// A sample rate of 0 disables guarded objects, while 1 uses a guard page for every object. +// A seed of 0 uses a random start point. Only objects within the size bound are eligable for guard pages. +mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed); +mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max); + // ------------------------------------------------------ // Convenience diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 7809503b..d58bd9ca 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -611,17 +611,23 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* } static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) { - MI_UNUSED(heap); - if (heap->guarded_sample_rate==0 || - size > heap->guarded_size_max || - size < heap->guarded_size_min) { + // this code is written to result in fast assembly as it is on the hot path for allocation + const size_t count = heap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time.. + if mi_likely(count != 0) { + // no sample + heap->guarded_sample_count = count; return false; } - if (++heap->guarded_sample_count < heap->guarded_sample_rate) { - return false; + else if (size >= heap->guarded_size_min && size <= heap->guarded_size_max) { + // use guarded allocation + heap->guarded_sample_count = heap->guarded_sample_rate; // reset + return (heap->guarded_sample_rate != 0); } - heap->guarded_sample_count = 0; // reset - return true; + else { + // failed size criteria, rewind count (but don't write to an empty heap) + if (heap->guarded_sample_rate != 0) { heap->guarded_sample_count = 1; } + return false; + } } mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ec181e61..f7bca137 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -507,7 +507,7 @@ struct mi_heap_s { size_t guarded_size_max; // maximal size for guarded objects size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) size_t guarded_sample_seed; // starting sample count - size_t guarded_sample_count; // current sample count (wraps at `sample_rate`) + size_t guarded_sample_count; // current sample count (counting down to 0) #endif mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") diff --git a/src/alloc.c b/src/alloc.c index 25b05526..bc4f2aa5 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -661,16 +661,14 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo void* const p = mi_block_ptr_set_guarded(block, obj_size); // stats - const size_t usize = mi_usable_size(p); - mi_assert_internal(usize >= size); mi_track_malloc(p, size, zero); - #if MI_STAT>1 if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } + #if MI_STAT>1 mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); - mi_heap_stat_counter_increase(heap, guarded_alloc_count, 1); + #endif + _mi_stat_counter_increase(&heap->tld->stats.guarded_alloc_count, 1); } - #endif #if MI_DEBUG>3 if (p != NULL && zero) { mi_assert_expensive(mi_mem_is_zero(p, size)); diff --git a/src/free.c b/src/free.c index afbafae6..f2e30b65 100644 --- a/src/free.c +++ b/src/free.c @@ -544,6 +544,7 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { // Remove guard page when building with MI_GUARDED #if MI_GUARDED static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(p); mi_assert_internal(mi_block_ptr_is_guarded(block, p)); mi_assert_internal(mi_page_has_aligned(page)); mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); diff --git a/src/init.c b/src/init.c index 822d5a18..ed161831 100644 --- a/src/init.c +++ b/src/init.c @@ -113,7 +113,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { false, // can reclaim 0, // tag #if MI_GUARDED - 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY @@ -167,16 +167,39 @@ bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. mi_stats_t _mi_stats_main = { MI_STATS_NULL }; #if MI_GUARDED +mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { + heap->guarded_sample_seed = seed; + if (heap->guarded_sample_seed == 0) { + heap->guarded_sample_seed = _mi_heap_random_next(heap); + } + heap->guarded_sample_rate = sample_rate; + if (heap->guarded_sample_rate >= 1) { + heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; + } + heap->guarded_sample_count = heap->guarded_sample_seed; // count down samples +} + +mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) { + heap->guarded_size_min = min; + heap->guarded_size_max = (min > max ? min : max); +} + void _mi_heap_guarded_init(mi_heap_t* heap) { - heap->guarded_sample_rate = mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX); - heap->guarded_size_max = mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX); - heap->guarded_size_min = mi_option_get_clamp(mi_option_guarded_min, 0, (long)heap->guarded_size_max); - heap->guarded_sample_seed = (size_t)mi_option_get(mi_option_guarded_sample_seed); - if (heap->guarded_sample_seed == 0) { heap->guarded_sample_seed = _mi_heap_random_next(heap); } - heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; - heap->guarded_sample_count = heap->guarded_sample_seed; + mi_heap_guarded_set_sample_rate(heap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_heap_guarded_set_size_bound(heap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); } #else +mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) { + MI_UNUSED(heap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +} + +mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) { + MI_UNUSED(heap); MI_UNUSED(min); MI_UNUSED(max); +} void _mi_heap_guarded_init(mi_heap_t* heap) { MI_UNUSED(heap); } @@ -576,7 +599,7 @@ static void mi_detect_cpu_features(void) { } #else static void mi_detect_cpu_features(void) { - // nothing + // nothing } #endif diff --git a/src/options.c b/src/options.c index 2c6814c8..4f95e601 100644 --- a/src/options.c +++ b/src/options.c @@ -146,7 +146,7 @@ static mi_option_desc_t options[_mi_option_last] = { MI_GiB, UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects { 0, UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) #if MI_GUARDED - { 1000,UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded(= 1000) + { 4000,UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded(= 1000) #else { 0, UNINIT, MI_OPTION(guarded_sample_rate)}, #endif diff --git a/test/main-override.cpp b/test/main-override.cpp index 9c47d3a1..3f64117a 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -62,7 +62,7 @@ int main() { test_mt_shutdown(); */ //fail_aslr(); - // mi_stats_print(NULL); + mi_stats_print(NULL); return 0; } diff --git a/test/test-stress.c b/test/test-stress.c index 88c39f23..ba9ab459 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -22,6 +22,7 @@ terms of the MIT license. #include #include +// #define MI_GUARDED // #define USE_STD_MALLOC // > mimalloc-test-stress [THREADS] [SCALE] [ITER] @@ -35,7 +36,7 @@ static int ITER = 400; static int THREADS = 8; static int SCALE = 25; static int ITER = 20; -#elif defined(MI_XGUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +#elif defined(xMI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = 8; static int SCALE = 10; static int ITER = 10; From b8dc09e3d2adde248daabe1303b161c714bff298 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 17 Nov 2024 22:56:26 -0800 Subject: [PATCH 35/38] fix asan with MI_GUARDED --- src/alloc-aligned.c | 3 +++ src/alloc.c | 5 +++-- src/os.c | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 6aee38c3..b4da4ded 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -123,6 +123,9 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); + #if MI_GUARDED + mi_track_mem_defined(p, sizeof(mi_block_t)); + #endif } return aligned_p; } diff --git a/src/alloc.c b/src/alloc.c index bc4f2aa5..a093f108 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -640,8 +640,9 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) { // give up to place it right in front of the guard page if the offset is too large for unalignment offset = MI_BLOCK_ALIGNMENT_MAX; } - void* p = (uint8_t*)block + offset; + void* p = (uint8_t*)block + offset; mi_track_align(block, p, offset, obj_size); + mi_track_mem_defined(block, sizeof(mi_block_t)); return p; } @@ -661,7 +662,7 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo void* const p = mi_block_ptr_set_guarded(block, obj_size); // stats - mi_track_malloc(p, size, zero); + mi_track_malloc(p, size, zero); if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } #if MI_STAT>1 diff --git a/src/os.c b/src/os.c index 62c8c934..a7130b90 100644 --- a/src/os.c +++ b/src/os.c @@ -273,7 +273,9 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit } else { // if not aligned, free it, overallocate, and unmap around it + #if !MI_TRACK_ASAN _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); + #endif mi_os_prim_free(p, size, commit, stats); if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; From ff56f6acbc786ddf533d348b27d249256d14dc44 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 17 Nov 2024 23:01:16 -0800 Subject: [PATCH 36/38] update azure pipeline to use sample rate of 1000 for guarded objects --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e4361f98..1fd71663 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -130,7 +130,7 @@ jobs: workingDirectory: $(BuildType) displayName: CTest env: - MIMALLOC_DEBUG_GUARDED_MAX: 1024 + MIMALLOC_GUARDED_SAMPLE_RATE: 1000 # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) From e7198ce3977f26716c014819df92d48b6058f569 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 17 Nov 2024 23:16:59 -0800 Subject: [PATCH 37/38] Extend azure pipeline with Ubuntu 24 & 20, windows 2019, and macOS 15 --- azure-pipelines.yml | 163 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 36 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1fd71663..6a2544f8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,7 +15,7 @@ trigger: jobs: - job: - displayName: Windows + displayName: Windows 2022 pool: vmImage: windows-2022 @@ -52,7 +52,7 @@ jobs: # artifact: mimalloc-windows-$(BuildType) - job: - displayName: Linux + displayName: Ubuntu 22.04 pool: vmImage: ubuntu-22.04 @@ -135,10 +135,10 @@ jobs: # artifact: mimalloc-ubuntu-$(BuildType) - job: - displayName: macOS + displayName: macOS 14 (Sonoma) pool: vmImage: - macOS-latest + macOS-14 strategy: matrix: Debug: @@ -163,35 +163,126 @@ jobs: # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-macos-$(BuildType) -# - job: -# displayName: Windows-2017 -# pool: -# vmImage: -# vs2017-win2016 -# strategy: -# matrix: -# Debug: -# BuildType: debug -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# MSBuildConfiguration: Debug -# Release: -# BuildType: release -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Release -# MSBuildConfiguration: Release -# Secure: -# BuildType: secure -# cmakeExtraArgs: -A x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON -# MSBuildConfiguration: Release -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - task: MSBuild@1 -# inputs: -# solution: $(BuildType)/libmimalloc.sln -# configuration: '$(MSBuildConfiguration)' -# - script: | -# cd $(BuildType) -# ctest --verbose --timeout 180 -# displayName: CTest +# ---------------------------------------------------------- +# Other OS versions (just debug mode) +# ---------------------------------------------------------- + +- job: + displayName: Windows 2019 + pool: + vmImage: + windows-2019 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + MSBuildConfiguration: Debug + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - task: MSBuild@1 + inputs: + solution: $(BuildType)/libmimalloc.sln + configuration: '$(MSBuildConfiguration)' + msbuildArguments: -m + - script: ctest --verbose --timeout 180 -C $(MSBuildConfiguration) + workingDirectory: $(BuildType) + displayName: CTest + +- job: + displayName: Ubuntu 24.04 + pool: + vmImage: + ubuntu-24.04 + strategy: + matrix: + Debug: + CC: gcc + CXX: g++ + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++: + CC: gcc + CXX: g++ + BuildType: debug-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(nproc) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 180 + workingDirectory: $(BuildType) + displayName: CTest + +- job: + displayName: Ubuntu 20.04 + pool: + vmImage: + ubuntu-20.04 + strategy: + matrix: + Debug: + CC: gcc + CXX: g++ + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++: + CC: gcc + CXX: g++ + BuildType: debug-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(nproc) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 180 + workingDirectory: $(BuildType) + displayName: CTest + +- job: + displayName: macOS 15 (Sequia) + pool: + vmImage: + macOS-15 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Secure: + BuildType: secure + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 180 + workingDirectory: $(BuildType) + displayName: CTest From 41029d9d49439cb1191fcc984592d0e9e56e73af Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 17 Nov 2024 23:20:18 -0800 Subject: [PATCH 38/38] fix azure pipeline --- azure-pipelines.yml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6a2544f8..00bfe8e9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -178,6 +178,10 @@ jobs: BuildType: debug cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON MSBuildConfiguration: Debug + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + MSBuildConfiguration: Release steps: - task: CMake@1 inputs: @@ -214,7 +218,16 @@ jobs: CXX: clang++ BuildType: debug-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - + Debug++ Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Release Clang: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release steps: - task: CMake@1 inputs: @@ -248,7 +261,16 @@ jobs: CXX: clang++ BuildType: debug-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - + Debug++ Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Release Clang: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release steps: - task: CMake@1 inputs: @@ -273,9 +295,6 @@ jobs: Release: BuildType: release cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Secure: - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON steps: - task: CMake@1 inputs: