From 095f848ae85b57fd0d478b2db150cf1821bbde47 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 24 Dec 2024 17:22:58 -0800 Subject: [PATCH 1/4] fix issue #976 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e84283fa..cde8144f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -412,7 +412,7 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) endif() if(MINGW) - add_definitions(-D_WIN32_WINNT=0x600) + add_definitions(-D_WIN32_WINNT=0x601) # issue #976 endif() if(MI_OPT_ARCH_FLAGS) From 63f47569f2c3211d13b8acac932ef78f4445f771 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 25 Dec 2024 13:35:54 -0800 Subject: [PATCH 2/4] fix debug_show_arenas parameters --- include/mimalloc.h | 2 +- src/arena.c | 22 +++++++++++----------- test/test-stress.c | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 5916228b..95a76c2d 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -276,7 +276,7 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; -mi_decl_export void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept; +mi_decl_export void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept; // Experimental: heaps associated with specific memory arena's typedef int mi_arena_id_t; diff --git a/src/arena.c b/src/arena.c index 5952c544..0e508115 100644 --- a/src/arena.c +++ b/src/arena.c @@ -911,11 +911,11 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_ return inuse_count; } -void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept { +void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept { size_t max_arenas = mi_atomic_load_relaxed(&mi_arena_count); size_t inuse_total = 0; - size_t abandoned_total = 0; - size_t purge_total = 0; + //size_t abandoned_total = 0; + //size_t purge_total = 0; for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]); if (arena == NULL) break; @@ -926,16 +926,16 @@ void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) if (arena->blocks_committed != NULL) { mi_debug_show_bitmap(" ", "committed blocks", arena->block_count, arena->blocks_committed, arena->field_count); } - if (show_abandoned) { - abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count); - } - if (show_purge && arena->blocks_purge != NULL) { - purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count); - } + //if (show_abandoned) { + // abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count); + //} + //if (show_purge && arena->blocks_purge != NULL) { + // purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count); + //} } if (show_inuse) _mi_verbose_message("total inuse blocks : %zu\n", inuse_total); - if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total); - if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total); + //if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total); + //if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total); } diff --git a/test/test-stress.c b/test/test-stress.c index 9b8eb2d0..8c5fca9c 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -238,7 +238,7 @@ static void test_stress(void) { } #ifndef NDEBUG //mi_collect(false); - //mi_debug_show_arenas(); + //mi_debug_show_arenas(true); #endif #if !defined(NDEBUG) || defined(MI_TSAN) if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); } @@ -316,7 +316,7 @@ int main(int argc, char** argv) { #ifndef USE_STD_MALLOC #ifndef NDEBUG - mi_debug_show_arenas(true,true,true); + mi_debug_show_arenas(true); mi_collect(true); #endif #endif From 27e0c467aefb5b7591cb291e4456823966f58344 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 25 Dec 2024 14:56:11 -0800 Subject: [PATCH 3/4] fix c++ initializer warning --- src/init.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index 81aca206..cc96e993 100644 --- a/src/init.c +++ b/src/init.c @@ -97,7 +97,12 @@ const mi_page_t _mi_page_empty = { // may lead to allocation itself on some platforms) // -------------------------------------------------------- -static mi_decl_cache_align mi_subproc_t subproc_main = { 0 }; // note: empty initializer to prevent running the constructor (in C++ compilation) +static mi_decl_cache_align mi_subproc_t subproc_main +#if __cplusplus += { }; // empty initializer to prevent running the constructor (with msvc) +#else += { 0 }; // C zero initialize +#endif static mi_decl_cache_align mi_tld_t tld_empty = { 0, // thread_id From 47b5f48b2909055dfe0e3caea5ec13cad74369bb Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 26 Dec 2024 09:52:46 -0800 Subject: [PATCH 4/4] renamed vcxproj --- CMakeLists.txt | 16 ++- ...{mimalloc.vcxproj => mimalloc-lib.vcxproj} | 2 +- ....vcxproj => mimalloc-override-dll.vcxproj} | 2 +- ide/vs2022/mimalloc-override-test.vcxproj | 2 +- ide/vs2022/mimalloc-override.vcxproj.filters | 113 ------------------ ide/vs2022/mimalloc-test-api.vcxproj | 2 +- ide/vs2022/mimalloc-test-stress.vcxproj | 2 +- ide/vs2022/mimalloc-test.vcxproj | 2 +- ide/vs2022/mimalloc.sln | 4 +- ide/vs2022/mimalloc.vcxproj.filters | 105 ---------------- include/mimalloc/internal.h | 2 +- include/mimalloc/prim.h | 37 +++++- src/prim/windows/prim.c | 12 +- 13 files changed, 67 insertions(+), 234 deletions(-) rename ide/vs2022/{mimalloc.vcxproj => mimalloc-lib.vcxproj} (99%) rename ide/vs2022/{mimalloc-override.vcxproj => mimalloc-override-dll.vcxproj} (99%) delete mode 100644 ide/vs2022/mimalloc-override.vcxproj.filters delete mode 100644 ide/vs2022/mimalloc.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index cde8144f..ca69fc6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_WIN_REDIRECT "Use redirection module ('mimalloc-redirect') on Windows if compiling mimalloc as a DLL" ON) +option(MI_WIN_USE_FIXED_TLS "Use a fixed TLS slot on Windows to avoid extra tests in the malloc fast path" OFF) option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix)" OFF) option(MI_LIBC_MUSL "Set this when linking with musl libc" OFF) option(MI_BUILD_SHARED "Build shared library" ON) @@ -35,7 +36,7 @@ option(MI_NO_THP "Disable transparent huge pages support on Linux/And option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") # deprecated options -option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination" OFF) +option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination (deprecated)" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) @@ -90,7 +91,7 @@ endif() if (CMAKE_GENERATOR MATCHES "^Visual Studio.*$") message(STATUS "Note: when building with Visual Studio the build type is specified when building.") - message(STATUS "For example: 'cmake --build . --config=Release") + message(STATUS "For example: 'cmake --build . --config=Release") endif() if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") @@ -322,10 +323,15 @@ if(MI_LIBC_MUSL) endif() if(MI_WIN_USE_FLS) - message(STATUS "Use the Fiber API to detect thread termination (MI_WIN_USE_FLS=ON)") + message(STATUS "Use the Fiber API to detect thread termination (deprecated) (MI_WIN_USE_FLS=ON)") list(APPEND mi_defines MI_WIN_USE_FLS=1) endif() +if(MI_WIN_USE_FIXED_TLS) + message(STATUS "Use fixed TLS slot on Windows to avoid extra tests in the malloc fast path (MI_WIN_USE_FIXED_TLS=ON)") + list(APPEND mi_defines MI_WIN_USE_FIXED_TLS=1) +endif() + # Determine architecture set(MI_OPT_ARCH_FLAGS "") set(MI_ARCH "unknown") @@ -543,7 +549,7 @@ if(MI_BUILD_SHARED) elseif(MI_ARCH STREQUAL "x64") set(MIMALLOC_REDIRECT_SUFFIX "") if(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64") - message(STATUS "Note: x64 code emulated on Windows for arm64 should use an arm64ec build of 'mimalloc-override.dll'") + message(STATUS "Note: x64 code emulated on Windows for arm64 should use an arm64ec build of 'mimalloc-override.dll'") message(STATUS " with 'mimalloc-redirect-arm64ec.dll'. See the 'bin\\readme.md' for more information.") endif() elseif(MI_ARCH STREQUAL "x86") @@ -667,7 +673,7 @@ endif() # ----------------------------------------------------------------------------- if (MI_OVERRIDE) if (MI_BUILD_SHARED) - target_compile_definitions(mimalloc PRIVATE MI_MALLOC_OVERRIDE) + target_compile_definitions(mimalloc PRIVATE MI_MALLOC_OVERRIDE) endif() if(NOT WIN32) # It is only possible to override malloc on Windows when building as a DLL. diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj similarity index 99% rename from ide/vs2022/mimalloc.vcxproj rename to ide/vs2022/mimalloc-lib.vcxproj index 9964310d..6c652b8a 100644 --- a/ide/vs2022/mimalloc.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -37,7 +37,7 @@ 15.0 {ABB5EAE7-B3E6-432E-B636-333449892EA6} - mimalloc + mimalloc-lib 10.0 mimalloc-lib diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj similarity index 99% rename from ide/vs2022/mimalloc-override.vcxproj rename to ide/vs2022/mimalloc-override-dll.vcxproj index 1278cd0f..c1849bb2 100644 --- a/ide/vs2022/mimalloc-override.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -37,7 +37,7 @@ 15.0 {ABB5EAE7-B3E6-432E-B636-333449892EA7} - mimalloc-override + mimalloc-override-dll 10.0 mimalloc-override-dll diff --git a/ide/vs2022/mimalloc-override-test.vcxproj b/ide/vs2022/mimalloc-override-test.vcxproj index 0e87cf36..427a75ae 100644 --- a/ide/vs2022/mimalloc-override-test.vcxproj +++ b/ide/vs2022/mimalloc-override-test.vcxproj @@ -344,7 +344,7 @@ - + {abb5eae7-b3e6-432e-b636-333449892ea7} diff --git a/ide/vs2022/mimalloc-override.vcxproj.filters b/ide/vs2022/mimalloc-override.vcxproj.filters deleted file mode 100644 index a9f66c35..00000000 --- a/ide/vs2022/mimalloc-override.vcxproj.filters +++ /dev/null @@ -1,113 +0,0 @@ - - - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - - - {9ef1cf48-7bb2-4af1-8cc1-603486e08a7a} - - - {cfcf1674-81e3-487a-a8dd-5f956ae4007d} - - - - - Headers - - - \ No newline at end of file diff --git a/ide/vs2022/mimalloc-test-api.vcxproj b/ide/vs2022/mimalloc-test-api.vcxproj index 27247569..b7f97ad2 100644 --- a/ide/vs2022/mimalloc-test-api.vcxproj +++ b/ide/vs2022/mimalloc-test-api.vcxproj @@ -282,7 +282,7 @@ - + {abb5eae7-b3e6-432e-b636-333449892ea6} diff --git a/ide/vs2022/mimalloc-test-stress.vcxproj b/ide/vs2022/mimalloc-test-stress.vcxproj index fd88cd8e..cb761f94 100644 --- a/ide/vs2022/mimalloc-test-stress.vcxproj +++ b/ide/vs2022/mimalloc-test-stress.vcxproj @@ -279,7 +279,7 @@ - + {abb5eae7-b3e6-432e-b636-333449892ea6} diff --git a/ide/vs2022/mimalloc-test.vcxproj b/ide/vs2022/mimalloc-test.vcxproj index 6e4576fd..83202dbe 100644 --- a/ide/vs2022/mimalloc-test.vcxproj +++ b/ide/vs2022/mimalloc-test.vcxproj @@ -276,7 +276,7 @@ - + {abb5eae7-b3e6-432e-b636-333449892ea6} diff --git a/ide/vs2022/mimalloc.sln b/ide/vs2022/mimalloc.sln index 5a55c98b..040af3ac 100644 --- a/ide/vs2022/mimalloc.sln +++ b/ide/vs2022/mimalloc.sln @@ -3,11 +3,11 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.12.35527.113 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-lib", "mimalloc-lib.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-dll", "mimalloc-override-dll.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" EndProject diff --git a/ide/vs2022/mimalloc.vcxproj.filters b/ide/vs2022/mimalloc.vcxproj.filters deleted file mode 100644 index 54ee0fcb..00000000 --- a/ide/vs2022/mimalloc.vcxproj.filters +++ /dev/null @@ -1,105 +0,0 @@ - - - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - Sources - - - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - Headers - - - - - {dd2da697-c33c-4348-bf80-a802ebaa06fb} - - - {8027057b-4b93-4321-b93c-d51dd0c8077b} - - - \ No newline at end of file diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 48ec3f1c..ecb49a4e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -419,7 +419,7 @@ static inline bool mi_heap_is_backing(const mi_heap_t* heap) { static inline bool mi_heap_is_initialized(mi_heap_t* heap) { mi_assert_internal(heap != NULL); - return (heap != &_mi_heap_empty); + return (heap != NULL && heap != &_mi_heap_empty); } static inline uintptr_t _mi_ptr_cookie(const void* p) { diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index 56715df4..c21ffa63 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +Copyright (c) 2018-2024, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -129,6 +129,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap); // for each thread (unequal to zero). //------------------------------------------------------------------- + // On some libc + platform combinations we can directly access a thread-local storage (TLS) slot. // The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform. // If you test on another platform and it works please send a PR :-) @@ -206,8 +207,40 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce #endif } +#elif _WIN32 && MI_WIN_USE_FIXED_TLS && !defined(MI_WIN_USE_FLS) + +// On windows we can store the thread-local heap at a fixed TLS slot to avoid +// thread-local initialization checks in the fast path. This use a fixed location +// in the TCB though (last user-reserved slot by default) which may clash with other applications. + +#define MI_HAS_TLS_SLOT 2 // 2 = we can reliable initialize the slot (saving a test on each malloc) + +#if MI_WIN_USE_FIXED_TLS > 1 +#define MI_TLS_SLOT (MI_WIN_USE_FIXED_TLS) +#elif MI_SIZE_SIZE == 4 +#define MI_TLS_SLOT (0x710) // Last user-reserved slot +// #define MI_TLS_SLOT (0xF0C) // Last TlsSlot (might clash with other app reserved slot) +#else +#define MI_TLS_SLOT (0x888) // Last user-reserved slot +// #define MI_TLS_SLOT (0x1678) // Last TlsSlot (might clash with other app reserved slot) #endif +static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { + #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) + return (void*)__readgsqword((unsigned long)slot); // direct load at offset from gs + #elif _M_IX86 && !defined(_M_ARM64EC) + return (void*)__readfsdword((unsigned long)slot); // direct load at offset from fs + #else + return ((void**)NtCurrentTeb())[slot / sizeof(void*)]; + #endif +} +static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { + ((void**)NtCurrentTeb())[slot / sizeof(void*)] = value; +} + +#endif + + // Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id // but unfortunately, it seems we cannot test for this reliably at this time (see issue #883) // Nevertheless, it seems needed on older graviton platforms (see issue #851). @@ -322,12 +355,14 @@ static inline mi_heap_t* mi_prim_get_default_heap(void); static inline mi_heap_t* mi_prim_get_default_heap(void) { mi_heap_t* heap = (mi_heap_t*)mi_prim_tls_slot(MI_TLS_SLOT); + #if MI_TLS_SLOT == 1 // check if the TLS slot is initialized if mi_unlikely(heap == NULL) { #ifdef __GNUC__ __asm(""); // prevent conditional load of the address of _mi_heap_empty #endif heap = (mi_heap_t*)&_mi_heap_empty; } + #endif return heap; } diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 1d3d6f41..ac228b53 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -622,6 +622,11 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { MI_UNUSED(reserved); MI_UNUSED(module); + #if MI_TLS_SLOT >= 2 + if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) { + _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); + } + #endif if (reason==DLL_PROCESS_ATTACH) { _mi_process_load(); } @@ -630,7 +635,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { } else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) { _mi_thread_done(NULL); - } + } } @@ -783,6 +788,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #endif mi_decl_export void _mi_redirect_entry(DWORD reason) { // called on redirection; careful as this may be called before DllMain + #if MI_TLS_SLOT >= 2 + if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) { + _mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty); + } + #endif if (reason == DLL_PROCESS_ATTACH) { mi_redirected = true; }