From bd6abd246330ed2870764b99c0c442ec71bad876 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 15 Jul 2019 10:07:23 -0700 Subject: [PATCH 01/67] add comment about large pages --- src/options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index b27f714f..47fba61a 100644 --- a/src/options.c +++ b/src/options.c @@ -35,7 +35,7 @@ static mi_option_desc_t options[_mi_option_last] = { { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, { 0, UNINIT, "pool_commit" }, - { 0, UNINIT, "eager_commit" }, // secure must have eager commit + { 0, UNINIT, "eager_commit" }, // secure and large pages must have eager commit { 0, UNINIT, "large_os_pages" }, // use large OS pages { 0, UNINIT, "reset_decommits" }, { 0, UNINIT, "reset_discards" }, From c228ecefd8c183b5992aa9dd71922dff33a568b4 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 18:59:32 -0700 Subject: [PATCH 02/67] update for new dynamic direction on windows 64-bit --- ide/vs2017/mimalloc-override-test.vcxproj | 36 +++++++++- .../mimalloc-override-test.vcxproj.filters | 3 + ide/vs2017/mimalloc-override.vcxproj | 33 +++++++--- ide/vs2017/mimalloc-override.vcxproj.filters | 11 ++-- ide/vs2017/mimalloc.sln | 10 --- ide/vs2017/mimalloc.vcxproj | 4 ++ include/mimalloc-internal.h | 1 + include/mimalloc.h | 14 ++-- src/alloc-aligned.c | 11 ++++ src/alloc-override-win.c | 5 -- src/alloc-override.c | 3 +- src/alloc-posix.c | 11 ++++ src/alloc.c | 7 +- src/heap.c | 2 +- src/init.c | 66 +++++++++++++++---- src/memory.c | 1 + src/options.c | 50 ++++++++++---- test/main-override.c | 19 +++--- test/main-override.cpp | 19 +++--- 19 files changed, 218 insertions(+), 88 deletions(-) diff --git a/ide/vs2017/mimalloc-override-test.vcxproj b/ide/vs2017/mimalloc-override-test.vcxproj index c50f80dc..77752890 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj +++ b/ide/vs2017/mimalloc-override-test.vcxproj @@ -90,10 +90,18 @@ true ..\..\include MultiThreadedDebugDLL + false + Default + false Console + kernel32.lib;%(AdditionalDependencies) + + + COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect32.dll $(OutputPath) + @@ -103,14 +111,20 @@ true ..\..\include MultiThreadedDebugDLL + false + Default + false Console - kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) - false + kernel32.lib;%(AdditionalDependencies) + + + COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect.dll $(OutputPath) + @@ -128,7 +142,11 @@ true true Console + kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect32.dll $(OutputPath) + @@ -150,9 +168,21 @@ kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect.dll $(OutputPath) + - + + true + true + true + true + + + false + false + diff --git a/ide/vs2017/mimalloc-override-test.vcxproj.filters b/ide/vs2017/mimalloc-override-test.vcxproj.filters index eb5e70b7..80f1c9c0 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-override-test.vcxproj.filters @@ -18,5 +18,8 @@ Source Files + + Source Files + \ No newline at end of file diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 5fe9f10e..5fad3cf1 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -70,21 +70,25 @@ $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll + mimalloc $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll + mimalloc $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll + mimalloc $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll + mimalloc @@ -96,6 +100,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions); MultiThreadedDebugDLL false + Default @@ -106,8 +111,7 @@ - DllEntry - kernel32.lib;%(AdditionalDependencies) + ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -124,6 +128,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions); MultiThreadedDebugDLL false + Default @@ -134,8 +139,7 @@ - DllEntry - kernel32.lib;%(AdditionalDependencies) + ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) @@ -156,12 +160,12 @@ $(IntDir) false MultiThreadedDLL + Default true true - DllEntry - kernel32.lib;%(AdditionalDependencies) + ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -188,12 +192,12 @@ $(IntDir) false MultiThreadedDLL + Default true true - DllEntry - kernel32.lib;%(AdditionalDependencies) + ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) @@ -219,7 +223,18 @@ false false - + + true + true + true + true + + + true + true + true + true + diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters index d2892c32..a39667f3 100644 --- a/ide/vs2017/mimalloc-override.vcxproj.filters +++ b/ide/vs2017/mimalloc-override.vcxproj.filters @@ -46,9 +46,6 @@ Source Files - - Source Files - Source Files @@ -61,8 +58,14 @@ Source Files + + Source Files + Source Files + + Source Files + - + \ No newline at end of file diff --git a/ide/vs2017/mimalloc.sln b/ide/vs2017/mimalloc.sln index aeab6b88..f4860d93 100644 --- a/ide/vs2017/mimalloc.sln +++ b/ide/vs2017/mimalloc.sln @@ -11,8 +11,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimall EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -53,14 +51,6 @@ Global {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 - {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index bb1818b0..4a2069c6 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -70,21 +70,25 @@ $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .lib + mimalloc-static $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .lib + mimalloc-static $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .lib + mimalloc-static $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .lib + mimalloc-static diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 3b45ada4..1d380e8f 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -35,6 +35,7 @@ bool _mi_is_main_thread(void); uintptr_t _mi_ptr_cookie(const void* p); uintptr_t _mi_random_shuffle(uintptr_t x); uintptr_t _mi_random_init(uintptr_t seed /* can be zero */); +bool _mi_preloading(); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); diff --git a/include/mimalloc.h b/include/mimalloc.h index 3c10a65b..9828a6bb 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -197,6 +197,7 @@ mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_b mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; + // ------------------------------------------------------ // Convenience // ------------------------------------------------------ @@ -245,14 +246,15 @@ mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); -// ---------------------------------------------------------------------------------- -// mi prefixed implementations of various posix, unix, and C++ allocation functions. -// ----------------------------------------------------------------------------------- +// -------------------------------------------------------------------------------------------- +// mi prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. +// (This can be convenient when providing overrides of these functions.) +// -------------------------------------------------------------------------------------------- -mi_decl_export void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept; mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; mi_decl_export int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; @@ -263,6 +265,10 @@ mi_decl_export mi_decl_allocator void* mi_pvalloc(size_t size) mi_attr_noexcept mi_decl_export mi_decl_allocator void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_export mi_decl_allocator void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); +mi_decl_export void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept; +mi_decl_export void* mi_aligned_recalloc(void* p, size_t size, size_t newcount, size_t alignment) mi_attr_noexcept; +mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t size, size_t newcount, size_t alignment, size_t offset) mi_attr_noexcept; + mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 3ef93c83..175fa3e3 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -150,3 +150,14 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment); } + +void* mi_aligned_offset_recalloc(void* p, size_t size, size_t newcount, size_t alignment, size_t offset) mi_attr_noexcept { + size_t newsize; + if (mi_mul_overflow(size,newcount,&newsize)) return NULL; + return mi_heap_realloc_zero_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset, true ); +} +void* mi_aligned_recalloc(void* p, size_t size, size_t newcount, size_t alignment) mi_attr_noexcept { + size_t newsize; + if (mi_mul_overflow(size, newcount, &newsize)) return NULL; + return mi_heap_realloc_zero_aligned(mi_get_default_heap(), p, newsize, alignment, true ); +} diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c index f0a5959a..7b0fe69f 100644 --- a/src/alloc-override-win.c +++ b/src/alloc-override-win.c @@ -98,11 +98,6 @@ static int __cdecl mi_setmaxstdio(int newmax); // Microsoft allocation extensions // ------------------------------------------------------ -static void* mi__expand(void* p, size_t newsize) { - void* res = mi_expand(p, newsize); - if (res == NULL) errno = ENOMEM; - return res; -} typedef size_t mi_nothrow_t; diff --git a/src/alloc-override.c b/src/alloc-override.c index 5ca88af7..5378850b 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -194,4 +194,5 @@ int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_me #pragma GCC visibility pop #endif -#endif // MI_MALLOC_OVERRIDE & !_WIN32 +#endif // MI_MALLOC_OVERRIDE && !_WIN32 + diff --git a/src/alloc-posix.c b/src/alloc-posix.c index b3185f15..3d28f655 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -80,3 +80,14 @@ void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { return newp; } +void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft + void* res = mi_expand(p, newsize); + if (res == NULL) errno = ENOMEM; + return res; +} + +void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { // Microsoft + size_t total; + if (mi_mul_overflow(count, size, &total)) return NULL; + return _mi_heap_realloc_zero(mi_get_default_heap(), p, total, true); +} diff --git a/src/alloc.c b/src/alloc.c index da8c69b9..10a74ce3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -393,12 +393,6 @@ void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_realloc(mi_get_default_heap(),p,newsize); } -void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { - size_t total; - if (mi_mul_overflow(count, size, &total)) return NULL; - return _mi_heap_realloc_zero(mi_get_default_heap(),p,total,true); -} - void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { return mi_heap_reallocn(mi_get_default_heap(),p,count,size); } @@ -537,6 +531,7 @@ std_new_handler_t mi_get_new_handler() { return _ZSt15get_new_handlerv(); } #else +// note: on windows we could dynamically link to `?get_new_handler@std@@YAP6AXXZXZ`. std_new_handler_t mi_get_new_handler() { return NULL; } diff --git a/src/heap.c b/src/heap.c index 2b7b7a99..48bb9830 100644 --- a/src/heap.c +++ b/src/heap.c @@ -172,7 +172,7 @@ void mi_collect(bool force) mi_attr_noexcept { ----------------------------------------------------------- */ mi_heap_t* mi_heap_get_default(void) { - mi_thread_init(); + // mi_thread_init(); return mi_get_default_heap(); } diff --git a/src/init.c b/src/init.c index f55b7318..06aa28c5 100644 --- a/src/init.c +++ b/src/init.c @@ -373,6 +373,47 @@ void mi_thread_done(void) mi_attr_noexcept { // -------------------------------------------------------- static void mi_process_done(void); +static bool os_preloading = true; // true until this module is initialized +static bool mi_redirected = false; // true if malloc redirects to mi_malloc + +// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. +bool _mi_preloading() { + return os_preloading; +} + +// Communicate with the redirection module on Windows +#if defined(_WIN32) && defined(MI_SHARED_LIB) +mi_decl_export void _mi_redirect_init() { + // called on redirection + mi_redirected = true; +} +__declspec(dllimport) bool mi_allocator_init(const char** message); +__declspec(dllimport) void mi_allocator_done(); +#else +static bool mi_allocator_init(const char** message) { + if (message != NULL) *message = NULL; + return true; +} +static void mi_allocator_done() { + // nothing to do +} +#endif + +// Called once by the process loader +static void mi_process_load(void) { + os_preloading = false; + atexit(&mi_process_done); + mi_process_init(); + //mi_stats_reset(); + if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); + + // show message from the redirector (if present) + const char* msg = NULL; + mi_allocator_init(&msg); + if (msg != NULL) _mi_verbose_message(msg); +} + +// Initialize the process; called by thread_init or the process loader void mi_process_init(void) mi_attr_noexcept { // ensure we are called once if (_mi_process_is_initialized) return; @@ -381,7 +422,7 @@ void mi_process_init(void) mi_attr_noexcept { // when using dynamic linking with interpose. mi_heap_t* h = _mi_heap_default; _mi_process_is_initialized = true; - + _mi_heap_main.thread_id = _mi_thread_id(); _mi_verbose_message("process init: 0x%zx\n", _mi_heap_main.thread_id); uintptr_t random = _mi_random_init(_mi_heap_main.thread_id) ^ (uintptr_t)h; @@ -389,15 +430,16 @@ void mi_process_init(void) mi_attr_noexcept { _mi_heap_main.cookie = (uintptr_t)&_mi_heap_main ^ random; #endif _mi_heap_main.random = _mi_random_shuffle(random); + mi_process_setup_auto_thread_done(); + _mi_os_init(); #if (MI_DEBUG) _mi_verbose_message("debug level : %d\n", MI_DEBUG); #endif - atexit(&mi_process_done); - mi_process_setup_auto_thread_done(); - mi_stats_reset(); - _mi_os_init(); + mi_thread_init(); + mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) } +// Called when the process is done (through `at_exit`) static void mi_process_done(void) { // only shutdown if we were initialized if (!_mi_process_is_initialized) return; @@ -413,7 +455,9 @@ static void mi_process_done(void) { mi_option_is_enabled(mi_option_verbose)) { mi_stats_print(NULL); } + mi_allocator_done(); _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); + os_preloading = true; // don't call the C runtime anymore } @@ -425,8 +469,8 @@ static void mi_process_done(void) { __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { UNUSED(reserved); UNUSED(inst); - if (reason==DLL_PROCESS_ATTACH) { - mi_process_init(); + if (reason==DLL_PROCESS_ATTACH) { + mi_process_load(); } else if (reason==DLL_THREAD_DETACH) { mi_thread_done(); @@ -437,7 +481,7 @@ static void mi_process_done(void) { #elif defined(__cplusplus) // C++: use static initialization to detect process start static bool _mi_process_init(void) { - mi_process_init(); + mi_process_load(); return (_mi_heap_main.thread_id != 0); } static bool mi_initialized = _mi_process_init(); @@ -445,14 +489,14 @@ static void mi_process_done(void) { #elif defined(__GNUC__) || defined(__clang__) // GCC,Clang: use the constructor attribute static void __attribute__((constructor)) _mi_process_init(void) { - mi_process_init(); + mi_process_load(); } #elif defined(_MSC_VER) // MSVC: use data section magic for static libraries // See static int _mi_process_init(void) { - mi_process_init(); + mi_process_load(); return 0; } typedef int(*_crt_cb)(void); @@ -467,5 +511,5 @@ static void mi_process_done(void) { #pragma data_seg() #else -#pragma message("define a way to call mi_process_init/done on your platform") +#pragma message("define a way to call mi_process_load on your platform") #endif diff --git a/src/memory.c b/src/memory.c index 83e90b0d..74a9628c 100644 --- a/src/memory.c +++ b/src/memory.c @@ -114,6 +114,7 @@ bool mi_is_in_heap_region(const void* p) { return false; } + /* ---------------------------------------------------------------------------- Commit from a region -----------------------------------------------------------------------------*/ diff --git a/src/options.c b/src/options.c index ac0910a0..36997d80 100644 --- a/src/options.c +++ b/src/options.c @@ -16,6 +16,10 @@ int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; } +#ifdef _WIN32 +#include +#endif + // -------------------------------------------------------- // Options // -------------------------------------------------------- @@ -31,7 +35,7 @@ typedef struct mi_option_desc_s { const char* name; // option name without `mimalloc_` prefix } mi_option_desc_t; -static mi_option_desc_t options[_mi_option_last] = +static mi_option_desc_t options[_mi_option_last] = { // stable options { 0, UNINIT, "show_stats" }, @@ -41,15 +45,15 @@ static mi_option_desc_t options[_mi_option_last] = // the following options are experimental and not all combinations make sense. { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, - { 1, UNINIT, "eager_commit" }, - { 1, UNINIT, "eager_region_commit" }, // eager_commit should be on when eager_region_commit is on + { 1, UNINIT, "eager_commit" }, + { 1, UNINIT, "eager_region_commit" }, // eager_commit should be on when eager_region_commit is on { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { 0, UNINIT, "reset_decommits" }, - { 0, UNINIT, "reset_discards" }, + { 0, UNINIT, "reset_decommits" }, + { 0, UNINIT, "reset_discards" }, #if MI_SECURE { MI_SECURE, INITIALIZED, "secure" } // in a secure build the environment setting is ignored #else - { 0, UNINIT, "secure" } + { 0, UNINIT, "secure" } #endif }; @@ -104,7 +108,17 @@ static void mi_vfprintf( FILE* out, const char* prefix, const char* fmt, va_list char buf[256]; if (fmt==NULL) return; if (out==NULL) out = stdout; + if (_mi_preloading()) return; vsnprintf(buf,sizeof(buf)-1,fmt,args); + #ifdef _WIN32 + // on windows with redirection, the C runtime uses us and we cannot call `fputs` + // while called from the C runtime itself, so use a non-locking option + if (out==stderr) { + if (prefix != NULL) _cputs(prefix); + _cputs(buf); + return; + } + #endif if (prefix != NULL) fputs(prefix,out); fputs(buf,out); } @@ -174,21 +188,29 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest[dest_size - 1] = 0; } -static void mi_option_init(mi_option_desc_t* desc) { - desc->init = DEFAULTED; - // Read option value from the environment - char buf[32]; - mi_strlcpy(buf, "mimalloc_", sizeof(buf)); - mi_strlcat(buf, desc->name, sizeof(buf)); +static const char* mi_getenv(const char* name) { + if (_mi_preloading()) return NULL; // don't call getenv too early #pragma warning(suppress:4996) - char* s = getenv(buf); + const char* s = getenv(name); if (s == NULL) { + char buf[64+1]; + strncpy_s(buf,64,name,64); buf[64] = 0; for (size_t i = 0; i < strlen(buf); i++) { - buf[i] = toupper(buf[i]); + buf[i] = toupper(name[i]); } #pragma warning(suppress:4996) s = getenv(buf); } + return s; +} + +static void mi_option_init(mi_option_desc_t* desc) { + if (!_mi_preloading()) desc->init = DEFAULTED; + // Read option value from the environment + char buf[64]; + mi_strlcpy(buf, "mimalloc_", sizeof(buf)); + mi_strlcat(buf, desc->name, sizeof(buf)); + const char* s = mi_getenv(buf); if (s != NULL) { mi_strlcpy(buf, s, sizeof(buf)); for (size_t i = 0; i < strlen(buf); i++) { diff --git a/test/main-override.c b/test/main-override.c index 836ea58d..ddb2f16e 100644 --- a/test/main-override.c +++ b/test/main-override.c @@ -3,30 +3,29 @@ #include #include -#include +//#include int main() { - mi_stats_reset(); + //mi_stats_reset(); void* p1 = malloc(78); void* p2 = malloc(24); free(p1); p1 = malloc(8); - char* s = strdup("hello\n"); + //char* s = strdup("hello\n"); free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); free(p2); - free(s); - mi_collect(true); + //free(s); + //mi_collect(true); /* now test if override worked by allocating/freeing across the api's*/ - p1 = mi_malloc(32); - free(p1); - p2 = malloc(32); - mi_free(p2); + //p1 = mi_malloc(32); + //free(p1); + //p2 = malloc(32); + //mi_free(p2); - mi_stats_print(NULL); return 0; } diff --git a/test/main-override.cpp b/test/main-override.cpp index 3f2bc960..3c192fe3 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -22,15 +23,17 @@ public: ~Test() { } }; -int main() { - mi_stats_reset(); + +int main() { + //mi_malloc_override(); + mi_stats_reset(); atexit(free_p); void* p1 = malloc(78); - void* p2 = malloc(24); + void* p2 = _aligned_malloc(24,16); free(p1); p1 = malloc(8); - char* s = mi_strdup("hello\n"); - free(p2); + char* s = _strdup("hello\n"); + _aligned_free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); @@ -40,12 +43,8 @@ int main() { delete t; t = new (std::nothrow) Test(42); delete t; - int err = mi_posix_memalign(&p1,32,60); - if (!err) free(p1); free(p); - mi_collect(true); - mi_stats_print(NULL); // MIMALLOC_VERBOSE env is set to 2 - return 0; + return 0; } class Static { From e94143c47ce54a6e90b88d08309d8dc5518d4e88 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 19:26:44 -0700 Subject: [PATCH 03/67] add redirection dll and library --- .gitattributes | 2 ++ bin/mimalloc-redirect.dll | Bin 0 -> 34816 bytes bin/mimalloc-redirect.lib | Bin 0 -> 3050 bytes 3 files changed, 2 insertions(+) create mode 100644 bin/mimalloc-redirect.dll create mode 100644 bin/mimalloc-redirect.lib diff --git a/.gitattributes b/.gitattributes index 4a42e93d..acdbdbf4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,5 @@ *.suo binary *.vcproj binary *.patch binary +*.dll binary +*.lib binary diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll new file mode 100644 index 0000000000000000000000000000000000000000..d3427e9548492d44174d12a5c8144fd62b5d2bfe GIT binary patch literal 34816 zcmeHQ3vgRknLbMGLU@w7HyL#j{&81sM0!c+d4%gY4I>Awqh%>h%E^{ z61U?n?m7i$-7MajQZfrOI8Qro18r)cbY+{hhL|@jG}w#Eni#8 zo?G~x6+^bV@2uF-5{PQyNN7*Q*QPc3g27Ns+vV3H@t_t6YPBEQuC;}l{i~NOS>%wS zeeZbijjE4V^{ev0Qpd^Lc-zsx67YpeBaAJByu*WxIf*&JSPjYPV{8<7<1@-wD;eNr%xxeGs6g)FDacmG{OvLH|8gxZ zEl5LI#FG}44Rk6i*48kj#pF?~Fo&fgjWVU%J! zibDDipqM@$#_lyWi&aG6HTsO-h71|sd>t-zRQpG&t+V5@wE~xc<+!ZB1($zWip!~G zxHw92`Kx#1@+)NT_``GwBlhFJrpqJO;{Rnm ztz6V5N2uY^yE+`@ECvoDhM{E_7j}aft;vLvVCQuRNqv==T`z>%ab z=`nFB4Pq-Y^F~XyfB<>J$3;t(q`p!}%#8oQtf z15AOLWaMysk;o?aVCOY-9mJp^CPs>N)i_xE5~1wK?!4pIP1{G;l}hes*VWA(%Vq7O zpIjnp2@RrCSV18iwX%y)lwaUm+>d_yLF`9#K7E_}QC3>MMg7l{{sD|+q59Q)a(HGY3=N_->a>59VMp#g`w*@(Nihh0M<@BrogVV^^ohyb74pQ(?r^`u|)C z`BYvZQ}aJ0`9F}CpO2?$mZtMVDFoGSUGEcveou^0W*1a#4{hE6C>7F&eV1HXq5cN5@aNE!zyG!WpN zS(98Q{7R$V1DOj+CY^?oQ7IAkWE~C{i(0=Eb3m9 z{m-Lc?CAZOB(ndr)Q`Mp$xmnYTh=fA$tx`wEWPJ|BTTR>!EeHyG>R@Vv+fWUCVswO z|Go9EOja!W3}Z(4J8&a(z|l^Oz`55ab^~~ZN`Ro+y>wIWr7nF7mV9KHQ-?TD^7D9F zL9uYx^>}a~#2^vg%Tbf%`jan{P1eX{gf(wNzm%X8yTP2X3dzGZ4B~g$_1x+@7zzCq zuVaQIaFivN->ci7zJ9SdY*C6Ga(VAkK@^qh^W(u=Wt{(T^(p>mTqPBB!0L#Cm6f^MWrd4wsx&|-urZ!D?96u&303?L!&O9vdCfb=1Q z=>fQY=xIkcVDjP;LLPANJ0nC%b=&|V`ak|b`LkA$HQ;X<=rTm-$Hm?Agb(z5}hW2Q^+@}#A+otWBF3&uB#o3EqAJ5%lU^NgSwD(v@BGEb1DwVGCR+ zDN`X@$GjL%ox<&zthndTgd3V5wu5*bxkaLQ*7Cu_{^Vx+_|qq?>;+n|Fyr)|h58*M zcnFb7cMOk}@Nq1=cz@X)!}?)tH&Z>ToOUdTAicjJ>_6F?l>~XtOM3Fvacmv1w1p&S zwWu}H+E7o623FdUq;nBp9yVa1+raoj=fbKh9_8_+AAz8Un^P}mDqkvT%f%kOjN*rl z*3DuI@zjP(Tp%Cc@LPOZQE~c~ifF6g&kReTDeq0cDN?5Oh@XP7D+gKCP;bk*$m@s-zu0bpX~O+W7pV8Nhmf(#7n zT{Si=;Um<}?6`)e6^|Dm&8~JQD|_nmpi^fOI^2~!akJh;2?|-XBn;!Yx0oV6Nl(gL zsecwkVxiW1%89Nex$Wl|HS}Y@p=W(D-6Vsr3kP=$t8v2=$4Ke4Re%FuOW;u{Eg+2& zUyiRQ&hN&i_YnvkzYghwxYqx^*XZo{B0tFp*yR55PIAke{N>MtTk<3w=Lb!G@N$VU zaGG~No{JhzJWg0o0<`2OAb`)Hcm?p9fo}sI4{~oHzYz+`HBLVtXQk9?kd_*x_X^S* zfn*X{pM3qNG)qR3_mP3qEhW>X8t*sX-?=jm3m%NA&4ov?gSPSG$QBymd2zlTp$q4Q zxE%?k(}F`0^w4;2)SdB?b&1q|kq+zVw1<9{S_BG>rxL&SD0!lymM}a=!01WRZB6Qj zDHk{#cs4IM9Frp$76*hmSO@2Dq&c>64ny@p-8pDOOn%25fLxZ;pK%rihG3&ra@AR*srVWwlGwt2{dDqd9(8RATS>|ezreaPzr64L8g ze5DY(H^FZzy*QlOunLZn9%u5s;7IKxLR%cuk9Ji#ms}~(i#Fe7UDOgQBw3HGqbi5s zx}F4OAkvVefA00kNp2}07n7mSqk{)_>V2!AWXnOs_aFWpCUy#PVq@u}%{Ty26n}xh z5b9v*13-Nn@wCAmFN-~z_b?iO<7!E)=SyO2H7$oGBwr5UIKnrQ{4Jinh>2QC@>yyy z6(QkHsFv%kn(I!I;{&cI4~>a64tnXJ5ijG}oZM$Y(``-_<%x&-wPX$ABASPm4E}0zGTBQEaEI(K zVD4p{H{1n<8k-BYw%Fqt0oKJ=R(vaw>f0+>9D_=x^my%Lt|$4WpOAGhTtB$EQqhU8 zq`Aph#$x>2VB#qkJWrT-wDD%K5f$KC#$ZfVZTlqNXsWu2UvK7DjbB&ttAk%}!!?7g z*eFJz7=dC0iV-MApcsK-1d0(TMxYphVg!m2C`O8`hIt!uH zei-F_f<*N#MiamLV9%lNIoT&Diq~!1Yqz_<{*A4l{lVIY&b|2Hnjd`s&G*w|`TOqL z`Eb-9iSBIn`GPxZBcbrFQ2Wki|Gu4Vfi_<&ey_AD;%^Q_{7tc)p?GX(TXbL3s`mG8 z*tyN$>i0$c*<|>2)h-clda_;o6cz#7LK|DS67Nhcv9Totm)h7;#F4iNUTprTWJq>toT+X}DbEVr@cM5piylD}^U zf1Qn8M|}8(wj>v%{y#-O>E8uNZGwyEvBkEn%y#WOcCD>K*h6&W7r`6mu^ViwMBJ%O zei0wZSt;r6m3Zn`ddLr=Bl@F~PPIW-{1hYbyF>urFiaAE^GA?dfb(T>qU}McN1@*f z2T)2;%21v~iK0A?zAlu{qV%JD9p$?yBxxRgSzW?PS=MU5y1BKL)wZ^73C1Fip}T=^ z2EJG*($E|V`ZF5_Vl^xV!L?tax4QrZDVdSRL z&_*dP7)Y$C$>S~%f>Qd><{Q-4NWjlq`$;8&q0cCJt5M}}WKm`W)^iZ-SP~H(9{rX3(THNFNOag~p+I*B--e6&u1SuNowQ)GHP$#YFxJ+Z36Iw+Sbb zD9ym-@K^|($^k`T&I`g>+Qu=$$P=kq+GoH2v>2ok4kJ(0theYf_e+Nie=_pO2ybRS=b*9jn?jcDr;!2a|a1J`h)`1?_v5~_~Qo8{k?QivIby;A>x#g2SxJvYvbeM3_J zNP%_?R?dtam!fp(0|kR4a0e!H(QKg(xmTq>o0XMuJM&X650sozte9i1~|hga%v zDbS9w8M32Y>hCSk4$nIa%cFerJ|DhM>Q5ACNBIodaaro0DA11Z4B27dCE~0}m@jS$ z-#0^cxTXI30__-^Av?lSe{X?yXz!YtJROkwM+&r~afa*|Q~H|<^RsV;?6^tlcNS=e z%GcjF+#vOP=GYFe)L$=Qi^ZOa85##`JUgV`ZV7uWc8txE9eq;ofJML6-kEL>N&O>p zY{zA(e@wy&i#^&+h0W*RHD2~+p{G(pr-X*wS+b)>>eVHzx7gvDB|BQA-gb+ABObF< z>6ZEr&#@f?QvZ;IBNlsRYaJVtdM6~jYO!Os)-k(Z&KC)t5*leSOL^*%dc6`hTI`su z`4X0TJ1zP>vlIsv_g<;L&tk{mEZZ?8^^aKW2+y(|m!OQ=aZMkQ4FG*d7v^>#|wD`B71@13Q13`o6063&#r)5#r``jg6z zJt8lr5{BofL|$qVx+PTo zvjrQa-cAYo=E#l_sdsFS^s9JZmHNvAVtiZ@s{Yx6Ua7Z5!tfl~(JA#FmT+*6>===H z$CUn$&(XRr^?K$=zskQxslQXgzB#gIMCwgSSiW~|#zn1ytE5UFoyGoI&`4D*B=S`Ti35ss1N#M-bkp) zAB`G6T@Zqb*J1?z4@ba*f5Z1FlsmWMUsD1u$G`r&3H1cv?bkE57nFOygS4`hw?P)cz=hzhVl&Rc!!2v`!2@5 zfVu`)hjJYCM!>&7IZO0_e~0oM>iEuu?M8VSb$s8zzJT%?>i7Bj%{%Sv{*kDcONo0k)znMjh{% zvLwnf)XQ&S?3*YXP@e#_Ymi0q0r#WaMD&1viQ+)L5AbU!ccPvE+`9sDQ6B`XT8Vg~ z?ge}mr4IE8z$YAtKk8$ETdQCb>S4gOw_&_c?*)7VWhd(8)r@@%++T zq&~$euv%h(3s5##1k7k{I%@Hov|9B#Rzc&JBQ93hyroUyt7`E6f{K69a zJ!? zW?w7*6vU?L$NbUid)~ifQH?Je^|$S6eN2OZV02S;JQBPs+SKB2^F>#+1)3tEXlQqA zRa2<#E?=~5^}e;$TAMEz*zJ$T9x-c!C{3%0MdH!e{lVQKDQ)F6($=r!eBg}wo8pl` z>@nE}CgT4@99sO%-bi3ypw+*}A2mA7-CNqh8iU(){(b&dt(C5us(sP>gZo0b-LBT+ zfz3^HhrOwKx34wouh!NWa%$G(*HW`4!={=whI3%AS)<~HM)q(NF~tb{aYvvL_pgY6 z=Zt=){>;dk(KA=ic+b|K?L2$=?ATfRxyExX=X%d2&rO`GJRd&ae!lPg)${Cv^Frr^ z?hAt#z!=6o9e};BywBa&+jqDx(N}pyJE9-yJ2G%&^oaAQ>uCMa!J|V*$Bw#>d5*Ol zOB@?HrXTknA3c8g_}KA@fj08qc?IUpmhx&rfiF>=(ip+Amxse>!ph3c&j8eVu*X z@W_6o{7B`I-Xn+MQRPwXXy4HR_~Sh0I_5q$cx>pH`?%-0emrq}1Rm)pyeH~UjGnj* cZ|YAro*X-wgg-6)VR%DAi=Scy{;(qOzZ;=1!2kdN literal 0 HcmV?d00001 diff --git a/bin/mimalloc-redirect.lib b/bin/mimalloc-redirect.lib new file mode 100644 index 0000000000000000000000000000000000000000..cf2f95e45cd7c00414296f75284e5cb82c3db91e GIT binary patch literal 3050 zcmcIm%T60H6g_zWO{m(aFJ;lHsf5HsB0yfNsu~JNMLZS4dJ;0B44Ncpm`Zfj9b(ZH zpTVZTfYcx8uFEdF=#C$t*B*N^nQuA^ zZtPhtsB~d;s}lkX)gY|ywCr}@?Ct>b$-904NEL+w} z$FV<)&3fI)kuCHho1$1{lfdC9a4`m)Q1GrutZ_GxX9UQSQ23ET3~~BCz!;Jk!Kkm5 zfVZ4G2HW+AZ#EIwfHM?s;|})pAtHqT3>)Eq>2M5H9P{f2vdbmQ5d5AC1o^h?Efi!v z*VCcw$s3l1Ihp7Jc-xHSV$nMfjMD_Jsl1MXIN+AN(c1g;abibwu||!rvLk8@nfH0q zXgH0Dg8cQ28b3+?2t|Tw4fB@*R}pHwphk-9NgUH_$X&*%eo3rk<+*aJlX#jTYU2mT z0FRKtx%p(nY}(!9?!J?os!r$DS2x*;L**f9*pisg5y)E^`i}6H(jK&Cw|zwGKR08s z>udj58bm3?M7lu(_!bVNhHu!lC@>|4p+WF4<(`J=XV^y+HaoB+daR0LpT2j@${{G4 zj>@P)MN+VyHcsQ5QQ!H3*|Gr@qYU?#_w_NVD3y#+-(*MBj8XT@lwqR&ik36*nv3ec z;(R5Pk-y1+O98e2Zpg^8bQ#NNmXZFVL6aF-l6r>wB|D;KxO?>TFg~87r3^vWsNODn z?br%+iy`o$`loaLCcE*VUGb** zFLr%5mfyeZa7(y$Lsz=m)lnl`84%fNvYZT9_HJ_3mZQg5CU79ed9pmBv@BnpD)WB< DR)@x@ literal 0 HcmV?d00001 From 12e0a0405267a4e595a960be003aaaa117459ef8 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 19:52:29 -0700 Subject: [PATCH 04/67] compile as C++ under msvc --- ide/vs2017/mimalloc-test.vcxproj | 7 +------ ide/vs2017/mimalloc-test.vcxproj.filters | 2 +- ide/vs2017/mimalloc.sln | 10 ++++++++++ ide/vs2017/mimalloc.vcxproj | 6 ++++-- src/alloc-override.c | 9 ++++----- src/memory.c | 2 +- test/main-override.cpp | 3 +-- 7 files changed, 22 insertions(+), 17 deletions(-) diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index 7976af56..8e61a97f 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -145,12 +145,7 @@ - - AssemblyAndSourceCode - AssemblyAndSourceCode - AssemblyAndSourceCode - AssemblyAndSourceCode - + diff --git a/ide/vs2017/mimalloc-test.vcxproj.filters b/ide/vs2017/mimalloc-test.vcxproj.filters index 9254f6c0..eb5e70b7 100644 --- a/ide/vs2017/mimalloc-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-test.vcxproj.filters @@ -15,7 +15,7 @@ - + Source Files diff --git a/ide/vs2017/mimalloc.sln b/ide/vs2017/mimalloc.sln index f4860d93..aeab6b88 100644 --- a/ide/vs2017/mimalloc.sln +++ b/ide/vs2017/mimalloc.sln @@ -11,6 +11,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimall EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -51,6 +53,14 @@ Global {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index 4a2069c6..86268f74 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -98,8 +98,9 @@ true ../../include MI_DEBUG=3;%(PreprocessorDefinitions); - Default + CompileAsCpp false + stdcpp17 @@ -116,8 +117,9 @@ true ../../include MI_DEBUG=3;%(PreprocessorDefinitions); - Default + CompileAsCpp false + stdcpp17 diff --git a/src/alloc-override.c b/src/alloc-override.c index 5378850b..fba0c199 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -19,10 +19,6 @@ terms of the MIT license. A copy of the license can be found in the file // Override system malloc // ------------------------------------------------------ -#if defined(_MSC_VER) -#pragma warning(disable:4273) // inconsistent dll linking -#endif - #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) @@ -62,6 +58,9 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(strdup), MI_INTERPOSE_MI(strndup) }; +#elif defined(_MSC_VER) + // cannot override malloc unless using a dll. + // we just override new/delete which does work in a static library. #else // On all other systems forward to our API void* malloc(size_t size) mi_attr_noexcept MI_FORWARD1(mi_malloc, size); @@ -94,7 +93,7 @@ terms of the MIT license. A copy of the license can be found in the file void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } - #if (__cplusplus >= 201402L) + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) void operator delete (void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); void operator delete[](void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); #endif diff --git a/src/memory.c b/src/memory.c index 74a9628c..e7d1887e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -105,7 +105,7 @@ static size_t mi_good_commit_size(size_t size) { } // Return if a pointer points into a region reserved by us. -bool mi_is_in_heap_region(const void* p) { +bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { size_t count = mi_atomic_read(®ions_count); for (size_t i = 0; i < count; i++) { uint8_t* start = (uint8_t*)mi_atomic_read_ptr(®ions[i].start); diff --git a/test/main-override.cpp b/test/main-override.cpp index 3c192fe3..ec7854a2 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -43,8 +43,7 @@ int main() { delete t; t = new (std::nothrow) Test(42); delete t; - free(p); - return 0; + return 0; } class Static { From 306a5423364880b190736447267c7bb47e457d46 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 19:52:29 -0700 Subject: [PATCH 05/67] add mimalloc-override header file and use C++ compilation with msvc --- CMakeLists.txt | 1 + ide/vs2017/mimalloc-test.vcxproj | 7 +- ide/vs2017/mimalloc-test.vcxproj.filters | 2 +- ide/vs2017/mimalloc.vcxproj | 6 +- include/mimalloc-override.h | 88 ++++++++++++++++++++++++ src/alloc-override.c | 9 ++- src/memory.c | 2 +- test/main-override.cpp | 6 +- 8 files changed, 101 insertions(+), 20 deletions(-) create mode 100644 include/mimalloc-override.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c9de8618..d86d096b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,6 +149,7 @@ target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_dir} LIBRARY NAMELINK_SKIP) install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_dir}) install(FILES include/mimalloc.h DESTINATION ${mi_install_dir}/include) +install(FILES include/mimalloc-override.h DESTINATION ${mi_install_dir}/include) install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_dir}/cmake) install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_dir}/cmake) install(EXPORT mimalloc DESTINATION ${mi_install_dir}/cmake) diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index 7976af56..8e61a97f 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -145,12 +145,7 @@ - - AssemblyAndSourceCode - AssemblyAndSourceCode - AssemblyAndSourceCode - AssemblyAndSourceCode - + diff --git a/ide/vs2017/mimalloc-test.vcxproj.filters b/ide/vs2017/mimalloc-test.vcxproj.filters index 9254f6c0..eb5e70b7 100644 --- a/ide/vs2017/mimalloc-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-test.vcxproj.filters @@ -15,7 +15,7 @@ - + Source Files diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index bb1818b0..d3b84c45 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -94,8 +94,9 @@ true ../../include MI_DEBUG=3;%(PreprocessorDefinitions); - Default + CompileAsCpp false + stdcpp17 @@ -112,8 +113,9 @@ true ../../include MI_DEBUG=3;%(PreprocessorDefinitions); - Default + CompileAsCpp false + stdcpp17 diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h new file mode 100644 index 00000000..62036b55 --- /dev/null +++ b/include/mimalloc-override.h @@ -0,0 +1,88 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018,2019 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_OVERRIDE_H +#define MIMALLOC_OVERRIDE_H + +#include "mimalloc.h" + +// Standard C allocation +#define malloc(n) mi_malloc(n) +#define calloc(n,c) mi_calloc(n,c) +#define realloc(p,n) mi_realloc(p,n) +#define free(p) mi_free(p) + +#define strdup(s) mi_strdup(s) +#define strndup(s) mi_strndup(s) +#define realpath(f,n) mi_realpath(f,n) + +// Microsoft extensions +#define _expand(p,n) mi_expand(p,n) +#define _msize(p) mi_usable_size(p) +#define _recalloc(p,n,c) mi_recalloc(p,n,c) +#define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v) +#define _wdupenv_s(b,n,v) mi_wdupenv_s(b,n,v) + +// Various Posix and Unix variants +#define reallocf(p,n) mi_reallocf(p,n) +#define malloc_size(p) mi_usable_size(p) +#define malloc_usable_size(p) mi_usable_size(p) +#define cfree(p) mi_free(p) + +#define valloc(n) mi_valloc(n) +#define pvalloc(n) mi_pvalloc(n) +#define reallocarray(p,s,n) mi_reallocarray(p,s,n) +#define memalign(a,n) mi_memalign(a,n) +#define aligned_alloc(a,n) mi_aligned_alloc(a,n) +#define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) +#define _posix_memalign(p,a,n) mi_posix_memalign(p,a,n) + +// Microsoft aligned variants +#define _aligned_malloc(n,a) mi_malloc_aligned(n,a) +#define _aligned_realloc(p,n,a) mi_realloc_aligned(p,n,a) +#define _aligned_recalloc(p,s,n,a) mi_aligned_recalloc(p,s,n,a) +#define _aligned_msize(p,a,o) mi_usable_size(p) +#define _aligned_offset_malloc(n,a,o) mi_malloc_aligned_at(n,a,o) +#define _aligned_offset_realloc(p,n,a,o) mi_realloc_aligned_at(p,n,a,o) +#define _aligned_offset_recalloc(p,s,n,a,o) mi_recalloc_aligned_at(p,s,n,a,o) + + +// ------------------------------------------------------ +// With a C++ compiler we override the new/delete operators. +// see +// ------------------------------------------------------ +#ifdef __cplusplus + #include + + void operator delete(void* p) noexcept { mi_free(p); }; + void operator delete[](void* p) noexcept { mi_free(p); }; + + void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } + void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } + + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { (void)(tag); return mi_new_nothrow(n); } + + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) + void operator delete (void* p, std::size_t n) { mi_free_size(p,n); }; + void operator delete[](void* p, std::size_t n) { mi_free_size(p,n); }; + #endif + + #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) + void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } + void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + #endif +#endif + +#endif MIMALLOC_OVERRIDE_H diff --git a/src/alloc-override.c b/src/alloc-override.c index 5ca88af7..a1d671bb 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -19,10 +19,6 @@ terms of the MIT license. A copy of the license can be found in the file // Override system malloc // ------------------------------------------------------ -#if defined(_MSC_VER) -#pragma warning(disable:4273) // inconsistent dll linking -#endif - #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) @@ -62,6 +58,9 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(strdup), MI_INTERPOSE_MI(strndup) }; +#elif defined(_MSC_VER) + // cannot override malloc unless using a dll. + // we just override new/delete which does work in a static library. #else // On all other systems forward to our API void* malloc(size_t size) mi_attr_noexcept MI_FORWARD1(mi_malloc, size); @@ -94,7 +93,7 @@ terms of the MIT license. A copy of the license can be found in the file void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } - #if (__cplusplus >= 201402L) + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) void operator delete (void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); void operator delete[](void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); #endif diff --git a/src/memory.c b/src/memory.c index 83e90b0d..bfbb5e39 100644 --- a/src/memory.c +++ b/src/memory.c @@ -105,7 +105,7 @@ static size_t mi_good_commit_size(size_t size) { } // Return if a pointer points into a region reserved by us. -bool mi_is_in_heap_region(const void* p) { +bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { size_t count = mi_atomic_read(®ions_count); for (size_t i = 0; i < count; i++) { uint8_t* start = (uint8_t*)mi_atomic_read_ptr(®ions[i].start); diff --git a/test/main-override.cpp b/test/main-override.cpp index 3f2bc960..7b562d41 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -40,11 +41,6 @@ int main() { delete t; t = new (std::nothrow) Test(42); delete t; - int err = mi_posix_memalign(&p1,32,60); - if (!err) free(p1); - free(p); - mi_collect(true); - mi_stats_print(NULL); // MIMALLOC_VERBOSE env is set to 2 return 0; } From 33aa5d92fa91f063bc09ffdbb6cc1aba40a9b759 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 21:20:18 -0700 Subject: [PATCH 06/67] fix to overrides on windows --- ide/vs2017/mimalloc-override.vcxproj | 1 + ide/vs2017/mimalloc-override.vcxproj.filters | 3 +++ ide/vs2017/mimalloc.vcxproj | 1 + ide/vs2017/mimalloc.vcxproj.filters | 5 ++++- include/mimalloc-override.h | 4 ++-- include/mimalloc.h | 8 ++++---- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 5fad3cf1..6c1020ea 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -214,6 +214,7 @@ + diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters index a39667f3..df0bf5ed 100644 --- a/ide/vs2017/mimalloc-override.vcxproj.filters +++ b/ide/vs2017/mimalloc-override.vcxproj.filters @@ -23,6 +23,9 @@ Header Files + + Header Files + diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index 86268f74..3310e642 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -250,6 +250,7 @@ + diff --git a/ide/vs2017/mimalloc.vcxproj.filters b/ide/vs2017/mimalloc.vcxproj.filters index 8bdeccf9..bffbb57f 100644 --- a/ide/vs2017/mimalloc.vcxproj.filters +++ b/ide/vs2017/mimalloc.vcxproj.filters @@ -70,5 +70,8 @@ Header Files + + Header Files + - + \ No newline at end of file diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index 9d26ab28..f6149514 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -39,10 +39,10 @@ including this header is not necessary. #define _strdup(s) mi_strdup(s) #define _strndup(s) mi_strndup(s) -#define _wcsdup(s) mi_wcsdup(s) +#define _wcsdup(s) (wchar_t*)mi_wcsdup((const unsigned short*)(s)) #define _mbsdup(s) mi_mbsdup(s) #define _dupenv_s(b,n,v) mi_dupenv_s(b,n,v) -#define _wdupenv_s(b,n,v) mi_wdupenv_s(b,n,v) +#define _wdupenv_s(b,n,v) mi_wdupenv_s((unsigned short*)(b),n,(const unsigned short*)(v)) // Various Posix and Unix variants #define reallocf(p,n) mi_reallocf(p,n) diff --git a/include/mimalloc.h b/include/mimalloc.h index 4726c4b8..be784511 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -246,10 +246,10 @@ mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); -// -------------------------------------------------------------------------------------------- -// mi prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. -// (This can be convenient when providing overrides of these functions.) -// -------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------------- +// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. +// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) +// ------------------------------------------------------------------------------------------------------- mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; From 8dad858888eacfc0166f993c4ca95dfb994ba39c Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 21:41:28 -0700 Subject: [PATCH 07/67] update to correct project settings for visual studio --- ide/vs2017/mimalloc-override.vcxproj | 54 ++++++-------------- ide/vs2017/mimalloc-override.vcxproj.filters | 3 -- ide/vs2017/mimalloc.vcxproj | 12 ++--- ide/vs2017/mimalloc.vcxproj.filters | 9 ++-- src/alloc-override.c | 4 +- 5 files changed, 24 insertions(+), 58 deletions(-) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 6c1020ea..3ca8158a 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -97,25 +97,18 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions); + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false Default - - - - - - - - ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) + Default @@ -125,26 +118,25 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions); + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false Default - - - - - - - - ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) + Default + + COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect.dll $(OutputPath) + + + copy mimalloc-redirect.dll to the output directory + @@ -155,7 +147,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions);NDEBUG + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -168,15 +160,8 @@ ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) + Default - - - - - - - - @@ -187,7 +172,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions);NDEBUG + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -200,14 +185,13 @@ ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) + Default - - + COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect.dll $(OutputPath) - - + copy mimalloc-redirect.dll to the output directory @@ -224,12 +208,6 @@ false false - - true - true - true - true - true true diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters index df0bf5ed..ffabddac 100644 --- a/ide/vs2017/mimalloc-override.vcxproj.filters +++ b/ide/vs2017/mimalloc-override.vcxproj.filters @@ -67,8 +67,5 @@ Source Files - - Source Files - \ No newline at end of file diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index 3310e642..a8cb7566 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -217,12 +217,6 @@ true true - - true - true - true - true - true true @@ -249,9 +243,9 @@ - - - + + + diff --git a/ide/vs2017/mimalloc.vcxproj.filters b/ide/vs2017/mimalloc.vcxproj.filters index bffbb57f..28d94e99 100644 --- a/ide/vs2017/mimalloc.vcxproj.filters +++ b/ide/vs2017/mimalloc.vcxproj.filters @@ -41,9 +41,6 @@ Source Files - - Source Files - Source Files @@ -64,13 +61,13 @@ Header Files - + Header Files - + Header Files - + Header Files diff --git a/src/alloc-override.c b/src/alloc-override.c index fba0c199..345d396c 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -9,8 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file #error "this file should be included from 'alloc.c' (so aliases can work)" #endif -#if defined(MI_MALLOC_OVERRIDE) && defined(_WIN32) && !(defined(MI_SHARED_LIB) && defined(_DLL)) -#error "It is only possible to override malloc on Windows when building as a DLL (and linking the C runtime as a DLL)" +#if defined(MI_MALLOC_OVERRIDE) && defined(_WIN32) && !(defined(MI_SHARED_LIB) && defined(_DLL) && defined(_WIN64)) +#error "It is only possible to override "malloc" on Windows when building as a 64-bit DLL (and linking the C runtime as a DLL)" #endif #if defined(MI_MALLOC_OVERRIDE) && !defined(_WIN32) From fd6fd2347072f060c1893b89f8a93a46c4c83357 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 18 Jul 2019 21:48:58 -0700 Subject: [PATCH 08/67] make eager-region-commit false by default on windows --- src/options.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/options.c b/src/options.c index 30c00b2f..e0c5e654 100644 --- a/src/options.c +++ b/src/options.c @@ -42,19 +42,24 @@ static mi_option_desc_t options[_mi_option_last] = { MI_DEBUG, UNINIT, "show_errors" }, { 0, UNINIT, "verbose" }, + #if MI_SECURE + { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored + #else + { 0, UNINIT, "secure" }, + #endif + // the following options are experimental and not all combinations make sense. + { 1, UNINIT, "eager_commit" }, // note: if eager_region_commit is on, this should be on too. + #ifdef _WIN32 + { 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...) + #else + { 1, UNINIT, "eager_region_commit" }, + #endif + { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, - { 1, UNINIT, "eager_commit" }, - { 1, UNINIT, "eager_region_commit" }, // eager_commit should be on when eager_region_commit is on - { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, "reset_decommits" }, - { 0, UNINIT, "reset_discards" }, - #if MI_SECURE - { MI_SECURE, INITIALIZED, "secure" } // in a secure build the environment setting is ignored - #else - { 0, UNINIT, "secure" } - #endif + { 0, UNINIT, "reset_discards" } }; static void mi_option_init(mi_option_desc_t* desc); From 2f63964e5c7ccdcdbd5562f3176dc96247fbd19e Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 19 Jul 2019 08:55:02 -0700 Subject: [PATCH 09/67] update stress test to be more deterministic --- test/test-stress.c | 158 ++++++++++++++++++++++++++++++--------------- 1 file changed, 107 insertions(+), 51 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index 4e4d9c0d..b26dfd04 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -1,42 +1,71 @@ /* ---------------------------------------------------------------------------- Copyright (c) 2018,2019 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. +terms of the MIT license. -----------------------------------------------------------------------------*/ /* This is a stress test for the allocator, using multiple threads and transferring objects between threads. This is not a typical workload - but uses a random size distribution. Do not use this test as a benchmark! - Note: pthreads uses mimalloc to allocate stacks and thus not all - memory is freed at the end. (usually the 320 byte chunks). + but uses a random linear size distribution. Do not use this test as a benchmark! */ #include #include +#include +#include #include -#include "mimalloc.h" -#include "mimalloc-internal.h" -#include "mimalloc-atomic.h" +#include + +// argument defaults +static int THREADS = 32; // more repeatable if THREADS <= #processors +static int N = 10; // scaling factor + +// static int THREADS = 8; // more repeatable if THREADS <= #processors +// static int N = 100; // scaling factor -#define N (10) // scaling factor -#define THREADS (32) #define TRANSFERS (1000) static volatile void* transfer[TRANSFERS]; -#if (MI_INTPTR_SIZE==8) +#if (INTPTR_MAX != UINT32_MAX) const uintptr_t cookie = 0xbf58476d1ce4e5b9UL; #else const uintptr_t cookie = 0x1ce4e5b9UL; #endif +static void* atomic_exchange_ptr(volatile void** p, void* newval); -static void* alloc_items(size_t items) { - if ((rand()%100) == 0) items *= 100; // 1% huge objects; +typedef uintptr_t* random_t; + +static uintptr_t pick(random_t r) { + uintptr_t x = *r; + #if (INTPTR_MAX > UINT32_MAX) + // by Sebastiano Vigna, see: + x ^= x >> 30; + x *= 0xbf58476d1ce4e5b9UL; + x ^= x >> 27; + x *= 0x94d049bb133111ebUL; + x ^= x >> 31; + #else + // by Chris Wellons, see: + x ^= x >> 16; + x *= 0x7feb352dUL; + x ^= x >> 15; + x *= 0x846ca68bUL; + x ^= x >> 16; + #endif + *r = x; + return x; +} + +static bool chance(size_t perc, random_t r) { + return (pick(r) % 100 <= perc); +} + +static void* alloc_items(size_t items, random_t r) { + if (chance(1, r)) items *= 100; // 1% huge objects; if (items==40) items++; // pthreads uses that size for stack increases - uintptr_t* p = mi_mallocn_tp(uintptr_t,items); - if(p == NULL) return NULL; + uintptr_t* p = (uintptr_t*)mi_malloc(items*sizeof(uintptr_t)); for (uintptr_t i = 0; i < items; i++) p[i] = (items - i) ^ cookie; return p; } @@ -47,7 +76,7 @@ static void free_items(void* p) { uintptr_t items = (q[0] ^ cookie); for (uintptr_t i = 0; i < items; i++) { if ((q[i]^cookie) != items - i) { - fprintf(stderr,"memory corruption at block %p at %zu\n", p, i); + fprintf(stderr, "memory corruption at block %p at %zu\n", p, i); abort(); } } @@ -57,43 +86,45 @@ static void free_items(void* p) { static void stress(intptr_t tid) { + //bench_start_thread(); + uintptr_t r = tid ^ 42; const size_t max_item = 128; // in words const size_t max_item_retained = 10*max_item; - size_t allocs = 80*N*(tid%8 + 1); // some threads do more + size_t allocs = 25*N*(tid%8 + 1); // some threads do more size_t retain = allocs/2; void** data = NULL; size_t data_size = 0; size_t data_top = 0; - void** retained = mi_mallocn_tp(void*,retain); + void** retained = (void**)mi_malloc(retain*sizeof(void*)); size_t retain_top = 0; while (allocs>0 || retain>0) { - if (retain == 0 || ((rand()%4 == 0) && allocs > 0)) { - // 75% alloc + if (retain == 0 || (chance(50, &r) && allocs > 0)) { + // 50%+ alloc allocs--; if (data_top >= data_size) { data_size += 100000; - data = mi_reallocn_tp(data, void*, data_size); + data = (void**)mi_realloc(data, data_size*sizeof(void*)); } - data[data_top++] = alloc_items((rand() % max_item) + 1); + data[data_top++] = alloc_items((pick(&r) % max_item) + 1, &r); } else { // 25% retain - retained[retain_top++] = alloc_items( 10*((rand() % max_item_retained) + 1) ); + retained[retain_top++] = alloc_items(10*((pick(&r) % max_item_retained) + 1), &r); retain--; } - if ((rand()%3)!=0 && data_top > 0) { + if (chance(66, &r) && data_top > 0) { // 66% free previous alloc - size_t idx = rand() % data_top; + size_t idx = pick(&r) % data_top; free_items(data[idx]); - data[idx]=NULL; + data[idx] = NULL; } - if ((tid%2)==0 && (rand()%4)==0 && data_top > 0) { - // 25% transfer-swap of half the threads - size_t data_idx = rand() % data_top; - size_t transfer_idx = rand() % TRANSFERS; + if (chance(25, &r) && data_top > 0) { + // 25% transfer-swap + size_t data_idx = pick(&r) % data_top; + size_t transfer_idx = pick(&r) % TRANSFERS; void* p = data[data_idx]; - void* q = mi_atomic_exchange_ptr(&transfer[transfer_idx],p); + void* q = atomic_exchange_ptr(&transfer[transfer_idx], p); data[data_idx] = q; } } @@ -106,20 +137,33 @@ static void stress(intptr_t tid) { } mi_free(retained); mi_free(data); + //bench_end_thread(); } -static void run_os_threads(); +static void run_os_threads(size_t nthreads); -int main() { - srand(42); - memset((void*)transfer,0,TRANSFERS*sizeof(void*)); - run_os_threads(); +int main(int argc, char** argv) { + if (argc>=2) { + char* end; + long n = strtol(argv[1], &end, 10); + if (n > 0) THREADS = n; + } + if (argc>=3) { + char* end; + long n = (strtol(argv[2], &end, 10)); + if (n > 0) N = n; + } + printf("start with %i threads with a %i%% load-per-thread\n", THREADS, N); + //bench_start_program(); + memset((void*)transfer, 0, TRANSFERS*sizeof(void*)); + run_os_threads(THREADS); for (int i = 0; i < TRANSFERS; i++) { free_items((void*)transfer[i]); } - mi_collect(false); // ensures abandoned segments are reclaimed - mi_collect(true); // frees everything + mi_collect(false); + mi_collect(true); mi_stats_print(NULL); + //bench_end_program(); return 0; } @@ -133,36 +177,48 @@ static DWORD WINAPI thread_entry(LPVOID param) { return 0; } -static void run_os_threads() { - DWORD tids[THREADS]; - HANDLE thandles[THREADS]; - for(intptr_t i = 0; i < THREADS; i++) { - thandles[i] = CreateThread(0,4096,&thread_entry,(void*)(i),0,&tids[i]); +static void run_os_threads(size_t nthreads) { + DWORD* tids = (DWORD*)malloc(nthreads * sizeof(DWORD)); + HANDLE* thandles = (HANDLE*)malloc(nthreads * sizeof(HANDLE)); + for (intptr_t i = 0; i < nthreads; i++) { + thandles[i] = CreateThread(0, 4096, &thread_entry, (void*)(i), 0, &tids[i]); } - for (int i = 0; i < THREADS; i++) { + for (int i = 0; i < nthreads; i++) { WaitForSingleObject(thandles[i], INFINITE); } } +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + #if (INTPTR_MAX == UINT32_MAX) + return (void*)InterlockedExchange((volatile LONG*)p, (LONG)newval); + #else + return (void*)InterlockedExchange64((volatile LONG64*)p, (LONG64)newval); + #endif +} #else #include +#include -static void* thread_entry( void* param ) { +static void* thread_entry(void* param) { stress((uintptr_t)param); return NULL; } -static void run_os_threads() { - pthread_t threads[THREADS]; - memset(threads,0,sizeof(pthread_t)*THREADS); - //pthread_setconcurrency(THREADS); - for(uintptr_t i = 0; i < THREADS; i++) { +static void run_os_threads(size_t nthreads) { + pthread_t* threads = (pthread_t*)mi_malloc(nthreads*sizeof(pthread_t)); + memset(threads, 0, sizeof(pthread_t)*nthreads); + //pthread_setconcurrency(nthreads); + for (uintptr_t i = 0; i < nthreads; i++) { pthread_create(&threads[i], NULL, &thread_entry, (void*)i); } - for (size_t i = 0; i < THREADS; i++) { + for (size_t i = 0; i < nthreads; i++) { pthread_join(threads[i], NULL); } } +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + return atomic_exchange_explicit((volatile _Atomic(void*)*)p, newval, memory_order_acquire); +} + #endif From 81a7ae33e7e2c1a132d87afe4ecfd022f64bb5c1 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 19 Jul 2019 09:26:22 -0700 Subject: [PATCH 10/67] fix order of options --- include/mimalloc.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index be784511..d92c2866 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -225,14 +225,14 @@ typedef enum mi_option_e { mi_option_show_errors, mi_option_verbose, // the following options are experimental - mi_option_page_reset, - mi_option_cache_reset, + mi_option_secure, mi_option_eager_commit, mi_option_eager_region_commit, mi_option_large_os_pages, // implies eager commit + mi_option_page_reset, + mi_option_cache_reset, mi_option_reset_decommits, mi_option_reset_discards, - mi_option_secure, _mi_option_last } mi_option_t; From dd5fa88c459b4c2066b3fcc8d5fbe453156f6f8f Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 19 Jul 2019 09:26:51 -0700 Subject: [PATCH 11/67] check better for valid pointers in free in debug mode --- src/alloc.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 10a74ce3..bac925ee 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -198,28 +198,29 @@ static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, mi_pag // Free a block void mi_free(void* p) mi_attr_noexcept -{ - // optimize: merge null check with the segment masking (below) - //if (p == NULL) return; - +{ #if (MI_DEBUG>0) if (mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)) { _mi_error_message("trying to free an invalid (unaligned) pointer: %p\n", p); return; } #endif - + const mi_segment_t* const segment = _mi_ptr_segment(p); - if (segment == NULL) return; // checks for (p==NULL) - bool local = (_mi_thread_id() == segment->thread_id); // preload, note: putting the thread_id in the page->flags does not improve performance + if (segment == NULL) return; // checks for (p==NULL) #if (MI_DEBUG>0) + if (mi_unlikely(!mi_is_in_heap_region(p))) { + _mi_warning_message("possibly trying to mi_free a pointer that does not point to a valid heap region: %p\n" + "(this may still be a valid very large allocation (over 64MiB))\n", p); + } if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { _mi_error_message("trying to mi_free a pointer that does not point to a valid heap space: %p\n", p); return; } #endif + bool local = (_mi_thread_id() == segment->thread_id); // preload, note: putting the thread_id in the page->flags does not improve performance mi_page_t* page = _mi_segment_page_of(segment, p); #if (MI_STAT>1) From fcd0253ac9567a2cd7e03c5473430ed23ebcc479 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 19 Jul 2019 09:27:14 -0700 Subject: [PATCH 12/67] limit number of error messages to 10 --- src/options.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/options.c b/src/options.c index e0c5e654..46f0a36e 100644 --- a/src/options.c +++ b/src/options.c @@ -6,6 +6,7 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" #include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include #include // strcmp @@ -50,7 +51,7 @@ static mi_option_desc_t options[_mi_option_last] = // the following options are experimental and not all combinations make sense. { 1, UNINIT, "eager_commit" }, // note: if eager_region_commit is on, this should be on too. - #ifdef _WIN32 + #ifdef _WIN32 // and BSD? { 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...) #else { 1, UNINIT, "eager_region_commit" }, @@ -58,8 +59,8 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, "page_reset" }, { 0, UNINIT, "cache_reset" }, - { 0, UNINIT, "reset_decommits" }, - { 0, UNINIT, "reset_discards" } + { 0, UNINIT, "reset_decommits" }, // note: cannot enable this if secure is on + { 0, UNINIT, "reset_discards" } // note: cannot enable this if secure is on }; static void mi_option_init(mi_option_desc_t* desc); @@ -106,6 +107,8 @@ void mi_option_enable_default(mi_option_t option, bool enable) { // -------------------------------------------------------- // Messages // -------------------------------------------------------- +#define MAX_ERROR_COUNT (10) +static uintptr_t error_count = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings // Define our own limited `fprintf` that avoids memory allocation. // We do this using `snprintf` with a limited buffer. @@ -153,6 +156,7 @@ void _mi_verbose_message(const char* fmt, ...) { void _mi_error_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; + if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return; va_list args; va_start(args,fmt); mi_vfprintf(stderr, "mimalloc: error: ", fmt, args); @@ -162,6 +166,7 @@ void _mi_error_message(const char* fmt, ...) { void _mi_warning_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; + if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return; va_list args; va_start(args,fmt); mi_vfprintf(stderr, "mimalloc: warning: ", fmt, args); From 2ac2c6a1c37617bb190e4833ff509b9ef6afd5a1 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 19 Jul 2019 09:27:32 -0700 Subject: [PATCH 13/67] fix format of error messages --- src/os.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/os.c b/src/os.c index 4279bf8d..3527c94d 100644 --- a/src/os.c +++ b/src/os.c @@ -471,7 +471,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); #endif if (err != 0) { - _mi_warning_message("commit/decommit error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err); + _mi_warning_message("commit/decommit error: start: 0x%p, csize: 0x%x, err: %i\n", start, csize, err); } mi_assert_internal(err == 0); return (err == 0); @@ -503,8 +503,10 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! - #if MI_DEBUG>1 - memset(start, 0, csize); // pretend it is eagerly reset + #if (MI_DEBUG>1) + if (!mi_option_is_enabled(mi_option_secure)) { + memset(start, 0, csize); // pretend it is eagerly reset + } #endif #if defined(_WIN32) @@ -535,7 +537,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) int err = madvise(start, csize, MADV_DONTNEED); #endif if (err != 0) { - _mi_warning_message("madvise reset error: start: 0x%8p, csize: 0x%8zux, errno: %i\n", start, csize, errno); + _mi_warning_message("madvise reset error: start: 0x%p, csize: 0x%x, errno: %i\n", start, csize, errno); } //mi_assert(err == 0); if (err != 0) return false; @@ -584,7 +586,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); #endif if (err != 0) { - _mi_warning_message("mprotect error: start: 0x%8p, csize: 0x%8zux, err: %i\n", start, csize, err); + _mi_warning_message("mprotect error: start: 0x%p, csize: 0x%x, err: %i\n", start, csize, err); } return (err == 0); } From f0de0b6f6852a345c0cb86fc264679accbcf6b5f Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 19 Jul 2019 09:33:14 -0700 Subject: [PATCH 14/67] remove old windows override code --- src/alloc-override-win.c | 702 --------------------------------------- 1 file changed, 702 deletions(-) delete mode 100644 src/alloc-override-win.c diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c deleted file mode 100644 index 7b0fe69f..00000000 --- a/src/alloc-override-win.c +++ /dev/null @@ -1,702 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ - -#include "mimalloc.h" -#include "mimalloc-internal.h" - -#if !defined(_WIN32) -#error "this file should only be included on Windows" -#endif - -#include -#include - - -/* -To override the C runtime `malloc` on Windows we need to patch the allocation -functions at runtime initialization. Unfortunately we can never patch before the -runtime initializes itself, because as soon as we call `GetProcAddress` on the -runtime module (a DLL or EXE in Windows speak), it will first load and initialize -(by the OS calling `DllMain` on it). - -This means that some things might be already allocated by the C runtime itself -(and possibly other DLL's) before we get to resolve runtime adresses. This is -no problem if everyone unwinds in order: when we unload, we unpatch and restore -the original crt `free` routines and crt malloc'd memory is freed correctly. - -But things go wrong if such early CRT alloc'd memory is freed or re-allocated -_after_ we patch, but _before_ we unload (and unpatch), or if any memory allocated -by us is freed after we unpatched. - -There are two tricky situations to deal with: - -1. The Thread Local Storage (TLS): when the main thread stops it will call registered - callbacks on TLS entries (allocated by `FlsAlloc`). This is done by the OS - before any DLL's are unloaded. Unfortunately, the C runtime registers such - TLS entries with CRT allocated memory which is freed in the callback. - -2. Inside the CRT: - a. Some variables might get initialized by patched allocated - blocks but freed during CRT unloading after we unpatched - (like temporary file buffers). - b. Some blocks are allocated at CRT and freed by the CRT (like the - environment storage). - c. And some blocks are allocated by the CRT and then reallocated - while patched, and finally freed after unpatching! This - happens with the `atexit` functions for example to grow the array - of registered functions. - -In principle situation 2 is hopeless: since we cannot patch before CRT initialization, -we can never be sure how to free or reallocate a pointer during CRT unloading. -However, in practice there is a good solution: when terminating, we just patch -the reallocation and free routines to no-ops -- we are winding down anyway! This leaves -just the reallocation problm of CRT alloc'd memory once we are patched. Here, a study of the -CRT reveals that there seem to be just three such situations: - -1. When registering `atexit` routines (to grow the exit function table), -2. When calling `_setmaxstdio` (to grow the file handle table), -3. and `_popen`/`_wpopen` (to grow handle pairs). These turn out not to be - a problem as these are NULL initialized. - -We fix these by providing wrappers: - -1. We first register a _global_ `atexit` routine ourselves (`mi_patches_at_exit`) before patching, - and then patch the `_crt_atexit` function to implement our own global exit list (and the - same for `_crt_at_quick_exit`). All module local lists are no problem since they are always fully - (un)patched from initialization to end. We can register in the global list by dynamically - getting the global `_crt_atexit` entry from `ucrtbase.dll`. - -2. The `_setmaxstdio` is _detoured_: we patch it by a stub that unpatches first, - calls the original routine and repatches again. - -That leaves us to reliably shutdown and enter "termination mode": - -1. Using our trick to get the global exit list entry point, we register an exit function `mi_patches_atexit` - that first executes all our home brew list of exit functions, and then enters a _termination_ - phase that patches realloc/free variants with no-ops. Patching later again with special no-ops for - `free` also improves efficiency during the program run since no flags need to be checked. - -2. That is not quite good enough yet since after executing exit routines after us on the - global exit list (registered by the CRT), - the OS starts to unwind the TLS callbacks and we would like to run callbacks registered after loading - our DLL to be done in patched mode. So, we also allocate a TLS entry when our DLL is loaded and when its - callback is called, we re-enable the original patches again. Since TLS is destroyed in FIFO order - this runs any callbacks in later DLL's in patched mode. - -3. Finally the DLL's get unloaded by the OS in order (still patched) until our DLL gets unloaded - and then we start a termination phase again, and patch realloc/free with no-ops for good this time. - -*/ - -static int __cdecl mi_setmaxstdio(int newmax); - -// ------------------------------------------------------ -// Microsoft allocation extensions -// ------------------------------------------------------ - - -typedef size_t mi_nothrow_t; - -static void mi_free_nothrow(void* p, mi_nothrow_t tag) { - UNUSED(tag); - mi_free(p); -} - -// Versions of `free`, `realloc`, `recalloc`, `expand` and `msize` -// that are used during termination and are no-ops. -static void mi_free_term(void* p) { - UNUSED(p); -} - -static void mi_free_size_term(void* p, size_t size) { - UNUSED(size); - UNUSED(p); -} - -static void mi_free_nothrow_term(void* p, mi_nothrow_t tag) { - UNUSED(tag); - UNUSED(p); -} - -static void* mi_realloc_term(void* p, size_t newsize) { - UNUSED(p); UNUSED(newsize); - return NULL; -} - -static void* mi__recalloc_term(void* p, size_t newcount, size_t newsize) { - UNUSED(p); UNUSED(newcount); UNUSED(newsize); - return NULL; -} - -static void* mi__expand_term(void* p, size_t newsize) { - UNUSED(p); UNUSED(newsize); - return NULL; -} - -static size_t mi__msize_term(void* p) { - UNUSED(p); - return 0; -} - - -// Debug versions, forward to base versions (that get patched) - -static void* mi__malloc_dbg(size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return _malloc_base(size); -} - -static void* mi__calloc_dbg(size_t count, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return _calloc_base(count, size); -} - -static void* mi__realloc_dbg(void* p, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return _realloc_base(p, size); -} - -static void mi__free_dbg(void* p, int block_type) { - UNUSED(block_type); - _free_base(p); -} - - -// the `recalloc`,`expand`, and `msize` don't have base versions and thus need a separate term version - -static void* mi__recalloc_dbg(void* p, size_t count, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi_recalloc(p, count, size); -} - -static void* mi__expand_dbg(void* p, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi__expand(p, size); -} - -static size_t mi__msize_dbg(void* p, int block_type) { - UNUSED(block_type); - return mi_usable_size(p); -} - -static void* mi__recalloc_dbg_term(void* p, size_t count, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi__recalloc_term(p, count, size); -} - -static void* mi__expand_dbg_term(void* p, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi__expand_term(p, size); -} - -static size_t mi__msize_dbg_term(void* p, int block_type) { - UNUSED(block_type); - return mi__msize_term(p); -} - - -// ------------------------------------------------------ -// implement our own global atexit handler -// ------------------------------------------------------ -typedef void (cbfun_t)(void); -typedef int (atexit_fun_t)(cbfun_t* fn); -typedef uintptr_t encoded_t; - -typedef struct exit_list_s { - encoded_t functions; // encoded pointer to array of encoded function pointers - size_t count; - size_t capacity; -} exit_list_t; - -#define MI_EXIT_INC (64) - -static exit_list_t atexit_list = { 0, 0, 0 }; -static exit_list_t at_quick_exit_list = { 0, 0, 0 }; -static CRITICAL_SECTION atexit_lock; - -// encode/decode function pointers with a random canary for security -static encoded_t canary; - -static inline void *decode(encoded_t x) { - return (void*)(x^canary); -} - -static inline encoded_t encode(void* p) { - return ((uintptr_t)p ^ canary); -} - - -static void init_canary() -{ - canary = _mi_random_init(0); - atexit_list.functions = at_quick_exit_list.functions = encode(NULL); -} - - -// initialize the list -static void mi_initialize_atexit(void) { - InitializeCriticalSection(&atexit_lock); - init_canary(); -} - -// register an exit function -static int mi_register_atexit(exit_list_t* list, cbfun_t* fn) { - if (fn == NULL) return EINVAL; - EnterCriticalSection(&atexit_lock); - encoded_t* functions = (encoded_t*)decode(list->functions); - if (list->count >= list->capacity) { // at first `functions == decode(0) == NULL` - encoded_t* newf = (encoded_t*)mi_recalloc(functions, list->capacity + MI_EXIT_INC, sizeof(cbfun_t*)); - if (newf != NULL) { - list->capacity += MI_EXIT_INC; - list->functions = encode(newf); - functions = newf; - } - } - int result; - if (list->count < list->capacity && functions != NULL) { - functions[list->count] = encode(fn); - list->count++; - result = 0; // success - } - else { - result = ENOMEM; - } - LeaveCriticalSection(&atexit_lock); - return result; -} - -// Register a global `atexit` function -static int mi_atexit(cbfun_t* fn) { - return mi_register_atexit(&atexit_list,fn); -} - -static int mi_at_quick_exit(cbfun_t* fn) { - return mi_register_atexit(&at_quick_exit_list,fn); -} - -static int mi_register_onexit(void* table, cbfun_t* fn) { - // TODO: how can we distinguish a quick_exit from atexit? - return mi_atexit(fn); -} - -// Execute exit functions in a list -static void mi_execute_exit_list(exit_list_t* list) { - // copy and zero the list structure - EnterCriticalSection(&atexit_lock); - exit_list_t clist = *list; - memset(list,0,sizeof(*list)); - LeaveCriticalSection(&atexit_lock); - - // now execute the functions outside of the lock - encoded_t* functions = (encoded_t*)decode(clist.functions); - if (functions != NULL) { - for (size_t i = clist.count; i > 0; i--) { // careful with unsigned count down.. - cbfun_t* fn = (cbfun_t*)decode(functions[i-1]); - if (fn==NULL) break; // corrupted! - fn(); - } - mi_free(functions); - } -} - - - -// ------------------------------------------------------ -// Jump assembly instructions for patches -// ------------------------------------------------------ - -#if defined(_M_IX86) || defined(_M_X64) - -#define MI_JUMP_SIZE 14 // at most 2+4+8 for a long jump or 1+5 for a short one - -typedef struct mi_jump_s { - uint8_t opcodes[MI_JUMP_SIZE]; -} mi_jump_t; - -void mi_jump_restore(void* current, const mi_jump_t* saved) { - memcpy(current, &saved->opcodes, MI_JUMP_SIZE); -} - -void mi_jump_write(void* current, void* target, mi_jump_t* save) { - if (save != NULL) { - memcpy(&save->opcodes, current, MI_JUMP_SIZE); - } - uint8_t* opcodes = ((mi_jump_t*)current)->opcodes; - ptrdiff_t diff = (uint8_t*)target - (uint8_t*)current; - uint32_t ofs32 = (uint32_t)diff; - #ifdef _M_X64 - uint64_t ofs64 = (uint64_t)diff; - if (ofs64 != (uint64_t)ofs32) { - // use long jump - opcodes[0] = 0xFF; - opcodes[1] = 0x25; - *((uint32_t*)&opcodes[2]) = 0; - *((uint64_t*)&opcodes[6]) = (uint64_t)target; - } - else - #endif - { - // use short jump - opcodes[0] = 0xE9; - *((uint32_t*)&opcodes[1]) = ofs32 - 5 /* size of the short jump instruction */; - } -} - -#elif defined(_M_ARM64) - -#define MI_JUMP_SIZE 16 - -typedef struct mi_jump_s { - uint8_t opcodes[MI_JUMP_SIZE]; -} mi_jump_t; - -void mi_jump_restore(void* current, const mi_jump_t* saved) { - memcpy(current, &saved->opcodes, MI_JUMP_SIZE); -} - -void mi_jump_write(void* current, void* target, mi_jump_t* save) { - if (save != NULL) { - memcpy(&save->opcodes, current, MI_JUMP_SIZE); - } - uint8_t* opcodes = ((mi_jump_t*)current)->opcodes; - uint64_t diff = (uint8_t*)target - (uint8_t*)current; - - // 0x50 0x00 0x00 0x58 ldr x16, .+8 # load PC relative +8 - // 0x00 0x02 0x3F 0xD6 blr x16 # and jump - //
- //
- static const uint8_t jump_opcodes[8] = { 0x50, 0x00, 0x00, 0x58, 0x00, 0x02, 0x3F, 0xD6 }; - memcpy(&opcodes[0], jump_opcodes, sizeof(jump_opcodes)); - *((uint64_t*)&opcodes[8]) = diff; -} - -#else -#error "define jump instructions for this platform" -#endif - - -// ------------------------------------------------------ -// Patches -// ------------------------------------------------------ -typedef enum patch_apply_e { - PATCH_NONE, - PATCH_TARGET, - PATCH_TARGET_TERM -} patch_apply_t; - -#define MAX_ENTRIES 4 // maximum number of patched entry points (like `malloc` in ucrtbase and msvcrt) - -typedef struct mi_patch_s { - const char* name; // name of the function to patch - void* target; // the address of the new target (never NULL) - void* target_term; // the address of the target during termination (or NULL) - patch_apply_t applied; // what target has been applied? - void* originals[MAX_ENTRIES]; // the resolved addresses of the function (or NULLs) - mi_jump_t saves[MAX_ENTRIES]; // the saved instructions in case it was applied -} mi_patch_t; - -#define MI_PATCH_NAME3(name,target,term) { name, &target, &term, PATCH_NONE, {NULL,NULL,NULL,NULL} } -#define MI_PATCH_NAME2(name,target) { name, &target, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} } -#define MI_PATCH3(name,target,term) MI_PATCH_NAME3(#name, target, term) -#define MI_PATCH2(name,target) MI_PATCH_NAME2(#name, target) -#define MI_PATCH1(name) MI_PATCH2(name,mi_##name) - -static mi_patch_t patches[] = { - // we implement our own global exit handler (as the CRT versions do a realloc internally) - //MI_PATCH2(_crt_atexit, mi_atexit), - //MI_PATCH2(_crt_at_quick_exit, mi_at_quick_exit), - MI_PATCH2(_setmaxstdio, mi_setmaxstdio), - MI_PATCH2(_register_onexit_function, mi_register_onexit), - - // override higher level atexit functions so we can implement at_quick_exit correcty - MI_PATCH2(atexit, mi_atexit), - MI_PATCH2(at_quick_exit, mi_at_quick_exit), - - // regular entries - MI_PATCH2(malloc, mi_malloc), - MI_PATCH2(calloc, mi_calloc), - MI_PATCH3(realloc, mi_realloc,mi_realloc_term), - MI_PATCH3(free, mi_free,mi_free_term), - - // extended api - MI_PATCH2(_strdup, mi_strdup), - MI_PATCH2(_strndup, mi_strndup), - MI_PATCH3(_expand, mi__expand,mi__expand_term), - MI_PATCH3(_recalloc, mi_recalloc,mi__recalloc_term), - MI_PATCH3(_msize, mi_usable_size,mi__msize_term), - - // base versions - MI_PATCH2(_malloc_base, mi_malloc), - MI_PATCH2(_calloc_base, mi_calloc), - MI_PATCH3(_realloc_base, mi_realloc,mi_realloc_term), - MI_PATCH3(_free_base, mi_free,mi_free_term), - - // these base versions are in the crt but without import records - MI_PATCH_NAME3("_recalloc_base", mi_recalloc,mi__recalloc_term), - MI_PATCH_NAME3("_msize_base", mi_usable_size,mi__msize_term), - - // debug - MI_PATCH2(_malloc_dbg, mi__malloc_dbg), - MI_PATCH2(_realloc_dbg, mi__realloc_dbg), - MI_PATCH2(_calloc_dbg, mi__calloc_dbg), - MI_PATCH2(_free_dbg, mi__free_dbg), - - MI_PATCH3(_expand_dbg, mi__expand_dbg, mi__expand_dbg_term), - MI_PATCH3(_recalloc_dbg, mi__recalloc_dbg, mi__recalloc_dbg_term), - MI_PATCH3(_msize_dbg, mi__msize_dbg, mi__msize_dbg_term), - -#if 0 - // override new/delete variants for efficiency (?) -#ifdef _WIN64 - // 64 bit new/delete - MI_PATCH_NAME2("??2@YAPEAX_K@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPEAX_K@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPEAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPEAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??3@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized - MI_PATCH_NAME3("??_V@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized - MI_PATCH_NAME2("??2@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - MI_PATCH_NAME3("??_V@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - - -#else - // 32 bit new/delete - MI_PATCH_NAME2("??2@YAPAXI@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPAXI@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??3@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized - MI_PATCH_NAME3("??_V@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized - - MI_PATCH_NAME2("??2@YAPAXIABUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPAXIABUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - -#endif -#endif - { NULL, NULL, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} } -}; - - -// Apply a patch -static bool mi_patch_apply(mi_patch_t* patch, patch_apply_t apply) -{ - if (patch->originals[0] == NULL) return true; // unresolved - if (apply == PATCH_TARGET_TERM && patch->target_term == NULL) apply = PATCH_TARGET; // avoid re-applying non-term variants - if (patch->applied == apply) return false; - - for (int i = 0; i < MAX_ENTRIES; i++) { - void* original = patch->originals[i]; - if (original == NULL) break; // no more - - DWORD protect = PAGE_READWRITE; - if (!VirtualProtect(original, MI_JUMP_SIZE, PAGE_EXECUTE_READWRITE, &protect)) return false; - if (apply == PATCH_NONE) { - mi_jump_restore(original, &patch->saves[i]); - } - else { - void* target = (apply == PATCH_TARGET ? patch->target : patch->target_term); - mi_assert_internal(target != NULL); - if (target != NULL) mi_jump_write(original, target, &patch->saves[i]); - } - VirtualProtect(original, MI_JUMP_SIZE, protect, &protect); - } - patch->applied = apply; - return true; -} - -// Apply all patches -static bool _mi_patches_apply(patch_apply_t apply, patch_apply_t* previous) { - static patch_apply_t current = PATCH_NONE; - if (previous != NULL) *previous = current; - if (current == apply) return true; - current = apply; - bool ok = true; - for (size_t i = 0; patches[i].name != NULL; i++) { - if (!mi_patch_apply(&patches[i], apply)) ok = false; - } - return ok; -} - -// Export the following three functions just in case -// a user needs that level of control. - -// Disable all patches -mi_decl_export void mi_patches_disable(void) { - _mi_patches_apply(PATCH_NONE, NULL); -} - -// Enable all patches normally -mi_decl_export bool mi_patches_enable(void) { - return _mi_patches_apply( PATCH_TARGET, NULL ); -} - -// Enable all patches in termination phase where free is a no-op -mi_decl_export bool mi_patches_enable_term(void) { - return _mi_patches_apply(PATCH_TARGET_TERM, NULL); -} - -// ------------------------------------------------------ -// Stub for _setmaxstdio -// ------------------------------------------------------ - -static int __cdecl mi_setmaxstdio(int newmax) { - patch_apply_t previous; - _mi_patches_apply(PATCH_NONE, &previous); // disable patches - int result = _setmaxstdio(newmax); // call original function (that calls original CRT recalloc) - _mi_patches_apply(previous,NULL); // and re-enable patches - return result; -} - - -// ------------------------------------------------------ -// Resolve addresses dynamically -// ------------------------------------------------------ - -// Try to resolve patches for a given module (DLL) -static void mi_module_resolve(const char* fname, HMODULE mod, int priority) { - // see if any patches apply - for (size_t i = 0; patches[i].name != NULL; i++) { - mi_patch_t* patch = &patches[i]; - if (patch->applied == PATCH_NONE) { - // find an available entry - int i = 0; - while (i < MAX_ENTRIES && patch->originals[i] != NULL) i++; - if (i < MAX_ENTRIES) { - void* addr = GetProcAddress(mod, patch->name); - if (addr != NULL) { - // found it! set the address - patch->originals[i] = addr; - _mi_trace_message(" override %s at %s!%p (entry %i)\n", patch->name, fname, addr, i); - } - } - } - } -} - -#define MIMALLOC_NAME "mimalloc-override.dll" -#define UCRTBASE_NAME "ucrtbase.dll" -#define UCRTBASED_NAME "ucrtbased.dll" - -// Resolve addresses of all patches by inspecting the loaded modules -static atexit_fun_t* crt_atexit = NULL; -static atexit_fun_t* crt_at_quick_exit = NULL; - - -static bool mi_patches_resolve(void) { - // get all loaded modules - HANDLE process = GetCurrentProcess(); // always -1, no need to release - DWORD needed = 0; - HMODULE modules[400]; // try to stay under 4k to not trigger the guard page - EnumProcessModules(process, modules, sizeof(modules), &needed); - if (needed == 0) return false; - int count = needed / sizeof(HMODULE); - int ucrtbase_index = 0; - int mimalloc_index = 0; - // iterate through the loaded modules - _mi_trace_message("overriding malloc dynamically...\n"); - for (int i = 0; i < count; i++) { - HMODULE mod = modules[i]; - char filename[MAX_PATH] = { 0 }; - DWORD slen = GetModuleFileName(mod, filename, MAX_PATH); - if (slen > 0 && slen < MAX_PATH) { - // filter out potential crt modules only - filename[slen] = 0; - const char* lastsep = strrchr(filename, '\\'); - const char* basename = (lastsep==NULL ? filename : lastsep+1); - _mi_trace_message(" %i: dynamic module %s\n", i, filename); - - // remember indices so we can check load order (in debug mode) - if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i; - if (_stricmp(basename, UCRTBASE_NAME) == 0) ucrtbase_index = i; - if (_stricmp(basename, UCRTBASED_NAME) == 0) ucrtbase_index = i; - - // see if we potentially patch in this module - int priority = 0; - if (i == 0) priority = 2; // main module to allow static crt linking - else if (_strnicmp(basename, "ucrt", 4) == 0) priority = 3; // new ucrtbase.dll in windows 10 - // NOTE: don't override msvcr -- leads to crashes in setlocale (needs more testing) - // else if (_strnicmp(basename, "msvcr", 5) == 0) priority = 1; // older runtimes - - if (priority > 0) { - // probably found a crt module, try to patch it - mi_module_resolve(basename,mod,priority); - - // try to find the atexit functions for the main process (in `ucrtbase.dll`) - if (crt_atexit==NULL) crt_atexit = (atexit_fun_t*)GetProcAddress(mod, "_crt_atexit"); - if (crt_at_quick_exit == NULL) crt_at_quick_exit = (atexit_fun_t*)GetProcAddress(mod, "_crt_at_quick_exit"); - } - } - } - int diff = mimalloc_index - ucrtbase_index; - if (diff > 1) { - _mi_warning_message("warning: the \"mimalloc-override\" DLL seems not to load before or right after the C runtime (\"ucrtbase\").\n" - " Try to fix this by changing the linking order.\n"); - } - return true; -} - - -// ------------------------------------------------------ -// Dll Entry -// ------------------------------------------------------ - -extern BOOL WINAPI _DllMainCRTStartup(HINSTANCE inst, DWORD reason, LPVOID reserved); - -static DWORD mi_fls_unwind_entry; -static void NTAPI mi_fls_unwind(PVOID value) { - if (value != NULL) mi_patches_enable(); // and re-enable normal patches again for DLL's loaded after us - return; -} - -static void mi_patches_atexit(void) { - mi_execute_exit_list(&atexit_list); - mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op -} - -static void mi_patches_at_quick_exit(void) { - mi_execute_exit_list(&at_quick_exit_list); - mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op -} - -__declspec(dllexport) BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) { - if (reason == DLL_PROCESS_ATTACH) { - __security_init_cookie(); - } - else if (reason == DLL_PROCESS_DETACH) { - // enter termination phase for good now - mi_patches_enable_term(); - } - // C runtime main - BOOL ok = _DllMainCRTStartup(inst, reason, reserved); - if (reason == DLL_PROCESS_ATTACH && ok) { - // Now resolve patches - ok = mi_patches_resolve(); - if (ok) { - // and register our unwind entry (this must be after resolving due to possible delayed DLL initialization from GetProcAddress) - mi_fls_unwind_entry = FlsAlloc(&mi_fls_unwind); - if (mi_fls_unwind_entry != FLS_OUT_OF_INDEXES) { - FlsSetValue(mi_fls_unwind_entry, (void*)1); - } - - // register our patch disabler in the global exit list - mi_initialize_atexit(); - if (crt_atexit != NULL) (*crt_atexit)(&mi_patches_atexit); - if (crt_at_quick_exit != NULL) (*crt_at_quick_exit)(&mi_patches_at_quick_exit); - - // and patch ! this also redirects the `atexit` handling for the global exit list - mi_patches_enable(); - - // hide internal allocation - mi_stats_reset(); - } - } - return ok; -} From 40cb631683bc5864a704b088021350b422af9f8c Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 13:03:51 -0700 Subject: [PATCH 15/67] re-add missing thread_init needed when running in debug mode --- src/heap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heap.c b/src/heap.c index 48bb9830..e55fdf34 100644 --- a/src/heap.c +++ b/src/heap.c @@ -172,7 +172,7 @@ void mi_collect(bool force) mi_attr_noexcept { ----------------------------------------------------------- */ mi_heap_t* mi_heap_get_default(void) { - // mi_thread_init(); + mi_thread_init(); return mi_get_default_heap(); } From df33efb19a3d6498e727441f0fd119f556df44c9 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 13:09:34 -0700 Subject: [PATCH 16/67] improved debug warning for freeing invalid pointers --- src/alloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index bac925ee..099bdbad 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -211,8 +211,11 @@ void mi_free(void* p) mi_attr_noexcept #if (MI_DEBUG>0) if (mi_unlikely(!mi_is_in_heap_region(p))) { - _mi_warning_message("possibly trying to mi_free a pointer that does not point to a valid heap region: %p\n" + _mi_warning_message("possibly trying to mi_free a pointer that does not point to a valid heap region: 0x%p\n" "(this may still be a valid very large allocation (over 64MiB))\n", p); + if (mi_likely(_mi_ptr_cookie(segment) == segment->cookie)) { + _mi_warning_message("(yes, the previous pointer 0x%p was valid after all)\n", p); + } } if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { _mi_error_message("trying to mi_free a pointer that does not point to a valid heap space: %p\n", p); From 1e27cef873f6db6b8548a2278323f34e665d52d2 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 21 Jul 2019 23:21:14 +0800 Subject: [PATCH 17/67] Enforce strict include-what-you-use policy The include-what-you-use (IWYU) policy is beneficial to faster compilation and fewer recompilations. Many build tools, such as GNU make, provide a mechanism for automatically figuring out what .h files a .cc file depends on. These mechanisms typically look at #include lines. When unnecessary #includes are listed, the build system is more likely to recompile in cases where it is not necessary. With the enforcement, header file no longer includes . Reference: https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/WhyIWYU.md --- include/mimalloc-types.h | 1 - include/mimalloc.h | 1 - src/alloc-aligned.c | 2 +- src/alloc.c | 5 +++-- src/init.c | 3 ++- src/options.c | 3 ++- src/os.c | 2 +- src/page.c | 2 -- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 2f16020f..613aae82 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -8,7 +8,6 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_TYPES_H #define MIMALLOC_TYPES_H -#include // size_t etc. #include // ptrdiff_t #include // uintptr_t, uint16_t, etc diff --git a/include/mimalloc.h b/include/mimalloc.h index ec09f119..e8a40ab6 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -69,7 +69,6 @@ terms of the MIT license. A copy of the license can be found in the file // Includes // ------------------------------------------------------ -#include // size_t, malloc etc. #include // bool #include // FILE diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 3ef93c83..4fcf433a 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memset +#include // memset, memcpy // ------------------------------------------------------ // Aligned Allocation diff --git a/src/alloc.c b/src/alloc.c index d5050b03..a5078dc5 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -8,7 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include // memset +#include // memset, memcpy, strlen +#include // malloc, exit #define MI_IN_ALLOC_C #include "alloc-override.c" @@ -463,7 +464,7 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) } } #else -#include +#include // pathconf static size_t mi_path_max() { static size_t path_max = 0; if (path_max <= 0) { diff --git a/src/init.c b/src/init.c index 5b2d3c8e..12dcd5dc 100644 --- a/src/init.c +++ b/src/init.c @@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memcpy +#include // memcpy, memset +#include // atexit // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { diff --git a/src/options.c b/src/options.c index 0f588740..02ca4800 100644 --- a/src/options.c +++ b/src/options.c @@ -8,7 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include -#include // strcmp +#include // strtol +#include // strncpy, strncat, strlen, strstr #include // toupper #include diff --git a/src/os.c b/src/os.c index f9705992..950676ed 100644 --- a/src/os.c +++ b/src/os.c @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memset +#include // strerror #include #if defined(_WIN32) diff --git a/src/page.c b/src/page.c index fc7c4f01..1ac026f3 100644 --- a/src/page.c +++ b/src/page.c @@ -15,8 +15,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include // memset, memcpy - /* ----------------------------------------------------------- Definition of page queues for each block size ----------------------------------------------------------- */ From 069f52523f148c5d5d157f54481024befdba74b9 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 17:08:09 -0700 Subject: [PATCH 18/67] use hinted address to mmap to reduce mmap calls --- src/os.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/os.c b/src/os.c index 3527c94d..3b92e641 100644 --- a/src/os.c +++ b/src/os.c @@ -10,6 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // memset #include @@ -242,6 +243,23 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { return (void*) aligned_base; } #else +static void* mi_unix_mmapx(size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + void* p = NULL; + #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + // on 64-bit systems, use a special area for 4MiB aligned allocations + static volatile intptr_t aligned_base = ((intptr_t)1 << 42); // starting at 4TiB + if (try_alignment <= MI_SEGMENT_SIZE && (size%MI_SEGMENT_SIZE)==0 && (aligned_base%try_alignment)==0) { + intptr_t hint = mi_atomic_add(&aligned_base,size) - size; + p = mmap((void*)hint,size,protect_flags,flags,fd,0); + if (p==MAP_FAILED) p = NULL; // fall back to regular mmap + } + #endif + if (p==NULL) { + p = mmap(NULL,size,protect_flags,flags,fd,0); + } + return p; +} + static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) { void* p = NULL; #if !defined(MAP_ANONYMOUS) @@ -278,12 +296,12 @@ static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) // try large page allocation // TODO: if always failing due to permissions or no huge pages, try to avoid repeatedly trying? // Should we check this in _mi_os_init? (as on Windows) - p = mmap(NULL, size, protect_flags, lflags, fd, 0); + p = mi_unix_mmapx(size, try_alignment, protect_flags, lflags, fd); if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission } } if (p == NULL) { - p = mmap(NULL, size, protect_flags, flags, -1, 0); + p = mi_unix_mmapx(size, try_alignment, protect_flags, flags, -1); if (p == MAP_FAILED) p = NULL; } return p; @@ -439,7 +457,7 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -// Commit/Decommit memory. +// Commit/Decommit memory. // Usuelly commit is aligned liberal, while decommit is aligned conservative. // (but not for the reset version where we want commit to be conservative as well) static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, mi_stats_t* stats) { @@ -503,7 +521,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! - #if (MI_DEBUG>1) + #if (MI_DEBUG>1) if (!mi_option_is_enabled(mi_option_secure)) { memset(start, 0, csize); // pretend it is eagerly reset } @@ -521,7 +539,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); mi_assert_internal(p == start); if (p != start) return false; - } + } #else #if defined(MADV_FREE) static int advice = MADV_FREE; From 272a8e03e4ac01d517de36ecf0026e3153eb3187 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 17:13:36 -0700 Subject: [PATCH 19/67] use atomic ops to guard large page tries on windows --- src/os.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index 3b92e641..78e638c2 100644 --- a/src/os.c +++ b/src/os.c @@ -206,20 +206,21 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment } static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) { - static size_t large_page_try_ok = 0; + static volatile uintptr_t large_page_try_ok = 0; void* p = NULL; if (use_large_os_page(size, try_alignment)) { - if (large_page_try_ok > 0) { + uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); + if (try_ok > 0) { // if a large page page allocation fails, it seems the calls to VirtualAlloc get very expensive. // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. - large_page_try_ok--; + mi_atomic_compare_exchange(&large_page_try_ok, try_ok - 1, try_ok); } else { // large OS pages must always reserve and commit. p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE | flags); // fall back to non-large page allocation on error (`p == NULL`). if (p == NULL) { - large_page_try_ok = 10; // on error, don't try again for the next N allocations + mi_atomic_write(&large_page_try_ok,10); // on error, don't try again for the next N allocations } } } From 7091670c22f5c5e0bbd42a9b2cbf1c98df7cfc94 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 17:14:13 -0700 Subject: [PATCH 20/67] trailing id after #endif --- include/mimalloc-override.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index f6149514..d81063ab 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -75,7 +75,7 @@ including this header is not necessary. // ------------------------------------------------------ #ifdef __cplusplus #include - + void operator delete(void* p) noexcept { mi_free(p); }; void operator delete[](void* p) noexcept { mi_free(p); }; @@ -103,4 +103,4 @@ including this header is not necessary. #endif #endif -#endif MIMALLOC_OVERRIDE_H +#endif // MIMALLOC_OVERRIDE_H From e4caee5f55f8a71e07890577d2095248911aec33 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 01:36:16 -0700 Subject: [PATCH 21/67] update test files and overriding --- ide/vs2017/mimalloc-override-test.vcxproj | 15 ++------- .../mimalloc-override-test.vcxproj.filters | 3 -- ide/vs2017/mimalloc-test.vcxproj | 6 ++-- ide/vs2017/mimalloc-test.vcxproj.filters | 2 +- include/mimalloc-override.h | 12 ++++--- test/CMakeLists.txt | 19 ++++++++---- test/main-override-static.c | 31 +++++++++++++++++++ test/main-override.c | 7 ++--- test/main-override.cpp | 13 +++----- 9 files changed, 67 insertions(+), 41 deletions(-) create mode 100644 test/main-override-static.c diff --git a/ide/vs2017/mimalloc-override-test.vcxproj b/ide/vs2017/mimalloc-override-test.vcxproj index 77752890..7df1e79a 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj +++ b/ide/vs2017/mimalloc-override-test.vcxproj @@ -172,23 +172,14 @@ COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect.dll $(OutputPath) - - - true - true - true - true - - - false - false - - {abb5eae7-b3e6-432e-b636-333449892ea7} + + + diff --git a/ide/vs2017/mimalloc-override-test.vcxproj.filters b/ide/vs2017/mimalloc-override-test.vcxproj.filters index 80f1c9c0..eb5e70b7 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-override-test.vcxproj.filters @@ -18,8 +18,5 @@ Source Files - - Source Files - \ No newline at end of file diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index 8e61a97f..c1539aeb 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -144,14 +144,14 @@ Console - - - {abb5eae7-b3e6-432e-b636-333449892ea6} + + + diff --git a/ide/vs2017/mimalloc-test.vcxproj.filters b/ide/vs2017/mimalloc-test.vcxproj.filters index eb5e70b7..fca75e1c 100644 --- a/ide/vs2017/mimalloc-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-test.vcxproj.filters @@ -15,7 +15,7 @@ - + Source Files diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index d81063ab..c3348068 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -69,11 +69,15 @@ including this header is not necessary. #define _aligned_offset_recalloc(p,s,n,a,o) mi_recalloc_aligned_at(p,s,n,a,o) -// ------------------------------------------------------ -// With a C++ compiler we override the new/delete operators. +// ----------------------------------------------------------------- +// With a C++ compiler we can override all the new/delete operators +// by defining 'MIMALLOC_DEFINE_NEW_DELETE' in some source file and +// then including this header file. This is not needed when linking +// statically with the mimalloc library, but it can be more performant +// on Windows when using dynamic overiding as well. // see -// ------------------------------------------------------ -#ifdef __cplusplus +// ----------------------------------------------------------------- +#if defined(__cplusplus) && defined(MIMALLOC_DEFINE_NEW_DELETE) #include void operator delete(void* p) noexcept { mi_free(p); }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 42d4a2f4..8a830073 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,14 +24,21 @@ target_link_libraries(dynamic-override PUBLIC mimalloc) add_executable(dynamic-override-cxx main-override.cpp) target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) -# with a static library +# overriding with a static object file works reliable as the symbols in the +# object file have priority over those in library files +add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) +target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) +target_link_libraries(static-override-obj PUBLIC pthread) + +# overriding with a static library works too if using the `mimalloc-override.h` +# header to redefine malloc/free. +add_executable(static-override-static main-override-static.c) +target_link_libraries(static-override-static PUBLIC mimalloc-static) + + +# overriding with a static library: this may not work if the library is linked too late add_executable(static-override main-override.c) target_link_libraries(static-override PUBLIC mimalloc-static) add_executable(static-override-cxx main-override.cpp) target_link_libraries(static-override-cxx PUBLIC mimalloc-static) - -# and with a static object file -add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) -target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) -target_link_libraries(static-override-obj PUBLIC pthread) diff --git a/test/main-override-static.c b/test/main-override-static.c new file mode 100644 index 00000000..6ddf4f37 --- /dev/null +++ b/test/main-override-static.c @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +#include +#include // redefines malloc etc. + +int main() { + mi_version(); + void* p1 = malloc(78); + void* p2 = malloc(24); + free(p1); + p1 = malloc(8); + //char* s = strdup("hello\n"); + free(p2); + p2 = malloc(16); + p1 = realloc(p1, 32); + free(p1); + free(p2); + //free(s); + //mi_collect(true); + + /* now test if override worked by allocating/freeing across the api's*/ + //p1 = mi_malloc(32); + //free(p1); + //p2 = malloc(32); + //mi_free(p2); + mi_stats_print(NULL); + return 0; +} diff --git a/test/main-override.c b/test/main-override.c index ddb2f16e..1bec1179 100644 --- a/test/main-override.c +++ b/test/main-override.c @@ -3,11 +3,10 @@ #include #include -//#include - +#include int main() { - //mi_stats_reset(); + mi_version(); // ensure mimalloc library is linked void* p1 = malloc(78); void* p2 = malloc(24); free(p1); @@ -26,6 +25,6 @@ int main() { //free(p1); //p2 = malloc(32); //mi_free(p2); - + mi_stats_print(NULL); return 0; } diff --git a/test/main-override.cpp b/test/main-override.cpp index 8f47dcd1..fb7ab7a1 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -4,8 +4,6 @@ #include #include -#include - #include static void* p = malloc(8); @@ -24,16 +22,15 @@ public: }; -int main() { - //mi_malloc_override(); - mi_stats_reset(); +int main() { + mi_version(); atexit(free_p); void* p1 = malloc(78); - void* p2 = _aligned_malloc(24,16); + void* p2 = mi_malloc_aligned(16,24); free(p1); p1 = malloc(8); - char* s = _strdup("hello\n"); - _aligned_free(p2); + char* s = mi_strdup("hello\n"); + mi_free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); From e90938fb4bf9f7a3ce259d98f3e85576297a0a72 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 10:10:45 -0700 Subject: [PATCH 22/67] merge --- src/alloc-posix.c | 4 ++-- test/test-stress.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 4e844ba3..1f55b3a8 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -19,6 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #include // memcpy +#include // getenv #ifndef EINVAL #define EINVAL 22 @@ -115,7 +116,7 @@ int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { #pragma warning(suppress:4996) char* p = getenv(name); if (p==NULL) { - *buf = NULL; + *buf = NULL; } else { *buf = mi_strdup(p); @@ -146,4 +147,3 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) return 0; #endif } - diff --git a/test/test-stress.c b/test/test-stress.c index b26dfd04..298e5a17 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -180,7 +180,7 @@ static DWORD WINAPI thread_entry(LPVOID param) { static void run_os_threads(size_t nthreads) { DWORD* tids = (DWORD*)malloc(nthreads * sizeof(DWORD)); HANDLE* thandles = (HANDLE*)malloc(nthreads * sizeof(HANDLE)); - for (intptr_t i = 0; i < nthreads; i++) { + for (uintptr_t i = 0; i < nthreads; i++) { thandles[i] = CreateThread(0, 4096, &thread_entry, (void*)(i), 0, &tids[i]); } for (int i = 0; i < nthreads; i++) { From 86cadca059b2bf80cd94aa05d6f04813d144c482 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 16:11:06 -0700 Subject: [PATCH 23/67] more comments --- test/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8a830073..8bf36521 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,29 +14,31 @@ endif() # Import mimalloc (if installed) find_package(mimalloc 1.0 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) +message(STATUS "Found mimalloc installed at: ${MIMALLOC_TARGET_DIR}") -message(STATUS "${MIMALLOC_INCLUDE_DIR}") - -# Tests +# overriding with a dynamic library add_executable(dynamic-override main-override.c) target_link_libraries(dynamic-override PUBLIC mimalloc) add_executable(dynamic-override-cxx main-override.cpp) target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) + # overriding with a static object file works reliable as the symbols in the # object file have priority over those in library files add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) target_link_libraries(static-override-obj PUBLIC pthread) + # overriding with a static library works too if using the `mimalloc-override.h` -# header to redefine malloc/free. +# header to redefine malloc/free. (the library already overrides new/delete) add_executable(static-override-static main-override-static.c) target_link_libraries(static-override-static PUBLIC mimalloc-static) # overriding with a static library: this may not work if the library is linked too late +# on the command line after the C runtime library; but we cannot control that well in CMake add_executable(static-override main-override.c) target_link_libraries(static-override PUBLIC mimalloc-static) From 7c26ce9280e306d9d476bbbc26fa2a95ad71cadc Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 10:27:14 -0700 Subject: [PATCH 24/67] ensure C++ compilation on windows --- ide/vs2017/mimalloc-override.vcxproj | 8 ++++---- ide/vs2017/mimalloc.vcxproj | 4 ++-- include/mimalloc-override.h | 2 +- src/init.c | 6 ++++++ test/main-override.cpp | 6 +++--- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 3ca8158a..f41b2efc 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -100,7 +100,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -121,7 +121,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) @@ -152,7 +152,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp true @@ -177,7 +177,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp true diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index a8cb7566..3e453471 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -154,7 +154,7 @@ Neither false false - Default + CompileAsCpp true @@ -185,7 +185,7 @@ Neither false false - Default + CompileAsCpp true diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index c3348068..56b41e6b 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file This header can be used to statically redirect malloc/free and new/delete to the mimalloc variants. This can be useful if one can include this file on each source file in a project (but be careful when using external code to -not accidentally mix pointer from different allocators). +not accidentally mix pointers from different allocators). On windows it can still be good to always try to include this header even when dynamically overriding since this will give better performance especially diff --git a/src/init.c b/src/init.c index d00d7c05..152e906b 100644 --- a/src/init.c +++ b/src/init.c @@ -384,12 +384,18 @@ bool _mi_preloading() { // Communicate with the redirection module on Windows #if defined(_WIN32) && defined(MI_SHARED_LIB) +#ifdef __cplusplus +extern "C" { +#endif mi_decl_export void _mi_redirect_init() { // called on redirection mi_redirected = true; } __declspec(dllimport) bool mi_allocator_init(const char** message); __declspec(dllimport) void mi_allocator_done(); +#ifdef __cplusplus +} +#endif #else static bool mi_allocator_init(const char** message) { if (message != NULL) *message = NULL; diff --git a/test/main-override.cpp b/test/main-override.cpp index fb7ab7a1..6c7fc0d5 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -23,14 +23,14 @@ public: int main() { - mi_version(); + mi_stats_reset(); // ignore earlier allocations atexit(free_p); void* p1 = malloc(78); void* p2 = mi_malloc_aligned(16,24); free(p1); p1 = malloc(8); char* s = mi_strdup("hello\n"); - mi_free(p2); + free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); @@ -39,7 +39,7 @@ int main() { Test* t = new Test(42); delete t; t = new (std::nothrow) Test(42); - delete t; + delete t; return 0; } From f0530b6a83b6b060a36f92d7e82f2d72264af646 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 20:51:12 -0700 Subject: [PATCH 25/67] small optimizations, use bitwise aligne --- CMakeLists.txt | 1 + include/mimalloc-internal.h | 26 +++++++++++++++++++++++++- include/mimalloc-types.h | 11 ++++++----- include/mimalloc.h | 6 +++--- src/alloc.c | 6 +++--- src/init.c | 8 +++++--- src/os.c | 7 ------- src/page.c | 14 ++++++++++---- src/segment.c | 12 ++++++------ 9 files changed, 59 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d86d096b..ec0fd99a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) list(APPEND mi_cflags -fvisibility=hidden) + list(APPEND mi_cflags -fbranch-target-load-optimize ) endif() endif() diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cbed5909..e261dba2 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -39,7 +39,6 @@ bool _mi_preloading(); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); -uintptr_t _mi_align_up(uintptr_t sz, size_t alignment); void _mi_os_init(void); // called from process init void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data @@ -165,6 +164,20 @@ static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) { #endif } +// Align upwards +static inline uintptr_t _mi_is_power_of_two(uintptr_t x) { + return ((x & (x - 1)) == 0); +} +static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((sz + mask) & ~mask); + } + else { + return (((sz + mask)/alignment)*alignment); + } +} + // Align a byte size to a size in _machine words_, // i.e. byte size == `wsize*sizeof(void*)`. static inline size_t _mi_wsize_from_size(size_t size) { @@ -324,12 +337,23 @@ static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, mi_bl } static inline mi_block_t* mi_block_next(mi_page_t* page, mi_block_t* block) { + #if MI_SECURE return mi_block_nextx(page->cookie,block); + #else + UNUSED(page); + return mi_block_nextx(0, block); + #endif } static inline void mi_block_set_next(mi_page_t* page, mi_block_t* block, mi_block_t* next) { + #if MI_SECURE mi_block_set_nextx(page->cookie,block,next); + #else + UNUSED(page); + mi_block_set_nextx(0, block, next); + #endif } + // ------------------------------------------------------------------- // Getting the thread id should be performant // as it is called in the fast path of `_mi_free`, diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 7221f5b8..5c14ffd4 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -132,10 +132,9 @@ typedef union mi_page_flags_u { } mi_page_flags_t; // Thread free list. -// We use bottom 2 bits of the pointer for mi_delayed_t flags +// We use the bottom 2 bits of the pointer for mi_delayed_t flags typedef uintptr_t mi_thread_free_t; - // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: // `free` for blocks that can be allocated, @@ -165,9 +164,11 @@ typedef struct mi_page_s { mi_page_flags_t flags; uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory - + mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) + #if MI_SECURE uintptr_t cookie; // random cookie to encode the free lists + #endif size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) @@ -182,9 +183,9 @@ typedef struct mi_page_s { // improve page index calculation #if MI_INTPTR_SIZE==8 - //void* padding[1]; // 10 words on 64-bit + //void* padding[1]; // 12 words on 64-bit #elif MI_INTPTR_SIZE==4 - void* padding[1]; // 12 words on 32-bit + void* padding[1]; // 12 words on 32-bit #endif } mi_page_t; diff --git a/include/mimalloc.h b/include/mimalloc.h index e7e83791..c6b7b5f8 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -52,8 +52,8 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #else #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) - #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) - #define mi_cdecl // leads to warnings... __attribute__((cdecl)) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_cdecl // leads to warnings... __attribute__((cdecl)) #endif #else #define mi_decl_thread __thread @@ -62,7 +62,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_malloc #define mi_attr_alloc_size(s) #define mi_attr_alloc_size2(s1,s2) - #define mi_cdecl + #define mi_cdecl #endif // ------------------------------------------------------ diff --git a/src/alloc.c b/src/alloc.c index 649b6e95..6a91c0ad 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -237,9 +237,9 @@ void mi_free(void* p) mi_attr_noexcept #endif // adjust if it might be an un-aligned block - if (mi_likely(page->flags.value==0)) { // note: merging both tests (local | value) does not matter for performance + if (mi_likely(page->flags.value==0)) { // not full or aligned mi_block_t* block = (mi_block_t*)p; - if (mi_likely(local)) { + if (mi_likely(local)) { // note: merging both tests (local | value) does not matter for performance // owning thread can free a block directly mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance page->local_free = block; @@ -248,7 +248,7 @@ void mi_free(void* p) mi_attr_noexcept } else { // use atomic operations for a multi-threaded free - _mi_free_block_mt(page, block); + _mi_free_block_mt(page, block); } } else { diff --git a/src/init.c b/src/init.c index 152e906b..44e3c9cb 100644 --- a/src/init.c +++ b/src/init.c @@ -12,9 +12,11 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { - 0, false, false, false, {0}, - 0, 0, - NULL, 0, 0, // free, used, cookie + 0, false, false, false, {0}, 0, 0, + NULL, 0, // free, used + #if MI_SECURE + 0, + #endif NULL, 0, 0, 0, NULL, NULL, NULL #if (MI_INTPTR_SIZE==4) diff --git a/src/os.c b/src/os.c index bcdd0ea4..b0eab87d 100644 --- a/src/os.c +++ b/src/os.c @@ -34,13 +34,6 @@ terms of the MIT license. A copy of the license can be found in the file ----------------------------------------------------------- */ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); -uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { - uintptr_t x = (sz / alignment) * alignment; - if (x < sz) x += alignment; - if (x < sz) return 0; // overflow - return x; -} - static void* mi_align_up_ptr(void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } diff --git a/src/page.c b/src/page.c index b0c0b382..d46a5aad 100644 --- a/src/page.c +++ b/src/page.c @@ -93,7 +93,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) { bool _mi_page_is_valid(mi_page_t* page) { mi_assert_internal(mi_page_is_valid_init(page)); + #if MI_SECURE mi_assert_internal(page->cookie != 0); + #endif if (page->heap!=NULL) { mi_segment_t* segment = _mi_page_segment(page); mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id); @@ -119,7 +121,7 @@ void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay ) { else if (mi_unlikely(mi_tf_delayed(tfree) == MI_DELAYED_FREEING)) { mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. continue; // and try again - } + } } while((mi_tf_delayed(tfreex) != mi_tf_delayed(tfree)) && // avoid atomic operation if already equal !mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex, tfree)); @@ -258,7 +260,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) { mi_block_t* next = mi_block_nextx(heap->cookie,block); // use internal free instead of regular one to keep stats etc correct if (!_mi_free_delayed_block(block)) { - // we might already start delayed freeing while another thread has not yet + // we might already start delayed freeing while another thread has not yet // reset the delayed_freeing flag; in that case delay it further by reinserting. mi_block_t* dfree; do { @@ -498,7 +500,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st if (page->capacity >= page->reserved) return; size_t page_size; - _mi_page_start(_mi_page_segment(page), page, &page_size); + _mi_page_start(_mi_page_segment(page), page, &page_size); _mi_stat_increase(&stats->pages_extended, 1); // calculate the extend count @@ -533,7 +535,9 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi page->block_size = block_size; mi_assert_internal(page_size / block_size < (1L<<16)); page->reserved = (uint16_t)(page_size / block_size); + #if MI_SECURE page->cookie = _mi_heap_random(heap) | 1; + #endif mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); @@ -543,7 +547,9 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page->next == NULL); mi_assert_internal(page->prev == NULL); mi_assert_internal(page->flags.has_aligned == false); + #if MI_SECURE mi_assert_internal(page->cookie != 0); + #endif mi_assert_expensive(mi_page_is_valid_init(page)); // initialize an initial free list @@ -683,7 +689,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); mi_heap_stat_increase( heap, huge, block_size); - } + } return page; } diff --git a/src/segment.c b/src/segment.c index 7f7bedd7..8f254a26 100644 --- a/src/segment.c +++ b/src/segment.c @@ -235,8 +235,8 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se // The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use, -// and no more than 4. -#define MI_SEGMENT_CACHE_MAX (4) +// and no more than 2. +#define MI_SEGMENT_CACHE_MAX (2) #define MI_SEGMENT_CACHE_FRACTION (8) // note: returned segment may be partially reset @@ -252,7 +252,7 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t } static bool mi_segment_cache_full(mi_segments_tld_t* tld) { - if (tld->cache_count < MI_SEGMENT_CACHE_MAX && + if (tld->cache_count < MI_SEGMENT_CACHE_MAX && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { // always allow 1 element cache return false; } @@ -318,7 +318,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_size = (page_kind == MI_PAGE_HUGE ? segment_size : (size_t)1 << page_shift); // Try to get it from our thread local cache first - bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM); + bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM); bool protection_still_good = false; mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld); if (segment != NULL) { @@ -702,10 +702,10 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; - if (block_size <= (MI_SMALL_PAGE_SIZE/16)*3) { + if (block_size <= (MI_SMALL_PAGE_SIZE/4)) { page = mi_segment_small_page_alloc(tld,os_tld); } - else if (block_size <= (MI_MEDIUM_PAGE_SIZE/16)*3) { + else if (block_size <= (MI_MEDIUM_PAGE_SIZE/4)) { page = mi_segment_medium_page_alloc(tld, os_tld); } else if (block_size < (MI_LARGE_SIZE_MAX - sizeof(mi_segment_t))) { From 6ca8b3fd89a5192ae794a8ea9483695c94b0d1e7 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 21:25:57 -0700 Subject: [PATCH 26/67] remove old comment --- src/page.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/page.c b/src/page.c index d46a5aad..69d32bfe 100644 --- a/src/page.c +++ b/src/page.c @@ -733,11 +733,4 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept // and try again, this time succeeding! (i.e. this should never recurse) return _mi_page_malloc(heap, page, size); - /* - if (page->used == page->reserved) { - // needed for huge pages to free reliably from other threads. - mi_page_to_full(page,mi_page_queue_of(page)); - } - return p; - */ } From 2b911b0b10d2c4b36d5c15b0f21e3a80f0a48a76 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 09:36:58 -0700 Subject: [PATCH 27/67] fix cmake build on windows --- CMakeLists.txt | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec0fd99a..45d5f988 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,14 +121,28 @@ add_library(mimalloc SHARED ${mi_sources}) set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} NO_SONAME "YES" OUTPUT_NAME ${mi_basename} ) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) +target_link_libraries(mimalloc PUBLIC ${mi_libraries}) target_include_directories(mimalloc PUBLIC $ $ ) -target_link_libraries(mimalloc PUBLIC ${mi_libraries}) +if(WIN32) + # On windows copy the mimalloc redirection dll too. + target_link_libraries(mimalloc PRIVATE ../../bin/mimalloc-redirect) + add_custom_command(TARGET mimalloc POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy "../../bin/mimalloc-redirect.dll" $ + COMMENT "Copy mimalloc-redirect.dll to output directory") +endif() # static library add_library(mimalloc-static STATIC ${mi_sources}) +target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) +target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) +target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) +target_include_directories(mimalloc-static PUBLIC + $ + $ +) if(WIN32) # When building both static and shared libraries on Windows, a static library should use a # different output name to avoid the conflict with the import library of a shared one. @@ -137,14 +151,6 @@ if(WIN32) else() set_target_properties(mimalloc-static PROPERTIES OUTPUT_NAME ${mi_basename}) endif() -target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) -target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) - -target_include_directories(mimalloc-static PUBLIC - $ - $ -) -target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) # install static and shared library, and the include files install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_dir} LIBRARY NAMELINK_SKIP) From 48a3d0c6e288ee7c0ed3d6fdd9d917b01bc4000a Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 09:37:36 -0700 Subject: [PATCH 28/67] fix 32-bit build of stress test --- test/test-stress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index 298e5a17..511679ac 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -27,7 +27,7 @@ static int N = 10; // scaling factor static volatile void* transfer[TRANSFERS]; -#if (INTPTR_MAX != UINT32_MAX) +#if (UINTPTR_MAX != UINT32_MAX) const uintptr_t cookie = 0xbf58476d1ce4e5b9UL; #else const uintptr_t cookie = 0x1ce4e5b9UL; @@ -39,7 +39,7 @@ typedef uintptr_t* random_t; static uintptr_t pick(random_t r) { uintptr_t x = *r; - #if (INTPTR_MAX > UINT32_MAX) + #if (UINTPTR_MAX > UINT32_MAX) // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; @@ -183,7 +183,7 @@ static void run_os_threads(size_t nthreads) { for (uintptr_t i = 0; i < nthreads; i++) { thandles[i] = CreateThread(0, 4096, &thread_entry, (void*)(i), 0, &tids[i]); } - for (int i = 0; i < nthreads; i++) { + for (size_t i = 0; i < nthreads; i++) { WaitForSingleObject(thandles[i], INFINITE); } } From 663e63aac20639ecf0af27c2d26e66821f942a0b Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 09:59:20 -0700 Subject: [PATCH 29/67] ensure cmake uses C++ compilation with MSVC --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45d5f988..980ab542 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,6 @@ set(mi_sources src/options.c src/init.c) - # Set default build type if (NOT CMAKE_BUILD_TYPE) if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$") @@ -44,6 +43,11 @@ if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") set(MI_SECURE "ON") endif() +if(CMAKE_C_COMPILER_ID MATCHES "MSVC") + set(MI_USE_CXX "ON") +endif() + + # Options if(MI_OVERRIDE MATCHES "ON") message(STATUS "Override standard malloc (MI_OVERRIDE=ON)") @@ -78,7 +82,7 @@ endif() if(MI_USE_CXX MATCHES "ON") message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) - set_source_files_properties(src/static.c test/test-api.c PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c test/test-stress.c PROPERTIES LANGUAGE CXX ) endif() # Compiler flags From 0a81d26c83b4a5cceb47bc212826df6f34940559 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 12:40:42 -0700 Subject: [PATCH 30/67] fix mimalloc-redirect path on windows cmake build --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 980ab542..8b37e579 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,9 +132,9 @@ target_include_directories(mimalloc PUBLIC ) if(WIN32) # On windows copy the mimalloc redirection dll too. - target_link_libraries(mimalloc PRIVATE ../../bin/mimalloc-redirect) + target_link_libraries(mimalloc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect.lib) add_custom_command(TARGET mimalloc POST_BUILD - COMMAND "${CMAKE_COMMAND}" -E copy "../../bin/mimalloc-redirect.dll" $ + COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect.dll" $ COMMENT "Copy mimalloc-redirect.dll to output directory") endif() From 46b11fa0a48ed34bb753470668bff5c6551dfab9 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 15 Jul 2019 10:07:23 -0700 Subject: [PATCH 31/67] add comment about large pages --- src/options.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/options.c b/src/options.c index 46f0a36e..5c4548c2 100644 --- a/src/options.c +++ b/src/options.c @@ -36,6 +36,7 @@ typedef struct mi_option_desc_s { const char* name; // option name without `mimalloc_` prefix } mi_option_desc_t; +<<<<<<< HEAD static mi_option_desc_t options[_mi_option_last] = { // stable options @@ -43,6 +44,16 @@ static mi_option_desc_t options[_mi_option_last] = { MI_DEBUG, UNINIT, "show_errors" }, { 0, UNINIT, "verbose" }, +======= +static mi_option_desc_t options[_mi_option_last] = { + { 0, UNINIT, "page_reset" }, + { 0, UNINIT, "cache_reset" }, + { 0, UNINIT, "pool_commit" }, + { 0, UNINIT, "eager_commit" }, // secure and large pages must have eager commit + { 0, UNINIT, "large_os_pages" }, // use large OS pages + { 0, UNINIT, "reset_decommits" }, + { 0, UNINIT, "reset_discards" }, +>>>>>>> add comment about large pages #if MI_SECURE { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored #else From 8dba36bcecf33aaa743e75baf5996bfb9f898ead Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 7 Jul 2019 12:56:40 +0800 Subject: [PATCH 32/67] Use checked unsigned multiplication extension of GCC/Clang Most processors have carry flags which they set on addition overflow, so it is a good idea to access them whenever possible. Most of them also have widening multiply instructions that can be used to detect overflow of the non-widening version. Both GCC and Clang offer a way to detect an overflow for security critical applications. Reference: https://clang.llvm.org/docs/LanguageExtensions.html#checked-arithmetic-builtins --- include/mimalloc-internal.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1d380e8f..cbed5909 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -117,6 +117,9 @@ bool _mi_page_is_valid(mi_page_t* page); #define mi_likely(x) (x) #endif +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif #if defined(_MSC_VER) #define mi_decl_noinline __declspec(noinline) @@ -149,9 +152,17 @@ bool _mi_page_is_valid(mi_page_t* page); // Overflow detecting multiply #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) { +#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 +#if (MI_INTPTR_SIZE == 4) + return __builtin_umul_overflow(size, count, total); +#else + return __builtin_umull_overflow(size, count, total); +#endif +#else /* __builtin_umul_overflow is unavailable */ *total = size * count; return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); +#endif } // Align a byte size to a size in _machine words_, From 7266e7006a8a8d8a79d1fc99796599f1f5967daf Mon Sep 17 00:00:00 2001 From: caixiangyue Date: Fri, 19 Jul 2019 16:23:14 +0800 Subject: [PATCH 33/67] fix typo --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 3527c94d..6b99e9a8 100644 --- a/src/os.c +++ b/src/os.c @@ -209,7 +209,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, void* p = NULL; if (use_large_os_page(size, try_alignment)) { if (large_page_try_ok > 0) { - // if a large page page allocation fails, it seems the calls to VirtualAlloc get very expensive. + // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. large_page_try_ok--; } From 3e9c953eea501d9ab1796f7dbc50ab4957d3b25f Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 21 Jul 2019 21:42:00 +0800 Subject: [PATCH 34/67] Add branch prediction hint for mi_option_get mi_option_get is called frequently in stress tests, and the patch adds extra hint to the compiler to emit instructions that will cause branch prediction to favour the "likely" side of a jump instruction. --- src/options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 5c4548c2..826df946 100644 --- a/src/options.c +++ b/src/options.c @@ -79,7 +79,7 @@ static void mi_option_init(mi_option_desc_t* desc); long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); mi_option_desc_t* desc = &options[option]; - if (desc->init == UNINIT) { + if (mi_unlikely(desc->init == UNINIT)) { mi_option_init(desc); if (option != mi_option_verbose) { _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value); From 28b874129ad3ce0eab03385c797d16245a5af9fb Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 21 Jul 2019 22:20:05 +0800 Subject: [PATCH 35/67] Avoid using strlen function in loop --- src/options.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 826df946..63f470ea 100644 --- a/src/options.c +++ b/src/options.c @@ -214,10 +214,16 @@ static const char* mi_getenv(const char* name) { #pragma warning(suppress:4996) const char* s = getenv(name); if (s == NULL) { +<<<<<<< HEAD char buf[64+1]; mi_strlcpy(buf,name,64); for (size_t i = 0; i < strlen(buf); i++) { buf[i] = toupper(name[i]); +======= + size_t buf_size = strlen(buf); + for (size_t i = 0; i < buf_size; i++) { + buf[i] = toupper(buf[i]); +>>>>>>> Avoid using strlen function in loop } #pragma warning(suppress:4996) s = getenv(buf); @@ -234,7 +240,8 @@ static void mi_option_init(mi_option_desc_t* desc) { const char* s = mi_getenv(buf); if (s != NULL) { mi_strlcpy(buf, s, sizeof(buf)); - for (size_t i = 0; i < strlen(buf); i++) { + size_t buf_size = strlen(buf); // TODO: use strnlen? + for (size_t i = 0; i < buf_size; i++) { buf[i] = toupper(buf[i]); } if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) { From 2eb607d03fe57f74da253c397461daef035b2bde Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 13:03:51 -0700 Subject: [PATCH 36/67] re-add missing thread_init needed when running in debug mode --- src/heap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heap.c b/src/heap.c index 48bb9830..e55fdf34 100644 --- a/src/heap.c +++ b/src/heap.c @@ -172,7 +172,7 @@ void mi_collect(bool force) mi_attr_noexcept { ----------------------------------------------------------- */ mi_heap_t* mi_heap_get_default(void) { - // mi_thread_init(); + mi_thread_init(); return mi_get_default_heap(); } From 65037d051bf257435d95b21a606fe6bdf5f91cf3 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 13:09:34 -0700 Subject: [PATCH 37/67] improved debug warning for freeing invalid pointers --- src/alloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index bac925ee..099bdbad 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -211,8 +211,11 @@ void mi_free(void* p) mi_attr_noexcept #if (MI_DEBUG>0) if (mi_unlikely(!mi_is_in_heap_region(p))) { - _mi_warning_message("possibly trying to mi_free a pointer that does not point to a valid heap region: %p\n" + _mi_warning_message("possibly trying to mi_free a pointer that does not point to a valid heap region: 0x%p\n" "(this may still be a valid very large allocation (over 64MiB))\n", p); + if (mi_likely(_mi_ptr_cookie(segment) == segment->cookie)) { + _mi_warning_message("(yes, the previous pointer 0x%p was valid after all)\n", p); + } } if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { _mi_error_message("trying to mi_free a pointer that does not point to a valid heap space: %p\n", p); From fe229f8fad24eca5dc4953c93bbc87795e6d819a Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 22 Jul 2019 04:45:40 +0800 Subject: [PATCH 38/67] Fix path name in documentation about macOS --- doc/mimalloc-doc.h | 2 +- docs/overrides.html | 2 +- readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 16327d2c..57b7dd4c 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -808,7 +808,7 @@ library so all calls to the standard `malloc` interface are resolved to the _mimalloc_ library. - `env LD_PRELOAD=/usr/lib/libmimalloc.so myprogram` (on Linux, BSD, etc.) -- `env DYLD_INSERT_LIBRARIES=usr/lib/libmimalloc.dylib myprogram` (On macOS) +- `env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram` (On macOS) Note certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). diff --git a/docs/overrides.html b/docs/overrides.html index 16400375..2360a936 100644 --- a/docs/overrides.html +++ b/docs/overrides.html @@ -109,7 +109,7 @@ $(document).ready(function(){initNavTree('overrides.html','');});

On these systems we preload the mimalloc shared library so all calls to the standard malloc interface are resolved to the mimalloc library.

  • env LD_PRELOAD=/usr/lib/libmimalloc.so myprogram (on Linux, BSD, etc.)
    • -

  • env DYLD_INSERT_LIBRARIES=usr/lib/libmimalloc.dylib myprogram (On macOS)

    +

  • env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram (On macOS)

    Note certain security restrictions may apply when doing this from the shell.

diff --git a/readme.md b/readme.md index 85234c24..8ff19deb 100644 --- a/readme.md +++ b/readme.md @@ -191,7 +191,7 @@ library so all calls to the standard `malloc` interface are resolved to the _mimalloc_ library. - `env LD_PRELOAD=/usr/lib/libmimalloc.so myprogram` (on Linux, BSD, etc.) -- `env DYLD_INSERT_LIBRARIES=usr/lib/libmimalloc.dylib myprogram` (On macOS) +- `env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram` (On macOS) Note certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). From b5e26bedb1be3a8bd110b451bd1427fa737be1bb Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 21 Jul 2019 23:21:14 +0800 Subject: [PATCH 39/67] Enforce strict include-what-you-use policy The include-what-you-use (IWYU) policy is beneficial to faster compilation and fewer recompilations. Many build tools, such as GNU make, provide a mechanism for automatically figuring out what .h files a .cc file depends on. These mechanisms typically look at #include lines. When unnecessary #includes are listed, the build system is more likely to recompile in cases where it is not necessary. With the enforcement, header file no longer includes . Reference: https://github.com/include-what-you-use/include-what-you-use/blob/master/docs/WhyIWYU.md --- include/mimalloc-types.h | 1 - include/mimalloc.h | 1 - src/alloc-aligned.c | 2 +- src/alloc.c | 5 +++-- src/init.c | 3 ++- src/options.c | 3 ++- src/os.c | 2 +- src/page.c | 2 -- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index d591ff86..7221f5b8 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -8,7 +8,6 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_TYPES_H #define MIMALLOC_TYPES_H -#include // size_t etc. #include // ptrdiff_t #include // uintptr_t, uint16_t, etc diff --git a/include/mimalloc.h b/include/mimalloc.h index d92c2866..e7e83791 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -69,7 +69,6 @@ terms of the MIT license. A copy of the license can be found in the file // Includes // ------------------------------------------------------ -#include // size_t, malloc etc. #include // bool #include // FILE diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 175fa3e3..2f44f317 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memset +#include // memset, memcpy // ------------------------------------------------------ // Aligned Allocation diff --git a/src/alloc.c b/src/alloc.c index 099bdbad..649b6e95 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -8,7 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include // memset +#include // memset, memcpy, strlen +#include // malloc, exit #define MI_IN_ALLOC_C #include "alloc-override.c" @@ -465,7 +466,7 @@ char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) } } #else -#include +#include // pathconf static size_t mi_path_max() { static size_t path_max = 0; if (path_max <= 0) { diff --git a/src/init.c b/src/init.c index 06aa28c5..d00d7c05 100644 --- a/src/init.c +++ b/src/init.c @@ -7,7 +7,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memcpy +#include // memcpy, memset +#include // atexit // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { diff --git a/src/options.c b/src/options.c index 63f470ea..349599ad 100644 --- a/src/options.c +++ b/src/options.c @@ -9,7 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-atomic.h" #include -#include // strcmp +#include // strtol +#include // strncpy, strncat, strlen, strstr #include // toupper #include diff --git a/src/os.c b/src/os.c index 6b99e9a8..52fd4206 100644 --- a/src/os.c +++ b/src/os.c @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" -#include // memset +#include // strerror #include #if defined(_WIN32) diff --git a/src/page.c b/src/page.c index 685b6b4a..b0c0b382 100644 --- a/src/page.c +++ b/src/page.c @@ -15,8 +15,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc-internal.h" #include "mimalloc-atomic.h" -#include // memset, memcpy - /* ----------------------------------------------------------- Definition of page queues for each block size ----------------------------------------------------------- */ From 7b7c36c8c7511d6d8a718cd5d2e36db5d96a4812 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 17:08:09 -0700 Subject: [PATCH 40/67] use hinted address to mmap to reduce mmap calls --- src/os.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/os.c b/src/os.c index 52fd4206..a5f12c17 100644 --- a/src/os.c +++ b/src/os.c @@ -10,6 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-internal.h" +#include "mimalloc-atomic.h" #include // strerror #include @@ -242,6 +243,23 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { return (void*) aligned_base; } #else +static void* mi_unix_mmapx(size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + void* p = NULL; + #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + // on 64-bit systems, use a special area for 4MiB aligned allocations + static volatile intptr_t aligned_base = ((intptr_t)1 << 42); // starting at 4TiB + if (try_alignment <= MI_SEGMENT_SIZE && (size%MI_SEGMENT_SIZE)==0 && (aligned_base%try_alignment)==0) { + intptr_t hint = mi_atomic_add(&aligned_base,size) - size; + p = mmap((void*)hint,size,protect_flags,flags,fd,0); + if (p==MAP_FAILED) p = NULL; // fall back to regular mmap + } + #endif + if (p==NULL) { + p = mmap(NULL,size,protect_flags,flags,fd,0); + } + return p; +} + static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) { void* p = NULL; #if !defined(MAP_ANONYMOUS) @@ -278,12 +296,12 @@ static void* mi_unix_mmap(size_t size, size_t try_alignment, int protect_flags) // try large page allocation // TODO: if always failing due to permissions or no huge pages, try to avoid repeatedly trying? // Should we check this in _mi_os_init? (as on Windows) - p = mmap(NULL, size, protect_flags, lflags, fd, 0); + p = mi_unix_mmapx(size, try_alignment, protect_flags, lflags, fd); if (p == MAP_FAILED) p = NULL; // fall back to regular mmap if large is exhausted or no permission } } if (p == NULL) { - p = mmap(NULL, size, protect_flags, flags, -1, 0); + p = mi_unix_mmapx(size, try_alignment, protect_flags, flags, -1); if (p == MAP_FAILED) p = NULL; } return p; @@ -439,7 +457,7 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -// Commit/Decommit memory. +// Commit/Decommit memory. // Usuelly commit is aligned liberal, while decommit is aligned conservative. // (but not for the reset version where we want commit to be conservative as well) static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, mi_stats_t* stats) { @@ -503,7 +521,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! - #if (MI_DEBUG>1) + #if (MI_DEBUG>1) if (!mi_option_is_enabled(mi_option_secure)) { memset(start, 0, csize); // pretend it is eagerly reset } @@ -521,7 +539,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); mi_assert_internal(p == start); if (p != start) return false; - } + } #else #if defined(MADV_FREE) static int advice = MADV_FREE; From b611c7fb34a14ccea4038c1955e169f550ecdab5 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 17:13:36 -0700 Subject: [PATCH 41/67] use atomic ops to guard large page tries on windows --- src/os.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index a5f12c17..f7b36258 100644 --- a/src/os.c +++ b/src/os.c @@ -206,20 +206,26 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment } static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags) { - static size_t large_page_try_ok = 0; + static volatile uintptr_t large_page_try_ok = 0; void* p = NULL; if (use_large_os_page(size, try_alignment)) { +<<<<<<< HEAD if (large_page_try_ok > 0) { // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. +======= + uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); + if (try_ok > 0) { + // if a large page page allocation fails, it seems the calls to VirtualAlloc get very expensive. +>>>>>>> use atomic ops to guard large page tries on windows // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. - large_page_try_ok--; + mi_atomic_compare_exchange(&large_page_try_ok, try_ok - 1, try_ok); } else { // large OS pages must always reserve and commit. p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE | flags); // fall back to non-large page allocation on error (`p == NULL`). if (p == NULL) { - large_page_try_ok = 10; // on error, don't try again for the next N allocations + mi_atomic_write(&large_page_try_ok,10); // on error, don't try again for the next N allocations } } } From 6d9fab5af41878deb98889e3e4ab7d59c4ae3c46 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 21 Jul 2019 17:14:13 -0700 Subject: [PATCH 42/67] trailing id after #endif --- include/mimalloc-override.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index f6149514..d81063ab 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -75,7 +75,7 @@ including this header is not necessary. // ------------------------------------------------------ #ifdef __cplusplus #include - + void operator delete(void* p) noexcept { mi_free(p); }; void operator delete[](void* p) noexcept { mi_free(p); }; @@ -103,4 +103,4 @@ including this header is not necessary. #endif #endif -#endif MIMALLOC_OVERRIDE_H +#endif // MIMALLOC_OVERRIDE_H From 219d46ff0c38e83a0153876fe19ae8dbf994dfdc Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 01:36:16 -0700 Subject: [PATCH 43/67] update test files and overriding --- ide/vs2017/mimalloc-override-test.vcxproj | 15 ++------- .../mimalloc-override-test.vcxproj.filters | 3 -- ide/vs2017/mimalloc-test.vcxproj | 6 ++-- ide/vs2017/mimalloc-test.vcxproj.filters | 2 +- include/mimalloc-override.h | 12 ++++--- test/CMakeLists.txt | 19 ++++++++---- test/main-override-static.c | 31 +++++++++++++++++++ test/main-override.c | 7 ++--- test/main-override.cpp | 13 +++----- 9 files changed, 67 insertions(+), 41 deletions(-) create mode 100644 test/main-override-static.c diff --git a/ide/vs2017/mimalloc-override-test.vcxproj b/ide/vs2017/mimalloc-override-test.vcxproj index 77752890..7df1e79a 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj +++ b/ide/vs2017/mimalloc-override-test.vcxproj @@ -172,23 +172,14 @@ COPY /Y $(SolutionDir)..\..\bin\mimalloc-redirect.dll $(OutputPath) - - - true - true - true - true - - - false - false - - {abb5eae7-b3e6-432e-b636-333449892ea7} + + + diff --git a/ide/vs2017/mimalloc-override-test.vcxproj.filters b/ide/vs2017/mimalloc-override-test.vcxproj.filters index 80f1c9c0..eb5e70b7 100644 --- a/ide/vs2017/mimalloc-override-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-override-test.vcxproj.filters @@ -18,8 +18,5 @@ Source Files - - Source Files - \ No newline at end of file diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index 8e61a97f..c1539aeb 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -144,14 +144,14 @@ Console - - - {abb5eae7-b3e6-432e-b636-333449892ea6} + + + diff --git a/ide/vs2017/mimalloc-test.vcxproj.filters b/ide/vs2017/mimalloc-test.vcxproj.filters index eb5e70b7..fca75e1c 100644 --- a/ide/vs2017/mimalloc-test.vcxproj.filters +++ b/ide/vs2017/mimalloc-test.vcxproj.filters @@ -15,7 +15,7 @@ - + Source Files diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index d81063ab..c3348068 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -69,11 +69,15 @@ including this header is not necessary. #define _aligned_offset_recalloc(p,s,n,a,o) mi_recalloc_aligned_at(p,s,n,a,o) -// ------------------------------------------------------ -// With a C++ compiler we override the new/delete operators. +// ----------------------------------------------------------------- +// With a C++ compiler we can override all the new/delete operators +// by defining 'MIMALLOC_DEFINE_NEW_DELETE' in some source file and +// then including this header file. This is not needed when linking +// statically with the mimalloc library, but it can be more performant +// on Windows when using dynamic overiding as well. // see -// ------------------------------------------------------ -#ifdef __cplusplus +// ----------------------------------------------------------------- +#if defined(__cplusplus) && defined(MIMALLOC_DEFINE_NEW_DELETE) #include void operator delete(void* p) noexcept { mi_free(p); }; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 42d4a2f4..8a830073 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,14 +24,21 @@ target_link_libraries(dynamic-override PUBLIC mimalloc) add_executable(dynamic-override-cxx main-override.cpp) target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) -# with a static library +# overriding with a static object file works reliable as the symbols in the +# object file have priority over those in library files +add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) +target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) +target_link_libraries(static-override-obj PUBLIC pthread) + +# overriding with a static library works too if using the `mimalloc-override.h` +# header to redefine malloc/free. +add_executable(static-override-static main-override-static.c) +target_link_libraries(static-override-static PUBLIC mimalloc-static) + + +# overriding with a static library: this may not work if the library is linked too late add_executable(static-override main-override.c) target_link_libraries(static-override PUBLIC mimalloc-static) add_executable(static-override-cxx main-override.cpp) target_link_libraries(static-override-cxx PUBLIC mimalloc-static) - -# and with a static object file -add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) -target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) -target_link_libraries(static-override-obj PUBLIC pthread) diff --git a/test/main-override-static.c b/test/main-override-static.c new file mode 100644 index 00000000..6ddf4f37 --- /dev/null +++ b/test/main-override-static.c @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +#include +#include // redefines malloc etc. + +int main() { + mi_version(); + void* p1 = malloc(78); + void* p2 = malloc(24); + free(p1); + p1 = malloc(8); + //char* s = strdup("hello\n"); + free(p2); + p2 = malloc(16); + p1 = realloc(p1, 32); + free(p1); + free(p2); + //free(s); + //mi_collect(true); + + /* now test if override worked by allocating/freeing across the api's*/ + //p1 = mi_malloc(32); + //free(p1); + //p2 = malloc(32); + //mi_free(p2); + mi_stats_print(NULL); + return 0; +} diff --git a/test/main-override.c b/test/main-override.c index ddb2f16e..1bec1179 100644 --- a/test/main-override.c +++ b/test/main-override.c @@ -3,11 +3,10 @@ #include #include -//#include - +#include int main() { - //mi_stats_reset(); + mi_version(); // ensure mimalloc library is linked void* p1 = malloc(78); void* p2 = malloc(24); free(p1); @@ -26,6 +25,6 @@ int main() { //free(p1); //p2 = malloc(32); //mi_free(p2); - + mi_stats_print(NULL); return 0; } diff --git a/test/main-override.cpp b/test/main-override.cpp index 8f47dcd1..fb7ab7a1 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -4,8 +4,6 @@ #include #include -#include - #include static void* p = malloc(8); @@ -24,16 +22,15 @@ public: }; -int main() { - //mi_malloc_override(); - mi_stats_reset(); +int main() { + mi_version(); atexit(free_p); void* p1 = malloc(78); - void* p2 = _aligned_malloc(24,16); + void* p2 = mi_malloc_aligned(16,24); free(p1); p1 = malloc(8); - char* s = _strdup("hello\n"); - _aligned_free(p2); + char* s = mi_strdup("hello\n"); + mi_free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); From 0b4d74a5668b2869701a66a398f27d9bf13d163f Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 10:10:45 -0700 Subject: [PATCH 44/67] merge --- src/alloc-posix.c | 4 ++-- test/test-stress.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 4e844ba3..1f55b3a8 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -19,6 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #include // memcpy +#include // getenv #ifndef EINVAL #define EINVAL 22 @@ -115,7 +116,7 @@ int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept { #pragma warning(suppress:4996) char* p = getenv(name); if (p==NULL) { - *buf = NULL; + *buf = NULL; } else { *buf = mi_strdup(p); @@ -146,4 +147,3 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) return 0; #endif } - diff --git a/test/test-stress.c b/test/test-stress.c index b26dfd04..298e5a17 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -180,7 +180,7 @@ static DWORD WINAPI thread_entry(LPVOID param) { static void run_os_threads(size_t nthreads) { DWORD* tids = (DWORD*)malloc(nthreads * sizeof(DWORD)); HANDLE* thandles = (HANDLE*)malloc(nthreads * sizeof(HANDLE)); - for (intptr_t i = 0; i < nthreads; i++) { + for (uintptr_t i = 0; i < nthreads; i++) { thandles[i] = CreateThread(0, 4096, &thread_entry, (void*)(i), 0, &tids[i]); } for (int i = 0; i < nthreads; i++) { From 598ed19c614cdf8c3a2b967df8b4b5793111c95e Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 16:11:06 -0700 Subject: [PATCH 45/67] more comments --- test/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8a830073..8bf36521 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,29 +14,31 @@ endif() # Import mimalloc (if installed) find_package(mimalloc 1.0 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) +message(STATUS "Found mimalloc installed at: ${MIMALLOC_TARGET_DIR}") -message(STATUS "${MIMALLOC_INCLUDE_DIR}") - -# Tests +# overriding with a dynamic library add_executable(dynamic-override main-override.c) target_link_libraries(dynamic-override PUBLIC mimalloc) add_executable(dynamic-override-cxx main-override.cpp) target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) + # overriding with a static object file works reliable as the symbols in the # object file have priority over those in library files add_executable(static-override-obj main-override.c ${MIMALLOC_TARGET_DIR}/mimalloc.o) target_include_directories(static-override-obj PUBLIC ${MIMALLOC_TARGET_DIR}/include) target_link_libraries(static-override-obj PUBLIC pthread) + # overriding with a static library works too if using the `mimalloc-override.h` -# header to redefine malloc/free. +# header to redefine malloc/free. (the library already overrides new/delete) add_executable(static-override-static main-override-static.c) target_link_libraries(static-override-static PUBLIC mimalloc-static) # overriding with a static library: this may not work if the library is linked too late +# on the command line after the C runtime library; but we cannot control that well in CMake add_executable(static-override main-override.c) target_link_libraries(static-override PUBLIC mimalloc-static) From 66b8c37ab396553ad43f332ebf6d1abc55c98ef3 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 10:27:14 -0700 Subject: [PATCH 46/67] ensure C++ compilation on windows --- ide/vs2017/mimalloc-override.vcxproj | 8 ++++---- ide/vs2017/mimalloc.vcxproj | 4 ++-- include/mimalloc-override.h | 2 +- src/init.c | 6 ++++++ test/main-override.cpp | 6 +++--- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 3ca8158a..f41b2efc 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -100,7 +100,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -121,7 +121,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - Default + CompileAsCpp ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) @@ -152,7 +152,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp true @@ -177,7 +177,7 @@ $(IntDir) false MultiThreadedDLL - Default + CompileAsCpp true diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index a8cb7566..3e453471 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -154,7 +154,7 @@ Neither false false - Default + CompileAsCpp true @@ -185,7 +185,7 @@ Neither false false - Default + CompileAsCpp true diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index c3348068..56b41e6b 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file This header can be used to statically redirect malloc/free and new/delete to the mimalloc variants. This can be useful if one can include this file on each source file in a project (but be careful when using external code to -not accidentally mix pointer from different allocators). +not accidentally mix pointers from different allocators). On windows it can still be good to always try to include this header even when dynamically overriding since this will give better performance especially diff --git a/src/init.c b/src/init.c index d00d7c05..152e906b 100644 --- a/src/init.c +++ b/src/init.c @@ -384,12 +384,18 @@ bool _mi_preloading() { // Communicate with the redirection module on Windows #if defined(_WIN32) && defined(MI_SHARED_LIB) +#ifdef __cplusplus +extern "C" { +#endif mi_decl_export void _mi_redirect_init() { // called on redirection mi_redirected = true; } __declspec(dllimport) bool mi_allocator_init(const char** message); __declspec(dllimport) void mi_allocator_done(); +#ifdef __cplusplus +} +#endif #else static bool mi_allocator_init(const char** message) { if (message != NULL) *message = NULL; diff --git a/test/main-override.cpp b/test/main-override.cpp index fb7ab7a1..6c7fc0d5 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -23,14 +23,14 @@ public: int main() { - mi_version(); + mi_stats_reset(); // ignore earlier allocations atexit(free_p); void* p1 = malloc(78); void* p2 = mi_malloc_aligned(16,24); free(p1); p1 = malloc(8); char* s = mi_strdup("hello\n"); - mi_free(p2); + free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); @@ -39,7 +39,7 @@ int main() { Test* t = new Test(42); delete t; t = new (std::nothrow) Test(42); - delete t; + delete t; return 0; } From 189ad0f81dc3b029c05a3b7cf19d8ba78c4e0429 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 20:51:12 -0700 Subject: [PATCH 47/67] small optimizations, use bitwise aligne --- CMakeLists.txt | 1 + include/mimalloc-internal.h | 26 +++++++++++++++++++++++++- include/mimalloc-types.h | 11 ++++++----- include/mimalloc.h | 6 +++--- src/alloc.c | 6 +++--- src/init.c | 8 +++++--- src/os.c | 7 ------- src/page.c | 14 ++++++++++---- src/segment.c | 12 ++++++------ 9 files changed, 59 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d86d096b..ec0fd99a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) list(APPEND mi_cflags -fvisibility=hidden) + list(APPEND mi_cflags -fbranch-target-load-optimize ) endif() endif() diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cbed5909..e261dba2 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -39,7 +39,6 @@ bool _mi_preloading(); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); -uintptr_t _mi_align_up(uintptr_t sz, size_t alignment); void _mi_os_init(void); // called from process init void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data @@ -165,6 +164,20 @@ static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) { #endif } +// Align upwards +static inline uintptr_t _mi_is_power_of_two(uintptr_t x) { + return ((x & (x - 1)) == 0); +} +static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { + uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? + return ((sz + mask) & ~mask); + } + else { + return (((sz + mask)/alignment)*alignment); + } +} + // Align a byte size to a size in _machine words_, // i.e. byte size == `wsize*sizeof(void*)`. static inline size_t _mi_wsize_from_size(size_t size) { @@ -324,12 +337,23 @@ static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, mi_bl } static inline mi_block_t* mi_block_next(mi_page_t* page, mi_block_t* block) { + #if MI_SECURE return mi_block_nextx(page->cookie,block); + #else + UNUSED(page); + return mi_block_nextx(0, block); + #endif } static inline void mi_block_set_next(mi_page_t* page, mi_block_t* block, mi_block_t* next) { + #if MI_SECURE mi_block_set_nextx(page->cookie,block,next); + #else + UNUSED(page); + mi_block_set_nextx(0, block, next); + #endif } + // ------------------------------------------------------------------- // Getting the thread id should be performant // as it is called in the fast path of `_mi_free`, diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 7221f5b8..5c14ffd4 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -132,10 +132,9 @@ typedef union mi_page_flags_u { } mi_page_flags_t; // Thread free list. -// We use bottom 2 bits of the pointer for mi_delayed_t flags +// We use the bottom 2 bits of the pointer for mi_delayed_t flags typedef uintptr_t mi_thread_free_t; - // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: // `free` for blocks that can be allocated, @@ -165,9 +164,11 @@ typedef struct mi_page_s { mi_page_flags_t flags; uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory - + mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) + #if MI_SECURE uintptr_t cookie; // random cookie to encode the free lists + #endif size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) @@ -182,9 +183,9 @@ typedef struct mi_page_s { // improve page index calculation #if MI_INTPTR_SIZE==8 - //void* padding[1]; // 10 words on 64-bit + //void* padding[1]; // 12 words on 64-bit #elif MI_INTPTR_SIZE==4 - void* padding[1]; // 12 words on 32-bit + void* padding[1]; // 12 words on 32-bit #endif } mi_page_t; diff --git a/include/mimalloc.h b/include/mimalloc.h index e7e83791..c6b7b5f8 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -52,8 +52,8 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #else #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) - #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) - #define mi_cdecl // leads to warnings... __attribute__((cdecl)) + #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) + #define mi_cdecl // leads to warnings... __attribute__((cdecl)) #endif #else #define mi_decl_thread __thread @@ -62,7 +62,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_malloc #define mi_attr_alloc_size(s) #define mi_attr_alloc_size2(s1,s2) - #define mi_cdecl + #define mi_cdecl #endif // ------------------------------------------------------ diff --git a/src/alloc.c b/src/alloc.c index 649b6e95..6a91c0ad 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -237,9 +237,9 @@ void mi_free(void* p) mi_attr_noexcept #endif // adjust if it might be an un-aligned block - if (mi_likely(page->flags.value==0)) { // note: merging both tests (local | value) does not matter for performance + if (mi_likely(page->flags.value==0)) { // not full or aligned mi_block_t* block = (mi_block_t*)p; - if (mi_likely(local)) { + if (mi_likely(local)) { // note: merging both tests (local | value) does not matter for performance // owning thread can free a block directly mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance page->local_free = block; @@ -248,7 +248,7 @@ void mi_free(void* p) mi_attr_noexcept } else { // use atomic operations for a multi-threaded free - _mi_free_block_mt(page, block); + _mi_free_block_mt(page, block); } } else { diff --git a/src/init.c b/src/init.c index 152e906b..44e3c9cb 100644 --- a/src/init.c +++ b/src/init.c @@ -12,9 +12,11 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { - 0, false, false, false, {0}, - 0, 0, - NULL, 0, 0, // free, used, cookie + 0, false, false, false, {0}, 0, 0, + NULL, 0, // free, used + #if MI_SECURE + 0, + #endif NULL, 0, 0, 0, NULL, NULL, NULL #if (MI_INTPTR_SIZE==4) diff --git a/src/os.c b/src/os.c index f7b36258..f6358912 100644 --- a/src/os.c +++ b/src/os.c @@ -34,13 +34,6 @@ terms of the MIT license. A copy of the license can be found in the file ----------------------------------------------------------- */ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); -uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { - uintptr_t x = (sz / alignment) * alignment; - if (x < sz) x += alignment; - if (x < sz) return 0; // overflow - return x; -} - static void* mi_align_up_ptr(void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } diff --git a/src/page.c b/src/page.c index b0c0b382..d46a5aad 100644 --- a/src/page.c +++ b/src/page.c @@ -93,7 +93,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) { bool _mi_page_is_valid(mi_page_t* page) { mi_assert_internal(mi_page_is_valid_init(page)); + #if MI_SECURE mi_assert_internal(page->cookie != 0); + #endif if (page->heap!=NULL) { mi_segment_t* segment = _mi_page_segment(page); mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id); @@ -119,7 +121,7 @@ void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay ) { else if (mi_unlikely(mi_tf_delayed(tfree) == MI_DELAYED_FREEING)) { mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. continue; // and try again - } + } } while((mi_tf_delayed(tfreex) != mi_tf_delayed(tfree)) && // avoid atomic operation if already equal !mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex, tfree)); @@ -258,7 +260,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) { mi_block_t* next = mi_block_nextx(heap->cookie,block); // use internal free instead of regular one to keep stats etc correct if (!_mi_free_delayed_block(block)) { - // we might already start delayed freeing while another thread has not yet + // we might already start delayed freeing while another thread has not yet // reset the delayed_freeing flag; in that case delay it further by reinserting. mi_block_t* dfree; do { @@ -498,7 +500,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st if (page->capacity >= page->reserved) return; size_t page_size; - _mi_page_start(_mi_page_segment(page), page, &page_size); + _mi_page_start(_mi_page_segment(page), page, &page_size); _mi_stat_increase(&stats->pages_extended, 1); // calculate the extend count @@ -533,7 +535,9 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi page->block_size = block_size; mi_assert_internal(page_size / block_size < (1L<<16)); page->reserved = (uint16_t)(page_size / block_size); + #if MI_SECURE page->cookie = _mi_heap_random(heap) | 1; + #endif mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); @@ -543,7 +547,9 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page->next == NULL); mi_assert_internal(page->prev == NULL); mi_assert_internal(page->flags.has_aligned == false); + #if MI_SECURE mi_assert_internal(page->cookie != 0); + #endif mi_assert_expensive(mi_page_is_valid_init(page)); // initialize an initial free list @@ -683,7 +689,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); mi_heap_stat_increase( heap, huge, block_size); - } + } return page; } diff --git a/src/segment.c b/src/segment.c index 7f7bedd7..8f254a26 100644 --- a/src/segment.c +++ b/src/segment.c @@ -235,8 +235,8 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se // The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use, -// and no more than 4. -#define MI_SEGMENT_CACHE_MAX (4) +// and no more than 2. +#define MI_SEGMENT_CACHE_MAX (2) #define MI_SEGMENT_CACHE_FRACTION (8) // note: returned segment may be partially reset @@ -252,7 +252,7 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t } static bool mi_segment_cache_full(mi_segments_tld_t* tld) { - if (tld->cache_count < MI_SEGMENT_CACHE_MAX && + if (tld->cache_count < MI_SEGMENT_CACHE_MAX && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { // always allow 1 element cache return false; } @@ -318,7 +318,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_size = (page_kind == MI_PAGE_HUGE ? segment_size : (size_t)1 << page_shift); // Try to get it from our thread local cache first - bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM); + bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM); bool protection_still_good = false; mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld); if (segment != NULL) { @@ -702,10 +702,10 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; - if (block_size <= (MI_SMALL_PAGE_SIZE/16)*3) { + if (block_size <= (MI_SMALL_PAGE_SIZE/4)) { page = mi_segment_small_page_alloc(tld,os_tld); } - else if (block_size <= (MI_MEDIUM_PAGE_SIZE/16)*3) { + else if (block_size <= (MI_MEDIUM_PAGE_SIZE/4)) { page = mi_segment_medium_page_alloc(tld, os_tld); } else if (block_size < (MI_LARGE_SIZE_MAX - sizeof(mi_segment_t))) { From a02204970513fa48183e8f89736e148cd837cbda Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 22 Jul 2019 21:25:57 -0700 Subject: [PATCH 48/67] remove old comment --- src/page.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/page.c b/src/page.c index d46a5aad..69d32bfe 100644 --- a/src/page.c +++ b/src/page.c @@ -733,11 +733,4 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept // and try again, this time succeeding! (i.e. this should never recurse) return _mi_page_malloc(heap, page, size); - /* - if (page->used == page->reserved) { - // needed for huge pages to free reliably from other threads. - mi_page_to_full(page,mi_page_queue_of(page)); - } - return p; - */ } From f2f45ad5df2d138beee086e46931bf454cc11527 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 09:36:58 -0700 Subject: [PATCH 49/67] fix cmake build on windows --- CMakeLists.txt | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec0fd99a..45d5f988 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,14 +121,28 @@ add_library(mimalloc SHARED ${mi_sources}) set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} NO_SONAME "YES" OUTPUT_NAME ${mi_basename} ) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) +target_link_libraries(mimalloc PUBLIC ${mi_libraries}) target_include_directories(mimalloc PUBLIC $ $ ) -target_link_libraries(mimalloc PUBLIC ${mi_libraries}) +if(WIN32) + # On windows copy the mimalloc redirection dll too. + target_link_libraries(mimalloc PRIVATE ../../bin/mimalloc-redirect) + add_custom_command(TARGET mimalloc POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy "../../bin/mimalloc-redirect.dll" $ + COMMENT "Copy mimalloc-redirect.dll to output directory") +endif() # static library add_library(mimalloc-static STATIC ${mi_sources}) +target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) +target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) +target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) +target_include_directories(mimalloc-static PUBLIC + $ + $ +) if(WIN32) # When building both static and shared libraries on Windows, a static library should use a # different output name to avoid the conflict with the import library of a shared one. @@ -137,14 +151,6 @@ if(WIN32) else() set_target_properties(mimalloc-static PROPERTIES OUTPUT_NAME ${mi_basename}) endif() -target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) -target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) - -target_include_directories(mimalloc-static PUBLIC - $ - $ -) -target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) # install static and shared library, and the include files install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_dir} LIBRARY NAMELINK_SKIP) From 13364b50b8017b1c7357db1e6bb3a97fa313cd63 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 09:37:36 -0700 Subject: [PATCH 50/67] fix 32-bit build of stress test --- test/test-stress.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index 298e5a17..511679ac 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -27,7 +27,7 @@ static int N = 10; // scaling factor static volatile void* transfer[TRANSFERS]; -#if (INTPTR_MAX != UINT32_MAX) +#if (UINTPTR_MAX != UINT32_MAX) const uintptr_t cookie = 0xbf58476d1ce4e5b9UL; #else const uintptr_t cookie = 0x1ce4e5b9UL; @@ -39,7 +39,7 @@ typedef uintptr_t* random_t; static uintptr_t pick(random_t r) { uintptr_t x = *r; - #if (INTPTR_MAX > UINT32_MAX) + #if (UINTPTR_MAX > UINT32_MAX) // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; @@ -183,7 +183,7 @@ static void run_os_threads(size_t nthreads) { for (uintptr_t i = 0; i < nthreads; i++) { thandles[i] = CreateThread(0, 4096, &thread_entry, (void*)(i), 0, &tids[i]); } - for (int i = 0; i < nthreads; i++) { + for (size_t i = 0; i < nthreads; i++) { WaitForSingleObject(thandles[i], INFINITE); } } From c02a0c9b49771cdfef95fde4a577c8d92423de15 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 09:59:20 -0700 Subject: [PATCH 51/67] ensure cmake uses C++ compilation with MSVC --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 45d5f988..980ab542 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,6 @@ set(mi_sources src/options.c src/init.c) - # Set default build type if (NOT CMAKE_BUILD_TYPE) if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$") @@ -44,6 +43,11 @@ if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") set(MI_SECURE "ON") endif() +if(CMAKE_C_COMPILER_ID MATCHES "MSVC") + set(MI_USE_CXX "ON") +endif() + + # Options if(MI_OVERRIDE MATCHES "ON") message(STATUS "Override standard malloc (MI_OVERRIDE=ON)") @@ -78,7 +82,7 @@ endif() if(MI_USE_CXX MATCHES "ON") message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) - set_source_files_properties(src/static.c test/test-api.c PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c test/test-stress.c PROPERTIES LANGUAGE CXX ) endif() # Compiler flags From ab022e4271c51fb164ae31cb530f65f6884ca17d Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 12:40:42 -0700 Subject: [PATCH 52/67] fix mimalloc-redirect path on windows cmake build --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 980ab542..8b37e579 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,9 +132,9 @@ target_include_directories(mimalloc PUBLIC ) if(WIN32) # On windows copy the mimalloc redirection dll too. - target_link_libraries(mimalloc PRIVATE ../../bin/mimalloc-redirect) + target_link_libraries(mimalloc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect.lib) add_custom_command(TARGET mimalloc POST_BUILD - COMMAND "${CMAKE_COMMAND}" -E copy "../../bin/mimalloc-redirect.dll" $ + COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect.dll" $ COMMENT "Copy mimalloc-redirect.dll to output directory") endif() From fa5dc176213b557bcd0bc0b2c7bd953b576547f1 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 15:09:54 -0700 Subject: [PATCH 53/67] fix merge conflicts --- src/options.c | 20 ++------------------ src/os.c | 5 ----- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/options.c b/src/options.c index 349599ad..56fb4851 100644 --- a/src/options.c +++ b/src/options.c @@ -37,7 +37,6 @@ typedef struct mi_option_desc_s { const char* name; // option name without `mimalloc_` prefix } mi_option_desc_t; -<<<<<<< HEAD static mi_option_desc_t options[_mi_option_last] = { // stable options @@ -45,16 +44,6 @@ static mi_option_desc_t options[_mi_option_last] = { MI_DEBUG, UNINIT, "show_errors" }, { 0, UNINIT, "verbose" }, -======= -static mi_option_desc_t options[_mi_option_last] = { - { 0, UNINIT, "page_reset" }, - { 0, UNINIT, "cache_reset" }, - { 0, UNINIT, "pool_commit" }, - { 0, UNINIT, "eager_commit" }, // secure and large pages must have eager commit - { 0, UNINIT, "large_os_pages" }, // use large OS pages - { 0, UNINIT, "reset_decommits" }, - { 0, UNINIT, "reset_discards" }, ->>>>>>> add comment about large pages #if MI_SECURE { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored #else @@ -215,16 +204,11 @@ static const char* mi_getenv(const char* name) { #pragma warning(suppress:4996) const char* s = getenv(name); if (s == NULL) { -<<<<<<< HEAD - char buf[64+1]; - mi_strlcpy(buf,name,64); - for (size_t i = 0; i < strlen(buf); i++) { - buf[i] = toupper(name[i]); -======= + char buf[64]; + mi_strlcpy(buf,name,sizeof(buf)); size_t buf_size = strlen(buf); for (size_t i = 0; i < buf_size; i++) { buf[i] = toupper(buf[i]); ->>>>>>> Avoid using strlen function in loop } #pragma warning(suppress:4996) s = getenv(buf); diff --git a/src/os.c b/src/os.c index f6358912..56abd55d 100644 --- a/src/os.c +++ b/src/os.c @@ -202,14 +202,9 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, static volatile uintptr_t large_page_try_ok = 0; void* p = NULL; if (use_large_os_page(size, try_alignment)) { -<<<<<<< HEAD - if (large_page_try_ok > 0) { - // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. -======= uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); if (try_ok > 0) { // if a large page page allocation fails, it seems the calls to VirtualAlloc get very expensive. ->>>>>>> use atomic ops to guard large page tries on windows // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. mi_atomic_compare_exchange(&large_page_try_ok, try_ok - 1, try_ok); } From 89da085b6718ec55ffb263c4d7aae1f162180f2f Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 15:10:25 -0700 Subject: [PATCH 54/67] rename to mimalloc-override.dll and use C compilation --- ide/vs2017/mimalloc-override.vcxproj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index f41b2efc..82f0b432 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -70,25 +70,25 @@ $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc + mimalloc-override $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc + mimalloc-override $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc + mimalloc-override $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ .dll - mimalloc + mimalloc-override @@ -100,7 +100,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - CompileAsCpp + Default ../../bin/mimalloc-redirect32.lib;%(AdditionalDependencies) @@ -121,7 +121,7 @@ MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false - CompileAsCpp + Default ../../bin/mimalloc-redirect.lib;%(AdditionalDependencies) @@ -152,7 +152,7 @@ $(IntDir) false MultiThreadedDLL - CompileAsCpp + Default true @@ -177,7 +177,7 @@ $(IntDir) false MultiThreadedDLL - CompileAsCpp + Default true From 3d6feead60daf63decdba9c5a0ae4a7dee0f037b Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 15:54:47 -0700 Subject: [PATCH 55/67] add heap region check to cfree --- src/alloc-posix.c | 4 +++- src/memory.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 1f55b3a8..672b73b3 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -38,7 +38,9 @@ size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { } void mi_cfree(void* p) mi_attr_noexcept { - mi_free(p); + if (mi_is_in_heap_region(p)) { + mi_free(p); + } } int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { diff --git a/src/memory.c b/src/memory.c index e7d1887e..7f8cfb14 100644 --- a/src/memory.c +++ b/src/memory.c @@ -106,6 +106,7 @@ static size_t mi_good_commit_size(size_t size) { // Return if a pointer points into a region reserved by us. bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { + if (p==NULL) return false; size_t count = mi_atomic_read(®ions_count); for (size_t i = 0; i < count; i++) { uint8_t* start = (uint8_t*)mi_atomic_read_ptr(®ions[i].start); From 095a87be2eab8b58dd24df87bcb7df26dcbc46de Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 17:57:27 -0700 Subject: [PATCH 56/67] add recursion guard for overriding in windows --- src/options.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/options.c b/src/options.c index 919884bd..01620e75 100644 --- a/src/options.c +++ b/src/options.c @@ -111,25 +111,34 @@ void mi_option_enable_default(mi_option_t option, bool enable) { #define MAX_ERROR_COUNT (10) static uintptr_t error_count = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings +// When overriding malloc, we may recurse into mi_vfprintf if an allocation +// inside the C runtime causes another message. +static mi_decl_thread bool recurse = false; + // Define our own limited `fprintf` that avoids memory allocation. // We do this using `snprintf` with a limited buffer. static void mi_vfprintf( FILE* out, const char* prefix, const char* fmt, va_list args ) { char buf[256]; if (fmt==NULL) return; + if (_mi_preloading() || recurse) return; + recurse = true; if (out==NULL) out = stdout; - if (_mi_preloading()) return; vsnprintf(buf,sizeof(buf)-1,fmt,args); #ifdef _WIN32 - // on windows with redirection, the C runtime uses us and we cannot call `fputs` - // while called from the C runtime itself, so use a non-locking option + // on windows with redirection, the C runtime cannot handle locale dependent output + // after the main thread closes so use direct console output. if (out==stderr) { if (prefix != NULL) _cputs(prefix); _cputs(buf); - return; } + else #endif - if (prefix != NULL) fputs(prefix,out); - fputs(buf,out); + { + if (prefix != NULL) fputs(prefix,out); + fputs(buf,out); + } + recurse = false; + return; } void _mi_fprintf( FILE* out, const char* fmt, ... ) { From 613d0c1993c1b00c4b4039e4144a791cfbf0d57f Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Jul 2019 17:57:27 -0700 Subject: [PATCH 57/67] merge 095a87b --- src/options.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/options.c b/src/options.c index 56fb4851..a68ddbdf 100644 --- a/src/options.c +++ b/src/options.c @@ -111,25 +111,34 @@ void mi_option_enable_default(mi_option_t option, bool enable) { #define MAX_ERROR_COUNT (10) static uintptr_t error_count = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings +// When overriding malloc, we may recurse into mi_vfprintf if an allocation +// inside the C runtime causes another message. +static mi_decl_thread bool recurse = false; + // Define our own limited `fprintf` that avoids memory allocation. // We do this using `snprintf` with a limited buffer. static void mi_vfprintf( FILE* out, const char* prefix, const char* fmt, va_list args ) { char buf[256]; if (fmt==NULL) return; + if (_mi_preloading() || recurse) return; + recurse = true; if (out==NULL) out = stdout; - if (_mi_preloading()) return; vsnprintf(buf,sizeof(buf)-1,fmt,args); #ifdef _WIN32 - // on windows with redirection, the C runtime uses us and we cannot call `fputs` - // while called from the C runtime itself, so use a non-locking option - if (out==stderr) { - if (prefix != NULL) _cputs(prefix); - _cputs(buf); - return; + // on windows with redirection, the C runtime cannot handle locale dependent output + // after the main thread closes so use direct console output. + if (out==stderr) { + if (prefix != NULL) _cputs(prefix); + _cputs(buf); } + else #endif - if (prefix != NULL) fputs(prefix,out); - fputs(buf,out); + { + if (prefix != NULL) fputs(prefix,out); + fputs(buf,out); + } + recurse = false; + return; } void _mi_fprintf( FILE* out, const char* fmt, ... ) { From 0e8241c1400004ba7a4b0a780aba747207559e8d Mon Sep 17 00:00:00 2001 From: Jakub Szymanski Date: Tue, 30 Jul 2019 10:36:00 -0700 Subject: [PATCH 58/67] change from MIMALLOC_OVERRIDE to MIMALLOC_DISABLE_OVERRIDE --- src/alloc-override-win.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c index 93886982..d1d51b9a 100644 --- a/src/alloc-override-win.c +++ b/src/alloc-override-win.c @@ -685,8 +685,8 @@ __declspec(dllexport) BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID if (ok) { // check if patching is not disabled #pragma warning(suppress:4996) - const char* s = getenv("MIMALLOC_OVERRIDE"); - bool enabled = (s == NULL || strstr("1;TRUE;YES;ON", s) != NULL); + const char* s = getenv("MIMALLOC_DISABLE_OVERRIDE"); + bool enabled = (s == NULL || !(strstr("1;TRUE;YES;ON", s) != NULL)); if (!enabled) { _mi_verbose_message("override is disabled\n"); } From 6313c21d936c7368930837bf9f1501fcebdb3e45 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 6 Aug 2019 18:25:57 -0700 Subject: [PATCH 59/67] fix output directory in test projects --- ide/vs2017/mimalloc-test-stress.vcxproj | 8 ++++---- ide/vs2017/mimalloc-test.vcxproj | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ide/vs2017/mimalloc-test-stress.vcxproj b/ide/vs2017/mimalloc-test-stress.vcxproj index e8cc5045..b8267d0b 100644 --- a/ide/vs2017/mimalloc-test-stress.vcxproj +++ b/ide/vs2017/mimalloc-test-stress.vcxproj @@ -67,19 +67,19 @@ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ diff --git a/ide/vs2017/mimalloc-test.vcxproj b/ide/vs2017/mimalloc-test.vcxproj index c1539aeb..27c7bb6e 100644 --- a/ide/vs2017/mimalloc-test.vcxproj +++ b/ide/vs2017/mimalloc-test.vcxproj @@ -67,19 +67,19 @@ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ - $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ From 56778fe7d210ffbb95e830e2bbcba04eb84351af Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 6 Aug 2019 18:57:53 -0700 Subject: [PATCH 60/67] split normal and secure extend in a separate routine --- src/page.c | 130 +++++++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 59 deletions(-) diff --git a/src/page.c b/src/page.c index 69d32bfe..b2bbc2ad 100644 --- a/src/page.c +++ b/src/page.c @@ -404,6 +404,59 @@ void _mi_page_retire(mi_page_t* page) { #define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT) #define MI_MIN_SLICES (2) +static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats) { + UNUSED(stats); + mi_assert_internal(page->free == NULL); + mi_assert_internal(page->local_free == NULL); + mi_assert_internal(page->capacity + extend <= page->reserved); + void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL); + size_t bsize = page->block_size; + + // initialize a randomized free list + // set up `slice_count` slices to alternate between + size_t shift = MI_MAX_SLICE_SHIFT; + while ((extend >> shift) == 0) { + shift--; + } + size_t slice_count = (size_t)1U << shift; + size_t slice_extend = extend / slice_count; + mi_assert_internal(slice_extend >= 1); + mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice + size_t counts[MI_MAX_SLICES]; // available objects in the slice + for (size_t i = 0; i < slice_count; i++) { + blocks[i] = mi_page_block_at(page, page_area, page->capacity + i*slice_extend); + counts[i] = slice_extend; + } + counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?) + + // and initialize the free list by randomly threading through them + // set up first element + size_t current = _mi_heap_random(heap) % slice_count; + counts[current]--; + page->free = blocks[current]; + // and iterate through the rest + uintptr_t rnd = heap->random; + for (size_t i = 1; i < extend; i++) { + // call random_shuffle only every INTPTR_SIZE rounds + size_t round = i%MI_INTPTR_SIZE; + if (round == 0) rnd = _mi_random_shuffle(rnd); + // select a random next slice index + size_t next = ((rnd >> 8*round) & (slice_count-1)); + while (counts[next]==0) { // ensure it still has space + next++; + if (next==slice_count) next = 0; + } + // and link the current block to it + counts[next]--; + mi_block_t* block = blocks[current]; + blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block + mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next` + current = next; + } + mi_block_set_next(page, blocks[current], NULL); // end of the list + heap->random = _mi_random_shuffle(rnd); +} + static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats) { UNUSED(stats); @@ -413,66 +466,17 @@ static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t e void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL ); size_t bsize = page->block_size; mi_block_t* start = mi_page_block_at(page, page_area, page->capacity); - if (extend < MI_MIN_SLICES || !mi_option_is_enabled(mi_option_secure)) { - // initialize a sequential free list - mi_block_t* end = mi_page_block_at(page, page_area, page->capacity + extend - 1); - mi_block_t* block = start; - for (size_t i = 0; i < extend; i++) { - mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize); - mi_block_set_next(page,block,next); - block = next; - } - mi_block_set_next(page, end, NULL); - page->free = start; - } - else { - // initialize a randomized free list - // set up `slice_count` slices to alternate between - size_t shift = MI_MAX_SLICE_SHIFT; - while ((extend >> shift) == 0) { - shift--; - } - size_t slice_count = (size_t)1U << shift; - size_t slice_extend = extend / slice_count; - mi_assert_internal(slice_extend >= 1); - mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice - size_t counts[MI_MAX_SLICES]; // available objects in the slice - for (size_t i = 0; i < slice_count; i++) { - blocks[i] = mi_page_block_at(page, page_area, page->capacity + i*slice_extend); - counts[i] = slice_extend; - } - counts[slice_count-1] += (extend % slice_count); // final slice holds the modulus too (todo: distribute evenly?) - // and initialize the free list by randomly threading through them - // set up first element - size_t current = _mi_heap_random(heap) % slice_count; - counts[current]--; - page->free = blocks[current]; - // and iterate through the rest - uintptr_t rnd = heap->random; - for (size_t i = 1; i < extend; i++) { - // call random_shuffle only every INTPTR_SIZE rounds - size_t round = i%MI_INTPTR_SIZE; - if (round == 0) rnd = _mi_random_shuffle(rnd); - // select a random next slice index - size_t next = ((rnd >> 8*round) & (slice_count-1)); - while (counts[next]==0) { // ensure it still has space - next++; - if (next==slice_count) next = 0; - } - // and link the current block to it - counts[next]--; - mi_block_t* block = blocks[current]; - blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block - mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next` - current = next; - } - mi_block_set_next( page, blocks[current], NULL); // end of the list - heap->random = _mi_random_shuffle(rnd); + // initialize a sequential free list + mi_block_t* last = mi_page_block_at(page, page_area, page->capacity + extend - 1); + mi_block_t* block = start; + while(block <= last) { + mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize); + mi_block_set_next(page,block,next); + block = next; } - // enable the new free list - page->capacity += (uint16_t)extend; - _mi_stat_increase(&stats->page_committed, extend * page->block_size); + mi_block_set_next(page, last, NULL); + page->free = start; } /* ----------------------------------------------------------- @@ -518,7 +522,15 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st mi_assert_internal(extend < (1UL<<16)); // and append the extend the free list - mi_page_free_list_extend(heap, page, extend, stats ); + if (extend < MI_MIN_SLICES || !mi_option_is_enabled(mi_option_secure)) { + mi_page_free_list_extend(heap, page, extend, stats ); + } + else { + mi_page_free_list_extend_secure(heap, page, extend, stats); + } + // enable the new free list + page->capacity += (uint16_t)extend; + _mi_stat_increase(&stats->page_committed, extend * page->block_size); mi_assert_expensive(mi_page_is_valid_init(page)); } From 55778d2fe41fe78dea4c3da3e89c5b7fd15c624a Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 8 Aug 2019 11:36:13 -0700 Subject: [PATCH 61/67] improved stats --- include/mimalloc-internal.h | 2 +- include/mimalloc-types.h | 4 ++-- src/init.c | 3 ++- src/page.c | 7 ++++--- src/segment.c | 13 +++++++++---- src/stats.c | 19 ++++++++++++------- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index e261dba2..cc487a21 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -307,7 +307,7 @@ static inline bool mi_page_all_used(mi_page_t* page) { static inline bool mi_page_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; - return (page->reserved - page->used + page->thread_freed < frac); + return (page->reserved - page->used + page->thread_freed <= frac); } static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 5c14ffd4..6c094091 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -324,12 +324,12 @@ typedef struct mi_stats_s { mi_stat_count_t pages_abandoned; mi_stat_count_t pages_extended; mi_stat_count_t mmap_calls; - mi_stat_count_t mmap_right_align; - mi_stat_count_t mmap_ensure_aligned; mi_stat_count_t commit_calls; mi_stat_count_t threads; mi_stat_count_t huge; mi_stat_count_t malloc; + mi_stat_count_t segments_cache; + mi_stat_counter_t page_no_retire; mi_stat_counter_t searches; #if MI_STAT>1 mi_stat_count_t normal[MI_BIN_HUGE+1]; diff --git a/src/init.c b/src/init.c index 8075ea35..3b060fa4 100644 --- a/src/init.c +++ b/src/init.c @@ -61,7 +61,8 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), \ + { 0, 0 }, \ { 0, 0 } \ MI_STAT_COUNT_END_NULL() diff --git a/src/page.c b/src/page.c index b2bbc2ad..aa8a8415 100644 --- a/src/page.c +++ b/src/page.c @@ -216,7 +216,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size mi_page_t* page = _mi_segment_page_alloc(block_size, &heap->tld->segments, &heap->tld->os); if (page == NULL) return NULL; mi_page_init(heap, page, block_size, &heap->tld->stats); - mi_heap_stat_increase( heap, pages, 1); + _mi_stat_increase( &heap->tld->stats.pages, 1); mi_page_queue_push(heap, pq, page); mi_assert_expensive(_mi_page_is_valid(page)); return page; @@ -352,7 +352,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // account for huge pages here if (page->block_size > MI_LARGE_SIZE_MAX) { - mi_heap_stat_decrease(page->heap, huge, page->block_size); + _mi_stat_decrease(&page->heap->tld->stats.huge, page->block_size); } // remove from the page list @@ -386,6 +386,7 @@ void _mi_page_retire(mi_page_t* page) { // if its neighbours are almost fully used. if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { + _mi_stat_counter_increase(&page->heap->tld->stats.page_no_retire,1); return; // dont't retire after all } } @@ -700,7 +701,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { if (page != NULL) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); - mi_heap_stat_increase( heap, huge, block_size); + _mi_stat_increase( &heap->tld->stats.huge, block_size); } return page; } diff --git a/src/segment.c b/src/segment.c index 8f254a26..a86c3bc0 100644 --- a/src/segment.c +++ b/src/segment.c @@ -248,17 +248,19 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t tld->cache = segment->next; segment->next = NULL; mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); + _mi_stat_decrease(&tld->stats->segments_cache, 1); return segment; } static bool mi_segment_cache_full(mi_segments_tld_t* tld) { - if (tld->cache_count < MI_SEGMENT_CACHE_MAX && - tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { // always allow 1 element cache + if (tld->cache_count < MI_SEGMENT_CACHE_MAX + && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION)) + ) { // always allow 1 element cache return false; } // take the opportunity to reduce the segment cache if it is too large (now) // TODO: this never happens as we check against peak usage, should we use current usage instead? - while (tld->cache_count > (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { + while (tld->cache_count > MI_SEGMENT_CACHE_MAX ) { //(1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { mi_segment_t* segment = mi_segment_cache_pop(0,tld); mi_assert_internal(segment != NULL); if (segment != NULL) mi_segment_os_free(segment, segment->segment_size, tld); @@ -269,7 +271,9 @@ static bool mi_segment_cache_full(mi_segments_tld_t* tld) { static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) { mi_assert_internal(!mi_segment_is_in_free_queue(segment, tld)); mi_assert_internal(segment->next == NULL); - if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) return false; + if (segment->segment_size != MI_SEGMENT_SIZE || mi_segment_cache_full(tld)) { + return false; + } mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); if (mi_option_is_enabled(mi_option_cache_reset)) { _mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); @@ -277,6 +281,7 @@ static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) segment->next = tld->cache; tld->cache = segment; tld->cache_count++; + _mi_stat_increase(&tld->stats->segments_cache,1); return true; } diff --git a/src/stats.c b/src/stats.c index 2b15bf9e..8725e48c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -99,14 +99,14 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); mi_stat_add(&stats->mmap_calls, &src->mmap_calls, 1); - mi_stat_add(&stats->mmap_ensure_aligned, &src->mmap_ensure_aligned, 1); - mi_stat_add(&stats->mmap_right_align, &src->mmap_right_align, 1); mi_stat_add(&stats->commit_calls, &src->commit_calls, 1); mi_stat_add(&stats->threads, &src->threads, 1); mi_stat_add(&stats->pages_extended, &src->pages_extended, 1); mi_stat_add(&stats->malloc, &src->malloc, 1); + mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); mi_stat_add(&stats->huge, &src->huge, 1); + mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); mi_stat_counter_add(&stats->searches, &src->searches, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { @@ -172,10 +172,15 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t } static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, FILE* out ) { - double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); - _mi_fprintf(out,"%10s: %7.1f avg\n", msg, avg); + _mi_fprintf(out, "%10s:", msg); + mi_print_amount(stat->total, -1, out); + _mi_fprintf(out, "\n"); } +static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, FILE* out) { + double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); + _mi_fprintf(out, "%10s: %7.1f avg\n", msg, avg); +} static void mi_print_header( FILE* out ) { @@ -229,15 +234,15 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_n mi_stat_print(&stats->page_committed, "touched", 1, out); mi_stat_print(&stats->segments, "segments", -1, out); mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out); + mi_stat_print(&stats->segments_cache, "-cached", -1, out); mi_stat_print(&stats->pages, "pages", -1, out); mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out); mi_stat_print(&stats->pages_extended, "-extended", 0, out); + mi_stat_counter_print(&stats->page_no_retire, "-noretire", out); mi_stat_print(&stats->mmap_calls, "mmaps", 0, out); - mi_stat_print(&stats->mmap_right_align, "mmap fast", 0, out); - mi_stat_print(&stats->mmap_ensure_aligned, "mmap slow", 0, out); mi_stat_print(&stats->commit_calls, "commits", 0, out); mi_stat_print(&stats->threads, "threads", 0, out); - mi_stat_counter_print(&stats->searches, "searches", out); + mi_stat_counter_print_avg(&stats->searches, "searches", out); if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs); From 6596e970a52d28a5e449e9765c33f88787a96bb5 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 8 Aug 2019 15:23:18 -0700 Subject: [PATCH 62/67] move in_full and has_aligned into page threadid for a single test in mi_free --- include/mimalloc-internal.h | 4 ++++ include/mimalloc-types.h | 32 ++++++++++++++++++++++---------- src/alloc.c | 25 +++++++++---------------- src/init.c | 9 +++++---- src/page.c | 7 ++++--- src/segment.c | 8 ++++++-- 6 files changed, 50 insertions(+), 35 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cc487a21..2c3ccffd 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -314,6 +314,10 @@ static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) return &((mi_heap_t*)heap)->pages[_mi_bin(size)]; } +static inline uintptr_t mi_page_thread_id(const mi_page_t* page) { + return (page->flags.padding << MI_PAGE_FLAGS_BITS); +} + // ------------------------------------------------------------------- // Encoding/Decoding the free list next pointers // ------------------------------------------------------------------- diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 6c094091..073d23d3 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -123,14 +123,26 @@ typedef enum mi_delayed_e { } mi_delayed_t; +// Use the lowest two bits of a thread id for the `in_full` and `has_aligned` flags +// This allows a single test in `mi_free` to check for unlikely cases +// (namely, non-local free, aligned free, or freeing in a full page) +#define MI_PAGE_FLAGS_BITS (2) typedef union mi_page_flags_u { - uint16_t value; + uintptr_t threadidx; struct { - bool has_aligned; - bool in_full; + #ifdef MI_BIG_ENDIAN + uintptr_t padding : (MI_INTPTR_SIZE*8 - MI_PAGE_FLAGS_BITS); + uintptr_t in_full : 1; + uintptr_t has_aligned : 1; + #else + uintptr_t in_full : 1; + uintptr_t has_aligned : 1; + uintptr_t padding : (MI_INTPTR_SIZE*8 - MI_PAGE_FLAGS_BITS); + #endif }; } mi_page_flags_t; + // Thread free list. // We use the bottom 2 bits of the pointer for mi_delayed_t flags typedef uintptr_t mi_thread_free_t; @@ -160,16 +172,16 @@ typedef struct mi_page_s { bool is_reset:1; // `true` if the page memory was reset bool is_committed:1; // `true` if the page virtual memory is committed - // layout like this to optimize access in `mi_malloc` and `mi_free` - mi_page_flags_t flags; + // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory - + // 16 bits padding mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) #if MI_SECURE uintptr_t cookie; // random cookie to encode the free lists #endif size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) + mi_page_flags_t flags; // threadid:62 | has_aligned:1 | in_full:1 mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) volatile uintptr_t thread_freed; // at least this number of blocks are in `thread_free` @@ -182,10 +194,10 @@ typedef struct mi_page_s { struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` // improve page index calculation -#if MI_INTPTR_SIZE==8 - //void* padding[1]; // 12 words on 64-bit +#if (MI_INTPTR_SIZE==8 && MI_SECURE==0) + void* padding[1]; // 12 words on 64-bit #elif MI_INTPTR_SIZE==4 - void* padding[1]; // 12 words on 32-bit + // void* padding[1]; // 12 words on 32-bit #endif } mi_page_t; @@ -215,7 +227,7 @@ typedef struct mi_segment_s { // layout like this to optimize access in `mi_free` size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). - uintptr_t thread_id; // unique id of the thread owning this segment + volatile uintptr_t thread_id; // unique id of the thread owning this segment mi_page_kind_t page_kind; // kind of pages: small, large, or huge mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages } mi_segment_t; diff --git a/src/alloc.c b/src/alloc.c index 6a91c0ad..f23ed896 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -223,8 +223,7 @@ void mi_free(void* p) mi_attr_noexcept return; } #endif - - bool local = (_mi_thread_id() == segment->thread_id); // preload, note: putting the thread_id in the page->flags does not improve performance + mi_page_t* page = _mi_segment_page_of(segment, p); #if (MI_STAT>1) @@ -237,23 +236,17 @@ void mi_free(void* p) mi_attr_noexcept #endif // adjust if it might be an un-aligned block - if (mi_likely(page->flags.value==0)) { // not full or aligned + uintptr_t tid = _mi_thread_id(); + if (mi_likely(tid == page->flags.threadidx)) { // local, and not full or aligned mi_block_t* block = (mi_block_t*)p; - if (mi_likely(local)) { // note: merging both tests (local | value) does not matter for performance - // owning thread can free a block directly - mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance - page->local_free = block; - page->used--; - if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); } - } - else { - // use atomic operations for a multi-threaded free - _mi_free_block_mt(page, block); - } + mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance + page->local_free = block; + page->used--; + if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); } } else { - // aligned blocks, or a full page; use the more generic path - mi_free_generic(segment, page, local, p); + // non-local, aligned blocks, or a full page; use the more generic path + mi_free_generic(segment, page, tid == mi_page_thread_id(page), p); } } diff --git a/src/init.c b/src/init.c index 3b060fa4..13ed9561 100644 --- a/src/init.c +++ b/src/init.c @@ -12,15 +12,16 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { - 0, false, false, false, {0}, 0, 0, - NULL, 0, // free, used + 0, false, false, false, 0, 0, + NULL, // free #if MI_SECURE 0, #endif + 0, {0}, // used, flags NULL, 0, 0, 0, NULL, NULL, NULL - #if (MI_INTPTR_SIZE==4) - , { NULL } + #if (MI_INTPTR_SIZE==8 && MI_SECURE==0) + , { NULL } #endif }; diff --git a/src/page.c b/src/page.c index aa8a8415..4ff797c0 100644 --- a/src/page.c +++ b/src/page.c @@ -71,10 +71,11 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(page->block_size > 0); mi_assert_internal(page->used <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); - + mi_segment_t* segment = _mi_page_segment(page); uint8_t* start = _mi_page_start(segment,page,NULL); mi_assert_internal(start == _mi_segment_page_start(segment,page,page->block_size,NULL)); + mi_assert_internal(segment->thread_id == mi_page_thread_id(page)); //mi_assert_internal(start + page->capacity*page->block_size == page->top); mi_assert_internal(mi_page_list_is_valid(page,page->free)); @@ -458,7 +459,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si heap->random = _mi_random_shuffle(rnd); } -static void mi_page_free_list_extend( mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats) +static void mi_page_free_list_extend( mi_page_t* page, size_t extend, mi_stats_t* stats) { UNUSED(stats); mi_assert_internal(page->free == NULL); @@ -524,7 +525,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st // and append the extend the free list if (extend < MI_MIN_SLICES || !mi_option_is_enabled(mi_option_secure)) { - mi_page_free_list_extend(heap, page, extend, stats ); + mi_page_free_list_extend(page, extend, stats ); } else { mi_page_free_list_extend_secure(heap, page, extend, stats); diff --git a/src/segment.c b/src/segment.c index a86c3bc0..256c30eb 100644 --- a/src/segment.c +++ b/src/segment.c @@ -226,6 +226,7 @@ static void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) { static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_segments_tld_t* tld) { + segment->thread_id = 0; mi_segments_track_size(-((long)segment_size),tld); if (mi_option_is_enabled(mi_option_secure)) { _mi_mem_unprotect(segment, segment->segment_size); // ensure no more guard pages are set @@ -412,8 +413,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t mi_assert_expensive(!mi_segment_queue_contains(&tld->medium_free, segment)); mi_assert(segment->next == NULL); mi_assert(segment->prev == NULL); - _mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size); - segment->thread_id = 0; + _mi_stat_decrease(&tld->stats->page_committed, segment->segment_info_size); // update reset memory statistics /* @@ -618,6 +618,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen } else { // otherwise reclaim it + page->flags.threadidx = segment->thread_id; _mi_page_reclaim(heap,page); } } @@ -648,6 +649,7 @@ static mi_page_t* mi_segment_page_alloc_in(mi_segment_t* segment, mi_segments_tl mi_assert_internal(mi_segment_has_free(segment)); mi_page_t* page = mi_segment_find_free(segment, tld->stats); page->segment_in_use = true; + page->flags.threadidx = segment->thread_id; segment->used++; mi_assert_internal(segment->used <= segment->capacity); if (segment->used == segment->capacity) { @@ -687,6 +689,7 @@ static mi_page_t* mi_segment_large_page_alloc(mi_segments_tld_t* tld, mi_os_tld_ segment->used = 1; mi_page_t* page = &segment->pages[0]; page->segment_in_use = true; + page->flags.threadidx = segment->thread_id; return page; } @@ -698,6 +701,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld segment->used = 1; mi_page_t* page = &segment->pages[0]; page->segment_in_use = true; + page->flags.threadidx = segment->thread_id; return page; } From 5e56b40fe6446bd6f1bb583805b6fb8230911ab6 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 8 Aug 2019 17:18:49 -0700 Subject: [PATCH 63/67] improve page flags handling --- include/mimalloc-internal.h | 8 +++++++- include/mimalloc-types.h | 14 +++++++------- src/alloc.c | 2 +- src/page.c | 4 ++-- src/segment.c | 8 ++++---- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 2c3ccffd..ad9b3ecf 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -315,7 +315,13 @@ static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) } static inline uintptr_t mi_page_thread_id(const mi_page_t* page) { - return (page->flags.padding << MI_PAGE_FLAGS_BITS); + return (page->flags.xthread_id << MI_PAGE_FLAGS_BITS); +} + +static inline void mi_page_init_flags(mi_page_t* page, uintptr_t thread_id) { + page->flags.value = 0; + page->flags.xthread_id = (thread_id >> MI_PAGE_FLAGS_BITS); + mi_assert(page->flags.value == thread_id); } // ------------------------------------------------------------------- diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 073d23d3..3af664cf 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -126,18 +126,18 @@ typedef enum mi_delayed_e { // Use the lowest two bits of a thread id for the `in_full` and `has_aligned` flags // This allows a single test in `mi_free` to check for unlikely cases // (namely, non-local free, aligned free, or freeing in a full page) -#define MI_PAGE_FLAGS_BITS (2) +#define MI_PAGE_FLAGS_BITS (2) +#define MI_PAGE_FLAGS_TID_BITS (MI_INTPTR_SIZE*8 - MI_PAGE_FLAGS_BITS) typedef union mi_page_flags_u { - uintptr_t threadidx; + uintptr_t value; struct { #ifdef MI_BIG_ENDIAN - uintptr_t padding : (MI_INTPTR_SIZE*8 - MI_PAGE_FLAGS_BITS); + uintptr_t xthread_id : MI_PAGE_FLAGS_TID_BITS; + #endif uintptr_t in_full : 1; uintptr_t has_aligned : 1; - #else - uintptr_t in_full : 1; - uintptr_t has_aligned : 1; - uintptr_t padding : (MI_INTPTR_SIZE*8 - MI_PAGE_FLAGS_BITS); + #ifndef MI_BIG_ENDIAN + uintptr_t xthread_id : MI_PAGE_FLAGS_TID_BITS; #endif }; } mi_page_flags_t; diff --git a/src/alloc.c b/src/alloc.c index f23ed896..fe9d5fb0 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -237,7 +237,7 @@ void mi_free(void* p) mi_attr_noexcept // adjust if it might be an un-aligned block uintptr_t tid = _mi_thread_id(); - if (mi_likely(tid == page->flags.threadidx)) { // local, and not full or aligned + if (mi_likely(tid == page->flags.value)) { // local, and not full or aligned mi_block_t* block = (mi_block_t*)p; mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance page->local_free = block; diff --git a/src/page.c b/src/page.c index 4ff797c0..9be0372d 100644 --- a/src/page.c +++ b/src/page.c @@ -75,7 +75,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_segment_t* segment = _mi_page_segment(page); uint8_t* start = _mi_page_start(segment,page,NULL); mi_assert_internal(start == _mi_segment_page_start(segment,page,page->block_size,NULL)); - mi_assert_internal(segment->thread_id == mi_page_thread_id(page)); + mi_assert_internal(segment->thread_id==0 || segment->thread_id == mi_page_thread_id(page)); //mi_assert_internal(start + page->capacity*page->block_size == page->top); mi_assert_internal(mi_page_list_is_valid(page,page->free)); @@ -387,7 +387,7 @@ void _mi_page_retire(mi_page_t* page) { // if its neighbours are almost fully used. if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { - _mi_stat_counter_increase(&page->heap->tld->stats.page_no_retire,1); + _mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1); return; // dont't retire after all } } diff --git a/src/segment.c b/src/segment.c index 256c30eb..f2fd09ad 100644 --- a/src/segment.c +++ b/src/segment.c @@ -618,7 +618,7 @@ bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segmen } else { // otherwise reclaim it - page->flags.threadidx = segment->thread_id; + mi_page_init_flags(page,segment->thread_id); _mi_page_reclaim(heap,page); } } @@ -649,7 +649,7 @@ static mi_page_t* mi_segment_page_alloc_in(mi_segment_t* segment, mi_segments_tl mi_assert_internal(mi_segment_has_free(segment)); mi_page_t* page = mi_segment_find_free(segment, tld->stats); page->segment_in_use = true; - page->flags.threadidx = segment->thread_id; + mi_page_init_flags(page,segment->thread_id); segment->used++; mi_assert_internal(segment->used <= segment->capacity); if (segment->used == segment->capacity) { @@ -689,7 +689,7 @@ static mi_page_t* mi_segment_large_page_alloc(mi_segments_tld_t* tld, mi_os_tld_ segment->used = 1; mi_page_t* page = &segment->pages[0]; page->segment_in_use = true; - page->flags.threadidx = segment->thread_id; + mi_page_init_flags(page,segment->thread_id); return page; } @@ -701,7 +701,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld segment->used = 1; mi_page_t* page = &segment->pages[0]; page->segment_in_use = true; - page->flags.threadidx = segment->thread_id; + mi_page_init_flags(page,segment->thread_id); return page; } From 442bad91904532bdc7fe45370e676c4aa46fc7f7 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 9 Aug 2019 11:18:38 -0700 Subject: [PATCH 64/67] add good-fit for allowing larger blocks in smaller segments --- include/mimalloc-types.h | 26 +++++-- src/alloc.c | 9 ++- src/init.c | 37 ++++++--- src/page-queue.c | 11 ++- src/page.c | 6 +- src/segment.c | 14 ++-- test/main-override-static.c | 146 ++++++++++++++++++++++++++++++++++++ 7 files changed, 214 insertions(+), 35 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 3af664cf..45307c15 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -91,21 +91,31 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE) #define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) -#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/8) // 64kb on 64-bit -#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/8) // 512kb on 64-bit +#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 64kb on 64-bit +#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 512kb on 64-bit #define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT) -// Maximum number of size classes. (spaced exponentially in 16.7% increments) -#define MI_BIN_HUGE (64U) - // Minimal alignment necessary. On most platforms 16 bytes are needed // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) -#if (MI_LARGE_WSIZE_MAX > 131072) +#define MI_BIN4 +#ifdef MI_BIN4 +// Maximum number of size classes. (spaced exponentially in 25% increments) +#define MI_BIN_HUGE (40U) + +#if (MI_LARGE_WSIZE_MAX > 524287) #error "define more bins" #endif +#else +// Maximum number of size classes. (spaced exponentially in 12.5% increments) +#define MI_BIN_HUGE (70U) + +#if (MI_LARGE_WSIZE_MAX > 393216) +#error "define more bins" +#endif +#endif typedef uintptr_t mi_encoded_t; @@ -172,10 +182,10 @@ typedef struct mi_page_s { bool is_reset:1; // `true` if the page memory was reset bool is_committed:1; // `true` if the page virtual memory is committed - // layout like this to optimize access in `mi_malloc` and `mi_free` + // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory - // 16 bits padding + // 16 bits padding mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) #if MI_SECURE uintptr_t cookie; // random cookie to encode the free lists diff --git a/src/alloc.c b/src/alloc.c index fe9d5fb0..bfb37d19 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -57,6 +57,7 @@ extern inline void* mi_malloc_small(size_t size) mi_attr_noexcept { return mi_heap_malloc_small(mi_get_default_heap(), size); } + // zero initialized small block void* mi_zalloc_small(size_t size) mi_attr_noexcept { void* p = mi_malloc_small(size); @@ -71,7 +72,7 @@ extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcep void* p; if (mi_likely(size <= MI_SMALL_SIZE_MAX)) { p = mi_heap_malloc_small(heap, size); - } + } else { p = _mi_malloc_generic(heap, size); } @@ -235,11 +236,11 @@ void mi_free(void* p) mi_attr_noexcept // huge page stat is accounted for in `_mi_page_retire` #endif - // adjust if it might be an un-aligned block uintptr_t tid = _mi_thread_id(); - if (mi_likely(tid == page->flags.value)) { // local, and not full or aligned + if (mi_likely(tid == page->flags.value)) { + // local, and not full or aligned mi_block_t* block = (mi_block_t*)p; - mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance + mi_block_set_next(page, block, page->local_free); page->local_free = block; page->used--; if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); } diff --git a/src/init.c b/src/init.c index 13ed9561..1ea510b2 100644 --- a/src/init.c +++ b/src/init.c @@ -32,24 +32,37 @@ const mi_page_t _mi_page_empty = { // Empty page queues for every bin #define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } +#ifdef MI_BIN4 #define MI_PAGE_QUEUES_EMPTY \ { QNULL(1), \ - QNULL(1), QNULL(2), QNULL(3), QNULL(4), QNULL(5), QNULL(6), QNULL(7), QNULL(8), \ - QNULL(10), QNULL(12), QNULL(14), QNULL(16), QNULL(20), QNULL(24), QNULL(28), QNULL(32), \ - QNULL(40), QNULL(48), QNULL(56), QNULL(64), QNULL(80), QNULL(96), QNULL(112), QNULL(128), \ - QNULL(160), QNULL(192), QNULL(224), QNULL(256), QNULL(320), QNULL(384), QNULL(448), QNULL(512), \ - QNULL(640), QNULL(768), QNULL(896), QNULL(1024), QNULL(1280), QNULL(1536), QNULL(1792), QNULL(2048), \ - QNULL(2560), QNULL(3072), QNULL(3584), QNULL(4096), QNULL(5120), QNULL(6144), QNULL(7168), QNULL(8192), \ - QNULL(10240), QNULL(12288), QNULL(14336), QNULL(16384), QNULL(20480), QNULL(24576), QNULL(28672), QNULL(32768), \ - QNULL(40960), QNULL(49152), QNULL(57344), QNULL(65536), QNULL(81920), QNULL(98304), QNULL(114688), \ - QNULL(MI_LARGE_WSIZE_MAX + 1 /*131072, Huge queue */), \ + QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ + QNULL( 11), QNULL( 15), QNULL( 23), QNULL( 31), QNULL( 47), QNULL( 63), QNULL( 95), QNULL( 127), /* 16 */ \ + QNULL( 191), QNULL( 255), QNULL( 383), QNULL( 511), QNULL( 767), QNULL( 1023), QNULL( 1535), QNULL( 2047), /* 24 */ \ + QNULL( 3071), QNULL( 4095), QNULL( 6143), QNULL( 8191), QNULL( 12287), QNULL( 16383), QNULL( 24575), QNULL( 32767), /* 32 */ \ + QNULL( 49151), QNULL( 65535), QNULL( 98303), QNULL(131071), QNULL(196607), QNULL(262143), QNULL(393215), /* 39 */ \ + QNULL(MI_LARGE_WSIZE_MAX + 1 /* 524287, Huge queue */), \ QNULL(MI_LARGE_WSIZE_MAX + 2) /* Full queue */ } +#else +#define MI_PAGE_QUEUES_EMPTY \ + { QNULL(1), \ + QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ + QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \ + QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \ + QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \ + QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \ + QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \ + QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ + QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ + QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), /* 69 */ \ + QNULL(MI_LARGE_WSIZE_MAX + 1 /* 393216, Huge queue */), \ + QNULL(MI_LARGE_WSIZE_MAX + 2) /* Full queue */ } +#endif #define MI_STAT_COUNT_NULL() {0,0,0,0} // Empty statistics #if MI_STAT>1 -#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT64(MI_STAT_COUNT_NULL) } +#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } #else #define MI_STAT_COUNT_END_NULL() #endif @@ -97,8 +110,8 @@ static mi_tld_t tld_main = { 0, &_mi_heap_main, { { NULL, NULL }, {NULL ,NULL}, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats }, // segments - { 0, NULL, NULL, 0, tld_main_stats }, // os - { MI_STATS_NULL } // stats + { 0, NULL, NULL, 0, tld_main_stats }, // os + { MI_STATS_NULL } // stats }; mi_heap_t _mi_heap_main = { diff --git a/src/page-queue.c b/src/page-queue.c index fd388113..69ebcc75 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -97,7 +97,7 @@ uint8_t _mi_bsr(uintptr_t x) { // Returns MI_BIN_HUGE if the size is too large. // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. -inline uint8_t _mi_bin(size_t size) { +extern inline uint8_t _mi_bin(size_t size) { size_t wsize = _mi_wsize_from_size(size); uint8_t bin; if (wsize <= 1) { @@ -120,16 +120,21 @@ inline uint8_t _mi_bin(size_t size) { bin = MI_BIN_HUGE; } else { - #if defined(MI_ALIGN4W) + #if defined(MI_ALIGN4W) if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes #endif + #ifdef MI_BIN4 + uint8_t b = mi_bsr32((uint32_t)wsize); + bin = ((b << 1) + (uint8_t)((wsize >> (b - 1)) & 0x01)) + 3; + #else wsize--; // find the highest bit uint8_t b = mi_bsr32((uint32_t)wsize); - // and use the top 3 bits to determine the bin (~16% worst internal fragmentation). + // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). // - adjust with 3 because we use do not round the first 8 sizes // which each get an exact bin bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; + #endif } mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE); return bin; diff --git a/src/page.c b/src/page.c index 9be0372d..e6be8df6 100644 --- a/src/page.c +++ b/src/page.c @@ -385,7 +385,7 @@ void _mi_page_retire(mi_page_t* page) { // is the only page left with free blocks. It is not clear // how to check this efficiently though... for now we just check // if its neighbours are almost fully used. - if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) { + if (mi_likely(page->block_size <= MI_MEDIUM_SIZE_MAX)) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { _mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1); return; // dont't retire after all @@ -722,10 +722,10 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept // call potential deferred free routines _mi_deferred_free(heap, false); - + // free delayed frees from other threads _mi_heap_delayed_free(heap); - + // huge allocation? mi_page_t* page; if (mi_unlikely(size > MI_LARGE_SIZE_MAX)) { diff --git a/src/segment.c b/src/segment.c index f2fd09ad..736345bf 100644 --- a/src/segment.c +++ b/src/segment.c @@ -236,8 +236,8 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se // The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use, -// and no more than 2. -#define MI_SEGMENT_CACHE_MAX (2) +// and no more than 4. +#define MI_SEGMENT_CACHE_MAX (4) #define MI_SEGMENT_CACHE_FRACTION (8) // note: returned segment may be partially reset @@ -708,16 +708,20 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld /* ----------------------------------------------------------- Page allocation and free ----------------------------------------------------------- */ +static bool mi_is_good_fit(size_t bsize, size_t size) { + // good fit if no more than 25% wasted + return (bsize > 0 && size > 0 && bsize < size && (size - (size % bsize)) < (size/4)); +} mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; - if (block_size <= (MI_SMALL_PAGE_SIZE/4)) { + if (block_size <= MI_SMALL_SIZE_MAX || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) { page = mi_segment_small_page_alloc(tld,os_tld); } - else if (block_size <= (MI_MEDIUM_PAGE_SIZE/4)) { + else if (block_size <= MI_MEDIUM_SIZE_MAX || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { page = mi_segment_medium_page_alloc(tld, os_tld); } - else if (block_size < (MI_LARGE_SIZE_MAX - sizeof(mi_segment_t))) { + else if (block_size < MI_LARGE_SIZE_MAX || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t))) { page = mi_segment_large_page_alloc(tld, os_tld); } else { diff --git a/test/main-override-static.c b/test/main-override-static.c index 6ddf4f37..83aa388a 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -6,8 +6,154 @@ #include #include // redefines malloc etc. +#include +#include + +#define MI_INTPTR_SIZE 8 +#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_INTPTR_SIZE) + +#define MI_BIN_HUGE 100 +//#define MI_ALIGN2W + +// Bit scan reverse: return the index of the highest bit. +static inline uint8_t mi_bsr32(uint32_t x); + +#if defined(_MSC_VER) +#include +#include +static inline uint8_t mi_bsr32(uint32_t x) { + uint32_t idx; + _BitScanReverse((DWORD*)&idx, x); + return idx; +} +#elif defined(__GNUC__) || defined(__clang__) +static inline uint8_t mi_bsr32(uint32_t x) { + return (31 - __builtin_clz(x)); +} +#else +static inline uint8_t mi_bsr32(uint32_t x) { + // de Bruijn multiplication, see + static const uint8_t debruijn[32] = { + 31, 0, 22, 1, 28, 23, 18, 2, 29, 26, 24, 10, 19, 7, 3, 12, + 30, 21, 27, 17, 25, 9, 6, 11, 20, 16, 8, 5, 15, 4, 14, 13, + }; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + x++; + return debruijn[(x*0x076be629) >> 27]; +} +#endif + +// Bit scan reverse: return the index of the highest bit. +uint8_t _mi_bsr(uintptr_t x) { + if (x == 0) return 0; + #if MI_INTPTR_SIZE==8 + uint32_t hi = (x >> 32); + return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); + #elif MI_INTPTR_SIZE==4 + return mi_bsr32(x); + #else + # error "define bsr for non-32 or 64-bit platforms" + #endif +} + +static inline size_t _mi_wsize_from_size(size_t size) { + return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); +} + +// Return the bin for a given field size. +// Returns MI_BIN_HUGE if the size is too large. +// We use `wsize` for the size in "machine word sizes", +// i.e. byte size == `wsize*sizeof(void*)`. +extern inline uint8_t _mi_bin8(size_t size) { + size_t wsize = _mi_wsize_from_size(size); + uint8_t bin; + if (wsize <= 1) { + bin = 1; + } + #if defined(MI_ALIGN4W) + else if (wsize <= 4) { + bin = (uint8_t)((wsize+1)&~1); // round to double word sizes + } + #elif defined(MI_ALIGN2W) + else if (wsize <= 8) { + bin = (uint8_t)((wsize+1)&~1); // round to double word sizes + } + #else + else if (wsize <= 8) { + bin = (uint8_t)wsize; + } + #endif + else if (wsize > MI_LARGE_WSIZE_MAX) { + bin = MI_BIN_HUGE; + } + else { + #if defined(MI_ALIGN4W) + if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes + #endif + wsize--; + // find the highest bit + uint8_t b = mi_bsr32((uint32_t)wsize); + // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). + // - adjust with 3 because we use do not round the first 8 sizes + // which each get an exact bin + bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; + } + return bin; +} + +extern inline uint8_t _mi_bin4(size_t size) { + size_t wsize = _mi_wsize_from_size(size); + uint8_t bin; + if (wsize <= 1) { + bin = 1; + } + #if defined(MI_ALIGN4W) + else if (wsize <= 4) { + bin = (uint8_t)((wsize+1)&~1); // round to double word sizes + } + #elif defined(MI_ALIGN2W) + else if (wsize <= 8) { + bin = (uint8_t)((wsize+1)&~1); // round to double word sizes + } + #else + else if (wsize <= 8) { + bin = (uint8_t)wsize; + } + #endif + else if (wsize > MI_LARGE_WSIZE_MAX) { + bin = MI_BIN_HUGE; + } + else { + uint8_t b = mi_bsr32((uint32_t)wsize); + bin = ((b << 1) + (uint8_t)((wsize >> (b - 1)) & 0x01)) + 3; + } + return bin; +} + +void mi_bins() { + printf(" QNULL(1), /* 0 */ \\\n "); + size_t last_bin = 0; + for (size_t size = 1; size < (MI_INTPTR_SIZE*MI_LARGE_WSIZE_MAX); size++) { + size_t bin = _mi_bin4(size); + if (bin != last_bin) { + size_t wsize = (size-1)/sizeof(intptr_t); + // printf("size: %6zd, wsize: %6d, bin: %6zd\n", size - 1, (size-1)/sizeof(intptr_t), last_bin); + printf("QNULL(%6zd), ", wsize); + if (last_bin%8 == 0) printf("/* %i */ \\\n ", last_bin); + last_bin = bin; + } + } +} + + + int main() { mi_version(); + mi_bins(); void* p1 = malloc(78); void* p2 = malloc(24); free(p1); From 7b16aa9302e104ebc4fc74ad8bea450ca58f6829 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 9 Aug 2019 11:22:38 -0700 Subject: [PATCH 65/67] remove accidental commit of different size bins experiment --- include/mimalloc-types.h | 10 --- src/init.c | 12 --- src/page-queue.c | 5 -- test/main-override-static.c | 145 ------------------------------------ 4 files changed, 172 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 45307c15..6aa82210 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -100,22 +100,12 @@ terms of the MIT license. A copy of the license can be found in the file // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) -#define MI_BIN4 -#ifdef MI_BIN4 -// Maximum number of size classes. (spaced exponentially in 25% increments) -#define MI_BIN_HUGE (40U) - -#if (MI_LARGE_WSIZE_MAX > 524287) -#error "define more bins" -#endif -#else // Maximum number of size classes. (spaced exponentially in 12.5% increments) #define MI_BIN_HUGE (70U) #if (MI_LARGE_WSIZE_MAX > 393216) #error "define more bins" #endif -#endif typedef uintptr_t mi_encoded_t; diff --git a/src/init.c b/src/init.c index 1ea510b2..f807d74a 100644 --- a/src/init.c +++ b/src/init.c @@ -32,17 +32,6 @@ const mi_page_t _mi_page_empty = { // Empty page queues for every bin #define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } -#ifdef MI_BIN4 -#define MI_PAGE_QUEUES_EMPTY \ - { QNULL(1), \ - QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ - QNULL( 11), QNULL( 15), QNULL( 23), QNULL( 31), QNULL( 47), QNULL( 63), QNULL( 95), QNULL( 127), /* 16 */ \ - QNULL( 191), QNULL( 255), QNULL( 383), QNULL( 511), QNULL( 767), QNULL( 1023), QNULL( 1535), QNULL( 2047), /* 24 */ \ - QNULL( 3071), QNULL( 4095), QNULL( 6143), QNULL( 8191), QNULL( 12287), QNULL( 16383), QNULL( 24575), QNULL( 32767), /* 32 */ \ - QNULL( 49151), QNULL( 65535), QNULL( 98303), QNULL(131071), QNULL(196607), QNULL(262143), QNULL(393215), /* 39 */ \ - QNULL(MI_LARGE_WSIZE_MAX + 1 /* 524287, Huge queue */), \ - QNULL(MI_LARGE_WSIZE_MAX + 2) /* Full queue */ } -#else #define MI_PAGE_QUEUES_EMPTY \ { QNULL(1), \ QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ @@ -56,7 +45,6 @@ const mi_page_t _mi_page_empty = { QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), /* 69 */ \ QNULL(MI_LARGE_WSIZE_MAX + 1 /* 393216, Huge queue */), \ QNULL(MI_LARGE_WSIZE_MAX + 2) /* Full queue */ } -#endif #define MI_STAT_COUNT_NULL() {0,0,0,0} diff --git a/src/page-queue.c b/src/page-queue.c index 69ebcc75..a386f8a1 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -123,10 +123,6 @@ extern inline uint8_t _mi_bin(size_t size) { #if defined(MI_ALIGN4W) if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes #endif - #ifdef MI_BIN4 - uint8_t b = mi_bsr32((uint32_t)wsize); - bin = ((b << 1) + (uint8_t)((wsize >> (b - 1)) & 0x01)) + 3; - #else wsize--; // find the highest bit uint8_t b = mi_bsr32((uint32_t)wsize); @@ -134,7 +130,6 @@ extern inline uint8_t _mi_bin(size_t size) { // - adjust with 3 because we use do not round the first 8 sizes // which each get an exact bin bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; - #endif } mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE); return bin; diff --git a/test/main-override-static.c b/test/main-override-static.c index 83aa388a..94891cc3 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -6,154 +6,9 @@ #include #include // redefines malloc etc. -#include -#include - -#define MI_INTPTR_SIZE 8 -#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_INTPTR_SIZE) - -#define MI_BIN_HUGE 100 -//#define MI_ALIGN2W - -// Bit scan reverse: return the index of the highest bit. -static inline uint8_t mi_bsr32(uint32_t x); - -#if defined(_MSC_VER) -#include -#include -static inline uint8_t mi_bsr32(uint32_t x) { - uint32_t idx; - _BitScanReverse((DWORD*)&idx, x); - return idx; -} -#elif defined(__GNUC__) || defined(__clang__) -static inline uint8_t mi_bsr32(uint32_t x) { - return (31 - __builtin_clz(x)); -} -#else -static inline uint8_t mi_bsr32(uint32_t x) { - // de Bruijn multiplication, see - static const uint8_t debruijn[32] = { - 31, 0, 22, 1, 28, 23, 18, 2, 29, 26, 24, 10, 19, 7, 3, 12, - 30, 21, 27, 17, 25, 9, 6, 11, 20, 16, 8, 5, 15, 4, 14, 13, - }; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - x |= x >> 8; - x |= x >> 16; - x++; - return debruijn[(x*0x076be629) >> 27]; -} -#endif - -// Bit scan reverse: return the index of the highest bit. -uint8_t _mi_bsr(uintptr_t x) { - if (x == 0) return 0; - #if MI_INTPTR_SIZE==8 - uint32_t hi = (x >> 32); - return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); - #elif MI_INTPTR_SIZE==4 - return mi_bsr32(x); - #else - # error "define bsr for non-32 or 64-bit platforms" - #endif -} - -static inline size_t _mi_wsize_from_size(size_t size) { - return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); -} - -// Return the bin for a given field size. -// Returns MI_BIN_HUGE if the size is too large. -// We use `wsize` for the size in "machine word sizes", -// i.e. byte size == `wsize*sizeof(void*)`. -extern inline uint8_t _mi_bin8(size_t size) { - size_t wsize = _mi_wsize_from_size(size); - uint8_t bin; - if (wsize <= 1) { - bin = 1; - } - #if defined(MI_ALIGN4W) - else if (wsize <= 4) { - bin = (uint8_t)((wsize+1)&~1); // round to double word sizes - } - #elif defined(MI_ALIGN2W) - else if (wsize <= 8) { - bin = (uint8_t)((wsize+1)&~1); // round to double word sizes - } - #else - else if (wsize <= 8) { - bin = (uint8_t)wsize; - } - #endif - else if (wsize > MI_LARGE_WSIZE_MAX) { - bin = MI_BIN_HUGE; - } - else { - #if defined(MI_ALIGN4W) - if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes - #endif - wsize--; - // find the highest bit - uint8_t b = mi_bsr32((uint32_t)wsize); - // and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation). - // - adjust with 3 because we use do not round the first 8 sizes - // which each get an exact bin - bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; - } - return bin; -} - -extern inline uint8_t _mi_bin4(size_t size) { - size_t wsize = _mi_wsize_from_size(size); - uint8_t bin; - if (wsize <= 1) { - bin = 1; - } - #if defined(MI_ALIGN4W) - else if (wsize <= 4) { - bin = (uint8_t)((wsize+1)&~1); // round to double word sizes - } - #elif defined(MI_ALIGN2W) - else if (wsize <= 8) { - bin = (uint8_t)((wsize+1)&~1); // round to double word sizes - } - #else - else if (wsize <= 8) { - bin = (uint8_t)wsize; - } - #endif - else if (wsize > MI_LARGE_WSIZE_MAX) { - bin = MI_BIN_HUGE; - } - else { - uint8_t b = mi_bsr32((uint32_t)wsize); - bin = ((b << 1) + (uint8_t)((wsize >> (b - 1)) & 0x01)) + 3; - } - return bin; -} - -void mi_bins() { - printf(" QNULL(1), /* 0 */ \\\n "); - size_t last_bin = 0; - for (size_t size = 1; size < (MI_INTPTR_SIZE*MI_LARGE_WSIZE_MAX); size++) { - size_t bin = _mi_bin4(size); - if (bin != last_bin) { - size_t wsize = (size-1)/sizeof(intptr_t); - // printf("size: %6zd, wsize: %6d, bin: %6zd\n", size - 1, (size-1)/sizeof(intptr_t), last_bin); - printf("QNULL(%6zd), ", wsize); - if (last_bin%8 == 0) printf("/* %i */ \\\n ", last_bin); - last_bin = bin; - } - } -} - - int main() { mi_version(); - mi_bins(); void* p1 = malloc(78); void* p2 = malloc(24); free(p1); From b1938530afcb70ed3567922c4d7005a76ecd9b69 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 9 Aug 2019 14:31:45 -0700 Subject: [PATCH 66/67] fix comment --- include/mimalloc-types.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 6aa82210..4002c12c 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -91,8 +91,8 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE) #define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) -#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 64kb on 64-bit -#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 512kb on 64-bit +#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit +#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1Mb on 64-bit #define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT) From 78a70ada2e44e3e3f6b9d89ea0600912c8b96a49 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 10 Aug 2019 15:17:40 -0700 Subject: [PATCH 67/67] Fix option reading even when preloading on Windows --- src/options.c | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/options.c b/src/options.c index 01620e75..cd7e5da1 100644 --- a/src/options.c +++ b/src/options.c @@ -208,8 +208,28 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest[dest_size - 1] = 0; } -static const char* mi_getenv(const char* name) { - if (_mi_preloading()) return NULL; // don't call getenv too early +#if defined _WIN32 +// On Windows use GetEnvironmentVariable instead of getenv to work +// reliably even when this is invoked before the C runtime is initialized. +// i.e. when `_mi_preloading() == true`. +#include +static bool mi_getenv(const char* name, char* result, size_t result_size) { + result[0] = 0; + bool ok = (GetEnvironmentVariableA(name, result, (DWORD)result_size) > 0); + if (!ok) { + char buf[64+1]; + size_t len = strlen(name); + if (len >= sizeof(buf)) len = sizeof(buf) - 1; + for (size_t i = 0; i < len; i++) { + buf[i] = toupper(name[i]); + } + buf[len] = 0; + ok = (GetEnvironmentVariableA(name, result, (DWORD)result_size) > 0); + } + return ok; +} +#else +static bool mi_getenv(const char* name, char* result, size_t result_size) { #pragma warning(suppress:4996) const char* s = getenv(name); if (s == NULL) { @@ -223,17 +243,23 @@ static const char* mi_getenv(const char* name) { #pragma warning(suppress:4996) s = getenv(buf); } - return s; + if (s != NULL && strlen(s) < result_size) { + mi_strlcpy(result, s, result_size); + return true; + } + else { + return false; + } } - +#endif static void mi_option_init(mi_option_desc_t* desc) { - if (!_mi_preloading()) desc->init = DEFAULTED; + desc->init = DEFAULTED; // Read option value from the environment char buf[64+1]; mi_strlcpy(buf, "mimalloc_", sizeof(buf)); mi_strlcat(buf, desc->name, sizeof(buf)); - const char* s = mi_getenv(buf); - if (s != NULL) { + char s[64+1]; + if (mi_getenv(buf, s, sizeof(s))) { size_t len = strlen(s); if (len >= sizeof(buf)) len = sizeof(buf) - 1; for (size_t i = 0; i < len; i++) {