From db3d8485d2f45a6f179d784f602f9eff4f60795c Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 21 Aug 2024 17:13:51 -0700 Subject: [PATCH 1/4] increase TSAN test to 400 iterations --- azure-pipelines.yml | 8 ++++---- test/test-stress.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 256f7088..4455dfeb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,7 +43,7 @@ jobs: solution: $(BuildType)/libmimalloc.sln configuration: '$(MSBuildConfiguration)' msbuildArguments: -m - - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) + - script: ctest --verbose --timeout 180 -C $(MSBuildConfiguration) workingDirectory: $(BuildType) displayName: CTest #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress @@ -126,7 +126,7 @@ jobs: cmakeArgs: .. $(cmakeExtraArgs) - script: make -j$(nproc) -C $(BuildType) displayName: Make - - script: ctest --verbose --timeout 240 + - script: ctest --verbose --timeout 180 workingDirectory: $(BuildType) displayName: CTest env: @@ -157,7 +157,7 @@ jobs: cmakeArgs: .. $(cmakeExtraArgs) - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) displayName: Make - - script: ctest --verbose --timeout 240 + - script: ctest --verbose --timeout 180 workingDirectory: $(BuildType) displayName: CTest # - upload: $(Build.SourcesDirectory)/$(BuildType) @@ -193,5 +193,5 @@ jobs: # configuration: '$(MSBuildConfiguration)' # - script: | # cd $(BuildType) -# ctest --verbose --timeout 240 +# ctest --verbose --timeout 180 # displayName: CTest diff --git a/test/test-stress.c b/test/test-stress.c index 25942299..30ad0e77 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -28,7 +28,7 @@ terms of the MIT license. #if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits static int THREADS = 8; static int SCALE = 25; -static int ITER = 200; +static int ITER = 400; #elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits static int THREADS = 8; static int SCALE = 25; From 394e8c27d8b924e8b0248adc2835f65451c4f2ab Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:02:24 -0700 Subject: [PATCH 2/4] add cmake option to add C pre processor definitions more easily --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e78ba9fd..b32542ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) option(MI_NO_THP "Disable transparent huge pages support on Linux/Android for the mimalloc process only" OFF) +option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") # deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) @@ -62,8 +63,8 @@ set(mi_sources set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build set(mi_cflags_dynamic "") # extra flags for a shared-object library build -set(mi_defines "") set(mi_libraries "") +set(mi_defines ${MI_EXTRA_CPPDEFS}) # ----------------------------------------------------------------------------- # Convenience: set default build type depending on the build directory From 638ea539de088390b36f646e6c342dd230d0c93a Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:04:01 -0700 Subject: [PATCH 3/4] allow certain options to have defaults set via the pre-processor at build time -- see issue #945 --- src/options.c | 62 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/src/options.c b/src/options.c index 27122c78..76d2cf82 100644 --- a/src/options.c +++ b/src/options.c @@ -47,6 +47,47 @@ typedef struct mi_option_desc_s { #define MI_OPTION(opt) mi_option_##opt, #opt, NULL #define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy +// Some options can be set at build time for statically linked libraries (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`) +// This is useful if we cannot pass them as environment variables +// (and setting them programmatically would be too late) + +#ifndef MI_DEFAULT_VERBOSE +#define MI_DEFAULT_VERBOSE 0 +#endif + +#ifndef MI_DEFAULT_EAGER_COMMIT +#define MI_DEFAULT_EAGER_COMMIT 1 +#endif + +#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT +#define MI_DEFAULT_ARENA_EAGER_COMMIT 2 +#endif + +#ifndef MI_DEFAULT_ARENA_RESERVE + #if (MI_INTPTR_SIZE>4) + #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L + #else + #define MI_DEFAULT_ARENA_RESERVE 128L*1024L + #endif +#endif + +#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC +#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0 +#endif + +#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES +#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES +#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0 +#endif + +#ifndef MI_DEFAULT_RESERVE_OS_MEMORY +#define MI_DEFAULT_RESERVE_OS_MEMORY 0 +#endif + + static mi_option_desc_t options[_mi_option_last] = { // stable options @@ -56,16 +97,16 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(show_errors) }, #endif { 0, UNINIT, MI_OPTION(show_stats) }, - { 0, UNINIT, MI_OPTION(verbose) }, + { MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) }, // the following options are experimental and not all combinations make sense. - { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) - { 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) + { MI_DEFAULT_EAGER_COMMIT, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) + { MI_DEFAULT_ARENA_EAGER_COMMIT, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux) { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit) - { 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { MI_DEFAULT_ALLOW_LARGE_OS_PAGES, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { MI_DEFAULT_RESERVE_HUGE_OS_PAGES, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages {-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N - { 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) + { MI_DEFAULT_RESERVE_OS_MEMORY, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`) { 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free { 0, UNINIT, MI_OPTION(abandoned_page_purge) }, // purge free page memory when a thread terminates @@ -83,16 +124,11 @@ static mi_option_desc_t options[_mi_option_last] = { 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output { 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! - #if (MI_INTPTR_SIZE>4) - { 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - #else - { 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit - #endif - + { MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free - { 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) + { MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. #if defined(MI_VISIT_ABANDONED) { 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandonded segments; requires taking locks during reclaim. From 50d3525a8c948a5943c89590629e869fc7cdbe0d Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 21 Oct 2024 05:04:27 -0700 Subject: [PATCH 4/4] add test for issue #944 --- test/main-override.cpp | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index fc7f70f0..50eb0267 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -37,6 +37,7 @@ static void tsan_numa_test(); // issue #414 static void strdup_test(); // issue #445 static void heap_thread_free_huge(); static void test_std_string(); // issue #697 +static void test_thread_local(); // issue #944 static void test_stl_allocators(); @@ -44,7 +45,8 @@ static void test_stl_allocators(); int main() { // mi_stats_reset(); // ignore earlier allocations - test_std_string(); + //test_std_string(); + test_thread_local(); // heap_thread_free_huge(); /* heap_thread_free_large(); @@ -312,3 +314,31 @@ static void tsan_numa_test() { dummy_worker(); t1.join(); } + + +class MTest +{ + char *data; +public: + MTest() { data = (char*)malloc(1024); } + ~MTest() { free(data); }; +}; + +thread_local MTest tlVariable; + +void threadFun( int i ) +{ + printf( "Thread %d\n", i ); + std::this_thread::sleep_for( std::chrono::milliseconds(100) ); +} + +void test_thread_local() +{ + for( int i=1; i < 100; ++i ) + { + std::thread t( threadFun, i ); + t.join(); + mi_stats_print(NULL); + } + return; +} \ No newline at end of file