From 6165177cb1461c49df3d3baca04d0da966544d51 Mon Sep 17 00:00:00 2001 From: Kirsten Lee Date: Tue, 20 Aug 2019 16:19:16 -0700 Subject: [PATCH 1/6] provide a set of solutions that use the vs2019 vcruntime --- ide/vs2019/mimalloc-override-test.vcxproj | 165 ++++++++++++++ ide/vs2019/mimalloc-override.vcxproj | 243 +++++++++++++++++++++ ide/vs2019/mimalloc-test-stress.vcxproj | 159 ++++++++++++++ ide/vs2019/mimalloc-test.vcxproj | 163 ++++++++++++++ ide/vs2019/mimalloc.sln | 71 ++++++ ide/vs2019/mimalloc.vcxproj | 252 ++++++++++++++++++++++ 6 files changed, 1053 insertions(+) create mode 100644 ide/vs2019/mimalloc-override-test.vcxproj create mode 100644 ide/vs2019/mimalloc-override.vcxproj create mode 100644 ide/vs2019/mimalloc-test-stress.vcxproj create mode 100644 ide/vs2019/mimalloc-test.vcxproj create mode 100644 ide/vs2019/mimalloc.sln create mode 100644 ide/vs2019/mimalloc.vcxproj diff --git a/ide/vs2019/mimalloc-override-test.vcxproj b/ide/vs2019/mimalloc-override-test.vcxproj new file mode 100644 index 00000000..d75a67e1 --- /dev/null +++ b/ide/vs2019/mimalloc-override-test.vcxproj @@ -0,0 +1,165 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {FEF7868F-750E-4C21-A04D-22707CC66879} + mimalloc-override-test + 10.0 + mimalloc-override-test + + + + Application + true + v142 + + + Application + false + v142 + true + + + Application + true + v142 + + + Application + false + v142 + true + + + + + + + + + + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + + Level3 + Disabled + true + true + ..\..\include + MultiThreadedDebugDLL + + + Console + + + + + Level3 + Disabled + true + true + ..\..\include + MultiThreadedDebugDLL + + + Console + + + kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + false + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + MultiThreadedDLL + + + true + true + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + MultiThreadedDLL + + + true + true + Console + + + kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + + + {abb5eae7-b3e6-432e-b636-333449892ea7} + + + + + + \ No newline at end of file diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj new file mode 100644 index 00000000..1f3b7ee2 --- /dev/null +++ b/ide/vs2019/mimalloc-override.vcxproj @@ -0,0 +1,243 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {ABB5EAE7-B3E6-432E-B636-333449892EA7} + mimalloc-override + 10.0 + mimalloc-override + + + + DynamicLibrary + true + v142 + + + DynamicLibrary + false + v142 + true + + + DynamicLibrary + true + v142 + + + DynamicLibrary + false + v142 + true + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .dll + + + + Level3 + Disabled + true + true + ../../include + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions); + MultiThreadedDebugDLL + false + + + + + + + + + + + DllEntry + kernel32.lib;%(AdditionalDependencies) + + + + + + + + + Level3 + Disabled + true + true + ../../include + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions); + MultiThreadedDebugDLL + false + + + + + + + + + + + DllEntry + kernel32.lib;%(AdditionalDependencies) + + + + + + + + + Level3 + MaxSpeed + true + true + true + true + ../../include + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + MultiThreadedDLL + + + true + true + DllEntry + kernel32.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + Level3 + MaxSpeed + true + true + true + true + ../../include + MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;_MBCS;%(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + MultiThreadedDLL + + + true + true + DllEntry + kernel32.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + false + false + false + false + + + + + + + + + + + true + true + true + true + + + + + + + + + \ No newline at end of file diff --git a/ide/vs2019/mimalloc-test-stress.vcxproj b/ide/vs2019/mimalloc-test-stress.vcxproj new file mode 100644 index 00000000..6aed1cc1 --- /dev/null +++ b/ide/vs2019/mimalloc-test-stress.vcxproj @@ -0,0 +1,159 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {FEF7958F-750E-4C21-A04D-22707CC66878} + mimalloc-test-stress + 10.0 + mimalloc-test-stress + + + + Application + true + v142 + + + Application + false + v142 + true + + + Application + true + v142 + + + Application + false + v142 + true + + + + + + + + + + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + + Level3 + Disabled + true + true + ..\..\include + + + Console + + + + + Level3 + Disabled + true + true + ..\..\include + + + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + %(PreprocessorDefinitions);NDEBUG + + + true + true + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + %(PreprocessorDefinitions);NDEBUG + + + true + true + Console + + + + + false + false + false + false + + + + + {abb5eae7-b3e6-432e-b636-333449892ea6} + + + + + + \ No newline at end of file diff --git a/ide/vs2019/mimalloc-test.vcxproj b/ide/vs2019/mimalloc-test.vcxproj new file mode 100644 index 00000000..1e901e45 --- /dev/null +++ b/ide/vs2019/mimalloc-test.vcxproj @@ -0,0 +1,163 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {FEF7858F-750E-4C21-A04D-22707CC66878} + mimalloctest + 10.0 + mimalloc-test + + + + Application + true + v142 + + + Application + false + v142 + true + + + Application + true + v142 + + + Application + false + v142 + true + + + + + + + + + + + + + + + + + + + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + $(ProjectDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + + + + Level3 + Disabled + true + true + ..\..\include + stdcpp17 + + + Console + + + + + Level3 + Disabled + true + true + ..\..\include + stdcpp17 + + + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + stdcpp17 + + + true + true + Console + + + + + Level3 + MaxSpeed + true + true + true + true + ..\..\include + _MBCS;%(PreprocessorDefinitions);NDEBUG + stdcpp17 + + + true + true + Console + + + + + AssemblyAndSourceCode + AssemblyAndSourceCode + AssemblyAndSourceCode + AssemblyAndSourceCode + + + + + {abb5eae7-b3e6-432e-b636-333449892ea6} + + + + + + \ No newline at end of file diff --git a/ide/vs2019/mimalloc.sln b/ide/vs2019/mimalloc.sln new file mode 100644 index 00000000..aeab6b88 --- /dev/null +++ b/ide/vs2019/mimalloc.sln @@ -0,0 +1,71 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.28010.2016 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc", "mimalloc.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA6}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test", "mimalloc-test.vcxproj", "{FEF7858F-750E-4C21-A04D-22707CC66878}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override", "mimalloc-override.vcxproj", "{ABB5EAE7-B3E6-432E-B636-333449892EA7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-override-test", "mimalloc-override-test.vcxproj", "{FEF7868F-750E-4C21-A04D-22707CC66879}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mimalloc-test-stress", "mimalloc-test-stress.vcxproj", "{FEF7958F-750E-4C21-A04D-22707CC66878}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.ActiveCfg = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x64.Build.0 = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.ActiveCfg = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Debug|x86.Build.0 = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.ActiveCfg = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x64.Build.0 = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.ActiveCfg = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA6}.Release|x86.Build.0 = Release|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7858F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.ActiveCfg = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x64.Build.0 = Debug|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.ActiveCfg = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Debug|x86.Build.0 = Debug|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.ActiveCfg = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x64.Build.0 = Release|x64 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.ActiveCfg = Release|Win32 + {ABB5EAE7-B3E6-432E-B636-333449892EA7}.Release|x86.Build.0 = Release|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x64.Build.0 = Debug|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Debug|x86.Build.0 = Debug|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.ActiveCfg = Release|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x64.Build.0 = Release|x64 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.ActiveCfg = Release|Win32 + {FEF7868F-750E-4C21-A04D-22707CC66879}.Release|x86.Build.0 = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.ActiveCfg = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x64.Build.0 = Debug|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.ActiveCfg = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Debug|x86.Build.0 = Debug|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.ActiveCfg = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x64.Build.0 = Release|x64 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.ActiveCfg = Release|Win32 + {FEF7958F-750E-4C21-A04D-22707CC66878}.Release|x86.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4297F93D-486A-4243-995F-7D32F59AE82A} + EndGlobalSection +EndGlobal diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj new file mode 100644 index 00000000..4d9563c2 --- /dev/null +++ b/ide/vs2019/mimalloc.vcxproj @@ -0,0 +1,252 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + 15.0 + {ABB5EAE7-B3E6-432E-B636-333449892EA6} + mimalloc + 10.0 + mimalloc + + + + StaticLibrary + true + v142 + + + StaticLibrary + false + v142 + true + + + StaticLibrary + true + v142 + + + StaticLibrary + false + v142 + true + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + + + $(SolutionDir)..\..\out\msvc-$(Platform)\$(Configuration)\ + $(SolutionDir)..\..\out\msvc-$(Platform)\$(ProjectName)\$(Configuration)\ + .lib + + + + Level3 + Disabled + true + true + ../../include + MI_DEBUG=3;%(PreprocessorDefinitions); + Default + false + + + + + + + + + + + Level3 + Disabled + true + true + ../../include + MI_DEBUG=3;%(PreprocessorDefinitions); + Default + false + + + + + + + + + + + + + + + + + + + Level3 + MaxSpeed + true + true + true + true + ../../include + %(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + false + AnySuitable + Neither + false + false + Default + + + true + true + + + + + + + + + + + Level3 + MaxSpeed + true + true + true + true + ../../include + %(PreprocessorDefinitions);NDEBUG + AssemblyAndSourceCode + $(IntDir) + false + false + AnySuitable + Neither + false + false + Default + + + true + true + + + + + + + + + + + + + + + + + false + false + false + false + + + true + true + true + true + + + true + true + true + true + + + true + true + true + true + + + + + + + + + true + true + true + true + + + + + + + + + + + + + + + + \ No newline at end of file From 23812cc0ac0e37fb2c123f1d391aecdfc372fbfc Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 24 Aug 2019 15:45:14 -0700 Subject: [PATCH 2/6] do not keep a queue of huge pages and free them directly --- src/alloc.c | 15 +++++++++++++++ src/page-queue.c | 1 + src/page.c | 30 ++++++++++++++++++------------ src/segment.c | 5 +++-- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index b7881ea5..76e093e7 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -115,6 +115,21 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc mi_thread_free_t tfreex; bool use_delayed; + mi_segment_t* segment = _mi_page_segment(page); + if (segment->page_kind==MI_PAGE_HUGE) { + // huge page segments are always abandoned and can be freed immediately + mi_assert_internal(segment->thread_id==0); + mi_assert_internal(segment->abandoned_next==NULL); + // claim it and free + mi_block_set_next(page, block, page->free); + page->free = block; + page->used--; + mi_heap_t* heap = mi_get_default_heap(); + segment->thread_id = heap->thread_id; + _mi_segment_page_free(page,true,&heap->tld->segments); + return; + } + do { tfree = page->thread_free; use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE || diff --git a/src/page-queue.c b/src/page-queue.c index 859b1d57..d613095f 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -268,6 +268,7 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) { mi_assert_internal(page->heap == NULL); mi_assert_internal(!mi_page_queue_contains(queue, page)); + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_LARGE_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); diff --git a/src/page.c b/src/page.c index a7b4a760..f7e0ce2c 100644 --- a/src/page.c +++ b/src/page.c @@ -98,11 +98,13 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif if (page->heap!=NULL) { mi_segment_t* segment = _mi_page_segment(page); - mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id); - mi_page_queue_t* pq = mi_page_queue_of(page); - mi_assert_internal(mi_page_queue_contains(pq, page)); - mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); - mi_assert_internal(mi_heap_contains_queue(page->heap,pq)); + mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == page->heap->thread_id || segment->thread_id==0); + if (segment->page_kind != MI_PAGE_HUGE) { + mi_page_queue_t* pq = mi_page_queue_of(page); + mi_assert_internal(mi_page_queue_contains(pq, page)); + mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); + mi_assert_internal(mi_heap_contains_queue(page->heap,pq)); + } } return true; } @@ -204,6 +206,7 @@ void _mi_page_free_collect(mi_page_t* page) { void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); mi_assert_internal(page->heap == NULL); + mi_assert_internal(_mi_page_segment(page)->page_kind != MI_PAGE_HUGE); _mi_page_free_collect(page); mi_page_queue_t* pq = mi_page_queue(heap, page->block_size); mi_page_queue_push(heap, pq, page); @@ -212,12 +215,13 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { // allocate a fresh page from a segment static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size_t block_size) { - mi_assert_internal(mi_heap_contains_queue(heap, pq)); + mi_assert_internal(pq==NULL||mi_heap_contains_queue(heap, pq)); mi_page_t* page = _mi_segment_page_alloc(block_size, &heap->tld->segments, &heap->tld->os); if (page == NULL) return NULL; + mi_assert_internal(pq==NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE); mi_page_init(heap, page, block_size, &heap->tld->stats); _mi_stat_increase( &heap->tld->stats.pages, 1); - mi_page_queue_push(heap, pq, page); + if (pq!=NULL) mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL mi_assert_expensive(_mi_page_is_valid(page)); return page; } @@ -699,13 +703,15 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn) mi_attr_noexcept { // A huge page is allocated directly without being in a queue static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { size_t block_size = _mi_wsize_from_size(size) * sizeof(uintptr_t); - mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); - mi_page_queue_t* pq = mi_page_queue(heap,block_size); - mi_assert_internal(mi_page_queue_is_huge(pq)); - mi_page_t* page = mi_page_fresh_alloc(heap,pq,block_size); + mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); + mi_page_t* page = mi_page_fresh_alloc(heap,NULL,block_size); if (page != NULL) { mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(page->block_size == block_size); + mi_assert_internal(_mi_page_segment(page)->page_kind==MI_PAGE_HUGE); + mi_assert_internal(_mi_page_segment(page)->used==1); + mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue + page->heap = NULL; if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_increase(&heap->tld->stats.giant, block_size); _mi_stat_counter_increase(&heap->tld->stats.giant_count, 1); @@ -714,7 +720,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { _mi_stat_increase(&heap->tld->stats.huge, block_size); _mi_stat_counter_increase(&heap->tld->stats.huge_count, 1); } - } + } return page; } diff --git a/src/segment.c b/src/segment.c index b1a5221c..3be703cf 100644 --- a/src/segment.c +++ b/src/segment.c @@ -134,7 +134,7 @@ static bool mi_segment_is_valid(mi_segment_t* segment) { if (!segment->pages[i].segment_in_use) nfree++; } mi_assert_internal(nfree + segment->used == segment->capacity); - mi_assert_internal(segment->thread_id == _mi_thread_id()); // or 0 + mi_assert_internal(segment->thread_id == _mi_thread_id() || (segment->thread_id==0)); // or 0 mi_assert_internal(segment->page_kind == MI_PAGE_HUGE || (mi_segment_pagesize(segment) * segment->capacity == segment->segment_size)); return true; @@ -700,6 +700,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld if (segment == NULL) return NULL; mi_assert_internal(segment->segment_size - segment->segment_info_size >= size); segment->used = 1; + segment->thread_id = 0; // huge pages are immediately abandoned mi_page_t* page = &segment->pages[0]; page->segment_in_use = true; return page; @@ -721,7 +722,7 @@ mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_ else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { page = mi_segment_medium_page_alloc(tld, os_tld); } - else if (block_size < MI_LARGE_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t))) { + else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) { page = mi_segment_large_page_alloc(tld, os_tld); } else { From a431d80fc38639fa4186baa0a0d13063fa7dc994 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 25 Aug 2019 10:01:11 -0700 Subject: [PATCH 3/6] better object sizes for large objects --- include/mimalloc-types.h | 12 +++++++----- src/page.c | 5 ++++- src/segment.c | 8 ++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index c20b663a..db39b9c4 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -91,11 +91,13 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE) #define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) -#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) -#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit -#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2Mb on 64-bit -#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT) -#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) +// The max object size are checked to not waste more than 12.5% internally over the page sizes. +// (Except for large pages since huge objects are allocated in 4MiB chunks) +#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 16kb +#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb +#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2mb +#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) +#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) // Minimal alignment necessary. On most platforms 16 bytes are needed // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` diff --git a/src/page.c b/src/page.c index f7e0ce2c..cb891d18 100644 --- a/src/page.c +++ b/src/page.c @@ -700,7 +700,10 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn) mi_attr_noexcept { General allocation ----------------------------------------------------------- */ -// A huge page is allocated directly without being in a queue +// A huge page is allocated directly without being in a queue. +// Because huge pages contain just one block, and the segment contains +// just that page, we always treat them as abandoned and any thread +// that frees the block can free the whole page and segment directly. static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { size_t block_size = _mi_wsize_from_size(size) * sizeof(uintptr_t); mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); diff --git a/src/segment.c b/src/segment.c index 3be703cf..85a63ca4 100644 --- a/src/segment.c +++ b/src/segment.c @@ -709,17 +709,13 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, mi_segments_tld_t* tld /* ----------------------------------------------------------- Page allocation and free ----------------------------------------------------------- */ -static bool mi_is_good_fit(size_t bsize, size_t size) { - // good fit if no more than 25% wasted - return (bsize > 0 && size > 0 && bsize < size && (size - (size % bsize)) < (size/4)); -} mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* page; - if (block_size <= MI_SMALL_OBJ_SIZE_MAX || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) { + if (block_size <= MI_SMALL_OBJ_SIZE_MAX) { page = mi_segment_small_page_alloc(tld,os_tld); } - else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { + else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) { page = mi_segment_medium_page_alloc(tld, os_tld); } else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) { From 7972b64a822dba03763d4fb9c28fa43765984da7 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 25 Aug 2019 11:18:16 -0700 Subject: [PATCH 4/6] resolve race on option initialization --- src/options.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index bc658ca9..8880c317 100644 --- a/src/options.c +++ b/src/options.c @@ -18,6 +18,9 @@ int mi_version(void) mi_attr_noexcept { // -------------------------------------------------------- // Options +// These can be accessed by multiple threads and may be +// concurrently initialized, but an initializing data race +// is ok since they resolve to the same value. // -------------------------------------------------------- typedef enum mi_init_e { UNINIT, // not yet initialized @@ -180,7 +183,6 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { } static void mi_option_init(mi_option_desc_t* desc) { - desc->init = DEFAULTED; // Read option value from the environment char buf[32]; mi_strlcpy(buf, "mimalloc_", sizeof(buf)); @@ -218,7 +220,12 @@ static void mi_option_init(mi_option_desc_t* desc) { } else { _mi_warning_message("environment option mimalloc_%s has an invalid value: %s\n", desc->name, buf); + desc->init = DEFAULTED; } } } + else { + desc->init = DEFAULTED; + } + mi_assert_internal(desc->init != UNINIT); } From 038e8fd7d68f8a328c0e728b102dccb00df2da0f Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 25 Aug 2019 12:10:18 -0700 Subject: [PATCH 5/6] reduce retire size --- src/page.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page.c b/src/page.c index cb891d18..54897af5 100644 --- a/src/page.c +++ b/src/page.c @@ -393,7 +393,7 @@ void _mi_page_retire(mi_page_t* page) { // is the only page left with free blocks. It is not clear // how to check this efficiently though... for now we just check // if its neighbours are almost fully used. - if (mi_likely(page->block_size <= MI_SMALL_SIZE_MAX)) { + if (mi_likely(page->block_size <= (MI_SMALL_SIZE_MAX/4))) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { _mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1); return; // dont't retire after all From b86c851cca62fc03233295178c2119d43e7ce486 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 25 Aug 2019 12:29:01 -0700 Subject: [PATCH 6/6] put segment cache behind an option and disable by default --- include/mimalloc.h | 1 + src/options.c | 1 + src/segment.c | 15 ++++++++------- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 9f27e463..4f13bc1f 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -228,6 +228,7 @@ typedef enum mi_option_e { mi_option_eager_commit, mi_option_eager_region_commit, mi_option_large_os_pages, // implies eager commit + mi_option_segment_cache, mi_option_page_reset, mi_option_cache_reset, mi_option_reset_decommits, diff --git a/src/options.c b/src/options.c index b48e45ef..b30ff1c6 100644 --- a/src/options.c +++ b/src/options.c @@ -65,6 +65,7 @@ static mi_option_desc_t options[_mi_option_last] = { 1, UNINIT, MI_OPTION(eager_region_commit) }, #endif { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(page_reset) }, { 0, UNINIT, MI_OPTION(cache_reset) }, { 0, UNINIT, MI_OPTION(reset_decommits) } // note: cannot enable this if secure is on diff --git a/src/segment.c b/src/segment.c index 85a63ca4..18c06fbc 100644 --- a/src/segment.c +++ b/src/segment.c @@ -236,8 +236,6 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se // The thread local segment cache is limited to be at most 1/8 of the peak size of segments in use, -// and no more than 4. -#define MI_SEGMENT_CACHE_MAX (4) #define MI_SEGMENT_CACHE_FRACTION (8) // note: returned segment may be partially reset @@ -253,15 +251,18 @@ static mi_segment_t* mi_segment_cache_pop(size_t segment_size, mi_segments_tld_t return segment; } -static bool mi_segment_cache_full(mi_segments_tld_t* tld) { - if (tld->cache_count < MI_SEGMENT_CACHE_MAX - && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION)) - ) { // always allow 1 element cache +static bool mi_segment_cache_full(mi_segments_tld_t* tld) +{ + if (tld->count == 1 && tld->cache_count==0) return false; // always cache at least the final segment of a thread + size_t max_cache = mi_option_get(mi_option_segment_cache); + if (tld->cache_count < max_cache + && tld->cache_count < (1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION)) // at least allow a 1 element cache + ) { return false; } // take the opportunity to reduce the segment cache if it is too large (now) // TODO: this never happens as we check against peak usage, should we use current usage instead? - while (tld->cache_count > MI_SEGMENT_CACHE_MAX ) { //(1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { + while (tld->cache_count > max_cache) { //(1 + (tld->peak_count / MI_SEGMENT_CACHE_FRACTION))) { mi_segment_t* segment = mi_segment_cache_pop(0,tld); mi_assert_internal(segment != NULL); if (segment != NULL) mi_segment_os_free(segment, segment->segment_size, tld);