diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj index 6847e388..ffa14a54 100644 --- a/ide/vs2017/mimalloc-override.vcxproj +++ b/ide/vs2017/mimalloc-override.vcxproj @@ -150,7 +150,6 @@ MaxSpeed true true - true true ../../include MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG @@ -159,6 +158,7 @@ false MultiThreadedDLL Default + false true @@ -182,7 +182,6 @@ MaxSpeed true true - true true ../../include MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG @@ -191,6 +190,7 @@ false MultiThreadedDLL Default + false true diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj index 3e453471..854bf921 100644 --- a/ide/vs2017/mimalloc.vcxproj +++ b/ide/vs2017/mimalloc.vcxproj @@ -141,8 +141,6 @@ Level3 MaxSpeed true - true - true true ../../include %(PreprocessorDefinitions);NDEBUG @@ -150,11 +148,9 @@ $(IntDir) false false - AnySuitable - Neither - false - false + Default CompileAsCpp + true true @@ -172,8 +168,6 @@ Level3 MaxSpeed true - true - true true ../../include %(PreprocessorDefinitions);NDEBUG @@ -181,11 +175,9 @@ $(IntDir) false false - AnySuitable - Neither - false - false + Default CompileAsCpp + true true diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 42720386..08f9ad51 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -151,15 +151,15 @@ bool _mi_page_is_valid(mi_page_t* page); // Overflow detecting multiply #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) -static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) { +static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 #if (MI_INTPTR_SIZE == 4) - return __builtin_umul_overflow(size, count, total); + return __builtin_umul_overflow(count, size, total); #else - return __builtin_umull_overflow(size, count, total); + return __builtin_umull_overflow(count, size, total); #endif #else /* __builtin_umul_overflow is unavailable */ - *total = size * count; + *total = count * size; return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 94188cc0..c0778f87 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -93,7 +93,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit -#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/4) // 1Mb on 64-bit +#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2Mb on 64-bit #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT) #define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) diff --git a/include/mimalloc.h b/include/mimalloc.h index c6b7b5f8..6615e2e2 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -220,8 +220,8 @@ mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; typedef enum mi_option_e { // stable options - mi_option_show_stats, mi_option_show_errors, + mi_option_show_stats, mi_option_verbose, // the following options are experimental mi_option_secure, @@ -231,14 +231,15 @@ typedef enum mi_option_e { mi_option_page_reset, mi_option_cache_reset, mi_option_reset_decommits, - mi_option_reset_discards, _mi_option_last } mi_option_t; mi_decl_export bool mi_option_is_enabled(mi_option_t option); -mi_decl_export void mi_option_enable(mi_option_t option, bool enable); -mi_decl_export void mi_option_enable_default(mi_option_t option, bool enable); +mi_decl_export void mi_option_enable(mi_option_t option); +mi_decl_export void mi_option_disable(mi_option_t option); +mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); +mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); mi_decl_export long mi_option_get(mi_option_t option); mi_decl_export void mi_option_set(mi_option_t option, long value); diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c index d1d51b9a..0bd05deb 100644 --- a/src/alloc-override-win.c +++ b/src/alloc-override-win.c @@ -666,7 +666,7 @@ static void mi_patches_at_quick_exit(void) { mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op } -__declspec(dllexport) BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) { +BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) { if (reason == DLL_PROCESS_ATTACH) { __security_init_cookie(); } diff --git a/src/heap.c b/src/heap.c index 35c1d460..78860125 100644 --- a/src/heap.c +++ b/src/heap.c @@ -303,14 +303,14 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; - // unfull all full pages - mi_page_t* page = heap->pages[MI_BIN_FULL].first; + // unfull all full pages in the `from` heap + mi_page_t* page = from->pages[MI_BIN_FULL].first; while (page != NULL) { mi_page_t* next = page->next; _mi_page_unfull(page); page = next; } - mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL); + mi_assert_internal(from->pages[MI_BIN_FULL].first == NULL); // free outstanding thread delayed free blocks _mi_heap_delayed_free(from); diff --git a/src/init.c b/src/init.c index c644359f..6dfb138d 100644 --- a/src/init.c +++ b/src/init.c @@ -106,14 +106,14 @@ mi_heap_t _mi_heap_main = { MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, NULL, - 0, - 0, + 0, // thread id #if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!) 0xCDCDCDCDCDCDCDCDUL, #else 0xCDCDCDCDUL, #endif - 0, + 0, // random + 0, // page count false // can reclaim }; diff --git a/src/memory.c b/src/memory.c index bf99ef42..26f87092 100644 --- a/src/memory.c +++ b/src/memory.c @@ -128,6 +128,7 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit size_t mask = mi_region_block_mask(blocks,bitidx); mi_assert_internal(mask != 0); mi_assert_internal((mask & mi_atomic_read(®ion->map)) == mask); + mi_assert_internal(®ions[idx] == region); // ensure the region is reserved void* start = mi_atomic_read_ptr(®ion->start); @@ -149,9 +150,23 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit mi_atomic_increment(®ions_count); } else { - // failed, another thread allocated just before us, free our allocated memory - // TODO: should we keep the allocated memory and assign it to some other region? - _mi_os_free(start, MI_REGION_SIZE, tld->stats); + // failed, another thread allocated just before us! + // we assign it to a later slot instead (up to 4 tries). + // note: we don't need to increment the region count, this will happen on another allocation + for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) { + void* s = mi_atomic_read_ptr(®ions[idx+i].start); + if (s == NULL) { // quick test + if (mi_atomic_compare_exchange_ptr(®ions[idx+i].start, start, s)) { + start = NULL; + break; + } + } + } + if (start != NULL) { + // free it if we didn't succeed to save it to some other region + _mi_os_free(start, MI_REGION_SIZE, tld->stats); + } + // and continue with the memory at our index start = mi_atomic_read_ptr(®ion->start); } } diff --git a/src/options.c b/src/options.c index 7e6e1a06..f44f10ce 100644 --- a/src/options.c +++ b/src/options.c @@ -34,34 +34,37 @@ typedef enum mi_init_e { typedef struct mi_option_desc_s { long value; // the value mi_init_t init; // is it initialized yet? (from the environment) + mi_option_t option; // for debugging: the option index should match the option const char* name; // option name without `mimalloc_` prefix } mi_option_desc_t; +#define MI_OPTION(opt) mi_option_##opt, #opt +#define MI_OPTION_DESC(opt) {0, UNINIT, MI_OPTION(opt) } + static mi_option_desc_t options[_mi_option_last] = { // stable options - { 0, UNINIT, "show_stats" }, - { MI_DEBUG, UNINIT, "show_errors" }, - { 0, UNINIT, "verbose" }, + { MI_DEBUG, UNINIT, MI_OPTION(show_errors) }, + { 0, UNINIT, MI_OPTION(show_stats) }, + { 0, UNINIT, MI_OPTION(verbose) }, #if MI_SECURE - { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored + { MI_SECURE, INITIALIZED, MI_OPTION(secure) }, // in a secure build the environment setting is ignored #else - { 0, UNINIT, "secure" }, + { 0, UNINIT, MI_OPTION(secure) }, #endif // the following options are experimental and not all combinations make sense. - { 1, UNINIT, "eager_commit" }, // note: if eager_region_commit is on, this should be on too. + { 1, UNINIT, MI_OPTION(eager_commit) }, // note: if eager_region_commit is on, this should be on too. #ifdef _WIN32 // and BSD? - { 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...) + { 0, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...) #else - { 1, UNINIT, "eager_region_commit" }, + { 1, UNINIT, MI_OPTION(eager_region_commit) }, #endif - { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's - { 0, UNINIT, "page_reset" }, - { 0, UNINIT, "cache_reset" }, - { 0, UNINIT, "reset_decommits" }, // note: cannot enable this if secure is on - { 0, UNINIT, "reset_discards" } // note: cannot enable this if secure is on + { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's + { 0, UNINIT, MI_OPTION(page_reset) }, + { 0, UNINIT, MI_OPTION(cache_reset) }, + { 0, UNINIT, MI_OPTION(reset_decommits) } // note: cannot enable this if secure is on }; static void mi_option_init(mi_option_desc_t* desc); @@ -69,6 +72,7 @@ static void mi_option_init(mi_option_desc_t* desc); long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); mi_option_desc_t* desc = &options[option]; + mi_assert(desc->option == option); // index should match the option if (mi_unlikely(desc->init == UNINIT)) { mi_option_init(desc); if (option != mi_option_verbose) { @@ -81,6 +85,7 @@ long mi_option_get(mi_option_t option) { void mi_option_set(mi_option_t option, long value) { mi_assert(option >= 0 && option < _mi_option_last); mi_option_desc_t* desc = &options[option]; + mi_assert(desc->option == option); // index should match the option desc->value = value; desc->init = INITIALIZED; } @@ -97,14 +102,23 @@ bool mi_option_is_enabled(mi_option_t option) { return (mi_option_get(option) != 0); } -void mi_option_enable(mi_option_t option, bool enable) { +void mi_option_set_enabled(mi_option_t option, bool enable) { mi_option_set(option, (enable ? 1 : 0)); } -void mi_option_enable_default(mi_option_t option, bool enable) { +void mi_option_set_enabled_default(mi_option_t option, bool enable) { mi_option_set_default(option, (enable ? 1 : 0)); } +void mi_option_enable(mi_option_t option) { + mi_option_set_enabled(option,true); +} + +void mi_option_disable(mi_option_t option) { + mi_option_set_enabled(option,false); +} + + // -------------------------------------------------------- // Messages // -------------------------------------------------------- diff --git a/src/os.c b/src/os.c index bee5ac64..7afe447e 100644 --- a/src/os.c +++ b/src/os.c @@ -22,6 +22,9 @@ terms of the MIT license. A copy of the license can be found in the file #else #include // mmap #include // sysconf +#if defined(__linux__) +#include // linux mmap flags +#endif #if defined(__APPLE__) #include #endif @@ -67,7 +70,7 @@ size_t _mi_os_large_page_size() { static bool use_large_os_page(size_t size, size_t alignment) { // if we have access, check the size and alignment requirements - if (large_os_page_size == 0) return false; + if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false; return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0); } @@ -81,11 +84,9 @@ static size_t mi_os_good_alloc_size(size_t size, size_t alignment) { #if defined(_WIN32) // We use VirtualAlloc2 for aligned allocation, but it is only supported on Windows 10 and Windows Server 2016. // So, we need to look it up dynamically to run on older systems. (use __stdcall for 32-bit compatibility) -// Same for DiscardVirtualMemory. (hide MEM_EXTENDED_PARAMETER to compile with older SDK's) +// (hide MEM_EXTENDED_PARAMETER to compile with older SDK's) typedef PVOID(__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG); -typedef DWORD(__stdcall *PDiscardVirtualMemory)(PVOID,SIZE_T); static PVirtualAlloc2 pVirtualAlloc2 = NULL; -static PDiscardVirtualMemory pDiscardVirtualMemory = NULL; void _mi_os_init(void) { // get the page size @@ -100,7 +101,6 @@ void _mi_os_init(void) { // use VirtualAlloc2FromApp if possible as it is available to Windows store apps pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2FromApp"); if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2"); - pDiscardVirtualMemory = (PDiscardVirtualMemory)GetProcAddress(hDll, "DiscardVirtualMemory"); FreeLibrary(hDll); } // Try to see if large OS pages are supported @@ -555,17 +555,9 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) #if defined(_WIN32) // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory - // (but this is for an access pattern that immediately reuses the memory) - if (mi_option_is_enabled(mi_option_reset_discards) && pDiscardVirtualMemory != NULL) { - DWORD ok = (*pDiscardVirtualMemory)(start, csize); - mi_assert_internal(ok == ERROR_SUCCESS); - if (ok != ERROR_SUCCESS) return false; - } - else { - void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); - mi_assert_internal(p == start); - if (p != start) return false; - } + void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); + mi_assert_internal(p == start); + if (p != start) return false; #else #if defined(MADV_FREE) static int advice = MADV_FREE; diff --git a/src/page.c b/src/page.c index 627cb728..41e41b89 100644 --- a/src/page.c +++ b/src/page.c @@ -741,7 +741,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept // huge allocation? mi_page_t* page; if (mi_unlikely(size > MI_LARGE_OBJ_SIZE_MAX)) { - if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) { + if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see ) page = NULL; } else { diff --git a/test/test-api.c b/test/test-api.c index f4e32746..a5f61074 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -66,7 +66,7 @@ bool test_heap2(); // Main testing // --------------------------------------------------------------------------- int main() { - mi_option_enable(mi_option_verbose,false); + mi_option_disable(mi_option_verbose); // --------------------------------------------------- // Malloc