diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b6ed554..26f5eed8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ option(MI_PADDING "Enable padding to detect heap block overflow (alway option(MI_OVERRIDE "Override the standard malloc interface (e.g. define entry points for malloc() etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) -option(MI_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) -option(MI_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) +option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) +option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) @@ -25,7 +25,7 @@ option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) option(MI_SKIP_COLLECT_ON_EXIT, "Skip collecting memory on program exit" OFF) -option(MI_NO_PADDING "Force no use of padding even in DEBUG mode ets." OFF) +option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) # deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) @@ -125,42 +125,39 @@ endif() if(MI_SECURE) message(STATUS "Set full secure build (MI_SECURE=ON)") - list(APPEND mi_defines MI_SECURE=4) - #if (MI_VALGRIND) - # message(WARNING "Secure mode is a bit weakened when compiling with Valgrind support as buffer overflow detection is no longer byte-precise (if running without valgrind)") - #endif() + list(APPEND mi_defines MI_SECURE=4) endif() -if(MI_VALGRIND) +if(MI_TRACK_VALGRIND) CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH) if (NOT MI_HAS_VALGRINDH) - set(MI_VALGRIND OFF) + set(MI_TRACK_VALGRIND OFF) message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first") - message(STATUS "Compile **without** Valgrind support (MI_VALGRIND=OFF)") + message(STATUS "Compile **without** Valgrind support (MI_TRACK_VALGRIND=OFF)") else() - message(STATUS "Compile with Valgrind support (MI_VALGRIND=ON)") - list(APPEND mi_defines MI_VALGRIND=1) + message(STATUS "Compile with Valgrind support (MI_TRACK_VALGRIND=ON)") + list(APPEND mi_defines MI_TRACK_VALGRIND=1) endif() endif() -if(MI_ASAN) +if(MI_TRACK_ASAN) if (APPLE AND MI_OVERRIDE) - set(MI_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_ASAN=OFF)") + set(MI_TRACK_ASAN OFF) + message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK_ASAN=OFF)") endif() - if (MI_VALGRIND) - set(MI_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_ASAN=OFF)") + if (MI_TRACK_VALGRIND) + set(MI_TRACK_ASAN OFF) + message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_TRACK_ASAN=OFF)") endif() - if(MI_ASAN) + if(MI_TRACK_ASAN) CHECK_INCLUDE_FILES("sanitizer/asan_interface.h" MI_HAS_ASANH) if (NOT MI_HAS_ASANH) - set(MI_ASAN OFF) + set(MI_TRACK_ASAN OFF) message(WARNING "Cannot find the 'sanitizer/asan_interface.h' -- install address sanitizer support first") - message(STATUS "Compile **without** address sanitizer support (MI_ASAN=OFF)") + message(STATUS "Compile **without** address sanitizer support (MI_TRACK_ASAN=OFF)") else() - message(STATUS "Compile with address sanitizer support (MI_ASAN=ON)") - list(APPEND mi_defines MI_ASAN=1) + message(STATUS "Compile with address sanitizer support (MI_TRACK_ASAN=ON)") + list(APPEND mi_defines MI_TRACK_ASAN=1) list(APPEND mi_cflags -fsanitize=address) list(APPEND mi_libraries -fsanitize=address) endif() @@ -327,10 +324,10 @@ set(mi_basename "mimalloc") if(MI_SECURE) set(mi_basename "${mi_basename}-secure") endif() -if(MI_VALGRIND) +if(MI_TRACK_VALGRIND) set(mi_basename "${mi_basename}-valgrind") endif() -if(MI_ASAN) +if(MI_TRACK_ASAN) set(mi_basename "${mi_basename}-asan") endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) diff --git a/include/mimalloc-track.h b/include/mimalloc-track.h index b2404f8d..272ca1b8 100644 --- a/include/mimalloc-track.h +++ b/include/mimalloc-track.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2021, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -9,20 +9,39 @@ terms of the MIT license. A copy of the license can be found in the file #define MIMALLOC_TRACK_H /* ------------------------------------------------------------------------------------------------------ -Track memory ranges with macros for tools like Valgrind -address sanitizer, or other memory checkers. +Track memory ranges with macros for tools like Valgrind address sanitizer, or other memory checkers. +These can be defined for tracking allocation: + + #define mi_track_malloc_size(p,reqsize,size,zero) + #define mi_track_free_size(p,_size) The macros are set up such that the size passed to `mi_track_free_size` -matches the size of the allocation, or the new size of a `mi_track_resize` (currently unused though). +always matches the size of `mi_track_malloc_size`. (currently, `size == mi_usable_size(p)`). +The `reqsize` is what the user requested, and `size >= reqsize`. +The `size` is either byte precise (and `size==reqsize`) if `MI_PADDING` is enabled, +or otherwise it is the usable block size which may be larger than the original request. +Use `_mi_block_size_of(void* p)` to get the full block size that was allocated (including padding etc). +The `zero` parameter is `true` if the allocated block is zero initialized. + +Optional: + + #define mi_track_align(p,alignedp,offset,size) + #define mi_track_resize(p,oldsize,newsize) + +The `mi_track_align` is called right after a `mi_track_malloc` for aligned pointers in a block. +The corresponding `mi_track_free` still uses the block start pointer and original size (corresponding to the `mi_track_malloc`). +The `mi_track_resize` is currently unused but could be called on reallocations within a block. + +The following macros are for tools like asan and valgrind to track whether memory is +defined, undefined, or not accessible at all: + + #define mi_track_mem_defined(p,size) + #define mi_track_mem_undefined(p,size) + #define mi_track_mem_noaccess(p,size) -The `size` is either byte precise (and what the user requested) if `MI_PADDING` is enabled, -or otherwise it is the full block size which may be larger than the original request. -Aligned pointers in a block are signaled right after a `mi_track_malloc` -with the `mi_track_align` macro. The corresponding `mi_track_free` still -uses the block start pointer and original size (corresponding to the `mi_track_malloc`). -------------------------------------------------------------------------------------------------------*/ -#if MI_VALGRIND +#if MI_TRACK_VALGRIND #define MI_TRACK_ENABLED 1 #define MI_TRACK_HEAP_DESTROY 1 // track free of individual blocks on heap_destroy @@ -38,7 +57,7 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #define mi_track_mem_undefined(p,size) VALGRIND_MAKE_MEM_UNDEFINED(p,size) #define mi_track_mem_noaccess(p,size) VALGRIND_MAKE_MEM_NOACCESS(p,size) -#elif MI_ASAN +#elif MI_TRACK_ASAN #define MI_TRACK_ENABLED 1 #define MI_TRACK_HEAP_DESTROY 0 @@ -68,6 +87,9 @@ uses the block start pointer and original size (corresponding to the `mi_track_m #endif +// ------------------- +// Utility definitions + #ifndef mi_track_resize #define mi_track_resize(p,oldsize,newsize) mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false) #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index e365a8f5..9b3f5972 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -29,8 +29,9 @@ terms of the MIT license. A copy of the license can be found in the file // Define NDEBUG in the release version to disable assertions. // #define NDEBUG -// Define MI_VALGRIND to enable valgrind support -// #define MI_VALGRIND 1 +// Define MI_TRACK_ to enable tracking support +// #define MI_TRACK_VALGRIND 1 +// #define MI_TRACK_ASAN 1 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 @@ -59,7 +60,7 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against heap block overflows. // The padding can detect buffer overflow on free. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_VALGRIND || MI_ASAN) +#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || MI_TRACK_VALGRIND || MI_TRACK_ASAN) #define MI_PADDING 1 #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index c13bda23..27f6b331 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2022, Microsoft Research, Daan Leijen +Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/readme.md b/readme.md index 364b974b..b102a50f 100644 --- a/readme.md +++ b/readme.md @@ -78,7 +78,7 @@ Note: the `v2.x` version has a new algorithm for managing internal mimalloc page and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. -* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with asan and improved [Valgrind] support. Support abitrary large +* 2022-12-23, `v1.7.9`, `v2.0.9`: Supports building with [#asan] and improved [#Valgrind] support. Support abitrary large alignments (in particular for `std::pmr` pools). Added C++ STL allocators attached to a specific heap (thanks @vmarkovtsev). Heap walks now visit all object (including huge objects). Support Windows nano server containers (by Johannes Schindelin,@dscho). @@ -347,16 +347,19 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to - Double free's, and freeing invalid heap pointers are detected. - Corrupted free-lists and some forms of use-after-free are detected. -## Valgrind +## Tools -Generally, we recommend using the standard allocator with the amazing [Valgrind] tool (and -also for other address sanitizers). -However, it is possible to build mimalloc with Valgrind support. This has a small performance -overhead but does allow detecting memory leaks and byte-precise buffer overflows directly on final -executables. To build with valgrind support, use the `MI_VALGRIND=ON` cmake option: +Generally, we recommend using the standard allocator with memory tracking tools, but mimalloc +can also be build to support the [address sanitizer][asan] or the excellent [Valgrind] tool. +This has a small performance overhead but does allow detecting memory leaks and byte-precise +buffer overflows directly on final executables. See also the `test/test-wrong.c` file to test with various tools. + +### Valgrind + +To build with valgrind support, use the `MI_TRACK_VALGRIND=ON` cmake option: ``` -> cmake ../.. -DMI_VALGRIND=ON +> cmake ../.. -DMI_TRACK_VALGRIND=ON ``` This can also be combined with secure mode or debug mode. @@ -385,6 +388,35 @@ Valgrind support is in its initial development -- please report any issues. [Valgrind]: https://valgrind.org/ [valgrind-soname]: https://valgrind.org/docs/manual/manual-core.html#opt.soname-synonyms +### ASAN + +To build with the address sanitizer, use the `-DMI_TRACK_ASAN=ON` cmake option: + +``` +> cmake ../.. -DMI_TRACK_ASAN=ON +``` + +This can also be combined with secure mode or debug mode. +You can then run your programs as:' + +``` +> ASAN_OPTIONS=verbosity=1 +``` + +When you link a program with an address sanitizer build of mimalloc, you should +generally compile that program too with the address sanitizer enabled. +For example, assuming you build mimalloc in `out/debug`: + +``` +clang -g -o test-wrong -Iinclude test/test-wrong.c out/debug/libmimalloc-asan-debug.a -lpthread -fsanitize=address -fsanitize-recover=address +``` + +Since the address sanitizer redirects the standard allocation functions, on some platforms (macOSX for example) +it is required to compile mimalloc with `-DMI_OVERRIDE=OFF`. +Adress sanitizer support is in its initial development -- please report any issues. + +[asan]: https://github.com/google/sanitizers/wiki/AddressSanitizer + # Overriding Standard Malloc diff --git a/test/test-api-fill.c b/test/test-api-fill.c index 85d8524f..a32dfa27 100644 --- a/test/test-api-fill.c +++ b/test/test-api-fill.c @@ -309,7 +309,7 @@ bool check_zero_init(uint8_t* p, size_t size) { #if MI_DEBUG >= 2 bool check_debug_fill_uninit(uint8_t* p, size_t size) { -#if MI_VALGRIND +#if MI_TRACK_VALGRIND (void)p; (void)size; return true; // when compiled with valgrind we don't init on purpose #else @@ -325,7 +325,7 @@ bool check_debug_fill_uninit(uint8_t* p, size_t size) { } bool check_debug_fill_freed(uint8_t* p, size_t size) { -#if MI_VALGRIND +#if MI_TRACK_VALGRIND (void)p; (void)size; return true; // when compiled with valgrind we don't fill on purpose #else diff --git a/test/test-wrong.c b/test/test-wrong.c index aaaf60b9..56a2339a 100644 --- a/test/test-wrong.c +++ b/test/test-wrong.c @@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file Compile in an "out/debug" folder: > cd out/debug - > cmake ../.. -DMI_VALGRIND=1 + > cmake ../.. -DMI_TRACK_VALGRIND=1 > make -j8 and then compile this file as: @@ -29,7 +29,7 @@ terms of the MIT license. A copy of the license can be found in the file Compile in an "out/debug" folder: > cd out/debug - > cmake ../.. -DMI_ASAN=1 + > cmake ../.. -DMI_TRACK_ASAN=1 > make -j8 and then compile this file as: