diff --git a/CMakeLists.txt b/CMakeLists.txt index 94bb6e1e..66626b3f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,8 @@ option(MI_PADDING "Enable padding to detect heap block overflow (only option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) option(MI_BUILD_OBJECT "Build object" ON) +option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) +option(MI_SHOW_ERRORS "Show error and warning messages by default" OFF) include("cmake/mimalloc-config-version.cmake") @@ -70,6 +72,7 @@ if(MI_OVERRIDE MATCHES "ON") # use zone's on macOS message(STATUS " Use malloc zone to override malloc (MI_OSX_ZONE=ON)") list(APPEND mi_sources src/alloc-override-osx.c) + list(APPEND mi_defines MI_OSX_ZONE=1) if(NOT MI_INTERPOSE MATCHES "ON") message(STATUS " (enabling INTERPOSE as well since zone's require this)") set(MI_INTERPOSE "ON") @@ -108,6 +111,16 @@ if(MI_PADDING MATCHES "OFF") list(APPEND mi_defines MI_PADDING=0) endif() +if(MI_XMALLOC MATCHES "ON") + message(STATUS "Enable abort() calls on memory allocation failure (MI_XMALLOC=ON)") + list(APPEND mi_defines MI_XMALLOC=1) +endif() + +if(MI_SHOW_ERRORS MATCHES "ON") + message(STATUS "Enable printing of error and warning messages by default (MI_SHOW_ERRORS=ON)") + list(APPEND mi_defines MI_SHOW_ERRORS=1) +endif() + if(MI_USE_CXX MATCHES "ON") message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) @@ -212,6 +225,7 @@ if (MI_BUILD_STATIC) message(STATUS "Static library will be built") add_library(mimalloc-static STATIC ${mi_sources}) + set_property(TARGET mimalloc-static PROPERTY POSITION_INDEPENDENT_CODE ON) target_compile_definitions(mimalloc-static PRIVATE ${mi_defines} MI_STATIC_LIB) target_compile_options(mimalloc-static PRIVATE ${mi_cflags}) target_link_libraries(mimalloc-static PUBLIC ${mi_libraries}) @@ -219,7 +233,7 @@ if (MI_BUILD_STATIC) $ $ ) - if(WIN32 AND MI_BUILD_SHARED) + if(WIN32) # When building both static and shared libraries on Windows, a static library should use a # different output name to avoid the conflict with the import library of a shared one. string(REPLACE "mimalloc" "mimalloc-static" mi_output_name ${mi_basename}) @@ -251,6 +265,7 @@ if (MI_BUILD_OBJECT) message(STATUS "Library object will be built") add_library(mimalloc-obj OBJECT src/static.c) + set_property(TARGET mimalloc-obj PROPERTY POSITION_INDEPENDENT_CODE ON) target_compile_definitions(mimalloc-obj PRIVATE ${mi_defines}) target_compile_options(mimalloc-obj PRIVATE ${mi_cflags}) target_include_directories(mimalloc-obj PUBLIC diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 954ec15d..c81e31bd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -40,8 +40,8 @@ jobs: cd $(BuildType) ctest displayName: CTest - - upload: $(Build.SourcesDirectory)/$(BuildType) - artifact: mimalloc-windows-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-windows-$(BuildType) - job: displayName: Linux @@ -99,8 +99,8 @@ jobs: displayName: Make - script: make test -C $(BuildType) displayName: CTest - - upload: $(Build.SourcesDirectory)/$(BuildType) - artifact: mimalloc-ubuntu-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-ubuntu-$(BuildType) - job: displayName: macOS @@ -127,5 +127,5 @@ jobs: displayName: Make - script: make test -C $(BuildType) displayName: CTest - - upload: $(Build.SourcesDirectory)/$(BuildType) - artifact: mimalloc-macos-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-macos-$(BuildType) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 7e50a2bc..449e2e41 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -61,7 +61,6 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif - // ------------------------------------------------------ // Platform specific values // ------------------------------------------------------ diff --git a/include/mimalloc.h b/include/mimalloc.h index 28d6d1b7..0af04a94 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -308,8 +308,7 @@ typedef enum mi_option_e { mi_option_use_numa_nodes, mi_option_os_tag, mi_option_max_errors, - _mi_option_last, - mi_option_eager_page_commit = mi_option_eager_commit + _mi_option_last } mi_option_t; diff --git a/readme.md b/readme.md index 583d54ed..fd600763 100644 --- a/readme.md +++ b/readme.md @@ -255,6 +255,32 @@ OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 +## Secure Mode + +_mimalloc_ can be build in secure mode by using the `-DMI_SECURE=ON` flags in `cmake`. This build enables various mitigations +to make mimalloc more robust against exploits. In particular: + +- All internal mimalloc pages are surrounded by guard pages and the heap metadata is behind a guard page as well (so a buffer overflow + exploit cannot reach into the metadata), +- All free list pointers are + [encoded](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396) + with per-page keys which is used both to prevent overwrites with a known pointer, as well as to detect heap corruption, +- Double free's are detected (and ignored), +- The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to + mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc + from the OS are also address randomized. + +As always, evaluate with care as part of an overall security strategy as all of the above are mitigations but not guarantees. + +## Debug Mode + +When _mimalloc_ is built using debug mode, various checks are done at runtime to catch development errors. + +- Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. +- All objects have padding at the end to detect (byte precise) heap block overflows. +- Double free's, and freeing invalid heap pointers are detected. +- Corrupted free-lists and some forms of use-after-free are detected. + # Overriding Malloc diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index cc03f5e2..c1c880ca 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -41,8 +41,11 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im ------------------------------------------------------ */ static size_t zone_size(malloc_zone_t* zone, const void* p) { - UNUSED(zone); UNUSED(p); - return 0; // as we cannot guarantee that `p` comes from us, just return 0 + UNUSED(zone); + if (!mi_is_in_heap_region(p)) + return 0; // not our pointer, bail out + + return mi_usable_size(p); } static void* zone_malloc(malloc_zone_t* zone, size_t size) { diff --git a/src/alloc-override.c b/src/alloc-override.c index 0e908f42..a09153c5 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -103,7 +103,7 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n); #endif - #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) + #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; @@ -165,7 +165,7 @@ extern "C" { void cfree(void* p) MI_FORWARD0(mi_free, p); void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); -size_t malloc_size(void* p) MI_FORWARD1(mi_usable_size,p); +size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p); #if !defined(__ANDROID__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); #else diff --git a/src/arena.c b/src/arena.c index 724fc52c..bb9fc174 100644 --- a/src/arena.c +++ b/src/arena.c @@ -36,6 +36,7 @@ of 256MiB in practice. // os.c void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld); +void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats); void _mi_os_free(void* p, size_t size, mi_stats_t* stats); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize); @@ -213,13 +214,13 @@ void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, siz Arena free ----------------------------------------------------------- */ -void _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats) { +void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats) { mi_assert_internal(size > 0 && stats != NULL); if (p==NULL) return; if (size==0) return; if (memid == MI_MEMID_OS) { // was a direct OS allocation, pass through - _mi_os_free(p, size, stats); + _mi_os_free_ex(p, size, all_committed, stats); } else { // allocated in an arena diff --git a/src/options.c b/src/options.c index 0af4a485..1a4633ee 100644 --- a/src/options.c +++ b/src/options.c @@ -51,7 +51,11 @@ typedef struct mi_option_desc_s { static mi_option_desc_t options[_mi_option_last] = { // stable options - { MI_DEBUG, UNINIT, MI_OPTION(show_errors) }, +#if MI_DEBUG || defined(MI_SHOW_ERRORS) + { 1, UNINIT, MI_OPTION(show_errors) }, +#else + { 0, UNINIT, MI_OPTION(show_errors) }, +#endif { 0, UNINIT, MI_OPTION(show_stats) }, { 0, UNINIT, MI_OPTION(verbose) }, @@ -260,13 +264,17 @@ static void mi_recurse_exit(void) { } void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) { - if (!mi_recurse_enter()) return; if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) { // TODO: use mi_out_stderr for stderr? + if (!mi_recurse_enter()) return; out = mi_out_get_default(&arg); + if (prefix != NULL) out(prefix, arg); + out(message, arg); + mi_recurse_exit(); + } + else { + if (prefix != NULL) out(prefix, arg); + out(message, arg); } - if (prefix != NULL) out(prefix,arg); - out(message,arg); - mi_recurse_exit(); } // Define our own limited `fprintf` that avoids memory allocation. @@ -348,6 +356,11 @@ static void mi_error_default(int err) { abort(); } #endif +#if defined(MI_XMALLOC) + if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode + abort(); + } +#endif } void mi_register_error(mi_error_fun* fun, void* arg) { diff --git a/src/os.c b/src/os.c index 1c7e03c8..f33cfbc3 100644 --- a/src/os.c +++ b/src/os.c @@ -618,7 +618,7 @@ static void mi_mprotect_hint(int err) { } // Commit/Decommit memory. -// Usuelly commit is aligned liberal, while decommit is aligned conservative. +// Usually commit is aligned liberal, while decommit is aligned conservative. // (but not for the reset version where we want commit to be conservative as well) static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, bool* is_zero, mi_stats_t* stats) { // page align in the range, commit liberally, decommit conservative diff --git a/src/page.c b/src/page.c index 2903b258..a7f95a80 100644 --- a/src/page.c +++ b/src/page.c @@ -381,7 +381,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { } #define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX -#define MI_RETIRE_CYCLES (16) +#define MI_RETIRE_CYCLES (8) // Retire a page with no more used blocks // Important to not retire too quickly though as new @@ -406,7 +406,7 @@ void _mi_page_retire(mi_page_t* page) { if (mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page))) { if (pq->last==page && pq->first==page) { // the only page in the queue? mi_stat_counter_increase(_mi_stats_main.page_no_retire,1); - page->retire_expire = MI_RETIRE_CYCLES; + page->retire_expire = (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(pq >= heap->pages); const size_t index = pq - heap->pages; @@ -570,7 +570,8 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) if (page->capacity >= page->reserved) return; size_t page_size; - uint8_t* page_start = _mi_page_start(_mi_page_segment(page), page, &page_size); + //uint8_t* page_start = + _mi_page_start(_mi_page_segment(page), page, &page_size); mi_stat_counter_increase(tld->stats.pages_extended, 1); // calculate the extend count @@ -588,12 +589,6 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved); mi_assert_internal(extend < (1UL<<16)); - // commit on-demand for large and huge pages? - if (_mi_page_segment(page)->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) { - uint8_t* start = page_start + (page->capacity * bsize); - _mi_mem_commit(start, extend * bsize, NULL, &tld->os); - } - // and append the extend the free list if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) { mi_page_free_list_extend(page, bsize, extend, &tld->stats ); diff --git a/src/region.c b/src/region.c index fd7d4544..ae3a799a 100644 --- a/src/region.c +++ b/src/region.c @@ -49,7 +49,7 @@ bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats); // arena.c -void _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats); +void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats); void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); @@ -187,7 +187,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, const uintptr_t idx = mi_atomic_increment(®ions_count); if (idx >= MI_REGION_MAX) { mi_atomic_decrement(®ions_count); - _mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats); + _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats); _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, GiB)); return false; } @@ -205,6 +205,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, // and share it mi_region_info_t info; + info.value = 0; // initialize the full union to zero info.x.valid = true; info.x.is_large = region_large; info.x.numa_node = (short)_mi_os_numa_node(tld); @@ -391,7 +392,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re mem_region_t* region; if (mi_memid_is_arena(id,®ion,&bit_idx,&arena_memid)) { // was a direct arena allocation, pass through - _mi_arena_free(p, size, arena_memid, tld->stats); + _mi_arena_free(p, size, arena_memid, full_commit, tld->stats); } else { // allocated in a region @@ -454,12 +455,13 @@ void _mi_mem_collect(mi_os_tld_t* tld) { // on success, free the whole region uint8_t* start = mi_atomic_read_ptr(uint8_t,®ions[i].start); size_t arena_memid = mi_atomic_read_relaxed(®ions[i].arena_memid); + uintptr_t commit = mi_atomic_read_relaxed(®ions[i].commit); memset(®ions[i], 0, sizeof(mem_region_t)); // and release the whole region mi_atomic_write(®ion->info, 0); if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { _mi_abandoned_await_readers(); // ensure no pending reads - _mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats); + _mi_arena_free(start, MI_REGION_SIZE, arena_memid, (~commit == 0), tld->stats); } } } diff --git a/src/segment.c b/src/segment.c index 8aa838a2..d36cf1c5 100644 --- a/src/segment.c +++ b/src/segment.c @@ -204,9 +204,9 @@ static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t* mi_segment_protect_range((uint8_t*)segment + segment->segment_info_size - os_page_size, os_page_size, protect); if (MI_SECURE <= 1 || segment->capacity == 1) { // and protect the last (or only) page too - mi_assert_internal(segment->page_kind >= MI_PAGE_LARGE); + mi_assert_internal(MI_SECURE <= 1 || segment->page_kind >= MI_PAGE_LARGE); uint8_t* start = (uint8_t*)segment + segment->segment_size - os_page_size; - if (protect && !mi_option_is_enabled(mi_option_eager_page_commit)) { + if (protect && !segment->mem_is_committed) { // ensure secure page is committed _mi_mem_commit(start, os_page_size, NULL, tld); } @@ -236,12 +236,8 @@ static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, m void* start = mi_segment_raw_page_start(segment, page, &psize); page->is_reset = true; mi_assert_internal(size <= psize); - size_t reset_size = (size == 0 || size > psize ? psize : size); - if (size == 0 && segment->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) { - mi_assert_internal(page->xblock_size > 0); - reset_size = page->capacity * mi_page_block_size(page); - } - _mi_mem_reset(start, reset_size, tld->os); + size_t reset_size = ((size == 0 || size > psize) ? psize : size); + if (reset_size > 0) _mi_mem_reset(start, reset_size, tld->os); } static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld) @@ -253,12 +249,8 @@ static void mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size, size_t psize; uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); size_t unreset_size = (size == 0 || size > psize ? psize : size); - if (size == 0 && segment->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) { - mi_assert_internal(page->xblock_size > 0); - unreset_size = page->capacity * mi_page_block_size(page); - } bool is_zero = false; - _mi_mem_unreset(start, unreset_size, &is_zero, tld->os); + if (unreset_size > 0) _mi_mem_unreset(start, unreset_size, &is_zero, tld->os); if (is_zero) page->is_zero_init = true; } @@ -475,10 +467,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se if (any_reset && mi_option_is_enabled(mi_option_reset_decommits)) { fully_committed = false; } - if (segment->page_kind >= MI_PAGE_LARGE && !mi_option_is_enabled(mi_option_eager_page_commit)) { - fully_committed = false; - } - + _mi_mem_free(segment, segment_size, segment->memid, fully_committed, any_reset, tld->os); } @@ -627,8 +616,13 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ if (!commit) { // ensure the initial info is committed bool commit_zero = false; - _mi_mem_commit(segment, pre_size, &commit_zero, tld->os); + bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld->os); if (commit_zero) is_zero = true; + if (!ok) { + // commit failed; we cannot touch the memory: free the segment directly and return `NULL` + _mi_mem_free(segment, MI_SEGMENT_SIZE, memid, false, false, os_tld); + return NULL; + } } segment->memid = memid; segment->mem_is_fixed = mem_large; @@ -714,28 +708,28 @@ static bool mi_segment_has_free(const mi_segment_t* segment) { return (segment->used < segment->capacity); } -static void mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) { +static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) { mi_assert_internal(_mi_page_segment(page) == segment); mi_assert_internal(!page->segment_in_use); - // set in-use before doing unreset to prevent delayed reset mi_pages_reset_remove(page, tld); - page->segment_in_use = true; - segment->used++; + // check commit if (!page->is_committed) { mi_assert_internal(!segment->mem_is_fixed); - mi_assert_internal(!page->is_reset); + mi_assert_internal(!page->is_reset); + size_t psize; + uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); + bool is_zero = false; + const size_t gsize = (MI_SECURE >= 2 ? _mi_os_page_size() : 0); + bool ok = _mi_mem_commit(start, psize + gsize, &is_zero, tld->os); + if (!ok) return false; // failed to commit! + if (gsize > 0) { mi_segment_protect_range(start + psize, gsize, true); } + if (is_zero) { page->is_zero_init = true; } page->is_committed = true; - if (segment->page_kind < MI_PAGE_LARGE - || !mi_option_is_enabled(mi_option_eager_page_commit)) { - size_t psize; - uint8_t* start = mi_segment_raw_page_start(segment, page, &psize); - bool is_zero = false; - const size_t gsize = (MI_SECURE >= 2 ? _mi_os_page_size() : 0); - _mi_mem_commit(start, psize + gsize, &is_zero, tld->os); - if (gsize > 0) { mi_segment_protect_range(start + psize, gsize, true); } - if (is_zero) { page->is_zero_init = true; } - } } + // set in-use before doing unreset to prevent delayed reset + page->segment_in_use = true; + segment->used++; + // check reset if (page->is_reset) { mi_page_unreset(segment, page, 0, tld); // todo: only unreset the part that was reset? } @@ -746,6 +740,7 @@ static void mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg mi_assert_internal(!mi_segment_has_free(segment)); mi_segment_remove_from_free_queue(segment, tld); } + return true; } @@ -1212,8 +1207,8 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_segments_tld_t* for (size_t i = 0; i < segment->capacity; i++) { // TODO: use a bitmap instead of search? mi_page_t* page = &segment->pages[i]; if (!page->segment_in_use) { - mi_segment_page_claim(segment, page, tld); - return page; + bool ok = mi_segment_page_claim(segment, page, tld); + if (ok) return page; } } mi_assert(false); diff --git a/src/static.c b/src/static.c index ec9370eb..bf86166d 100644 --- a/src/static.c +++ b/src/static.c @@ -24,5 +24,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "alloc.c" #include "alloc-aligned.c" #include "alloc-posix.c" +#if MI_OSX_ZONE +#include "alloc-override-osx.c" +#endif #include "init.c" #include "options.c" diff --git a/src/stats.c b/src/stats.c index a1404502..478f8229 100644 --- a/src/stats.c +++ b/src/stats.c @@ -237,9 +237,51 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin #endif + +//------------------------------------------------------------ +// Use an output wrapper for line-buffered output +// (which is nice when using loggers etc.) +//------------------------------------------------------------ +typedef struct buffered_s { + mi_output_fun* out; // original output function + void* arg; // and state + char* buf; // local buffer of at least size `count+1` + size_t used; // currently used chars `used <= count` + size_t count; // total chars available for output +} buffered_t; + +static void mi_buffered_flush(buffered_t* buf) { + buf->buf[buf->used] = 0; + _mi_fputs(buf->out, buf->arg, NULL, buf->buf); + buf->used = 0; +} + +static void mi_buffered_out(const char* msg, void* arg) { + buffered_t* buf = (buffered_t*)arg; + if (msg==NULL || buf==NULL) return; + for (const char* src = msg; *src != 0; src++) { + char c = *src; + if (buf->used >= buf->count) mi_buffered_flush(buf); + mi_assert_internal(buf->used < buf->count); + buf->buf[buf->used++] = c; + if (c == '\n') mi_buffered_flush(buf); + } +} + +//------------------------------------------------------------ +// Print statistics +//------------------------------------------------------------ + static void mi_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit); -static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out, void* arg) mi_attr_noexcept { +static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out0, void* arg0) mi_attr_noexcept { + // wrap the output function to be line buffered + char buf[256]; + buffered_t buffer = { out0, arg0, buf, 0, 255 }; + mi_output_fun* out = &mi_buffered_out; + void* arg = &buffer; + + // and print using that mi_print_header(out,arg); #if MI_STAT>1 mi_stat_count_t normal = { 0,0,0,0 }; @@ -287,7 +329,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun _mi_fprintf(out, arg, ", commit charge: "); mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); } - _mi_fprintf(out, arg, "\n"); + _mi_fprintf(out, arg, "\n"); } static mi_msecs_t mi_time_start; // = 0