diff --git a/include/mimalloc.h b/include/mimalloc.h index 4ecb8be0..8cbe265f 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -370,6 +370,7 @@ typedef enum mi_option_e { mi_option_guarded_max, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0) mi_option_guarded_precise, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0) mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) + mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index ae3a3358..7809503b 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -91,6 +91,7 @@ void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_heap_guarded_init(mi_heap_t* heap); // os.c void _mi_os_init(void); // called from process init @@ -610,12 +611,21 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* } static inline bool mi_heap_malloc_use_guarded(mi_heap_t* heap, size_t size) { - MI_UNUSED(heap); - return (size <= (size_t)_mi_option_get_fast(mi_option_guarded_max) - && size >= (size_t)_mi_option_get_fast(mi_option_guarded_min)); + MI_UNUSED(heap); + if (heap->guarded_sample_rate==0 || + size > heap->guarded_size_max || + size < heap->guarded_size_min) { + return false; + } + if (++heap->guarded_sample_count < heap->guarded_sample_rate) { + return false; + } + heap->guarded_sample_count = 0; // reset + return true; } mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; + #endif diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 29ba8564..ec181e61 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -502,6 +502,13 @@ struct mi_heap_s { mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages uint8_t tag; // custom tag, can be used for separating heaps based on the object types + #if MI_GUARDED + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) + size_t guarded_sample_seed; // starting sample count + size_t guarded_sample_count; // current sample count (wraps at `sample_rate`) + #endif mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") }; @@ -594,6 +601,7 @@ typedef struct mi_stats_s { mi_stat_counter_t arena_count; mi_stat_counter_t arena_crossover_count; mi_stat_counter_t arena_rollback_count; + mi_stat_counter_t guarded_alloc_count; #if MI_STAT>1 mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; #endif diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 86b13dea..6aee38c3 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -36,6 +36,18 @@ static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, si mi_assert_internal(_mi_is_aligned(p, alignment)); return p; } + +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { + const size_t rate = heap->guarded_sample_rate; + heap->guarded_sample_rate = 0; + void* p = _mi_heap_malloc_zero(heap, size, zero); + heap->guarded_sample_rate = rate; + return p; +} +#else +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { + return _mi_heap_malloc_zero(heap, size, zero); +} #endif // Fallback aligned allocation that over-allocates -- split out for better codegen @@ -58,6 +70,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t return NULL; } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); + // note: no guarded as alignment > 0 p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block // zero afterwards as only the area from the aligned_p may be committed! if (p == NULL) return NULL; @@ -65,7 +78,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t else { // otherwise over-allocate oversize = size + alignment - 1; - p = _mi_heap_malloc_zero(heap, oversize, zero); + p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero); if (p == NULL) return NULL; } mi_page_t* page = _mi_ptr_page(p); @@ -80,7 +93,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_page_set_has_aligned(page, true); #if MI_GUARDED // set tag to aligned so mi_usable_size works with guard pages - if (adjust > sizeof(mi_block_t)) { + if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; block->next = MI_BLOCK_TAG_ALIGNED; } @@ -93,7 +106,11 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); mi_assert_internal(mi_usable_size(aligned_p)>=size); mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); - mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_page(aligned_p), aligned_p)); + #if MI_DEBUG > 1 + mi_page_t* const apage = _mi_ptr_page(aligned_p); + void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == unalign_p); + #endif // now zero the block if needed if (alignment > MI_BLOCK_ALIGNMENT_MAX) { @@ -126,7 +143,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = _mi_heap_malloc_zero(heap, size, zero); + void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { diff --git a/src/alloc.c b/src/alloc.c index 561b0026..25b05526 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -668,6 +668,7 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo if (p != NULL) { if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); } mi_heap_stat_increase(heap, malloc, mi_usable_size(p)); + mi_heap_stat_counter_increase(heap, guarded_alloc_count, 1); } #endif #if MI_DEBUG>3 diff --git a/src/heap.c b/src/heap.c index 78ebcd1e..581b3f71 100644 --- a/src/heap.c +++ b/src/heap.c @@ -221,6 +221,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); + _mi_heap_guarded_init(heap); // push on the thread local heaps list heap->next = heap->tld->heaps; heap->tld->heaps = heap; diff --git a/src/init.c b/src/init.c index 75458a1f..822d5a18 100644 --- a/src/init.c +++ b/src/init.c @@ -86,7 +86,8 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ - { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ + { 0, 0 } \ MI_STAT_COUNT_END_NULL() // -------------------------------------------------------- @@ -111,6 +112,9 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { NULL, // next false, // can reclaim 0, // tag + #if MI_GUARDED + 0, 0, 0, 0, 0, + #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -151,6 +155,9 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = { NULL, // next heap false, // can reclaim 0, // tag + #if MI_GUARDED + 0, 0, 0, 0, 0, + #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY }; @@ -159,6 +166,22 @@ bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. mi_stats_t _mi_stats_main = { MI_STATS_NULL }; +#if MI_GUARDED +void _mi_heap_guarded_init(mi_heap_t* heap) { + heap->guarded_sample_rate = mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX); + heap->guarded_size_max = mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX); + heap->guarded_size_min = mi_option_get_clamp(mi_option_guarded_min, 0, (long)heap->guarded_size_max); + heap->guarded_sample_seed = (size_t)mi_option_get(mi_option_guarded_sample_seed); + if (heap->guarded_sample_seed == 0) { heap->guarded_sample_seed = _mi_heap_random_next(heap); } + heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate; + heap->guarded_sample_count = heap->guarded_sample_seed; +} +#else +void _mi_heap_guarded_init(mi_heap_t* heap) { + MI_UNUSED(heap); +} +#endif + static void mi_heap_main_init(void) { if (_mi_heap_main.cookie == 0) { @@ -174,6 +197,7 @@ static void mi_heap_main_init(void) { _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); mi_lock_init(&mi_subproc_default.abandoned_os_lock); mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock); + _mi_heap_guarded_init(&_mi_heap_main); } } diff --git a/src/options.c b/src/options.c index c5f1e2a1..2c6814c8 100644 --- a/src/options.c +++ b/src/options.c @@ -150,6 +150,7 @@ static mi_option_desc_t options[_mi_option_last] = #else { 0, UNINIT, MI_OPTION(guarded_sample_rate)}, #endif + { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, }; static void mi_option_init(mi_option_desc_t* desc); @@ -173,7 +174,7 @@ void _mi_options_init(void) { mi_max_error_count = mi_option_get(mi_option_max_errors); mi_max_warning_count = mi_option_get(mi_option_max_warnings); #if MI_GUARDED - if (mi_option_get(mi_option_guarded_max) > 0) { + if (mi_option_get(mi_option_guarded_sample_rate) > 0) { if (mi_option_is_enabled(mi_option_allow_large_os_pages)) { mi_option_disable(mi_option_allow_large_os_pages); _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n"); diff --git a/src/stats.c b/src/stats.c index 99cf89c5..29376ace 100644 --- a/src/stats.c +++ b/src/stats.c @@ -118,6 +118,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_counter_add(&stats->searches, &src->searches, 1); mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1); mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); + mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count, 1); #if MI_STAT>1 for (size_t i = 0; i <= MI_BIN_HUGE; i++) { if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) { @@ -342,6 +343,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_counter_print(&stats->reset_calls, "resets", out, arg); mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); + mi_stat_counter_print(&stats->guarded_alloc_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count()); diff --git a/test/test-stress.c b/test/test-stress.c index cb769dbf..88c39f23 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -22,7 +22,6 @@ terms of the MIT license. #include #include -// #define MI_GUARDED // #define USE_STD_MALLOC // > mimalloc-test-stress [THREADS] [SCALE] [ITER] @@ -36,7 +35,7 @@ static int ITER = 400; static int THREADS = 8; static int SCALE = 25; static int ITER = 20; -#elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits +#elif defined(MI_XGUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = 8; static int SCALE = 10; static int ITER = 10;