diff --git a/include/mimalloc.h b/include/mimalloc.h index 867c5c97..e5133c96 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -366,8 +366,8 @@ typedef enum mi_option_e { mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) - mi_option_debug_guarded_min, // only used when build with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) - mi_option_debug_guarded_max, // only used when build with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) + mi_option_debug_guarded_min, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) + mi_option_debug_guarded_max, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index bf75f8c7..d1e6e5d8 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -72,8 +72,9 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Use guard pages behind objects of a certain size -#define MI_DEBUG_GUARDED 1 +// Use guard pages behind objects of a certain size (set by the MIMALLOC_DEBUG_GUARDED_MIN/MAX options) +// Padding should be disabled when using guard pages +// #define MI_DEBUG_GUARDED 1 #if defined(MI_DEBUG_GUARDED) #define MI_PADDING 0 #endif diff --git a/src/alloc.c b/src/alloc.c index 8f47e541..6fc85952 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -607,26 +607,26 @@ static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t siz mi_assert(MI_PADDING_SIZE==0); #endif // allocate multiple of page size ending in a guard page - const size_t bsize = _mi_align_up(size, MI_MAX_ALIGN_SIZE); // ensure minimal alignment requirement - const size_t psize = _mi_os_page_size(); - const size_t gsize = _mi_align_up(bsize + psize, psize); - void* const base = _mi_malloc_generic(heap, gsize, zero, huge_alignment); - if (base==NULL) return NULL; - mi_page_t* page = _mi_ptr_page(base); + const size_t obj_size = _mi_align_up(size, MI_MAX_ALIGN_SIZE); // ensure minimal alignment requirement + const size_t os_page_size = _mi_os_page_size(); + const size_t req_size = _mi_align_up(obj_size + os_page_size, os_page_size); + void* const block = _mi_malloc_generic(heap, req_size, zero, huge_alignment); + if (block==NULL) return NULL; + mi_page_t* page = _mi_ptr_page(block); mi_segment_t* segment = _mi_page_segment(page); - const size_t fullsize = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` - void* const gpage = (uint8_t*)base + (fullsize - psize); - mi_assert_internal(_mi_is_aligned(gpage, psize)); + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + void* const guard_page = (uint8_t*)block + (block_size - os_page_size); + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); // place block in front of the guard page - size_t offset = fullsize - psize - bsize; + size_t offset = block_size - os_page_size - obj_size; if (offset > MI_BLOCK_ALIGNMENT_MAX) { // give up to place it right in front of the guard page if the offset is too large for unalignment offset = MI_BLOCK_ALIGNMENT_MAX; } - void* const p = (uint8_t*)base + offset; - mi_assert_internal(p >= base); + void* const p = (uint8_t*)block + offset; + mi_assert_internal(p>=block); // set page flags if (offset > 0) { @@ -636,7 +636,7 @@ static mi_decl_restrict void* mi_heap_malloc_guarded(mi_heap_t* heap, size_t siz // set guard page if (segment->allow_decommit) { mi_page_set_has_guarded(page, true); - _mi_os_protect(gpage, psize); + _mi_os_protect(guard_page, os_page_size); } else { _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", p, size); diff --git a/src/free.c b/src/free.c index 2137f7c9..f2e5f8e3 100644 --- a/src/free.c +++ b/src/free.c @@ -26,25 +26,6 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // forward declaration of multi-threaded free (`_mt`) (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON) static mi_decl_noinline void mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block); -#if !MI_DEBUG_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block) { - MI_UNUSED(page); - MI_UNUSED(block); -} -#else -static void mi_block_unguard(mi_page_t* page, mi_block_t* block) { - if (mi_page_has_guarded(page)) { - const size_t bsize = mi_page_block_size(page); - const size_t psize = _mi_os_page_size(); - mi_assert_internal(bsize > psize); - mi_assert_internal(_mi_page_segment(page)->allow_decommit); - void* gpage = (uint8_t*)block + (bsize - psize); - mi_assert_internal(_mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); - } -} -#endif - // regular free of a (thread local) block pointer // fast path written carefully to prevent spilling on the stack static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool track_stats, bool check_full) @@ -88,6 +69,9 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { return (mi_block_t*)((uintptr_t)p - adjust); } +// forward declaration for a MI_DEBUG_GUARDED build +static void mi_block_unguard(mi_page_t* page, mi_block_t* block); + // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept { MI_UNUSED(segment); @@ -546,3 +530,25 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); } #endif + + +// Remove guard page when building with MI_DEBUG_GUARDED +#if !MI_DEBUG_GUARDED +static void mi_block_unguard(mi_page_t* page, mi_block_t* block) { + MI_UNUSED(page); + MI_UNUSED(block); + // do nothing +} +#else +static void mi_block_unguard(mi_page_t* page, mi_block_t* block) { + if (mi_page_has_guarded(page)) { + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(_mi_page_segment(page)->allow_decommit); + void* gpage = (uint8_t*)block + (bsize - psize); + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); + } +} +#endif diff --git a/src/options.c b/src/options.c index b8859235..27122c78 100644 --- a/src/options.c +++ b/src/options.c @@ -99,8 +99,8 @@ static mi_option_desc_t options[_mi_option_last] = #else { 0, UNINIT, MI_OPTION(visit_abandoned) }, #endif - { 0, UNINIT, MI_OPTION(debug_guarded_min) }, // only used when built with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) - { 0, UNINIT, MI_OPTION(debug_guarded_max) }, // only used when built with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) + { 0, UNINIT, MI_OPTION(debug_guarded_min) }, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects + { 0, UNINIT, MI_OPTION(debug_guarded_max) }, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects }; static void mi_option_init(mi_option_desc_t* desc); @@ -130,6 +130,7 @@ void _mi_options_init(void) { _mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n"); } } + _mi_verbose_message("guarded build: %s\n", mi_option_get(mi_option_debug_guarded_max) > 0 ? "enabled" : "disabled"); #endif }