kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-08-25 16:54:47 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

set extra debug padding per-heap

Browse source
This commit is contained in:
daan 2020-04-20 17:09:28 -07:00
parent ae608cda2f
commit 9ebb94fe17
10 changed files with 65 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/alloc-aligned.c
View file

@ -24,17 +24,17 @@ static mi_decl_restrict void* mi_base_malloc_zero_aligned_at(mi_heap_t* const he
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
// try if there is a small block available with just the right alignment
const size_t __extra_padding = mi_extra_padding();
const size_t padsize = size + __extra_padding; // safe for overflow as size <= PTRDIFF_MAX
const size_t extra_padding = mi_extra_padding(heap);
const size_t padsize = size + extra_padding; // safe for overflow as size <= PTRDIFF_MAX
if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) {
mi_page_t* page = _mi_heap_get_free_small_page(heap,padsize);
const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0;
if (mi_likely(page->free != NULL && is_aligned))
{
#if MI_STAT>1
mi_heap_stat_increase( heap, malloc, size);
mi_heap_stat_increase( heap, malloc, padsize);
#endif
void* p = _mi_page_malloc(heap,page,padsize MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); // TODO: inline _mi_page_malloc
void* p = _mi_page_malloc(heap,page,padsize MI_SOURCE_XARG); // TODO: inline _mi_page_malloc
mi_assert_internal(p != NULL);
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
if (zero) _mi_block_zero_init(page,p,size);
@ -44,7 +44,7 @@ static mi_decl_restrict void* mi_base_malloc_zero_aligned_at(mi_heap_t* const he
// use regular allocation if it is guaranteed to fit the alignment constraints
if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) {
void* p = _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG);
void* p = _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG); // base malloc adds padding again to size
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
return p;
}

24
src/alloc.c
View file

@ -28,11 +28,11 @@ terms of the MIT license. A copy of the license can be found in the file
// Fast allocation in a page: just pop from the free list.
// Fall back to generic allocation only if the list is empty.
extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_EXTRA_PADDING_XPARAM MI_SOURCE_XPARAM) mi_attr_noexcept {
extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept {
mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size);
mi_block_t* block = page->free;
if (mi_unlikely(block == NULL)) {
return _mi_malloc_generic(heap, size MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); // slow path
return _mi_malloc_generic(heap, size MI_SOURCE_XARG); // slow path
}
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
// pop from the free list
@ -52,9 +52,11 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
}
#endif
#if (MI_PADDING>0) && defined(MI_ENCODE_FREELIST)
const size_t extra_padding = mi_extra_padding(heap);
mi_assert_internal(extra_padding <= size && extra_padding >= MI_PADDING_SIZE);
mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page));
ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - __extra_padding));
mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - __extra_padding + delta));
ptrdiff_t delta = extra_padding - MI_PADDING_SIZE;
mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - extra_padding + delta));
padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys));
padding->delta = (uint32_t)(delta);
padding->source = __mi_source;
@ -71,17 +73,17 @@ MI_ALLOC_API1(inline mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, siz
mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
mi_assert(size <= MI_SMALL_SIZE_MAX);
const size_t __extra_padding = mi_extra_padding();
const size_t extra_padding = mi_extra_padding(heap);
#if (MI_PADDING)
if (size == 0) {
size = sizeof(void*);
}
if ((size + __extra_padding) > MI_SMALL_SIZE_MAX) {
if ((size + extra_padding) > MI_SMALL_SIZE_MAX) {
return MI_SOURCE_ARG(mi_heap_malloc, heap, size); // call base malloc in case we were invoked directly
}
#endif
mi_page_t* page = _mi_heap_get_free_small_page(heap,size + __extra_padding);
void* p = _mi_page_malloc(heap, page, size + __extra_padding MI_EXTRA_PADDING_XARG MI_SOURCE_XARG);
mi_page_t* page = _mi_heap_get_free_small_page(heap,size + extra_padding);
void* p = _mi_page_malloc(heap, page, size + extra_padding MI_SOURCE_XARG);
mi_assert_internal(p==NULL || mi_usable_size(p) >= size);
#if MI_STAT>1
if (p != NULL) {
@ -96,14 +98,14 @@ MI_ALLOC_API1(inline mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, siz
// The main allocation function
MI_ALLOC_API1(inline mi_decl_restrict void*, malloc, mi_heap_t*, heap, size_t, size)
{
const size_t __extra_padding = mi_extra_padding();
if (mi_likely(size <= MI_SMALL_SIZE_MAX - __extra_padding && __extra_padding < MI_SMALL_SIZE_MAX)) { // careful for overflow
const size_t extra_padding = mi_extra_padding(heap);
if (mi_likely(size <= MI_SMALL_SIZE_MAX - extra_padding && extra_padding < MI_SMALL_SIZE_MAX)) { // careful for overflow
return mi_base_malloc_small(heap, size MI_SOURCE_XARG);
}
else {
mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
void* const p = _mi_malloc_generic(heap, size + __extra_padding MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); // note: size + __extra_padding can overflow but it is detected in malloc_generic
void* const p = _mi_malloc_generic(heap, size + extra_padding MI_SOURCE_XARG); // note: size + __extra_padding can overflow but it is detected in malloc_generic
mi_assert_internal(p == NULL || mi_usable_size(p) >= size);
#if MI_STAT>1
if (p != NULL) {

16
src/heap.c
View file

@ -200,6 +200,7 @@ mi_heap_t* mi_heap_new(void) {
heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
heap->extra_padding = bheap->extra_padding;
heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe
// push on the thread local heaps list
heap->next = heap->tld->heaps;
@ -211,6 +212,11 @@ uintptr_t _mi_heap_random_next(mi_heap_t* heap) {
return _mi_random_next(&heap->random);
}
void mi_heap_set_extra_padding(mi_heap_t* heap, size_t extra_padding) {
if (extra_padding > 1*MiB) extra_padding = 1*MiB;
heap->extra_padding = extra_padding;
}
// zero out the page queues
static void mi_heap_reset_pages(mi_heap_t* heap) {
mi_assert_internal(mi_heap_is_initialized(heap));
@ -580,13 +586,17 @@ static bool mi_heap_print_json_visit(const mi_heap_t* heap, const mi_heap_area_t
_mi_fprintf(varg->out, varg->out_arg, varg->area_count==0 ? " {" : " ]\n}\n,{");
varg->area_count++;
varg->block_count = 0;
_mi_fprintf(varg->out, varg->out_arg, "\"page\": %zu, \"start\": 0x%p, \"block_size\": %zu, \"used_size\": %zu,\n \"reserved\": %zu, \"committed\": %zu,", varg->area_count, area->blocks, area->block_size, area->used, area->reserved, area->committed);
_mi_fprintf(varg->out, varg->out_arg,
"\"page\": %zu, \"start\": 0x%p, \"block_size\": %zu, \"used_size\": %zu,\n \"reserved\": %zu, \"committed\": %zu,",
varg->area_count, area->blocks, area->block_size, area->used, area->reserved, area->committed);
_mi_fprintf(varg->out, varg->out_arg, " \"blocks\": [\n");
}
else {
_mi_fprintf(varg->out, varg->out_arg, varg->block_count==0 ? " {" : " ,{");
varg->block_count++;
_mi_fprintf(varg->out, varg->out_arg, "\"block\": 0x%p, \"valid\": %s, \"size\": %zu, \"usable_size\": %zu, \"allocated_size\": %zu,\n ", info->block, info->valid ? "true" : "false", info->size, info->usable_size, info->allocated_size);
_mi_fprintf(varg->out, varg->out_arg,
"\"block\": 0x%p, \"valid\": %s, \"size\": %zu, \"usable_size\": %zu, \"allocated_size\": %zu,\n ",
info->block, info->valid ? "true" : "false", info->size, info->usable_size, info->allocated_size);
int lineno;
const char* fname;
void* ret = mi_source_unpack(info->source, &fname, &lineno);
@ -600,7 +610,7 @@ static bool mi_heap_print_json_visit(const mi_heap_t* heap, const mi_heap_area_t
void mi_heap_print_json(mi_heap_t* heap, mi_output_fun* out, void* arg) {
if (heap==NULL) heap = mi_heap_get_default();
mi_print_json_t info = { 0, 0, out, arg };
_mi_fprintf(info.out, info.out_arg, "{ \"heap\": 0x%p, \"thread_id\": 0x%zx, \"page_count\": %zu, \"block_padding\": %zu", heap, heap->thread_id, heap->page_count, mi_extra_padding() );
_mi_fprintf(info.out, info.out_arg, "{ \"heap\": 0x%p, \"thread_id\": 0x%zx, \"page_count\": %zu, \"block_padding\": %zu", heap, heap->thread_id, heap->page_count, mi_extra_padding(heap) );
_mi_fprintf(info.out, info.out_arg, ", \"pages\": [\n");
mi_heap_visit_blocks(heap, true, &mi_heap_print_json_visit, &info);
_mi_fprintf(info.out, info.out_arg, info.area_count==0 ? "]\n" : " ] }\n] }\n");

4
src/init.c
View file

@ -98,6 +98,7 @@ const mi_heap_t _mi_heap_empty = {
{ {0}, {0}, 0 },
0, // page count
MI_BIN_FULL, 0, // page retired min/max
0, // extra padding
NULL, // next
false
};
@ -133,6 +134,7 @@ mi_heap_t _mi_heap_main = {
{ {0x846ca68b}, {0}, 0 }, // random
0, // page count
MI_BIN_FULL, 0, // page retired min/max
0, // extra_padding
NULL, // next heap
false // can reclaim
};
@ -149,6 +151,7 @@ static void mi_heap_main_init(void) {
_mi_random_init(&_mi_heap_main.random);
_mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main);
_mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main);
_mi_heap_main.extra_padding = mi_option_get(mi_option_debug_extra_padding);
}
}
@ -195,6 +198,7 @@ static bool _mi_heap_init(void) {
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
heap->tld = tld;
heap->extra_padding = _mi_heap_main.extra_padding;
tld->heap_backing = heap;
tld->heaps = heap;
tld->segments.stats = &tld->stats;

10
src/page.c
View file

@ -792,7 +792,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed.
// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM MI_SOURCE_XPARAM) mi_attr_noexcept
void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept
{
mi_assert_internal(heap != NULL);
@ -811,8 +811,8 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM
// huge allocation?
mi_page_t* page;
const size_t req_size = size - MI_EXTRA_PADDING_ARG; // correct for padding_size in case of an overflow on `size`
if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_EXTRA_PADDING_ARG) )) {
const size_t req_size = size - mi_extra_padding(heap); // correct for padding_size in case of an overflow on `size`
if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - mi_extra_padding(heap)) )) {
if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
_mi_error_message(EOVERFLOW, "allocation request is too large (%zu b requested)\n", req_size);
return NULL;
@ -823,7 +823,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM
}
else {
// otherwise find a page with free blocks in our size segregated queues
mi_assert_internal(size >= MI_EXTRA_PADDING_ARG);
mi_assert_internal(size >= mi_extra_padding(heap));
page = mi_find_free_page(heap,size);
}
if (mi_unlikely(page == NULL)) { // out of memory
@ -835,5 +835,5 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM
mi_assert_internal(mi_page_block_size(page) >= size);
// and try again, this time succeeding! (i.e. this should never recurse)
return _mi_page_malloc(heap, page, size MI_EXTRA_PADDING_XARG MI_SOURCE_XARG);
return _mi_page_malloc(heap, page, size MI_SOURCE_XARG);
}