kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-05 15:09:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve mi_realloc codepath

Browse source
This commit is contained in:
Daan Leijen 2022-04-07 19:09:31 -07:00
parent 25ecec3c3b
commit 3c7ce7d3c6
2 changed files with 39 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
include/mimalloc-internal.h
View file

@ -138,8 +138,8 @@ mi_msecs_t _mi_clock_start(void);
// "alloc.c" // "alloc.c"
void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic` void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic`
void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero); void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero); void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept;
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p);
bool _mi_free_delayed_block(mi_block_t* block); bool _mi_free_delayed_block(mi_block_t* block);
void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size); void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size);
@ -945,7 +945,7 @@ static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0));
void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE);
memcpy(adst, asrc, n); _mi_memcpy(adst, asrc, n);
} }
#else #else
// Default fallback on `_mi_memcpy` // Default fallback on `_mi_memcpy`

51
src/alloc.c
View file

@ -147,7 +147,7 @@ mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept {
return p; return p;
} }
void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) { void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
void* p = mi_heap_malloc(heap,size); void* p = mi_heap_malloc(heap,size);
if (zero && p != NULL) { if (zero && p != NULL) {
_mi_block_zero_init(_mi_ptr_page(p),p,size); // todo: can we avoid getting the page again? _mi_block_zero_init(_mi_ptr_page(p),p,size); // todo: can we avoid getting the page again?
@ -530,20 +530,25 @@ bool _mi_free_delayed_block(mi_block_t* block) {
} }
// Bytes available in a block // Bytes available in a block
static size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { mi_decl_noinline static size_t mi_page_usable_aligned_size_of(const mi_segment_t* segment, const mi_page_t* page, const void* p) mi_attr_noexcept {
const mi_segment_t* const segment = mi_checked_ptr_segment(p,msg); const mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
if (segment==NULL) return 0; const size_t size = mi_page_usable_size_of(page, block);
const mi_page_t* const page = _mi_segment_page_of(segment, p); const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
const mi_block_t* block = (const mi_block_t*)p;
if (mi_unlikely(mi_page_has_aligned(page))) {
block = _mi_page_ptr_unalign(segment, page, p);
size_t size = mi_page_usable_size_of(page, block);
ptrdiff_t const adjust = (uint8_t*)p - (uint8_t*)block;
mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
return (size - adjust); return (size - adjust);
}
static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept {
const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg);
if (segment==NULL) return 0; // also returns 0 if `p == NULL`
const mi_page_t* const page = _mi_segment_page_of(segment, p);
if (mi_likely(!mi_page_has_aligned(page))) {
const mi_block_t* block = (const mi_block_t*)p;
return mi_page_usable_size_of(page, block);
} }
else { else {
return mi_page_usable_size_of(page, block); // split out to separate routine for improved code generation
return mi_page_usable_aligned_size_of(segment, page, p);
} }
} }
@ -612,29 +617,37 @@ mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
return mi_heap_mallocn(mi_get_default_heap(),count,size); return mi_heap_mallocn(mi_get_default_heap(),count,size);
} }
// Expand in place or fail // Expand (or shrink) in place (or fail)
void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
#if MI_PADDING
// we do not shrink/expand with padding enabled
MI_UNUSED(p); MI_UNUSED(newsize);
return NULL;
#else
if (p == NULL) return NULL; if (p == NULL) return NULL;
size_t size = _mi_usable_size(p,"mi_expand"); const size_t size = _mi_usable_size(p,"mi_expand");
if (newsize > size) return NULL; if (newsize > size) return NULL;
return p; // it fits return p; // it fits
#endif
} }
void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) { void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept {
if (p == NULL) return _mi_heap_malloc_zero(heap,newsize,zero); const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL
size_t size = _mi_usable_size(p,"mi_realloc"); if (mi_unlikely(newsize <= size && newsize >= (size / 2))) {
if (newsize <= size && newsize >= (size / 2)) { // todo: adjust potential padding to reflect the new size?
return p; // reallocation still fits and not more than 50% waste return p; // reallocation still fits and not more than 50% waste
} }
void* newp = mi_heap_malloc(heap,newsize); void* newp = mi_heap_malloc(heap,newsize);
if (mi_likely(newp != NULL)) { if (mi_likely(newp != NULL)) {
if (zero && newsize > size) { if (zero && newsize > size) {
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
memset((uint8_t*)newp + start, 0, newsize - start); memset((uint8_t*)newp + start, 0, newsize - start);
} }
if (mi_likely(p != NULL)) {
_mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize));
mi_free(p); // only free if successful mi_free(p); // only free the original pointer if successful
}
} }
return newp; return newp;
} }