kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-07-06 19:38:41 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

take 16 bits from used field to create a fast unalign path

Browse source
This commit is contained in:
Daan 2024-03-18 01:40:03 -07:00
parent 355f44f373
commit cc809b0cd4
6 changed files with 47 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/alloc.c
View file

@ -37,8 +37,8 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
}
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
// pop from the free list
page->used++;
page->free = mi_block_next(page, block);
page->used++;
mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
#if MI_DEBUG>3
if (page->free_is_zero) {

17
src/free.c
View file

@ -249,25 +249,30 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool
// Adjust a block that was allocated aligned, to the actual start of the block in the page.
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) {
mi_assert_internal(page!=NULL && p!=NULL);
const size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL);
const size_t adjust = (diff % mi_page_block_size(page));
const size_t diff = (mi_likely(page->block_offset_adj != 0)
? (uint8_t*)p - (uint8_t*)page - 8*(page->block_offset_adj-1)
: (uint8_t*)p - _mi_page_start(segment, page, NULL));
const size_t adjust = (mi_likely(page->block_size_shift != 0)
? diff & (((size_t)1 << page->block_size_shift) - 1)
: diff % mi_page_block_size(page));
return (mi_block_t*)((uintptr_t)p - adjust);
}
// free a local pointer
static void mi_decl_noinline mi_free_generic_local(const mi_segment_t* segment, mi_page_t* page, void* p) mi_attr_noexcept {
static void mi_decl_noinline mi_free_generic_local(mi_segment_t* segment, mi_page_t* page, void* p) mi_attr_noexcept {
mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p);
mi_free_block_local(page, block, true);
}
// free a pointer owned by another thread
static void mi_decl_noinline mi_free_generic_mt(const mi_segment_t* segment, mi_page_t* page, void* p) mi_attr_noexcept {
static void mi_decl_noinline mi_free_generic_mt(mi_segment_t* segment, mi_page_t* page, void* p) mi_attr_noexcept {
mi_block_t* const block = _mi_page_ptr_unalign(segment, page, p); // don't check `has_aligned` flag to avoid a race (issue #865)
mi_free_block_mt(segment, page, block);
}
// generic free (for runtime integration)
void mi_decl_noinline _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
void mi_decl_noinline _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept {
if (is_local) mi_free_generic_local(segment,page,p);
else mi_free_generic_mt(segment,page,p);
}
@ -469,7 +474,7 @@ static mi_decl_noinline void mi_free_block_mt(mi_segment_t* segment, mi_page_t*
// ------------------------------------------------------
// Bytes available in a block
mi_decl_noinline static size_t mi_page_usable_aligned_size_of(const mi_segment_t* segment, const mi_page_t* page, const void* p) mi_attr_noexcept {
static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_segment_t* segment, const mi_page_t* page, const void* p) mi_attr_noexcept {
const mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
const size_t size = mi_page_usable_size_of(page, block);
const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;

18
src/init.c
View file

@ -21,9 +21,11 @@ const mi_page_t _mi_page_empty = {
false, // is_zero
0, // retire_expire
NULL, // free
0, // used
0, // xblock_size
NULL, // local_free
0, // used
0, // block size shift
0, // block offset adj
0, // xblock_size
#if (MI_PADDING || MI_ENCODE_FREELIST)
{ 0, 0 },
#endif
@ -93,8 +95,6 @@ const mi_page_t _mi_page_empty = {
mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
NULL,
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY,
MI_ATOMIC_VAR_INIT(NULL),
0, // tid
0, // cookie
@ -104,7 +104,9 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
0, // page count
MI_BIN_FULL, 0, // page retired min/max
NULL, // next
false
false,
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY
};
@ -130,8 +132,6 @@ static mi_tld_t tld_main = {
mi_heap_t _mi_heap_main = {
&tld_main,
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY,
MI_ATOMIC_VAR_INIT(NULL),
0, // thread id
0, // initial cookie
@ -141,7 +141,9 @@ mi_heap_t _mi_heap_main = {
0, // page count
MI_BIN_FULL, 0, // page retired min/max
NULL, // next heap
false // can reclaim
false, // can reclaim
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY
};
bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.

12
src/page.c
View file

@ -660,7 +660,6 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_page_set_heap(page, heap);
size_t page_size;
const void* page_start = _mi_segment_page_start(segment, page, block_size, &page_size, NULL);
MI_UNUSED(page_start);
mi_track_mem_noaccess(page_start,page_size);
page->xblock_size = (block_size < MI_HUGE_BLOCK_SIZE ? (uint32_t)block_size : MI_HUGE_BLOCK_SIZE);
mi_assert_internal(page_size / block_size < (1L<<16));
@ -677,6 +676,15 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert_expensive(!page->is_zero_init || mi_mem_is_zero(page_start, page_size));
}
#endif
if (_mi_is_power_of_two(block_size) && block_size > 0) {
page->block_size_shift = (uint32_t)(mi_ctz((uintptr_t)block_size));
}
const ptrdiff_t start_offset = (uint8_t*)page_start - (uint8_t*)page;
const ptrdiff_t start_adjust = start_offset % block_size;
if (start_offset >= 0 && (start_adjust % 8) == 0 && (start_adjust/8) < 255) {
page->block_offset_adj = (uint8_t)((start_adjust/8) + 1);
}
mi_assert_internal(page->capacity == 0);
mi_assert_internal(page->free == NULL);
@ -690,6 +698,8 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert_internal(page->keys[0] != 0);
mi_assert_internal(page->keys[1] != 0);
#endif
mi_assert_internal(page->block_size_shift == 0 || (block_size == (1UL << page->block_size_shift)));
mi_assert_internal(page->block_offset_adj == 0 || (((uint8_t*)page_start - (uint8_t*)page - 8*(page->block_offset_adj-1))) % block_size == 0);
mi_assert_expensive(mi_page_is_valid_init(page));
// initialize an initial free list