kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-08 00:09:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge from dev-win, fix small object size check

Browse source
This commit is contained in:
Daan Leijen 2019-08-16 19:14:08 -07:00
commit fb12f298ca
7 changed files with 33 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
include/mimalloc-types.h
View file

@ -90,12 +90,11 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_SMALL_PAGE_SIZE (1<<MI_SMALL_PAGE_SHIFT) #define MI_SMALL_PAGE_SIZE (1<<MI_SMALL_PAGE_SHIFT)
#define MI_MEDIUM_PAGE_SIZE (1<<MI_MEDIUM_PAGE_SHIFT) #define MI_MEDIUM_PAGE_SIZE (1<<MI_MEDIUM_PAGE_SHIFT)
#define MI_MEDIUM_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb on 64-bit #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 16kb
#define MI_MEDIUM_WSIZE_MAX (MI_MEDIUM_SIZE_MAX/MI_INTPTR_SIZE) // 64kb on 64-bit #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb
#define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
#define MI_LARGE_SIZE_MAX (MI_SEGMENT_SIZE/4) // 16mb on 64-bit #define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/4) // 16mb
#define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX/MI_INTPTR_SIZE) #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX>>MI_INTPTR_SHIFT)
// Minimal alignment necessary. On most platforms 16 bytes are needed // Minimal alignment necessary. On most platforms 16 bytes are needed
// due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE`
@ -104,7 +103,7 @@ terms of the MIT license. A copy of the license can be found in the file
// Maximum number of size classes. (spaced exponentially in 12.5% increments) // Maximum number of size classes. (spaced exponentially in 12.5% increments)
#define MI_BIN_HUGE (73U) #define MI_BIN_HUGE (73U)
#if (MI_MEDIUM_WSIZE_MAX >= 655360) #if (MI_MEDIUM_OBJ_WSIZE_MAX >= 655360)
#error "define more bins" #error "define more bins"
#endif #endif

4
src/alloc.c
View file

@ -38,7 +38,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
block->next = 0; block->next = 0;
#endif #endif
#if (MI_STAT>1) #if (MI_STAT>1)
if(size <= MI_LARGE_SIZE_MAX) { if(size <= MI_LARGE_OBJ_SIZE_MAX) {
size_t bin = _mi_bin(size); size_t bin = _mi_bin(size);
mi_heap_stat_increase(heap,normal[bin], 1); mi_heap_stat_increase(heap,normal[bin], 1);
} }
@ -230,7 +230,7 @@ void mi_free(void* p) mi_attr_noexcept
#if (MI_STAT>1) #if (MI_STAT>1)
mi_heap_t* heap = mi_heap_get_default(); mi_heap_t* heap = mi_heap_get_default();
mi_heap_stat_decrease( heap, malloc, mi_usable_size(p)); mi_heap_stat_decrease( heap, malloc, mi_usable_size(p));
if (page->block_size <= MI_LARGE_SIZE_MAX) { if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease( heap, normal[_mi_bin(page->block_size)], 1); mi_heap_stat_decrease( heap, normal[_mi_bin(page->block_size)], 1);
} }
// huge page stat is accounted for in `_mi_page_retire` // huge page stat is accounted for in `_mi_page_retire`

6
src/heap.c
View file

@ -245,8 +245,8 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE); _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE);
// stats // stats
if (page->block_size > MI_MEDIUM_SIZE_MAX) { if (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX) {
if (page->block_size <= MI_LARGE_SIZE_MAX) { if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&heap->tld->stats.large,page->block_size); _mi_stat_decrease(&heap->tld->stats.large,page->block_size);
} }
else { else {
@ -255,7 +255,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
} }
#if (MI_STAT>1) #if (MI_STAT>1)
size_t inuse = page->used - page->thread_freed; size_t inuse = page->used - page->thread_freed;
if (page->block_size <= MI_LARGE_SIZE_MAX) { if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse); mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse);
} }
mi_heap_stat_decrease(heap,malloc, page->block_size * inuse); // todo: off for aligned blocks... mi_heap_stat_decrease(heap,malloc, page->block_size * inuse); // todo: off for aligned blocks...

4
src/init.c
View file

@ -43,8 +43,8 @@ const mi_page_t _mi_page_empty = {
QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \
QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \
QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
QNULL(MI_MEDIUM_WSIZE_MAX + 1 /* 655360, Huge queue */), \ QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \
QNULL(MI_MEDIUM_WSIZE_MAX + 2) /* Full queue */ } QNULL(MI_MEDIUM_OBJ_WSIZE_MAX + 2) /* Full queue */ }
#define MI_STAT_COUNT_NULL() {0,0,0,0} #define MI_STAT_COUNT_NULL() {0,0,0,0}

18
src/page-queue.c
View file

@ -34,15 +34,15 @@ terms of the MIT license. A copy of the license can be found in the file
static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) {
return (pq->block_size == (MI_MEDIUM_SIZE_MAX+sizeof(uintptr_t))); return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+sizeof(uintptr_t)));
} }
static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) {
return (pq->block_size == (MI_MEDIUM_SIZE_MAX+(2*sizeof(uintptr_t)))); return (pq->block_size == (MI_MEDIUM_OBJ_SIZE_MAX+(2*sizeof(uintptr_t))));
} }
static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
return (pq->block_size > MI_MEDIUM_SIZE_MAX); return (pq->block_size > MI_MEDIUM_OBJ_SIZE_MAX);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
@ -116,7 +116,7 @@ extern inline uint8_t _mi_bin(size_t size) {
bin = (uint8_t)wsize; bin = (uint8_t)wsize;
} }
#endif #endif
else if (wsize > MI_MEDIUM_WSIZE_MAX) { else if (wsize > MI_MEDIUM_OBJ_WSIZE_MAX) {
bin = MI_BIN_HUGE; bin = MI_BIN_HUGE;
} }
else { else {
@ -147,7 +147,7 @@ size_t _mi_bin_size(uint8_t bin) {
// Good size for allocation // Good size for allocation
size_t mi_good_size(size_t size) mi_attr_noexcept { size_t mi_good_size(size_t size) mi_attr_noexcept {
if (size <= MI_MEDIUM_SIZE_MAX) { if (size <= MI_MEDIUM_OBJ_SIZE_MAX) {
return _mi_bin_size(_mi_bin(size)); return _mi_bin_size(_mi_bin(size));
} }
else { else {
@ -245,7 +245,7 @@ static bool mi_page_queue_is_empty(mi_page_queue_t* queue) {
static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) {
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
mi_assert_expensive(mi_page_queue_contains(queue, page)); mi_assert_expensive(mi_page_queue_contains(queue, page));
mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) || (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
if (page->prev != NULL) page->prev->next = page->next; if (page->prev != NULL) page->prev->next = page->next;
if (page->next != NULL) page->next->prev = page->prev; if (page->next != NULL) page->next->prev = page->prev;
if (page == queue->last) queue->last = page->prev; if (page == queue->last) queue->last = page->prev;
@ -268,7 +268,7 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_
mi_assert_internal(page->heap == NULL); mi_assert_internal(page->heap == NULL);
mi_assert_internal(!mi_page_queue_contains(queue, page)); mi_assert_internal(!mi_page_queue_contains(queue, page));
mi_assert_internal(page->block_size == queue->block_size || mi_assert_internal(page->block_size == queue->block_size ||
(page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_huge(queue)) || (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(queue)) ||
(mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); (mi_page_is_in_full(page) && mi_page_queue_is_full(queue)));
mi_page_set_in_full(page, mi_page_queue_is_full(queue)); mi_page_set_in_full(page, mi_page_queue_is_full(queue));
@ -297,8 +297,8 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro
mi_assert_internal((page->block_size == to->block_size && page->block_size == from->block_size) || mi_assert_internal((page->block_size == to->block_size && page->block_size == from->block_size) ||
(page->block_size == to->block_size && mi_page_queue_is_full(from)) || (page->block_size == to->block_size && mi_page_queue_is_full(from)) ||
(page->block_size == from->block_size && mi_page_queue_is_full(to)) || (page->block_size == from->block_size && mi_page_queue_is_full(to)) ||
(page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_huge(to)) || (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_huge(to)) ||
(page->block_size > MI_MEDIUM_SIZE_MAX && mi_page_queue_is_full(to))); (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX && mi_page_queue_is_full(to)));
if (page->prev != NULL) page->prev->next = page->next; if (page->prev != NULL) page->prev->next = page->next;
if (page->next != NULL) page->next->prev = page->prev; if (page->next != NULL) page->next->prev = page->prev;

12
src/page.c
View file

@ -102,7 +102,7 @@ bool _mi_page_is_valid(mi_page_t* page) {
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == page->heap->thread_id); mi_assert_internal(!_mi_process_is_initialized || segment->thread_id==0 || segment->thread_id == page->heap->thread_id);
mi_page_queue_t* pq = mi_page_queue_of(page); mi_page_queue_t* pq = mi_page_queue_of(page);
mi_assert_internal(mi_page_queue_contains(pq, page)); mi_assert_internal(mi_page_queue_contains(pq, page));
mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_MEDIUM_SIZE_MAX || mi_page_is_in_full(page)); mi_assert_internal(pq->block_size==page->block_size || page->block_size > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_in_full(page));
mi_assert_internal(mi_heap_contains_queue(page->heap,pq)); mi_assert_internal(mi_heap_contains_queue(page->heap,pq));
} }
return true; return true;
@ -356,8 +356,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
mi_page_set_has_aligned(page, false); mi_page_set_has_aligned(page, false);
// account for huge pages here // account for huge pages here
if (page->block_size > MI_MEDIUM_SIZE_MAX) { if (page->block_size > MI_MEDIUM_OBJ_SIZE_MAX) {
if (page->block_size <= MI_LARGE_SIZE_MAX) { if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
_mi_stat_decrease(&page->heap->tld->stats.large, page->block_size); _mi_stat_decrease(&page->heap->tld->stats.large, page->block_size);
} }
else { else {
@ -394,7 +394,7 @@ void _mi_page_retire(mi_page_t* page) {
// is the only page left with free blocks. It is not clear // is the only page left with free blocks. It is not clear
// how to check this efficiently though... for now we just check // how to check this efficiently though... for now we just check
// if its neighbours are almost fully used. // if its neighbours are almost fully used.
if (mi_likely(page->block_size <= MI_MEDIUM_SIZE_MAX)) { if (mi_likely(page->block_size <= MI_MEDIUM_OBJ_SIZE_MAX)) {
if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) {
_mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1); _mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1);
return; // dont't retire after all return; // dont't retire after all
@ -713,7 +713,7 @@ static mi_page_t* mi_large_page_alloc(mi_heap_t* heap, size_t size) {
if (page != NULL) { if (page != NULL) {
mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(mi_page_immediate_available(page));
mi_assert_internal(page->block_size == block_size); mi_assert_internal(page->block_size == block_size);
if (page->block_size <= MI_LARGE_SIZE_MAX) { if (page->block_size <= MI_LARGE_OBJ_SIZE_MAX) {
_mi_stat_increase(&heap->tld->stats.large, block_size); _mi_stat_increase(&heap->tld->stats.large, block_size);
_mi_stat_counter_increase(&heap->tld->stats.large_count, 1); _mi_stat_counter_increase(&heap->tld->stats.large_count, 1);
} }
@ -746,7 +746,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
// huge allocation? // huge allocation?
mi_page_t* page; mi_page_t* page;
if (mi_unlikely(size > MI_MEDIUM_SIZE_MAX)) { if (mi_unlikely(size > MI_MEDIUM_OBJ_SIZE_MAX)) {
if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) { if (mi_unlikely(size >= (SIZE_MAX - MI_MAX_ALIGN_SIZE))) {
page = NULL; page = NULL;
} }

11
src/segment.c
View file

@ -22,7 +22,6 @@ static void mi_segment_map_freed_at(const mi_segment_t* segment);
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Segment allocation Segment allocation
In any case the memory for a segment is virtual and only In any case the memory for a segment is virtual and only
committed on demand (i.e. we are careful to not touch the memory committed on demand (i.e. we are careful to not touch the memory
until we actually allocate a block there) until we actually allocate a block there)
@ -404,7 +403,7 @@ static void mi_segment_page_split(mi_page_t* page, size_t slice_count, mi_segmen
} }
static mi_page_t* mi_segment_page_find(size_t slice_count, mi_segments_tld_t* tld) { static mi_page_t* mi_segment_page_find(size_t slice_count, mi_segments_tld_t* tld) {
mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_SIZE_MAX); mi_assert_internal(slice_count*MI_SEGMENT_SLICE_SIZE <= MI_LARGE_OBJ_SIZE_MAX);
// search from best fit up // search from best fit up
mi_page_queue_t* pq = mi_page_queue_for(slice_count,tld); mi_page_queue_t* pq = mi_page_queue_for(slice_count,tld);
if (slice_count == 0) slice_count = 1; if (slice_count == 0) slice_count = 1;
@ -557,7 +556,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t
static mi_page_t* mi_segment_page_alloc(mi_page_kind_t page_kind, size_t required, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) static mi_page_t* mi_segment_page_alloc(mi_page_kind_t page_kind, size_t required, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{ {
mi_assert_internal(required <= MI_LARGE_SIZE_MAX && page_kind <= MI_PAGE_LARGE); mi_assert_internal(required <= MI_LARGE_OBJ_SIZE_MAX && page_kind <= MI_PAGE_LARGE);
// find a free page // find a free page
size_t page_size = _mi_align_up(required,(required > MI_MEDIUM_PAGE_SIZE ? MI_MEDIUM_PAGE_SIZE : MI_SEGMENT_SLICE_SIZE)); size_t page_size = _mi_align_up(required,(required > MI_MEDIUM_PAGE_SIZE ? MI_MEDIUM_PAGE_SIZE : MI_SEGMENT_SLICE_SIZE));
@ -873,13 +872,13 @@ static bool mi_is_good_fit(size_t bsize, size_t size) {
mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { mi_page_t* _mi_segment_page_alloc(size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
mi_page_t* page; mi_page_t* page;
if (block_size <= MI_SMALL_SIZE_MAX) {// || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) { if (block_size <= MI_SMALL_OBJ_SIZE_MAX) {// || mi_is_good_fit(block_size,MI_SMALL_PAGE_SIZE)) {
page = mi_segment_page_alloc(MI_PAGE_SMALL,block_size,tld,os_tld); page = mi_segment_page_alloc(MI_PAGE_SMALL,block_size,tld,os_tld);
} }
else if (block_size <= MI_MEDIUM_SIZE_MAX) {// || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) { else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) {// || mi_is_good_fit(block_size, MI_MEDIUM_PAGE_SIZE)) {
page = mi_segment_page_alloc(MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,tld, os_tld); page = mi_segment_page_alloc(MI_PAGE_MEDIUM,MI_MEDIUM_PAGE_SIZE,tld, os_tld);
} }
else if (block_size <= MI_LARGE_SIZE_MAX) { else if (block_size <= MI_LARGE_OBJ_SIZE_MAX) {
page = mi_segment_page_alloc(MI_PAGE_LARGE,block_size,tld, os_tld); page = mi_segment_page_alloc(MI_PAGE_LARGE,block_size,tld, os_tld);
} }
else { else {