kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-11 01:39:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix stats for delay purge commit

Browse source
This commit is contained in:
daanx 2024-12-25 11:45:01 -08:00
commit 8339cefdeb
5 changed files with 43 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
include/mimalloc/internal.h
View file

@ -708,8 +708,8 @@ static inline bool mi_page_is_huge(const mi_page_t* page) {
static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {
mi_page_queue_t* const pq = &((mi_heap_t*)heap)->pages[_mi_bin(size)]; mi_page_queue_t* const pq = &((mi_heap_t*)heap)->pages[_mi_bin(size)];
if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); } if (size <= MI_LARGE_MAX_OBJ_SIZE) { mi_assert_internal(pq->block_size <= MI_LARGE_MAX_OBJ_SIZE); }
return pq; return pq;
} }

10
include/mimalloc/types.h
View file

@ -49,11 +49,11 @@ terms of the MIT license. A copy of the license can be found in the file
// Define MI_SECURE to enable security mitigations. Level 1 has minimal performance impact, // Define MI_SECURE to enable security mitigations. Level 1 has minimal performance impact,
// but protects most metadata with guard pages: // but protects most metadata with guard pages:
// #define MI_SECURE 1 // guard page around metadata // #define MI_SECURE 1 // guard page around metadata
// //
// Level 2 has more performance impact but protect well against various buffer overflows // Level 2 has more performance impact but protect well against various buffer overflows
// by surrounding all mimalloc pages with guard pages: // by surrounding all mimalloc pages with guard pages:
// #define MI_SECURE 2 // guard page around each mimalloc page (can fragment VMA's with large heaps..) // #define MI_SECURE 2 // guard page around each mimalloc page (can fragment VMA's with large heaps..)
// //
// The next two levels can have more performance cost: // The next two levels can have more performance cost:
// #define MI_SECURE 3 // randomize allocations, encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) // #define MI_SECURE 3 // randomize allocations, encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
// #define MI_SECURE 4 // checks for double free. (may be more expensive) // #define MI_SECURE 4 // checks for double free. (may be more expensive)
@ -293,7 +293,7 @@ typedef struct mi_page_s {
_Atomic(mi_page_flags_t) xflags; // `in_full_queue` and `has_aligned` flags _Atomic(mi_page_flags_t) xflags; // `in_full_queue` and `has_aligned` flags
size_t block_size; // size available in each block (always `>0`) size_t block_size; // size available in each block (always `>0`)
uint8_t* page_start; // start of the blocks uint8_t* page_start; // start of the blocks
mi_heaptag_t heap_tag; // tag of the owning heap, used to separate heaps by object type mi_heaptag_t heap_tag; // tag of the owning heap, used to separate heaps by object type
bool free_is_zero; // `true` if the blocks in the free list are zero initialized bool free_is_zero; // `true` if the blocks in the free list are zero initialized
// padding // padding
@ -328,7 +328,7 @@ typedef struct mi_page_s {
// (Except for large pages since huge objects are allocated in 4MiB chunks) // (Except for large pages since huge objects are allocated in 4MiB chunks)
#define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 8 KiB #define MI_SMALL_MAX_OBJ_SIZE ((MI_SMALL_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 8 KiB
#define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 64 KiB #define MI_MEDIUM_MAX_OBJ_SIZE ((MI_MEDIUM_PAGE_SIZE-MI_PAGE_INFO_SIZE)/8) // < 64 KiB
#define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/4) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with _mi_bin #define MI_LARGE_MAX_OBJ_SIZE (MI_LARGE_PAGE_SIZE/4) // <= 512 KiB // note: this must be a nice power of 2 or we get rounding issues with _mi_bin
#define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE) #define MI_LARGE_MAX_OBJ_WSIZE (MI_LARGE_MAX_OBJ_SIZE/MI_SIZE_SIZE)

13
src/arena.c
View file

@ -246,7 +246,15 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
} }
else { else {
// no need to commit, but check if already fully committed // no need to commit, but check if already fully committed
// commit requested, but the range may not be committed as a whole: ensure it is committed now
memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
if (!memid->initially_committed) {
// partly committed.. adjust stats
size_t already_committed_count = 0;
mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
mi_os_stat_decrease(committed, mi_size_of_slices(already_committed_count));
}
} }
mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
@ -304,8 +312,8 @@ static bool mi_arena_reserve(mi_subproc_t* subproc, size_t req_size, bool allow_
// on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice // on an OS with overcommit (Linux) we don't count the commit yet as it is on-demand. Once a slice
// is actually allocated for the first time it will be counted. // is actually allocated for the first time it will be counted.
const bool adjust = (overcommit && arena_commit); const bool adjust = (overcommit && arena_commit);
if (adjust) { if (adjust) {
mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve, true /* on alloc */); mi_subproc_stat_adjust_decrease( subproc, committed, arena_reserve, true /* on alloc */);
} }
// and try to reserve the arena // and try to reserve the arena
int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id); int err = mi_reserve_os_memory_ex2(subproc, arena_reserve, arena_commit, allow_large, false /* exclusive? */, arena_id);
@ -824,6 +832,7 @@ void _mi_arenas_page_free(mi_page_t* page) {
const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative
//mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, page->memid.mem.arena.slice_index, total_slices)); //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, page->memid.mem.arena.slice_index, total_slices));
mi_assert_internal(page->memid.mem.arena.slice_count >= total_slices); mi_assert_internal(page->memid.mem.arena.slice_count >= total_slices);
mi_assert_internal(total_slices > 0);
if (total_slices > 0) { if (total_slices > 0) {
mi_bitmap_setN(arena->slices_committed, page->memid.mem.arena.slice_index, total_slices, NULL); mi_bitmap_setN(arena->slices_committed, page->memid.mem.arena.slice_index, total_slices, NULL);
} }

41
src/os.c
View file

@ -108,7 +108,7 @@ size_t _mi_os_secure_guard_page_size(void) {
#endif #endif
} }
// In secure mode, try to decommit an area and output a warning if this fails. // In secure mode, try to decommit an area and output a warning if this fails.
bool _mi_os_secure_guard_page_set_at(void* addr, bool is_pinned) { bool _mi_os_secure_guard_page_set_at(void* addr, bool is_pinned) {
if (addr == NULL) return true; if (addr == NULL) return true;
#if MI_SECURE > 0 #if MI_SECURE > 0
@ -123,7 +123,7 @@ bool _mi_os_secure_guard_page_set_at(void* addr, bool is_pinned) {
#endif #endif
} }
// In secure mode, try to decommit an area and output a warning if this fails. // In secure mode, try to decommit an area and output a warning if this fails.
bool _mi_os_secure_guard_page_set_before(void* addr, bool is_pinned) { bool _mi_os_secure_guard_page_set_before(void* addr, bool is_pinned) {
return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), is_pinned); return _mi_os_secure_guard_page_set_at((uint8_t*)addr - _mi_os_secure_guard_page_size(), is_pinned);
} }
@ -143,22 +143,21 @@ bool _mi_os_secure_guard_page_reset_before(void* addr) {
return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size()); return _mi_os_secure_guard_page_reset_at((uint8_t*)addr - _mi_os_secure_guard_page_size());
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Free memory Free memory
-------------------------------------------------------------- */ -------------------------------------------------------------- */
static void mi_os_free_huge_os_pages(void* p, size_t size); static void mi_os_free_huge_os_pages(void* p, size_t size);
static void mi_os_prim_free(void* addr, size_t size, bool still_committed) { static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) {
mi_assert_internal((size % _mi_os_page_size()) == 0); mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr) if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free(addr, size); int err = _mi_prim_free(addr, size);
if (err != 0) { if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
} }
if (still_committed) { if (commit_size > 0) {
mi_os_stat_decrease(committed, size); mi_os_stat_decrease(committed, commit_size);
} }
mi_os_stat_decrease(reserved, size); mi_os_stat_decrease(reserved, size);
} }
@ -167,13 +166,19 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
if (mi_memkind_is_os(memid.memkind)) { if (mi_memkind_is_os(memid.memkind)) {
size_t csize = memid.mem.os.size; size_t csize = memid.mem.os.size;
if (csize==0) { _mi_os_good_alloc_size(size); } if (csize==0) { _mi_os_good_alloc_size(size); }
size_t commit_size = (still_committed ? csize : 0);
void* base = addr; void* base = addr;
// different base? (due to alignment) // different base? (due to alignment)
if (memid.mem.os.base != base) { if (memid.mem.os.base != base) {
mi_assert(memid.mem.os.base <= addr); mi_assert(memid.mem.os.base <= addr);
// mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr);
base = memid.mem.os.base; base = memid.mem.os.base;
if (memid.mem.os.size==0) { csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base); } const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
if (memid.mem.os.size==0) {
csize += diff;
}
if (still_committed) {
commit_size -= diff; // the (addr-base) part was already un-committed
}
} }
// free it // free it
if (memid.memkind == MI_MEM_OS_HUGE) { if (memid.memkind == MI_MEM_OS_HUGE) {
@ -181,7 +186,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
mi_os_free_huge_os_pages(base, csize); mi_os_free_huge_os_pages(base, csize);
} }
else { else {
mi_os_prim_free(base, csize, still_committed); mi_os_prim_free(base, csize, (still_committed ? commit_size : 0));
} }
} }
else { else {
@ -266,7 +271,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
} }
#endif #endif
if (p != NULL) { mi_os_prim_free(p, size, commit); } if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); }
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment; const size_t over_size = size + alignment;
@ -297,8 +302,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
size_t mid_size = _mi_align_up(size, _mi_os_page_size()); size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size; size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit); } if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit); } if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); }
// we can return the aligned pointer on `mmap` systems // we can return the aligned pointer on `mmap` systems
p = aligned_p; p = aligned_p;
*base = aligned_p; // since we freed the pre part, `*base == p`. *base = aligned_p; // since we freed the pre part, `*base == p`.
@ -454,9 +459,9 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
return _mi_os_commit_ex(addr, size, is_zero, size); return _mi_os_commit_ex(addr, size, is_zero, size);
} }
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stats_size) { static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL); mi_assert_internal(needs_recommit!=NULL);
mi_os_stat_decrease(committed, stats_size); mi_os_stat_decrease(committed, stat_size);
// page align // page align
size_t csize; size_t csize;
@ -505,7 +510,7 @@ bool _mi_os_reset(void* addr, size_t size) {
// either resets or decommits memory, returns true if the memory needs // either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on. // to be recommitted if it is to be re-used later on.
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size) bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size)
{ {
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
mi_os_stat_counter_increase(purge_calls, 1); mi_os_stat_counter_increase(purge_calls, 1);
@ -515,7 +520,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stats_size)
!_mi_preloading()) // don't decommit during preloading (unsafe) !_mi_preloading()) // don't decommit during preloading (unsafe)
{ {
bool needs_recommit = true; bool needs_recommit = true;
mi_os_decommit_ex(p, size, &needs_recommit, stats_size); mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
return needs_recommit; return needs_recommit;
} }
else { else {
@ -636,7 +641,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// no success, issue a warning and break // no success, issue a warning and break
if (p != NULL) { if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, true); mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);
} }
break; break;
} }
@ -682,7 +687,7 @@ static void mi_os_free_huge_os_pages(void* p, size_t size) {
if (p==NULL || size==0) return; if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p; uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) { while (size >= MI_HUGE_OS_PAGE_SIZE) {
mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, true); mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);
size -= MI_HUGE_OS_PAGE_SIZE; size -= MI_HUGE_OS_PAGE_SIZE;
base += MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE;
} }

3
test/test-stress.c
View file

@ -48,7 +48,7 @@ static int ITER = 20;
static int THREADS = 32; static int THREADS = 32;
static int SCALE = 50; static int SCALE = 50;
static int ITER = 50; static int ITER = 50;
#elif 1 #elif 0
static int THREADS = 32; static int THREADS = 32;
static int SCALE = 25; static int SCALE = 25;
static int ITER = 50; static int ITER = 50;
@ -82,6 +82,7 @@ static bool main_participates = false; // main thread participates as a
#define custom_calloc(n,s) mi_calloc(n,s) #define custom_calloc(n,s) mi_calloc(n,s)
#define custom_realloc(p,s) mi_realloc(p,s) #define custom_realloc(p,s) mi_realloc(p,s)
#define custom_free(p) mi_free(p) #define custom_free(p) mi_free(p)
#ifndef NDEBUG #ifndef NDEBUG
#define xHEAP_WALK // walk the heap objects? #define xHEAP_WALK // walk the heap objects?
#endif #endif