kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-07-06 11:34:38 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' into dev

Browse source
This commit is contained in:
Daan 2024-12-30 12:27:46 -08:00 • committed by GitHub
commit f33aa58d88
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG key ID: B5690EEEBB952194
79 changed files with 3509 additions and 3833 deletions
Download patch file Download diff file Expand all files Collapse all files

77
src/alloc-aligned.c
View file

@ -24,6 +24,33 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) {
return (bsize <= MI_MAX_ALIGN_GUARANTEE && (bsize & (alignment-1)) == 0);
}
#if MI_GUARDED
static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, size_t size, size_t alignment, bool zero) mi_attr_noexcept {
// use over allocation for guarded blocksl
mi_assert_internal(alignment > 0 && alignment < MI_BLOCK_ALIGNMENT_MAX);
const size_t oversize = size + alignment - 1;
void* base = _mi_heap_malloc_guarded(heap, oversize, zero);
void* p = mi_align_up_ptr(base, alignment);
mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size);
mi_assert_internal(mi_usable_size(p) >= size);
mi_assert_internal(_mi_is_aligned(p, alignment));
return p;
}
static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) {
const size_t rate = heap->guarded_sample_rate;
// only write if `rate!=0` so we don't write to the constant `_mi_heap_empty`
if (rate != 0) { heap->guarded_sample_rate = 0; }
void* p = _mi_heap_malloc_zero(heap, size, zero);
if (rate != 0) { heap->guarded_sample_rate = rate; }
return p;
}
#else
static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) {
return _mi_heap_malloc_zero(heap, size, zero);
}
#endif
// Fallback aligned allocation that over-allocates -- split out for better codegen
static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept
{
@ -38,12 +65,13 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t
// first (and single) page such that the segment info is `MI_SEGMENT_SIZE` bytes before it (so it can be found by aligning the pointer down)
if mi_unlikely(offset != 0) {
// todo: cannot support offset alignment for very large alignments yet
#if MI_DEBUG > 0
#if MI_DEBUG > 0
_mi_error_message(EOVERFLOW, "aligned allocation with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset);
#endif
#endif
return NULL;
}
oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size);
// note: no guarded as alignment > 0
p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block
// zero afterwards as only the area from the aligned_p may be committed!
if (p == NULL) return NULL;
@ -51,9 +79,10 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t
else {
// otherwise over-allocate
oversize = size + alignment - 1;
p = _mi_heap_malloc_zero(heap, oversize, zero);
p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero);
if (p == NULL) return NULL;
}
mi_page_t* page = _mi_ptr_page(p);
// .. and align within the allocation
const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)`
@ -62,17 +91,27 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t
mi_assert_internal(adjust < alignment);
void* aligned_p = (void*)((uintptr_t)p + adjust);
if (aligned_p != p) {
mi_page_t* page = _mi_ptr_page(p);
mi_page_set_has_aligned(page, true);
#if MI_GUARDED
// set tag to aligned so mi_usable_size works with guard pages
if (adjust >= sizeof(mi_block_t)) {
mi_block_t* const block = (mi_block_t*)p;
block->next = MI_BLOCK_TAG_ALIGNED;
}
#endif
_mi_padding_shrink(page, (mi_block_t*)p, adjust + size);
}
// todo: expand padding if overallocated ?
mi_assert_internal(mi_page_usable_block_size(_mi_ptr_page(p)) >= adjust + size);
mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_page(aligned_p), aligned_p));
mi_assert_internal(mi_page_usable_block_size(page) >= adjust + size);
mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
mi_assert_internal(mi_usable_size(aligned_p)>=size);
mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust);
#if MI_DEBUG > 1
mi_page_t* const apage = _mi_ptr_page(aligned_p);
void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p);
mi_assert_internal(p == unalign_p);
#endif
// now zero the block if needed
if (alignment > MI_BLOCK_ALIGNMENT_MAX) {
@ -85,6 +124,9 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t
if (p != aligned_p) {
mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p));
#if MI_GUARDED
mi_track_mem_defined(p, sizeof(mi_block_t));
#endif
}
return aligned_p;
}
@ -94,27 +136,27 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t*
{
mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));
// we don't allocate more than MI_MAX_ALLOC_SIZE (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) {
#if MI_DEBUG > 0
_mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);
#endif
return NULL;
}
// use regular allocation if it is guaranteed to fit the alignment constraints.
// this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist
// a page with the right block size, and if we always use the over-alloc fallback that would never happen.
if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) {
void* p = _mi_heap_malloc_zero(heap, size, zero);
void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero);
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0;
if mi_likely(is_aligned_or_null) {
return p;
}
else {
// this should never happen if the `mi_malloc_is_naturally_aligned` check is correct..
mi_assert(false);
mi_free(p);
mi_free(p);
}
}
@ -122,6 +164,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t*
return mi_heap_malloc_zero_aligned_at_overalloc(heap,size,alignment,offset,zero);
}
// Primitive aligned allocation
static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept
{
@ -132,11 +175,17 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
#endif
return NULL;
}
#if MI_GUARDED
if (offset==0 && alignment < MI_BLOCK_ALIGNMENT_MAX && mi_heap_malloc_use_guarded(heap,size)) {
return mi_heap_malloc_guarded_aligned(heap, size, alignment, zero);
}
#endif
// try first if there happens to be a small block available with just the right alignment
if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) {
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
const size_t padsize = size + MI_PADDING_SIZE;
const size_t padsize = size + MI_PADDING_SIZE;
mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize);
if mi_likely(page->free != NULL) {
const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
@ -305,3 +354,5 @@ mi_decl_nodiscard void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t
mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment);
}

6
src/alloc-override.c
View file

@ -248,7 +248,7 @@ extern "C" {
// Forward Posix/Unix calls as well
void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize)
size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p)
#if !defined(__ANDROID__) && !defined(__FreeBSD__)
#if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__)
size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p)
#else
size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p)
@ -289,8 +289,8 @@ mi_decl_weak int reallocarr(void* p, size_t count, size_t size) { return mi_r
void __libc_free(void* p) MI_FORWARD0(mi_free, p)
void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); }
#elif defined(__GLIBC__) && defined(__linux__)
// forward __libc interface (needed for glibc-based Linux distributions)
#elif defined(__linux__)
// forward __libc interface (needed for glibc-based and musl-based Linux distributions)
void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size)
void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size)
void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size)

110
src/alloc.c
View file

@ -31,17 +31,22 @@ terms of the MIT license. A copy of the license can be found in the file
extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept
{
mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size);
// check the free list
mi_block_t* const block = page->free;
if mi_unlikely(block == NULL) {
return _mi_malloc_generic(heap, size, zero, 0);
}
mi_assert_internal(block != NULL && _mi_ptr_page(block) == page);
// pop from the free list
page->free = mi_block_next(page, block);
page->used++;
mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE));
#if MI_DEBUG>3
if (page->free_is_zero) {
if (page->free_is_zero && size > sizeof(*block)) {
mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block)));
}
#endif
@ -54,7 +59,10 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_
// zero the block? note: we need to zero the full block size (issue #63)
if mi_unlikely(zero) {
mi_assert_internal(page->block_size != 0); // do not call with zero'ing for huge blocks (see _mi_malloc_generic)
mi_assert_internal(!mi_page_is_huge(page));
#if MI_PADDING
mi_assert_internal(page->block_size >= MI_PADDING_SIZE);
#endif
if (page->free_is_zero) {
block->next = 0;
mi_track_mem_defined(block, page->block_size - MI_PADDING_SIZE);
@ -91,7 +99,7 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_
mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta));
#endif
mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess
padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys));
padding->canary = mi_ptr_encode_canary(page,block,page->keys);
padding->delta = (uint32_t)(delta);
#if MI_PADDING_CHECK
if (!mi_page_is_huge(page)) {
@ -113,17 +121,27 @@ extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t siz
return _mi_page_malloc_zero(heap,page,size,true);
}
#if MI_GUARDED
mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept;
#endif
static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
mi_assert(heap != NULL);
mi_assert(size <= MI_SMALL_SIZE_MAX);
#if MI_DEBUG
const uintptr_t tid = _mi_thread_id();
mi_assert(heap->thread_id == 0 || heap->thread_id == tid); // heaps are thread local
#endif
mi_assert(size <= MI_SMALL_SIZE_MAX);
#if (MI_PADDING)
#if (MI_PADDING || MI_GUARDED)
if (size == 0) { size = sizeof(void*); }
#endif
#if MI_GUARDED
if (mi_heap_malloc_use_guarded(heap,size)) {
return _mi_heap_malloc_guarded(heap, size, zero);
}
#endif
// get page in constant time, and allocate from it
mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE);
void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero);
mi_track_malloc(p,size,zero);
@ -153,15 +171,23 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t si
// The main allocation function
extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept {
// fast path for small objects
if mi_likely(size <= MI_SMALL_SIZE_MAX) {
mi_assert_internal(huge_alignment == 0);
return mi_heap_malloc_small_zero(heap, size, zero);
}
#if MI_GUARDED
else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) {
return _mi_heap_malloc_guarded(heap, size, zero);
}
#endif
else {
// regular allocation
mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic
mi_track_malloc(p,size,zero);
#if MI_STAT>1
if (p != NULL) {
if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }
@ -577,6 +603,82 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) {
}
}
#if MI_GUARDED
// We always allocate a guarded allocation at an offset (`mi_page_has_aligned` will be true).
// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`)
// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size`
static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) {
// TODO: we can still make padding work by moving it out of the guard page area
mi_page_t* const page = _mi_ptr_page(block);
mi_page_set_has_aligned(page, true);
block->next = MI_BLOCK_TAG_GUARDED;
// set guard page at the end of the block
mi_segment_t* const segment = _mi_page_segment(page);
const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local`
const size_t os_page_size = _mi_os_page_size();
mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t));
if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) {
// should never happen
mi_free(block);
return NULL;
}
uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size;
mi_assert_internal(_mi_is_aligned(guard_page, os_page_size));
if (segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) {
_mi_os_protect(guard_page, os_page_size);
}
else {
_mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size);
}
// align pointer just in front of the guard page
size_t offset = block_size - os_page_size - obj_size;
mi_assert_internal(offset > sizeof(mi_block_t));
if (offset > MI_BLOCK_ALIGNMENT_MAX) {
// give up to place it right in front of the guard page if the offset is too large for unalignment
offset = MI_BLOCK_ALIGNMENT_MAX;
}
void* p = (uint8_t*)block + offset;
mi_track_align(block, p, offset, obj_size);
mi_track_mem_defined(block, sizeof(mi_block_t));
return p;
}
mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept
{
#if defined(MI_PADDING_SIZE)
mi_assert(MI_PADDING_SIZE==0);
#endif
// allocate multiple of page size ending in a guard page
// ensure minimal alignment requirement?
const size_t os_page_size = _mi_os_page_size();
const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE));
const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE);
const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size);
mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */);
if (block==NULL) return NULL;
void* const p = mi_block_ptr_set_guarded(block, obj_size);
// stats
mi_track_malloc(p, size, zero);
if (p != NULL) {
if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }
#if MI_STAT>1
mi_heap_stat_increase(heap, malloc, mi_usable_size(p));
#endif
_mi_stat_counter_increase(&heap->tld->stats.guarded_alloc_count, 1);
}
#if MI_DEBUG>3
if (p != NULL && zero) {
mi_assert_expensive(mi_mem_is_zero(p, size));
}
#endif
return p;
}
#endif
// ------------------------------------------------------
// ensure explicit external inline definitions are emitted!
// ------------------------------------------------------

42
src/arena-abandon.c
View file

@ -120,11 +120,7 @@ static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) {
mi_assert(segment->memid.memkind != MI_MEM_ARENA);
// not in an arena; we use a list of abandoned segments
mi_subproc_t* const subproc = segment->subproc;
if (!mi_lock_acquire(&subproc->abandoned_os_lock)) {
_mi_error_message(EFAULT, "internal error: failed to acquire the abandoned (os) segment lock to mark abandonment");
// we can continue but cannot visit/reclaim such blocks..
}
else {
mi_lock(&subproc->abandoned_os_lock) {
// push on the tail of the list (important for the visitor)
mi_segment_t* prev = subproc->abandoned_os_list_tail;
mi_assert_internal(prev == NULL || prev->abandoned_os_next == NULL);
@ -138,7 +134,6 @@ static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) {
mi_atomic_increment_relaxed(&subproc->abandoned_os_list_count);
mi_atomic_increment_relaxed(&subproc->abandoned_count);
// and release the lock
mi_lock_release(&subproc->abandoned_os_lock);
}
return;
}
@ -148,7 +143,7 @@ static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) {
void _mi_arena_segment_mark_abandoned(mi_segment_t* segment)
{
mi_assert_internal(segment->used == segment->abandoned);
mi_atomic_store_release(&segment->thread_id, 0); // mark as abandoned for multi-thread free's
mi_atomic_store_release(&segment->thread_id, (uintptr_t)0); // mark as abandoned for multi-thread free's
if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) {
mi_arena_segment_os_mark_abandoned(segment);
return;
@ -237,7 +232,7 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_at(mi_arena_t* arena, mi_s
static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_cursor_t* previous) {
const size_t max_arena = mi_arena_get_count();
size_t field_idx = mi_bitmap_index_field(previous->bitmap_idx);
size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx) + 1;
size_t bit_idx = mi_bitmap_index_bit_in_field(previous->bitmap_idx);
// visit arena's (from the previous cursor)
for (; previous->start < previous->end; previous->start++, field_idx = 0, bit_idx = 0) {
// index wraps around
@ -251,7 +246,7 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_
if mi_unlikely(field != 0) { // skip zero fields quickly
// we only take the arena lock if there are actually abandoned segments present
if (!has_lock && mi_option_is_enabled(mi_option_visit_abandoned)) {
has_lock = (previous->visit_all ? mi_lock_acquire(&arena->abandoned_visit_lock) : mi_lock_try_acquire(&arena->abandoned_visit_lock));
has_lock = (previous->visit_all ? (mi_lock_acquire(&arena->abandoned_visit_lock),true) : mi_lock_try_acquire(&arena->abandoned_visit_lock));
if (!has_lock) {
if (previous->visit_all) {
_mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the visitor lock");
@ -266,11 +261,12 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_
// pre-check if the bit is set
size_t mask = ((size_t)1 << bit_idx);
if mi_unlikely((field & mask) == mask) {
previous->bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx);
mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, previous->bitmap_idx);
mi_bitmap_index_t bitmap_idx = mi_bitmap_index_create(field_idx, bit_idx);
mi_segment_t* const segment = mi_arena_segment_clear_abandoned_at(arena, previous->subproc, bitmap_idx);
if (segment != NULL) {
//mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx));
if (has_lock) { mi_lock_release(&arena->abandoned_visit_lock); }
previous->bitmap_idx = mi_bitmap_index_create_ex(field_idx, bit_idx + 1); // start at next one for the next iteration
return segment;
}
}
@ -288,8 +284,8 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_list(mi_arena_field_c
// we only allow one thread per sub-process to do to visit guarded by the `abandoned_os_visit_lock`.
// The lock is released when the cursor is released.
if (!previous->hold_visit_lock) {
previous->hold_visit_lock = (previous->visit_all ? mi_lock_acquire(&previous->subproc->abandoned_os_visit_lock)
: mi_lock_try_acquire(&previous->subproc->abandoned_os_visit_lock));
previous->hold_visit_lock = (previous->visit_all ? (mi_lock_acquire(&previous->subproc->abandoned_os_visit_lock),true)
: mi_lock_try_acquire(&previous->subproc->abandoned_os_visit_lock));
if (!previous->hold_visit_lock) {
if (previous->visit_all) {
_mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the OS visitor lock");
@ -300,21 +296,15 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_list(mi_arena_field_c
// One list entry at a time
while (previous->os_list_count > 0) {
previous->os_list_count--;
const bool has_lock = mi_lock_acquire(&previous->subproc->abandoned_os_lock); // this could contend with concurrent OS block abandonment and reclaim from `free`
if (has_lock) {
mi_segment_t* segment = previous->subproc->abandoned_os_list;
// pop from head of the list, a subsequent mark will push at the end (and thus we iterate through os_list_count entries)
if (segment == NULL || mi_arena_segment_os_clear_abandoned(segment, false /* we already have the lock */)) {
mi_lock_release(&previous->subproc->abandoned_os_lock);
return segment;
}
// already abandoned, try again
mi_lock_acquire(&previous->subproc->abandoned_os_lock); // this could contend with concurrent OS block abandonment and reclaim from `free`
mi_segment_t* segment = previous->subproc->abandoned_os_list;
// pop from head of the list, a subsequent mark will push at the end (and thus we iterate through os_list_count entries)
if (segment == NULL || mi_arena_segment_os_clear_abandoned(segment, false /* we already have the lock */)) {
mi_lock_release(&previous->subproc->abandoned_os_lock);
return segment;
}
else {
_mi_error_message(EFAULT, "failed to acquire abandoned OS list lock during abandoned block visit\n");
return NULL;
}
// already abandoned, try again
mi_lock_release(&previous->subproc->abandoned_os_lock);
}
// done
mi_assert_internal(previous->os_list_count == 0);

189
src/arena.c
View file

@ -33,7 +33,7 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
typedef struct mi_arena_s {
mi_arena_id_t id; // arena id; 0 for non-specific
mi_memid_t memid; // memid of the memory area
_Atomic(uint8_t*)start; // the start of the memory area
_Atomic(uint8_t*) start; // the start of the memory area
size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`)
size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`)
size_t meta_size; // size of the arena structure itself (including its bitmaps)
@ -42,12 +42,13 @@ typedef struct mi_arena_s {
bool exclusive; // only allow allocations if specifically for this arena
bool is_large; // memory area consists of large- or huge OS pages (always committed)
mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited
_Atomic(size_t)search_idx; // optimization to start the search for free blocks
_Atomic(mi_msecs_t)purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`.
mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero?
mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted)
mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted)
mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here)
_Atomic(size_t) search_idx; // optimization to start the search for free blocks
_Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be purged from `blocks_purge`.
mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero?
mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted)
mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted)
mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here)
mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)
// do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields.
} mi_arena_t;
@ -60,6 +61,7 @@ typedef struct mi_arena_s {
// The available arenas
static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS];
static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0
static mi_decl_cache_align _Atomic(int64_t) mi_arenas_purge_expire; // set if there exist purgeable arenas
#define MI_IN_ARENA_C
#include "arena-abandon.c"
@ -186,7 +188,7 @@ void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) {
if (p != NULL) return p;
// or fall back to the OS
p = _mi_os_alloc(size, memid, &_mi_stats_main);
p = _mi_os_alloc(size, memid);
if (p == NULL) return NULL;
// zero the OS memory if needed
@ -199,7 +201,7 @@ void* _mi_arena_meta_zalloc(size_t size, mi_memid_t* memid) {
void _mi_arena_meta_free(void* p, mi_memid_t memid, size_t size) {
if (mi_memkind_is_os(memid.memkind)) {
_mi_os_free(p, size, memid, &_mi_stats_main);
_mi_os_free(p, size, memid);
}
else {
mi_assert(memid.memkind == MI_MEM_STATIC);
@ -216,10 +218,10 @@ void* mi_arena_block_start(mi_arena_t* arena, mi_bitmap_index_t bindex) {
----------------------------------------------------------- */
// claim the `blocks_inuse` bits
static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats)
static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t* bitmap_idx)
{
size_t idx = 0; // mi_atomic_load_relaxed(&arena->search_idx); // start from last search; ok to be relaxed as the exact start does not matter
if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx, stats)) {
if (_mi_bitmap_try_find_from_claim_across(arena->blocks_inuse, arena->field_count, idx, blocks, bitmap_idx)) {
mi_atomic_store_relaxed(&arena->search_idx, mi_bitmap_index_field(*bitmap_idx)); // start search from found location next time around
return true;
};
@ -232,13 +234,13 @@ static bool mi_arena_try_claim(mi_arena_t* arena, size_t blocks, mi_bitmap_index
----------------------------------------------------------- */
static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t arena_index, size_t needed_bcount,
bool commit, mi_memid_t* memid, mi_os_tld_t* tld)
bool commit, mi_memid_t* memid)
{
MI_UNUSED(arena_index);
mi_assert_internal(mi_arena_id_index(arena->id) == arena_index);
mi_bitmap_index_t bitmap_index;
if (!mi_arena_try_claim(arena, needed_bcount, &bitmap_index, tld->stats)) return NULL;
if (!mi_arena_try_claim(arena, needed_bcount, &bitmap_index)) return NULL;
// claimed it!
void* p = mi_arena_block_start(arena, bitmap_index);
@ -268,7 +270,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar
_mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted);
if (any_uncommitted) {
bool commit_zero = false;
if (!_mi_os_commit(p, mi_arena_block_size(needed_bcount), &commit_zero, tld->stats)) {
if (!_mi_os_commit(p, mi_arena_block_size(needed_bcount), &commit_zero)) {
memid->initially_committed = false;
}
else {
@ -286,10 +288,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar
// allocate in a specific arena
static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_node, int numa_node, size_t size, size_t alignment,
bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld )
bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid )
{
MI_UNUSED_RELEASE(alignment);
mi_assert_internal(alignment <= MI_SEGMENT_ALIGN);
mi_assert(alignment <= MI_SEGMENT_ALIGN);
const size_t bcount = mi_block_count_of_size(size);
const size_t arena_index = mi_arena_id_index(arena_id);
mi_assert_internal(arena_index < mi_atomic_load_relaxed(&mi_arena_count));
@ -307,7 +309,7 @@ static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_no
}
// try to allocate
void* p = mi_arena_try_alloc_at(arena, arena_index, bcount, commit, memid, tld);
void* p = mi_arena_try_alloc_at(arena, arena_index, bcount, commit, memid);
mi_assert_internal(p == NULL || _mi_is_aligned(p, alignment));
return p;
}
@ -316,7 +318,7 @@ static void* mi_arena_try_alloc_at_id(mi_arena_id_t arena_id, bool match_numa_no
// allocate from an arena with fallback to the OS
static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, size_t alignment,
bool commit, bool allow_large,
mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld )
mi_arena_id_t req_arena_id, mi_memid_t* memid )
{
MI_UNUSED(alignment);
mi_assert_internal(alignment <= MI_SEGMENT_ALIGN);
@ -326,21 +328,21 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz
if (req_arena_id != _mi_arena_id_none()) {
// try a specific arena if requested
if (mi_arena_id_index(req_arena_id) < max_arena) {
void* p = mi_arena_try_alloc_at_id(req_arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld);
void* p = mi_arena_try_alloc_at_id(req_arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);
if (p != NULL) return p;
}
}
else {
// try numa affine allocation
for (size_t i = 0; i < max_arena; i++) {
void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld);
void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);
if (p != NULL) return p;
}
// try from another numa node instead..
if (numa_node >= 0) { // if numa_node was < 0 (no specific affinity requested), all arena's have been tried already
for (size_t i = 0; i < max_arena; i++) {
void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), false /* only proceed if not numa local */, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld);
void* p = mi_arena_try_alloc_at_id(mi_arena_id_create(i), false /* only proceed if not numa local */, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);
if (p != NULL) return p;
}
}
@ -349,11 +351,10 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz
}
// try to reserve a fresh arena space
static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t req_arena_id, mi_arena_id_t *arena_id)
static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t *arena_id)
{
if (_mi_preloading()) return false; // use OS only while pre loading
if (req_arena_id != _mi_arena_id_none()) return false;
const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count);
if (arena_count > (MI_MAX_ARENAS - 4)) return false;
@ -385,27 +386,28 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large,
mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld)
mi_arena_id_t req_arena_id, mi_memid_t* memid)
{
mi_assert_internal(memid != NULL && tld != NULL);
mi_assert_internal(memid != NULL);
mi_assert_internal(size > 0);
*memid = _mi_memid_none();
const int numa_node = _mi_os_numa_node(tld); // current numa node
const int numa_node = _mi_os_numa_node(); // current numa node
// try to allocate in an arena if the alignment is small enough and the object is not too small (as for heap meta data)
if (!mi_option_is_enabled(mi_option_disallow_arena_alloc) || req_arena_id != _mi_arena_id_none()) { // is arena allocation allowed?
if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) {
void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld);
if (!mi_option_is_enabled(mi_option_disallow_arena_alloc)) { // is arena allocation allowed?
if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0)
{
void* p = mi_arena_try_alloc(numa_node, size, alignment, commit, allow_large, req_arena_id, memid);
if (p != NULL) return p;
// otherwise, try to first eagerly reserve a new arena
if (req_arena_id == _mi_arena_id_none()) {
mi_arena_id_t arena_id = 0;
if (mi_arena_reserve(size, allow_large, req_arena_id, &arena_id)) {
if (mi_arena_reserve(size, allow_large, &arena_id)) {
// and try allocate in there
mi_assert_internal(req_arena_id == _mi_arena_id_none());
p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid, tld);
p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);
if (p != NULL) return p;
}
}
@ -420,16 +422,16 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset
// finally, fall back to the OS
if (align_offset > 0) {
return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid, tld->stats);
return _mi_os_alloc_aligned_at_offset(size, alignment, align_offset, commit, allow_large, memid);
}
else {
return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid, tld->stats);
return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
}
}
void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld)
void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid)
{
return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, 0, commit, allow_large, req_arena_id, memid, tld);
return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, 0, commit, allow_large, req_arena_id, memid);
}
@ -455,7 +457,7 @@ static long mi_arena_purge_delay(void) {
// reset or decommit in an arena and update the committed/decommit bitmaps
// assumes we own the area (i.e. blocks_in_use is claimed by us)
static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, mi_stats_t* stats) {
static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) {
mi_assert_internal(arena->blocks_committed != NULL);
mi_assert_internal(arena->blocks_purge != NULL);
mi_assert_internal(!arena->memid.is_pinned);
@ -464,7 +466,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks,
bool needs_recommit;
if (_mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx)) {
// all blocks are committed, we can purge freely
needs_recommit = _mi_os_purge(p, size, stats);
needs_recommit = _mi_os_purge(p, size);
}
else {
// some blocks are not committed -- this can happen when a partially committed block is freed
@ -472,8 +474,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks,
// we need to ensure we do not try to reset (as that may be invalid for uncommitted memory),
// and also undo the decommit stats (as it was already adjusted)
mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits));
needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, stats);
if (needs_recommit) { _mi_stat_increase(&_mi_stats_main.committed, size); }
needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, 0);
}
// clear the purged blocks
@ -486,23 +487,26 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks,
// Schedule a purge. This is usually delayed to avoid repeated decommit/commit calls.
// Note: assumes we (still) own the area as we may purge immediately
static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks, mi_stats_t* stats) {
static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks) {
mi_assert_internal(arena->blocks_purge != NULL);
const long delay = mi_arena_purge_delay();
if (delay < 0) return; // is purging allowed at all?
if (_mi_preloading() || delay == 0) {
// decommit directly
mi_arena_purge(arena, bitmap_idx, blocks, stats);
mi_arena_purge(arena, bitmap_idx, blocks);
}
else {
// schedule decommit
mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
if (expire != 0) {
mi_atomic_addi64_acq_rel(&arena->purge_expire, (mi_msecs_t)(delay/10)); // add smallish extra delay
// schedule purge
const mi_msecs_t expire = _mi_clock_now() + delay;
mi_msecs_t expire0 = 0;
if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {
// expiration was not yet set
// maybe set the global arenas expire as well (if it wasn't set already)
mi_atomic_casi64_strong_acq_rel(&mi_arenas_purge_expire, &expire0, expire);
}
else {
mi_atomic_storei64_release(&arena->purge_expire, _mi_clock_now() + delay);
// already an expiration was set
}
_mi_bitmap_claim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx, NULL);
}
@ -511,7 +515,7 @@ static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t
// purge a range of blocks
// return true if the full range was purged.
// assumes we own the area (i.e. blocks_in_use is claimed by us)
static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitlen, size_t purge, mi_stats_t* stats) {
static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx, size_t bitlen, size_t purge) {
const size_t endidx = startidx + bitlen;
size_t bitidx = startidx;
bool all_purged = false;
@ -524,7 +528,7 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx,
if (count > 0) {
// found range to be purged
const mi_bitmap_index_t range_idx = mi_bitmap_index_create(idx, bitidx);
mi_arena_purge(arena, range_idx, count, stats);
mi_arena_purge(arena, range_idx, count);
if (count == bitlen) {
all_purged = true;
}
@ -535,16 +539,18 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx,
}
// returns true if anything was purged
static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi_stats_t* stats)
static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
{
if (arena->memid.is_pinned || arena->blocks_purge == NULL) return false;
// check pre-conditions
if (arena->memid.is_pinned) return false;
// expired yet?
mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
if (expire == 0) return false;
if (!force && expire > now) return false;
if (!force && (expire == 0 || expire > now)) return false;
// reset expire (if not already set concurrently)
mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0);
// potential purges scheduled, walk through the bitmap
bool any_purged = false;
bool full_purge = true;
@ -571,7 +577,7 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi
if (bitlen > 0) {
// read purge again now that we have the in_use bits
purge = mi_atomic_load_acquire(&arena->blocks_purge[i]);
if (!mi_arena_purge_range(arena, i, bitidx, bitlen, purge, stats)) {
if (!mi_arena_purge_range(arena, i, bitidx, bitlen, purge)) {
full_purge = false;
}
any_purged = true;
@ -591,9 +597,15 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force, mi
return any_purged;
}
static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats ) {
static void mi_arenas_try_purge( bool force, bool visit_all )
{
if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled
// check if any arena needs purging?
const mi_msecs_t now = _mi_clock_now();
mi_msecs_t arenas_expire = mi_atomic_load_acquire(&mi_arenas_purge_expire);
if (!force && (arenas_expire == 0 || arenas_expire < now)) return;
const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count);
if (max_arena == 0) return;
@ -601,17 +613,26 @@ static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats )
static mi_atomic_guard_t purge_guard;
mi_atomic_guard(&purge_guard)
{
mi_msecs_t now = _mi_clock_now();
size_t max_purge_count = (visit_all ? max_arena : 1);
// increase global expire: at most one purge per delay cycle
mi_atomic_store_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay());
size_t max_purge_count = (visit_all ? max_arena : 2);
bool all_visited = true;
for (size_t i = 0; i < max_arena; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]);
if (arena != NULL) {
if (mi_arena_try_purge(arena, now, force, stats)) {
if (max_purge_count <= 1) break;
if (mi_arena_try_purge(arena, now, force)) {
if (max_purge_count <= 1) {
all_visited = false;
break;
}
max_purge_count--;
}
}
}
if (all_visited) {
// all arena's were visited and purged: reset global expire
mi_atomic_store_release(&mi_arenas_purge_expire, 0);
}
}
}
@ -620,8 +641,8 @@ static void mi_arenas_try_purge( bool force, bool visit_all, mi_stats_t* stats )
Arena free
----------------------------------------------------------- */
void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memid, mi_stats_t* stats) {
mi_assert_internal(size > 0 && stats != NULL);
void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memid) {
mi_assert_internal(size > 0);
mi_assert_internal(committed_size <= size);
if (p==NULL) return;
if (size==0) return;
@ -636,7 +657,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi
// if partially committed, adjust the committed stats (as `_mi_os_free` will increase decommit by the full size)
_mi_stat_decrease(&_mi_stats_main.committed, committed_size);
}
_mi_os_free(p, size, memid, stats);
_mi_os_free(p, size, memid);
}
else if (memid.memkind == MI_MEM_ARENA) {
// allocated in an arena
@ -681,7 +702,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi
// works (as we should never reset decommitted parts).
}
// (delay) purge the entire range
mi_arena_schedule_purge(arena, bitmap_idx, blocks, stats);
mi_arena_schedule_purge(arena, bitmap_idx, blocks);
}
// and make it available to others again
@ -697,7 +718,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi
}
// purge expired decommits
mi_arenas_try_purge(false, false, stats);
mi_arenas_try_purge(false, false);
}
// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
@ -711,7 +732,7 @@ static void mi_arenas_unsafe_destroy(void) {
mi_lock_done(&arena->abandoned_visit_lock);
if (arena->start != NULL && mi_memkind_is_os(arena->memid.memkind)) {
mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], NULL);
_mi_os_free(arena->start, mi_arena_size(arena), arena->memid, &_mi_stats_main);
_mi_os_free(arena->start, mi_arena_size(arena), arena->memid);
}
else {
new_max_arena = i;
@ -726,15 +747,15 @@ static void mi_arenas_unsafe_destroy(void) {
}
// Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired
void _mi_arenas_collect(bool force_purge, mi_stats_t* stats) {
mi_arenas_try_purge(force_purge, force_purge /* visit all? */, stats);
void _mi_arenas_collect(bool force_purge) {
mi_arenas_try_purge(force_purge, force_purge /* visit all? */);
}
// destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit`
// for dynamic libraries that are unloaded and need to release all their allocated memory.
void _mi_arena_unsafe_destroy_all(mi_stats_t* stats) {
void _mi_arena_unsafe_destroy_all(void) {
mi_arenas_unsafe_destroy();
_mi_arenas_collect(true /* force purge */, stats); // purge non-owned arenas
_mi_arenas_collect(true /* force purge */); // purge non-owned arenas
}
// Is a pointer inside any of our arenas?
@ -838,11 +859,11 @@ int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exc
if (arena_id != NULL) *arena_id = _mi_arena_id_none();
size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block
mi_memid_t memid;
void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &memid, &_mi_stats_main);
void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, allow_large, &memid);
if (start == NULL) return ENOMEM;
const bool is_large = memid.is_pinned; // todo: use separate is_large field?
if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) {
_mi_os_free_ex(start, size, commit, memid, &_mi_stats_main);
_mi_os_free_ex(start, size, commit, memid);
_mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
return ENOMEM;
}
@ -890,11 +911,11 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_
return inuse_count;
}
void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept {
void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept {
size_t max_arenas = mi_atomic_load_relaxed(&mi_arena_count);
size_t inuse_total = 0;
size_t abandoned_total = 0;
size_t purge_total = 0;
//size_t abandoned_total = 0;
//size_t purge_total = 0;
for (size_t i = 0; i < max_arenas; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]);
if (arena == NULL) break;
@ -905,16 +926,16 @@ void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge)
if (arena->blocks_committed != NULL) {
mi_debug_show_bitmap(" ", "committed blocks", arena->block_count, arena->blocks_committed, arena->field_count);
}
if (show_abandoned) {
abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count);
}
if (show_purge && arena->blocks_purge != NULL) {
purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count);
}
//if (show_abandoned) {
// abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count);
//}
//if (show_purge && arena->blocks_purge != NULL) {
// purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count);
//}
}
if (show_inuse) _mi_verbose_message("total inuse blocks : %zu\n", inuse_total);
if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total);
if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total);
//if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total);
//if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total);
}
@ -938,7 +959,7 @@ int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_m
_mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages);
if (!mi_manage_os_memory_ex2(p, hsize, true, numa_node, exclusive, memid, arena_id)) {
_mi_os_free(p, hsize, memid, &_mi_stats_main);
_mi_os_free(p, hsize, memid);
return ENOMEM;
}
return 0;

12
src/bitmap.c
View file

@ -182,7 +182,7 @@ bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t
// Try to atomically claim a sequence of `count` bits starting from the field
// at `idx` in `bitmap` and crossing into subsequent fields. Returns `true` on success.
// Only needs to consider crossing into the next fields (see `mi_bitmap_try_find_from_claim_across`)
static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats)
static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t idx, const size_t count, const size_t retries, mi_bitmap_index_t* bitmap_idx)
{
mi_assert_internal(bitmap_idx != NULL);
@ -242,7 +242,7 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit
} while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));
// claimed!
mi_stat_counter_increase(stats->arena_crossover_count,1);
mi_stat_counter_increase(_mi_stats_main.arena_crossover_count,1);
*bitmap_idx = mi_bitmap_index_create(idx, initial_idx);
return true;
@ -262,10 +262,10 @@ rollback:
newmap = (map & ~initial_mask);
} while (!mi_atomic_cas_strong_acq_rel(field, &map, newmap));
}
mi_stat_counter_increase(stats->arena_rollback_count,1);
mi_stat_counter_increase(_mi_stats_main.arena_rollback_count,1);
// retry? (we make a recursive call instead of goto to be able to use const declarations)
if (retries <= 2) {
return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx, stats);
return mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, retries+1, bitmap_idx);
}
else {
return false;
@ -275,7 +275,7 @@ rollback:
// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success.
// Starts at idx, and wraps around to search in all `bitmap_fields` fields.
bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats) {
bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx) {
mi_assert_internal(count > 0);
if (count <= 2) {
// we don't bother with crossover fields for small counts
@ -295,7 +295,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm
}
*/
// if that fails, then try to claim across fields
if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx, stats)) {
if (mi_bitmap_try_find_claim_field_across(bitmap, bitmap_fields, idx, count, 0, bitmap_idx)) {
return true;
}
}

8
src/bitmap.h
View file

@ -35,9 +35,13 @@ typedef mi_bitmap_field_t* mi_bitmap_t;
typedef size_t mi_bitmap_index_t;
// Create a bit index.
static inline mi_bitmap_index_t mi_bitmap_index_create_ex(size_t idx, size_t bitidx) {
mi_assert_internal(bitidx <= MI_BITMAP_FIELD_BITS);
return (idx*MI_BITMAP_FIELD_BITS) + bitidx;
}
static inline mi_bitmap_index_t mi_bitmap_index_create(size_t idx, size_t bitidx) {
mi_assert_internal(bitidx < MI_BITMAP_FIELD_BITS);
return (idx*MI_BITMAP_FIELD_BITS) + bitidx;
return mi_bitmap_index_create_ex(idx,bitidx);
}
// Get the field index from a bit index.
@ -90,7 +94,7 @@ bool _mi_bitmap_is_any_claimed(mi_bitmap_t bitmap, size_t bitmap_fields, size_t
// Find `count` bits of zeros and set them to 1 atomically; returns `true` on success.
// Starts at idx, and wraps around to search in all `bitmap_fields` fields.
bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx, mi_stats_t* stats);
bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitmap_fields, const size_t start_field_idx, const size_t count, mi_bitmap_index_t* bitmap_idx);
// Set `count` bits at `bitmap_idx` to 0 atomically
// Returns `true` if all `count` bits were 1 previously.

71
src/free.c
View file

@ -34,11 +34,11 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool
if mi_unlikely(mi_check_is_double_free(page, block)) return;
mi_check_padding(page, block);
if (track_stats) { mi_stat_free(page, block); }
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN
#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN && !MI_GUARDED
memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
#endif
if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block)); } // faster then mi_usable_size as we already know the page and that p is unaligned
// actual free: push on the local free list
mi_block_set_next(page, block, page->local_free);
page->local_free = block;
@ -51,8 +51,8 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool
}
// Adjust a block that was allocated aligned, to the actual start of the block in the page.
// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
// `page_start` and `block_size` fields; however these are constant and the page won't be
// note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the
// `page_start` and `block_size` fields; however these are constant and the page won't be
// deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently.
mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
mi_assert_internal(page!=NULL && p!=NULL);
@ -69,16 +69,30 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
return (mi_block_t*)((uintptr_t)p - adjust);
}
// forward declaration for a MI_GUARDED build
#if MI_GUARDED
static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard(page, block, p); }
}
#else
static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
}
#endif
// free a local pointer (page parameter comes first for better codegen)
static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept {
MI_UNUSED(segment);
mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(page, p) : (mi_block_t*)p);
mi_block_check_unguard(page, block, p);
mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */);
}
// free a pointer owned by another thread (page parameter comes first for better codegen)
static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, mi_segment_t* segment, void* p) mi_attr_noexcept {
mi_block_t* const block = _mi_page_ptr_unalign(page, p); // don't check `has_aligned` flag to avoid a race (issue #865)
mi_block_check_unguard(page, block, p);
mi_free_block_mt(page, segment, block);
}
@ -95,17 +109,17 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms
{
MI_UNUSED(msg);
#if (MI_DEBUG>0)
if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) {
#if (MI_DEBUG>0)
if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) {
_mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
return NULL;
}
#endif
#endif
mi_segment_t* const segment = _mi_ptr_segment(p);
if mi_unlikely(segment==NULL) return segment;
#if (MI_DEBUG>0)
#if (MI_DEBUG>0)
if mi_unlikely(!mi_is_in_heap_region(p)) {
_mi_warning_message("%s: pointer might not point to a valid heap region: %p\n"
"(this may still be a valid very large allocation (over 64MiB))\n", msg, p);
@ -113,13 +127,13 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms
_mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p);
}
}
#endif
#if (MI_DEBUG>0 || MI_SECURE>=4)
#endif
#if (MI_DEBUG>0 || MI_SECURE>=4)
if mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie) {
_mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p);
return NULL;
}
#endif
#endif
return segment;
}
@ -231,11 +245,12 @@ static void mi_decl_noinline mi_free_block_delayed_mt( mi_page_t* page, mi_block
static void mi_decl_noinline mi_free_block_mt(mi_page_t* page, mi_segment_t* segment, mi_block_t* block)
{
// first see if the segment was abandoned and if we can reclaim it into our thread
if (mi_option_is_enabled(mi_option_abandoned_reclaim_on_free) &&
if (_mi_option_get_fast(mi_option_abandoned_reclaim_on_free) != 0 &&
#if MI_HUGE_PAGE_ABANDON
segment->page_kind != MI_PAGE_HUGE &&
#endif
mi_atomic_load_relaxed(&segment->thread_id) == 0)
mi_atomic_load_relaxed(&segment->thread_id) == 0 && // segment is abandoned?
mi_prim_get_default_heap() != (mi_heap_t*)&_mi_heap_empty) // and we did not already exit this thread (without this check, a fresh heap will be initalized (issue #944))
{
// the segment is abandoned, try to reclaim it into our heap
if (_mi_segment_attempt_reclaim(mi_heap_get_default(), segment)) {
@ -291,7 +306,13 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p
const size_t size = mi_page_usable_size_of(page, block);
const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
return (size - adjust);
const size_t aligned_size = (size - adjust);
#if MI_GUARDED
if (mi_block_ptr_is_guarded(block, p)) {
return aligned_size - _mi_os_page_size();
}
#endif
return aligned_size;
}
static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept {
@ -401,7 +422,7 @@ static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* bloc
uintptr_t keys[2];
keys[0] = page->keys[0];
keys[1] = page->keys[1];
bool ok = ((uint32_t)mi_ptr_encode(page,block,keys) == canary && *delta <= *bsize);
bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize);
mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
return ok;
}
@ -518,3 +539,23 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
MI_UNUSED(page); MI_UNUSED(block);
}
#endif
// Remove guard page when building with MI_GUARDED
#if MI_GUARDED
static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) {
MI_UNUSED(p);
mi_assert_internal(mi_block_ptr_is_guarded(block, p));
mi_assert_internal(mi_page_has_aligned(page));
mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t));
mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED);
const size_t bsize = mi_page_block_size(page);
const size_t psize = _mi_os_page_size();
mi_assert_internal(bsize > psize);
mi_assert_internal(_mi_page_segment(page)->allow_decommit);
void* gpage = (uint8_t*)block + bsize - psize;
mi_assert_internal(_mi_is_aligned(gpage, psize));
_mi_os_unprotect(gpage, psize);
}
#endif

49
src/heap.c
View file

@ -32,7 +32,7 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void
#if MI_DEBUG>1
size_t total = heap->page_count;
size_t count = 0;
#endif
#endif
for (size_t i = 0; i <= MI_BIN_FULL; i++) {
mi_page_queue_t* pq = &heap->pages[i];
@ -59,7 +59,7 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
MI_UNUSED(pq);
mi_assert_internal(mi_page_heap(page) == heap);
mi_segment_t* segment = _mi_page_segment(page);
mi_assert_internal(segment->thread_id == heap->thread_id);
mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id) == heap->thread_id);
mi_assert_expensive(_mi_page_is_valid(page));
return true;
}
@ -164,9 +164,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
if (force && is_main_thread && mi_heap_is_backing(heap)) {
_mi_thread_data_collect(); // collect thread data cache
}
// collect arenas (this is program wide so don't force purges on abandonment of threads)
_mi_arenas_collect(collect == MI_FORCE /* force purge? */, &heap->tld->stats);
_mi_arenas_collect(collect == MI_FORCE /* force purge? */);
}
void _mi_heap_collect_abandon(mi_heap_t* heap) {
@ -221,6 +221,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool
heap->cookie = _mi_heap_random_next(heap) | 1;
heap->keys[0] = _mi_heap_random_next(heap);
heap->keys[1] = _mi_heap_random_next(heap);
_mi_heap_guarded_init(heap);
// push on the thread local heaps list
heap->next = heap->tld->heaps;
heap->tld->heaps = heap;
@ -240,7 +241,7 @@ mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) {
}
mi_decl_nodiscard mi_heap_t* mi_heap_new(void) {
// don't reclaim abandoned memory or otherwise destroy is unsafe
// don't reclaim abandoned memory or otherwise destroy is unsafe
return mi_heap_new_ex(0 /* default heap tag */, true /* no reclaim */, _mi_arena_id_none());
}
@ -369,7 +370,13 @@ void mi_heap_destroy(mi_heap_t* heap) {
mi_assert(heap->no_reclaim);
mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
#if MI_GUARDED
// _mi_warning_message("'mi_heap_destroy' called but MI_GUARDED is enabled -- using `mi_heap_delete` instead (heap at %p)\n", heap);
mi_heap_delete(heap);
return;
#else
if (!heap->no_reclaim) {
_mi_warning_message("'mi_heap_destroy' called but ignored as the heap was not created with 'allow_destroy' (heap at %p)\n", heap);
// don't free in case it may contain reclaimed pages
mi_heap_delete(heap);
}
@ -382,12 +389,14 @@ void mi_heap_destroy(mi_heap_t* heap) {
_mi_heap_destroy_pages(heap);
mi_heap_free(heap);
}
#endif
}
// forcefully destroy all heaps in the current thread
void _mi_heap_unsafe_destroy_all(void) {
mi_heap_t* bheap = mi_heap_get_backing();
mi_heap_t* curr = bheap->tld->heaps;
void _mi_heap_unsafe_destroy_all(mi_heap_t* heap) {
mi_assert_internal(heap != NULL);
if (heap == NULL) return;
mi_heap_t* curr = heap->tld->heaps;
while (curr != NULL) {
mi_heap_t* next = curr->next;
if (curr->no_reclaim) {
@ -438,6 +447,12 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
mi_heap_reset_pages(from);
}
// are two heaps compatible with respect to heap-tag, exclusive arena etc.
static bool mi_heaps_are_compatible(mi_heap_t* heap1, mi_heap_t* heap2) {
return (heap1->tag == heap2->tag && // store same kind of objects
heap1->arena_id == heap2->arena_id); // same arena preference
}
// Safe delete a heap without freeing any still allocated blocks in that heap.
void mi_heap_delete(mi_heap_t* heap)
{
@ -446,9 +461,10 @@ void mi_heap_delete(mi_heap_t* heap)
mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (!mi_heap_is_backing(heap)) {
mi_heap_t* bheap = heap->tld->heap_backing;
if (bheap != heap && mi_heaps_are_compatible(bheap,heap)) {
// transfer still used pages to the backing heap
mi_heap_absorb(heap->tld->heap_backing, heap);
mi_heap_absorb(bheap, heap);
}
else {
// the backing heap abandons its pages
@ -536,13 +552,14 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
*shift = 64 - mi_clz(divisor - 1);
*magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
*shift = MI_INTPTR_BITS - mi_clz(divisor - 1);
*magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
}
static size_t mi_fast_divide(size_t n, uint64_t magic, size_t shift) {
mi_assert_internal(n <= UINT32_MAX);
return ((((uint64_t)n * magic) >> 32) + n) >> shift;
const uint64_t hi = ((uint64_t)n * magic) >> 32;
return (size_t)((hi + n) >> shift);
}
bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg) {
@ -581,7 +598,7 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_
// create a bitmap of free blocks.
#define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*))
uintptr_t free_map[MI_MAX_BLOCKS / MI_INTPTR_BITS];
const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);
const uintptr_t bmapsize = _mi_divide_up(page->capacity, MI_INTPTR_BITS);
memset(free_map, 0, bmapsize * sizeof(intptr_t));
if (page->capacity % MI_INTPTR_BITS != 0) {
// mark left-over bits at the end as free
@ -591,7 +608,7 @@ bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_
}
// fast repeated division by the block size
uint64_t magic;
uint64_t magic;
size_t shift;
mi_get_fast_divisor(bsize, &magic, &shift);
@ -665,7 +682,7 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun;
mi_heap_area_ex_t xarea;
xarea.page = page;
_mi_heap_area_init(&xarea.area, page);
_mi_heap_area_init(&xarea.area, page);
return fun(heap, &xarea, arg);
}

194
src/init.c
View file

@ -86,7 +86,8 @@ const mi_page_t _mi_page_empty = {
MI_STAT_COUNT_NULL(), \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \
{ 0, 0 } \
MI_STAT_COUNT_END_NULL()
// --------------------------------------------------------
@ -111,6 +112,9 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
NULL, // next
false, // can reclaim
0, // tag
#if MI_GUARDED
0, 0, 0, 0, 1, // count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`)
#endif
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY
};
@ -132,9 +136,8 @@ static mi_decl_cache_align mi_tld_t tld_main = {
&_mi_heap_main, &_mi_heap_main,
{ { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0},
0, 0, 0, 0, 0, &mi_subproc_default,
&tld_main.stats, &tld_main.os
&tld_main.stats
}, // segments
{ 0, &tld_main.stats }, // os
{ MI_STATS_NULL } // stats
};
@ -151,6 +154,9 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = {
NULL, // next heap
false, // can reclaim
0, // tag
#if MI_GUARDED
0, 0, 0, 0, 0,
#endif
MI_SMALL_PAGES_EMPTY,
MI_PAGE_QUEUES_EMPTY
};
@ -159,6 +165,45 @@ bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
mi_stats_t _mi_stats_main = { MI_STATS_NULL };
#if MI_GUARDED
mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) {
heap->guarded_sample_seed = seed;
if (heap->guarded_sample_seed == 0) {
heap->guarded_sample_seed = _mi_heap_random_next(heap);
}
heap->guarded_sample_rate = sample_rate;
if (heap->guarded_sample_rate >= 1) {
heap->guarded_sample_seed = heap->guarded_sample_seed % heap->guarded_sample_rate;
}
heap->guarded_sample_count = heap->guarded_sample_seed; // count down samples
}
mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) {
heap->guarded_size_min = min;
heap->guarded_size_max = (min > max ? min : max);
}
void _mi_heap_guarded_init(mi_heap_t* heap) {
mi_heap_guarded_set_sample_rate(heap,
(size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX),
(size_t)mi_option_get(mi_option_guarded_sample_seed));
mi_heap_guarded_set_size_bound(heap,
(size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX),
(size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) );
}
#else
mi_decl_export void mi_heap_guarded_set_sample_rate(mi_heap_t* heap, size_t sample_rate, size_t seed) {
MI_UNUSED(heap); MI_UNUSED(sample_rate); MI_UNUSED(seed);
}
mi_decl_export void mi_heap_guarded_set_size_bound(mi_heap_t* heap, size_t min, size_t max) {
MI_UNUSED(heap); MI_UNUSED(min); MI_UNUSED(max);
}
void _mi_heap_guarded_init(mi_heap_t* heap) {
MI_UNUSED(heap);
}
#endif
static void mi_heap_main_init(void) {
if (_mi_heap_main.cookie == 0) {
@ -174,6 +219,7 @@ static void mi_heap_main_init(void) {
_mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main);
mi_lock_init(&mi_subproc_default.abandoned_os_lock);
mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock);
_mi_heap_guarded_init(&_mi_heap_main);
}
}
@ -211,11 +257,10 @@ void mi_subproc_delete(mi_subproc_id_t subproc_id) {
mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id);
// check if there are no abandoned segments still..
bool safe_to_delete = false;
if (mi_lock_acquire(&subproc->abandoned_os_lock)) {
mi_lock(&subproc->abandoned_os_lock) {
if (subproc->abandoned_os_list == NULL) {
safe_to_delete = true;
}
mi_lock_release(&subproc->abandoned_os_lock);
}
if (!safe_to_delete) return;
// safe to release
@ -273,10 +318,10 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) {
// if that fails, allocate as meta data
if (td == NULL) {
mi_memid_t memid;
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main);
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid);
if (td == NULL) {
// if this fails, try once more. (issue #257)
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main);
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid);
if (td == NULL) {
// really out of memory
_mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t));
@ -306,7 +351,7 @@ static void mi_thread_data_free( mi_thread_data_t* tdfree ) {
}
}
// if that fails, just free it directly
_mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid, &_mi_stats_main);
_mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid);
}
void _mi_thread_data_collect(void) {
@ -316,7 +361,7 @@ void _mi_thread_data_collect(void) {
if (td != NULL) {
td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL);
if (td != NULL) {
_mi_os_free(td, sizeof(mi_thread_data_t), td->memid, &_mi_stats_main);
_mi_os_free(td, sizeof(mi_thread_data_t), td->memid);
}
}
}
@ -353,8 +398,6 @@ void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) {
tld->heaps = NULL;
tld->segments.subproc = &mi_subproc_default;
tld->segments.stats = &tld->stats;
tld->segments.os = &tld->os;
tld->os.stats = &tld->stats;
}
// Free the thread local default heap (called from `mi_thread_done`)
@ -508,54 +551,15 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) {
// --------------------------------------------------------
// Run functions on process init/done, and thread init/done
// --------------------------------------------------------
static void mi_cdecl mi_process_done(void);
static bool os_preloading = true; // true until this module is initialized
static bool mi_redirected = false; // true if malloc redirects to mi_malloc
// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
bool mi_decl_noinline _mi_preloading(void) {
return os_preloading;
}
mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept {
return mi_redirected;
}
// Communicate with the redirection module on Windows
#if defined(_WIN32) && defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)
#ifdef __cplusplus
extern "C" {
#endif
mi_decl_export void _mi_redirect_entry(DWORD reason) {
// called on redirection; careful as this may be called before DllMain
if (reason == DLL_PROCESS_ATTACH) {
mi_redirected = true;
}
else if (reason == DLL_PROCESS_DETACH) {
mi_redirected = false;
}
else if (reason == DLL_THREAD_DETACH) {
mi_thread_done();
}
}
__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);
__declspec(dllimport) void mi_cdecl mi_allocator_done(void);
#ifdef __cplusplus
}
#endif
#else
static bool mi_allocator_init(const char** message) {
if (message != NULL) *message = NULL;
return true;
}
static void mi_allocator_done(void) {
// nothing to do
}
#endif
// Called once by the process loader
static void mi_process_load(void) {
// Called once by the process loader from `src/prim/prim.c`
void _mi_process_load(void) {
mi_heap_main_init();
#if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD)
volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true;
@ -563,17 +567,14 @@ static void mi_process_load(void) {
#endif
os_preloading = false;
mi_assert_internal(_mi_is_main_thread());
#if !(defined(_WIN32) && defined(MI_SHARED_LIB)) // use Dll process detach (see below) instead of atexit (issue #521)
atexit(&mi_process_done);
#endif
_mi_options_init();
mi_process_setup_auto_thread_done();
mi_process_init();
if (mi_redirected) _mi_verbose_message("malloc is redirected.\n");
if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n");
// show message from the redirector (if present)
const char* msg = NULL;
mi_allocator_init(&msg);
_mi_allocator_init(&msg);
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
_mi_fputs(NULL,NULL,NULL,msg);
}
@ -585,12 +586,15 @@ static void mi_process_load(void) {
#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64))
#include <intrin.h>
mi_decl_cache_align bool _mi_cpu_has_fsrm = false;
mi_decl_cache_align bool _mi_cpu_has_erms = false;
static void mi_detect_cpu_features(void) {
// FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
// FSRM for fast short rep movsb/stosb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
// EMRS for fast enhanced rep movsb/stosb support
int32_t cpu_info[4];
__cpuid(cpu_info, 7);
_mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
_mi_cpu_has_erms = ((cpu_info[2] & (1 << 9)) != 0); // bit 9 of ECX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
}
#else
static void mi_detect_cpu_features(void) {
@ -651,7 +655,7 @@ void mi_process_init(void) mi_attr_noexcept {
}
// Called when the process is done (through `at_exit`)
static void mi_cdecl mi_process_done(void) {
void mi_cdecl _mi_process_done(void) {
// only shutdown if we were initialized
if (!_mi_process_is_initialized) return;
// ensure we are called once
@ -659,15 +663,20 @@ static void mi_cdecl mi_process_done(void) {
if (process_done) return;
process_done = true;
// get the default heap so we don't need to acces thread locals anymore
mi_heap_t* heap = mi_prim_get_default_heap(); // use prim to not initialize any heap
mi_assert_internal(heap != NULL);
// release any thread specific resources and ensure _mi_thread_done is called on all but the main thread
_mi_prim_thread_done_auto_done();
#ifndef MI_SKIP_COLLECT_ON_EXIT
#if (MI_DEBUG || !defined(MI_SHARED_LIB))
// free all memory if possible on process exit. This is not needed for a stand-alone process
// but should be done if mimalloc is statically linked into another shared library which
// is repeatedly loaded/unloaded, see issue #281.
mi_collect(true /* force */ );
mi_heap_collect(heap, true /* force */ );
#endif
#endif
@ -675,72 +684,17 @@ static void mi_cdecl mi_process_done(void) {
// since after process_done there might still be other code running that calls `free` (like at_exit routines,
// or C-runtime termination code.
if (mi_option_is_enabled(mi_option_destroy_on_exit)) {
mi_collect(true /* force */);
_mi_heap_unsafe_destroy_all(); // forcefully release all memory held by all heaps (of this thread only!)
_mi_arena_unsafe_destroy_all(& _mi_heap_main_get()->tld->stats);
mi_heap_collect(heap, true /* force */);
_mi_heap_unsafe_destroy_all(heap); // forcefully release all memory held by all heaps (of this thread only!)
_mi_arena_unsafe_destroy_all();
_mi_segment_map_unsafe_destroy();
}
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
mi_stats_print(NULL);
}
mi_allocator_done();
_mi_allocator_done();
_mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id);
os_preloading = true; // don't call the C runtime anymore
}
#if defined(_WIN32) && defined(MI_SHARED_LIB)
// Windows DLL: easy to hook into process_init and thread_done
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
MI_UNUSED(reserved);
MI_UNUSED(inst);
if (reason==DLL_PROCESS_ATTACH) {
mi_process_load();
}
else if (reason==DLL_PROCESS_DETACH) {
mi_process_done();
}
else if (reason==DLL_THREAD_DETACH) {
if (!mi_is_redirected()) {
mi_thread_done();
}
}
return TRUE;
}
#elif defined(_MSC_VER)
// MSVC: use data section magic for static libraries
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
static int _mi_process_init(void) {
mi_process_load();
return 0;
}
typedef int(*_mi_crt_callback_t)(void);
#if defined(_M_X64) || defined(_M_ARM64)
__pragma(comment(linker, "/include:" "_mi_msvc_initu"))
#pragma section(".CRT$XIU", long, read)
#else
__pragma(comment(linker, "/include:" "__mi_msvc_initu"))
#endif
#pragma data_seg(".CRT$XIU")
mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init };
#pragma data_seg()
#elif defined(__cplusplus)
// C++: use static initialization to detect process start
static bool _mi_process_init(void) {
mi_process_load();
return (_mi_heap_main.thread_id != 0);
}
static bool mi_initialized = _mi_process_init();
#elif defined(__GNUC__) || defined(__clang__)
// GCC,Clang: use the constructor attribute
static void __attribute__((constructor)) _mi_process_init(void) {
mi_process_load();
}
#else
#pragma message("define a way to call mi_process_load on your platform")
#endif

20
src/libc.c
View file

@ -130,7 +130,7 @@ static void mi_out_alignright(char fill, char* start, size_t len, size_t extra,
}
static void mi_out_num(uintptr_t x, size_t base, char prefix, char** out, char* end)
static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* end)
{
if (x == 0 || base == 0 || base > 16) {
if (prefix != 0) { mi_outc(prefix, out, end); }
@ -206,12 +206,13 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
}
else if (c == 'p' || c == 'x' || c == 'u') {
// unsigned
uintptr_t x = 0;
uintmax_t x = 0;
if (c == 'x' || c == 'u') {
if (numtype == 'z') x = va_arg(args, size_t);
else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
else if (numtype == 'L') x = (uintptr_t)va_arg(args, unsigned long long);
else x = va_arg(args, unsigned long);
else if (numtype == 'L') x = va_arg(args, unsigned long long);
else if (numtype == 'l') x = va_arg(args, unsigned long);
else x = va_arg(args, unsigned int);
}
else if (c == 'p') {
x = va_arg(args, uintptr_t);
@ -228,20 +229,21 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
}
else if (c == 'i' || c == 'd') {
// signed
intptr_t x = 0;
intmax_t x = 0;
if (numtype == 'z') x = va_arg(args, intptr_t );
else if (numtype == 't') x = va_arg(args, ptrdiff_t);
else if (numtype == 'L') x = (intptr_t)va_arg(args, long long);
else x = va_arg(args, long);
else if (numtype == 'L') x = va_arg(args, long long);
else if (numtype == 'l') x = va_arg(args, long);
else x = va_arg(args, int);
char pre = 0;
if (x < 0) {
pre = '-';
if (x > INTPTR_MIN) { x = -x; }
if (x > INTMAX_MIN) { x = -x; }
}
else if (numplus != 0) {
pre = numplus;
}
mi_out_num((uintptr_t)x, 10, pre, &out, end);
mi_out_num((uintmax_t)x, 10, pre, &out, end);
}
else if (c >= ' ' && c <= '~') {
// unknown format

119
src/options.c
View file

@ -47,6 +47,58 @@ typedef struct mi_option_desc_s {
#define MI_OPTION(opt) mi_option_##opt, #opt, NULL
#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy
// Some options can be set at build time for statically linked libraries
// (use `-DMI_EXTRA_CPPDEFS="opt1=val1;opt2=val2"`)
//
// This is useful if we cannot pass them as environment variables
// (and setting them programmatically would be too late)
#ifndef MI_DEFAULT_VERBOSE
#define MI_DEFAULT_VERBOSE 0
#endif
#ifndef MI_DEFAULT_EAGER_COMMIT
#define MI_DEFAULT_EAGER_COMMIT 1
#endif
#ifndef MI_DEFAULT_ARENA_EAGER_COMMIT
#define MI_DEFAULT_ARENA_EAGER_COMMIT 2
#endif
// in KiB
#ifndef MI_DEFAULT_ARENA_RESERVE
#if (MI_INTPTR_SIZE>4)
#define MI_DEFAULT_ARENA_RESERVE 1024L*1024L
#else
#define MI_DEFAULT_ARENA_RESERVE 128L*1024L
#endif
#endif
#ifndef MI_DEFAULT_DISALLOW_ARENA_ALLOC
#define MI_DEFAULT_DISALLOW_ARENA_ALLOC 0
#endif
#ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES
#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0
#endif
#ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES
#define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0
#endif
#ifndef MI_DEFAULT_RESERVE_OS_MEMORY
#define MI_DEFAULT_RESERVE_OS_MEMORY 0
#endif
#ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE
#if MI_GUARDED
#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000
#else
#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0
#endif
#endif
static mi_option_desc_t options[_mi_option_last] =
{
// stable options
@ -56,16 +108,21 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(show_errors) },
#endif
{ 0, UNINIT, MI_OPTION(show_stats) },
{ 0, UNINIT, MI_OPTION(verbose) },
{ MI_DEFAULT_VERBOSE, UNINIT, MI_OPTION(verbose) },
// the following options are experimental and not all combinations make sense.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`)
{ 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)
// some of the following options are experimental and not all combinations are allowed.
{ MI_DEFAULT_EAGER_COMMIT,
UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`)
{ MI_DEFAULT_ARENA_EAGER_COMMIT,
UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) }, // eager commit arena's? 2 is used to enable this only on an OS that has overcommit (i.e. linux)
{ 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset) (note: on linux this uses MADV_DONTNEED for decommit)
{ 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
{ MI_DEFAULT_ALLOW_LARGE_OS_PAGES,
UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ MI_DEFAULT_RESERVE_HUGE_OS_PAGES,
UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
{-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
{ 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)
{ MI_DEFAULT_RESERVE_OS_MEMORY,
UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)
{ 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread
{ 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free
{ 0, UNINIT, MI_OPTION(abandoned_page_purge) }, // purge free page memory when a thread terminates
@ -83,22 +140,24 @@ static mi_option_desc_t options[_mi_option_last] =
{ 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
{ 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.
{ 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
#if (MI_INTPTR_SIZE>4)
{ 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
#else
{ 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit
#endif
{ MI_DEFAULT_ARENA_RESERVE, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
{ 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
{ 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) },
{ 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free
{ 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
{ MI_DEFAULT_DISALLOW_ARENA_ALLOC, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
{ 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
#if defined(MI_VISIT_ABANDONED)
{ 1, INITIALIZED, MI_OPTION(visit_abandoned) }, // allow visiting heap blocks in abandoned segments; requires taking locks during reclaim.
#else
{ 0, UNINIT, MI_OPTION(visit_abandoned) },
#endif
{ 0, UNINIT, MI_OPTION(guarded_min) }, // only used when building with MI_GUARDED: minimal rounded object size for guarded objects
{ MI_GiB, UNINIT, MI_OPTION(guarded_max) }, // only used when building with MI_GUARDED: maximal rounded object size for guarded objects
{ 0, UNINIT, MI_OPTION(guarded_precise) }, // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
{ MI_DEFAULT_GUARDED_SAMPLE_RATE,
UNINIT, MI_OPTION(guarded_sample_rate)}, // 1 out of N allocations in the min/max range will be guarded (=4000)
{ 0, UNINIT, MI_OPTION(guarded_sample_seed)},
{ 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable.
};
static void mi_option_init(mi_option_desc_t* desc);
@ -108,8 +167,7 @@ static bool mi_option_has_size_in_kib(mi_option_t option) {
}
void _mi_options_init(void) {
// called on process load; should not be called before the CRT is initialized!
// (e.g. do not call this from process_init as that may run before CRT initialization)
// called on process load
mi_add_stderr_output(); // now it safe to use stderr for output
for(int i = 0; i < _mi_option_last; i++ ) {
mi_option_t option = (mi_option_t)i;
@ -122,8 +180,26 @@ void _mi_options_init(void) {
}
mi_max_error_count = mi_option_get(mi_option_max_errors);
mi_max_warning_count = mi_option_get(mi_option_max_warnings);
#if MI_GUARDED
if (mi_option_get(mi_option_guarded_sample_rate) > 0) {
if (mi_option_is_enabled(mi_option_allow_large_os_pages)) {
mi_option_disable(mi_option_allow_large_os_pages);
_mi_warning_message("option 'allow_large_os_pages' is disabled to allow for guarded objects\n");
}
}
_mi_verbose_message("guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled");
#endif
}
long _mi_option_get_fast(mi_option_t option) {
mi_assert(option >= 0 && option < _mi_option_last);
mi_option_desc_t* desc = &options[option];
mi_assert(desc->option == option); // index should match the option
//mi_assert(desc->init != UNINIT);
return desc->value;
}
mi_decl_nodiscard long mi_option_get(mi_option_t option) {
mi_assert(option >= 0 && option < _mi_option_last);
if (option < 0 || option >= _mi_option_last) return 0;
@ -141,7 +217,6 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma
}
mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
mi_assert_internal(mi_option_has_size_in_kib(option));
const long x = mi_option_get(option);
size_t size = (x < 0 ? 0 : (size_t)x);
if (mi_option_has_size_in_kib(option)) {
@ -157,6 +232,13 @@ void mi_option_set(mi_option_t option, long value) {
mi_assert(desc->option == option); // index should match the option
desc->value = value;
desc->init = INITIALIZED;
// ensure min/max range; be careful to not recurse.
if (desc->option == mi_option_guarded_min && _mi_option_get_fast(mi_option_guarded_max) < value) {
mi_option_set(mi_option_guarded_max, value);
}
else if (desc->option == mi_option_guarded_max && _mi_option_get_fast(mi_option_guarded_min) > value) {
mi_option_set(mi_option_guarded_min, value);
}
}
void mi_option_set_default(mi_option_t option, long value) {
@ -506,8 +588,7 @@ static void mi_option_init(mi_option_desc_t* desc) {
value = (size > LONG_MAX ? LONG_MAX : (long)size);
}
if (*end == 0) {
desc->value = value;
desc->init = INITIALIZED;
mi_option_set(desc->option, value);
}
else {
// set `init` first to avoid recursion through _mi_warning_message on mimalloc_verbose.

208
src/os.c
View file

@ -9,18 +9,38 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/atomic.h"
#include "mimalloc/prim.h"
#define mi_os_stat_increase(stat,amount) _mi_stat_increase(&_mi_stats_main.stat, amount)
#define mi_os_stat_decrease(stat,amount) _mi_stat_decrease(&_mi_stats_main.stat, amount)
#define mi_os_stat_counter_increase(stat,inc) _mi_stat_counter_increase(&_mi_stats_main.stat, inc)
/* -----------------------------------------------------------
Initialization.
Initialization.
----------------------------------------------------------- */
#ifndef MI_DEFAULT_VIRTUAL_ADDRESS_BITS
#if MI_INTPTR_SIZE < 8
#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 32
#else
#define MI_DEFAULT_VIRTUAL_ADDRESS_BITS 48
#endif
#endif
#ifndef MI_DEFAULT_PHYSICAL_MEMORY
#if MI_INTPTR_SIZE < 8
#define MI_DEFAULT_PHYSICAL_MEMORY 4*MI_GiB
#else
#define MI_DEFAULT_PHYSICAL_MEMORY 32*MI_GiB
#endif
#endif
static mi_os_mem_config_t mi_os_mem_config = {
4096, // page size
0, // large page size (usually 2MiB)
4096, // allocation granularity
true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)
false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
4096, // page size
0, // large page size (usually 2MiB)
4096, // allocation granularity
MI_DEFAULT_PHYSICAL_MEMORY,
MI_DEFAULT_VIRTUAL_ADDRESS_BITS,
true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)
false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
};
bool _mi_os_has_overcommit(void) {
@ -68,8 +88,8 @@ void _mi_os_init(void) {
/* -----------------------------------------------------------
Util
-------------------------------------------------------------- */
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats);
bool _mi_os_decommit(void* addr, size_t size);
bool _mi_os_commit(void* addr, size_t size, bool* is_zero);
static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
mi_assert_internal(alignment != 0);
@ -91,9 +111,10 @@ static void* mi_align_down_ptr(void* p, size_t alignment) {
aligned hinting
-------------------------------------------------------------- */
// On 64-bit systems, we can do efficient aligned allocation by using
// the 2TiB to 30TiB area to allocate those.
#if (MI_INTPTR_SIZE >= 8)
// On systems with enough virtual address bits, we can do efficient aligned allocation by using
// the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address
// space (64TiB) we use this technique. (but see issue #939)
#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT)
static mi_decl_cache_align _Atomic(uintptr_t)aligned_base;
// Return a MI_SEGMENT_SIZE aligned address that is probably available.
@ -110,6 +131,7 @@ static mi_decl_cache_align _Atomic(uintptr_t)aligned_base;
void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
{
if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;
if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space
size = _mi_align_up(size, MI_SEGMENT_SIZE);
if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
#if (MI_SECURE>0)
@ -137,45 +159,50 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
}
#endif
/* -----------------------------------------------------------
Free memory
-------------------------------------------------------------- */
static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats);
static void mi_os_free_huge_os_pages(void* p, size_t size);
static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_stats_t* tld_stats) {
MI_UNUSED(tld_stats);
mi_stats_t* stats = &_mi_stats_main;
static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) {
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free(addr, size);
if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
if (still_committed) { _mi_stat_decrease(&stats->committed, size); }
_mi_stat_decrease(&stats->reserved, size);
if (commit_size > 0) {
mi_os_stat_decrease(committed, commit_size);
}
mi_os_stat_decrease(reserved, size);
}
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats) {
if (stats == NULL) stats = &_mi_stats_main;
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) {
if (mi_memkind_is_os(memid.memkind)) {
size_t csize = _mi_os_good_alloc_size(size);
size_t csize = memid.mem.os.size;
if (csize==0) { _mi_os_good_alloc_size(size); }
size_t commit_size = (still_committed ? csize : 0);
void* base = addr;
// different base? (due to alignment)
if (memid.mem.os.base != NULL) {
mi_assert(memid.mem.os.base <= addr);
mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr);
if (memid.mem.os.base != base) {
mi_assert(memid.mem.os.base <= addr);
base = memid.mem.os.base;
csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base);
const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
if (memid.mem.os.size==0) {
csize += diff;
}
if (still_committed) {
commit_size -= diff; // the (addr-base) part was already un-committed
}
}
// free it
if (memid.memkind == MI_MEM_OS_HUGE) {
mi_assert(memid.is_pinned);
mi_os_free_huge_os_pages(base, csize, stats);
mi_os_free_huge_os_pages(base, csize);
}
else {
mi_os_prim_free(base, csize, still_committed, stats);
mi_os_prim_free(base, csize, (still_committed ? commit_size : 0));
}
}
else {
@ -184,9 +211,8 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
}
}
void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats) {
if (stats == NULL) stats = &_mi_stats_main;
_mi_os_free_ex(p, size, true, memid, stats);
void _mi_os_free(void* p, size_t size, mi_memid_t memid) {
_mi_os_free_ex(p, size, true, memid);
}
@ -195,7 +221,8 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats) {
-------------------------------------------------------------- */
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* tld_stats) {
// Also `hint_addr` is a hint and may be ignored.
static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL);
@ -204,18 +231,18 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
if (try_alignment == 0) { try_alignment = 1; } // avoid 0 to ensure there will be no divide by zero when aligning
*is_zero = false;
void* p = NULL;
int err = _mi_prim_alloc(size, try_alignment, commit, allow_large, is_large, is_zero, &p);
int err = _mi_prim_alloc(hint_addr, size, try_alignment, commit, allow_large, is_large, is_zero, &p);
if (err != 0) {
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, size, try_alignment, commit, allow_large);
_mi_warning_message("unable to allocate OS memory (error: %d (0x%x), addr: %p, size: 0x%zx bytes, align: 0x%zx, commit: %d, allow large: %d)\n", err, err, hint_addr, size, try_alignment, commit, allow_large);
}
MI_UNUSED(tld_stats);
mi_stats_t* stats = &_mi_stats_main;
mi_stat_counter_increase(stats->mmap_calls, 1);
mi_os_stat_counter_increase(mmap_calls, 1);
if (p != NULL) {
_mi_stat_increase(&stats->reserved, size);
mi_os_stat_increase(reserved, size);
if (commit) {
_mi_stat_increase(&stats->committed, size);
mi_os_stat_increase(committed, size);
// seems needed for asan (or `mimalloc-test-api` fails)
#ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); }
@ -226,10 +253,14 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
return p;
}
static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero) {
return mi_os_prim_alloc_at(NULL, size, try_alignment, commit, allow_large, is_large, is_zero);
}
// Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned.
static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) {
static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base) {
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(is_large != NULL);
@ -239,8 +270,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = _mi_align_up(size, _mi_os_page_size());
// try first with a hint (this will be aligned directly on Win 10+ or BSD)
void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats);
// try first with a requested alignment hint (this will usually be aligned directly on Win 10+ or BSD)
void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero);
if (p == NULL) return NULL;
// aligned already?
@ -249,14 +280,16 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
}
else {
// if not aligned, free it, overallocate, and unmap around it
#if !MI_TRACK_ASAN
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
mi_os_prim_free(p, size, commit, stats);
#endif
if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); }
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment;
if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block
// over-allocate uncommitted (virtual) memory
p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats);
p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero);
if (p == NULL) return NULL;
// set p to the aligned part in the full region
@ -267,22 +300,22 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
// explicitly commit only the aligned part
if (commit) {
_mi_os_commit(p, size, NULL, stats);
_mi_os_commit(p, size, NULL);
}
}
else { // mmap can free inside an allocation
// overallocate...
p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats);
p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area.
// and selectively unmap parts around the over-allocated area.
void* aligned_p = mi_align_up_ptr(p, alignment);
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); }
if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); }
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
*base = aligned_p; // since we freed the pre part, `*base == p`.
@ -298,37 +331,36 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
OS API: alloc and alloc_aligned
----------------------------------------------------------- */
void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats) {
void* _mi_os_alloc(size_t size, mi_memid_t* memid) {
*memid = _mi_memid_none();
if (size == 0) return NULL;
if (stats == NULL) stats = &_mi_stats_main;
size = _mi_os_good_alloc_size(size);
bool os_is_large = false;
bool os_is_zero = false;
void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats);
void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero);
if (p != NULL) {
*memid = _mi_memid_create_os(true, os_is_zero, os_is_large);
}
return p;
}
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats)
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid)
{
MI_UNUSED(&_mi_os_get_aligned_hint); // suppress unused warnings
*memid = _mi_memid_none();
if (size == 0) return NULL;
if (stats == NULL) stats = &_mi_stats_main;
size = _mi_os_good_alloc_size(size);
alignment = _mi_align_up(alignment, _mi_os_page_size());
bool os_is_large = false;
bool os_is_zero = false;
void* os_base = NULL;
void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, stats );
void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base );
if (p != NULL) {
*memid = _mi_memid_create_os(commit, os_is_zero, os_is_large);
memid->mem.os.base = os_base;
memid->mem.os.alignment = alignment;
// memid->mem.os.alignment = alignment;
memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned
}
return p;
}
@ -341,29 +373,28 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
to use the actual start of the memory region.
----------------------------------------------------------- */
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats) {
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid) {
mi_assert(offset <= MI_SEGMENT_SIZE);
mi_assert(offset <= size);
mi_assert((alignment % _mi_os_page_size()) == 0);
*memid = _mi_memid_none();
if (stats == NULL) stats = &_mi_stats_main;
if (offset > MI_SEGMENT_SIZE) return NULL;
if (offset == 0) {
// regular aligned allocation
return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid, stats);
return _mi_os_alloc_aligned(size, alignment, commit, allow_large, memid);
}
else {
// overallocate to align at an offset
const size_t extra = _mi_align_up(offset, alignment) - offset;
const size_t oversize = size + extra;
void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, stats);
void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid);
if (start == NULL) return NULL;
void* const p = (uint8_t*)start + extra;
mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
// decommit the overallocation at the start
if (commit && extra > _mi_os_page_size()) {
_mi_os_decommit(start, extra, stats);
_mi_os_decommit(start, extra);
}
return p;
}
@ -397,12 +428,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
return mi_os_page_align_areax(true, addr, size, newsize);
}
bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) {
MI_UNUSED(tld_stats);
mi_stats_t* stats = &_mi_stats_main;
bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
if (is_zero != NULL) { *is_zero = false; }
_mi_stat_increase(&stats->committed, size); // use size for precise commit vs. decommit
_mi_stat_counter_increase(&stats->commit_calls, 1);
mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit
mi_os_stat_counter_increase(commit_calls, 1);
// page align range
size_t csize;
@ -428,11 +457,13 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats
return true;
}
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_stats_t* tld_stats) {
MI_UNUSED(tld_stats);
mi_stats_t* stats = &_mi_stats_main;
bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
return _mi_os_commit_ex(addr, size, is_zero, size);
}
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL);
_mi_stat_decrease(&stats->committed, size);
mi_os_stat_decrease(committed, stat_size);
// page align
size_t csize;
@ -449,9 +480,9 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_
return (err == 0);
}
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) {
bool _mi_os_decommit(void* addr, size_t size) {
bool needs_recommit;
return mi_os_decommit_ex(addr, size, &needs_recommit, tld_stats);
return mi_os_decommit_ex(addr, size, &needs_recommit, size);
}
@ -459,13 +490,13 @@ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) {
// but may be used later again. This will release physical memory
// pages and reduce swapping while keeping the memory committed.
// We page align to a conservative area inside the range to reset.
bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
bool _mi_os_reset(void* addr, size_t size) {
// page align conservatively within the range
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
_mi_stat_increase(&stats->reset, csize);
_mi_stat_counter_increase(&stats->reset_calls, 1);
mi_os_stat_increase(reset, csize);
mi_os_stat_counter_increase(reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
memset(start, 0, csize); // pretend it is eagerly reset
@ -481,22 +512,22 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats)
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
_mi_stat_counter_increase(&stats->purge_calls, 1);
_mi_stat_increase(&stats->purged, size);
mi_os_stat_counter_increase(purge_calls, 1);
mi_os_stat_increase(purged, size);
if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?
!_mi_preloading()) // don't decommit during preloading (unsafe)
{
bool needs_recommit = true;
mi_os_decommit_ex(p, size, &needs_recommit, stats);
mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
return needs_recommit;
}
else {
if (allow_reset) { // this can sometimes be not allowed if the range is not fully committed
_mi_os_reset(p, size, stats);
_mi_os_reset(p, size);
}
return false; // needs no recommit
}
@ -504,8 +535,8 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats)
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge(void* p, size_t size, mi_stats_t * stats) {
return _mi_os_purge_ex(p, size, true, stats);
bool _mi_os_purge(void* p, size_t size) {
return _mi_os_purge_ex(p, size, true, size);
}
@ -613,15 +644,15 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// no success, issue a warning and break
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, true, &_mi_stats_main);
mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);
}
break;
}
// success, record it
page++; // increase before timeout check (see issue #711)
_mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE);
_mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE);
mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
// check for timeout
if (max_msecs > 0) {
@ -655,11 +686,11 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// free every huge page in a range individually (as we allocated per page)
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats) {
static void mi_os_free_huge_os_pages(void* p, size_t size) {
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, true, stats);
mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);
size -= MI_HUGE_OS_PAGE_SIZE;
base += MI_HUGE_OS_PAGE_SIZE;
}
@ -688,8 +719,7 @@ size_t _mi_os_numa_node_count_get(void) {
return count;
}
int _mi_os_numa_node_get(mi_os_tld_t* tld) {
MI_UNUSED(tld);
int _mi_os_numa_node_get(void) {
size_t numa_count = _mi_os_numa_node_count();
if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0
// never more than the node count and >= 0

67
src/page-queue.c
View file

@ -259,8 +259,16 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_
heap->page_count++;
}
static void mi_page_queue_move_to_front(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) {
mi_assert_internal(mi_page_heap(page) == heap);
mi_assert_internal(mi_page_queue_contains(queue, page));
if (queue->first == page) return;
mi_page_queue_remove(queue, page);
mi_page_queue_push(heap, queue, page);
mi_assert_internal(queue->first == page);
}
static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_expensive(mi_page_queue_contains(from, page));
mi_assert_expensive(!mi_page_queue_contains(to, page));
@ -273,6 +281,8 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro
(mi_page_is_huge(page) && mi_page_queue_is_full(to)));
mi_heap_t* heap = mi_page_heap(page);
// delete from `from`
if (page->prev != NULL) page->prev->next = page->next;
if (page->next != NULL) page->next->prev = page->prev;
if (page == from->last) from->last = page->prev;
@ -283,22 +293,59 @@ static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* fro
mi_heap_queue_first_update(heap, from);
}
page->prev = to->last;
page->next = NULL;
if (to->last != NULL) {
mi_assert_internal(heap == mi_page_heap(to->last));
to->last->next = page;
to->last = page;
// insert into `to`
if (enqueue_at_end) {
// enqueue at the end
page->prev = to->last;
page->next = NULL;
if (to->last != NULL) {
mi_assert_internal(heap == mi_page_heap(to->last));
to->last->next = page;
to->last = page;
}
else {
to->first = page;
to->last = page;
mi_heap_queue_first_update(heap, to);
}
}
else {
to->first = page;
to->last = page;
mi_heap_queue_first_update(heap, to);
if (to->first != NULL) {
// enqueue at 2nd place
mi_assert_internal(heap == mi_page_heap(to->first));
mi_page_t* next = to->first->next;
page->prev = to->first;
page->next = next;
to->first->next = page;
if (next != NULL) {
next->prev = page;
}
else {
to->last = page;
}
}
else {
// enqueue at the head (singleton list)
page->prev = NULL;
page->next = NULL;
to->first = page;
to->last = page;
mi_heap_queue_first_update(heap, to);
}
}
mi_page_set_in_full(page, mi_page_queue_is_full(to));
}
static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end */, page);
}
static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) {
// note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`)
mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page);
}
// Only called from `mi_heap_absorb`.
size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) {
mi_assert_internal(mi_heap_contains_queue(heap,pq));

112
src/page.c
View file

@ -276,7 +276,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
mi_assert_internal(mi_heap_contains_queue(heap, pq));
mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_OBJ_SIZE_MAX || block_size == pq->block_size);
#endif
mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments, &heap->tld->os);
mi_page_t* page = _mi_segment_page_alloc(heap, block_size, page_alignment, &heap->tld->segments);
if (page == NULL) {
// this may be out-of-memory, or an abandoned page was reclaimed (and in our queue)
return NULL;
@ -357,7 +357,7 @@ void _mi_page_unfull(mi_page_t* page) {
mi_page_set_in_full(page, false); // to get the right queue
mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);
mi_page_set_in_full(page, true);
mi_page_queue_enqueue_from(pq, pqfull, page);
mi_page_queue_enqueue_from_full(pq, pqfull, page);
}
static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
@ -403,6 +403,27 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
_mi_segment_page_abandon(page,segments_tld);
}
// force abandon a page
void _mi_page_force_abandon(mi_page_t* page) {
mi_heap_t* heap = mi_page_heap(page);
// mark page as not using delayed free
_mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false);
// ensure this page is no longer in the heap delayed free list
_mi_heap_delayed_free_all(heap);
// We can still access the page meta-info even if it is freed as we ensure
// in `mi_segment_force_abandon` that the segment is not freed (yet)
if (page->capacity == 0) return; // it may have been freed now
// and now unlink it from the page queue and abandon (or free)
mi_page_queue_t* pq = mi_heap_page_queue_of(heap, page);
if (mi_page_all_free(page)) {
_mi_page_free(page, pq, false);
}
else {
_mi_page_abandon(page, pq);
}
}
// Free a page with no more free blocks
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
@ -448,6 +469,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
// how to check this efficiently though...
// for now, we don't retire if it is the only page left of this size class.
mi_page_queue_t* pq = mi_page_queue_of(page);
#if MI_RETIRE_CYCLES > 0
const size_t bsize = mi_page_block_size(page);
if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue?
if (pq->last==page && pq->first==page) { // the only page in the queue?
@ -463,7 +485,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept {
return; // don't free after all
}
}
#endif
_mi_page_free(page, pq, false);
}
@ -709,6 +731,17 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
Find pages with free blocks
-------------------------------------------------------------*/
// search for a best next page to use for at most N pages (often cut short if immediate blocks are available)
#define MI_MAX_CANDIDATE_SEARCH (4)
// is the page not yet used up to its reserved space?
static bool mi_page_is_expandable(const mi_page_t* page) {
mi_assert_internal(page != NULL);
mi_assert_internal(page->capacity <= page->reserved);
return (page->capacity < page->reserved);
}
// Find a page with free blocks of `page->block_size`.
static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* pq, bool first_try)
{
@ -716,39 +749,77 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
#if MI_STAT
size_t count = 0;
#endif
size_t candidate_count = 0; // we reset this on the first candidate to limit the search
mi_page_t* page_candidate = NULL; // a page with free space
mi_page_t* page = pq->first;
while (page != NULL)
{
mi_page_t* next = page->next; // remember next
#if MI_STAT
count++;
#endif
candidate_count++;
// 0. collect freed blocks by us and other threads
// collect freed blocks by us and other threads
_mi_page_free_collect(page, false);
// 1. if the page contains free blocks, we are done
if (mi_page_immediate_available(page)) {
#if MI_MAX_CANDIDATE_SEARCH > 1
// search up to N pages for a best candidate
// is the local free list non-empty?
const bool immediate_available = mi_page_immediate_available(page);
// if the page is completely full, move it to the `mi_pages_full`
// queue so we don't visit long-lived pages too often.
if (!immediate_available && !mi_page_is_expandable(page)) {
mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
mi_page_to_full(page, pq);
}
else {
// the page has free space, make it a candidate
// we prefer non-expandable pages with high usage as candidates (to reduce commit, and increase chances of free-ing up pages)
if (page_candidate == NULL) {
page_candidate = page;
candidate_count = 0;
}
// prefer to reuse fuller pages (in the hope the less used page gets freed)
else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page) && !mi_page_is_expandable(page)) {
page_candidate = page;
}
// if we find a non-expandable candidate, or searched for N pages, return with the best candidate
if (immediate_available || candidate_count > MI_MAX_CANDIDATE_SEARCH) {
mi_assert_internal(page_candidate!=NULL);
break;
}
}
#else
// first-fit algorithm
// If the page contains free blocks, we are done
if (mi_page_immediate_available(page) || mi_page_is_expandable(page)) {
break; // pick this one
}
// 2. Try to extend
if (page->capacity < page->reserved) {
mi_page_extend_free(heap, page, heap->tld);
mi_assert_internal(mi_page_immediate_available(page));
break;
}
// 3. If the page is completely full, move it to the `mi_pages_full`
// If the page is completely full, move it to the `mi_pages_full`
// queue so we don't visit long-lived pages too often.
mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page));
mi_page_to_full(page, pq);
#endif
page = next;
} // for each page
mi_heap_stat_counter_increase(heap, searches, count);
// set the page to the best candidate
if (page_candidate != NULL) {
page = page_candidate;
}
if (page != NULL && !mi_page_immediate_available(page)) {
mi_assert_internal(mi_page_is_expandable(page));
mi_page_extend_free(heap, page, heap->tld);
}
if (page == NULL) {
_mi_heap_collect_retired(heap, false); // perhaps make a page available
page = mi_page_fresh(heap, pq);
@ -758,10 +829,14 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
}
}
else {
mi_assert(pq->first == page);
// move the page to the front of the queue
mi_page_queue_move_to_front(heap, pq, page);
page->retire_expire = 0;
// _mi_heap_collect_retired(heap, false); // update retire counts; note: increases rss on MemoryLoad bench so don't do this
}
mi_assert_internal(page == NULL || mi_page_immediate_available(page));
return page;
}
@ -769,7 +844,9 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
// Find a page with free blocks of `size`.
static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
mi_page_queue_t* pq = mi_page_queue(heap,size);
mi_page_queue_t* pq = mi_page_queue(heap, size);
// check the first page: we even do this with candidate search or otherwise we re-search every time
mi_page_t* page = pq->first;
if (page != NULL) {
#if (MI_SECURE>=3) // in secure mode, we extend half the time to increase randomness
@ -788,6 +865,7 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
return page; // fast path
}
}
return mi_page_queue_find_free_ex(heap, pq, true);
}
@ -912,7 +990,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al
mi_assert_internal(mi_page_block_size(page) >= size);
// and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc)
if mi_unlikely(zero && page->block_size == 0) {
if mi_unlikely(zero && mi_page_is_huge(page)) {
// note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case.
void* p = _mi_page_malloc(heap, page, size);
mi_assert_internal(p != NULL);

4
src/prim/emscripten/prim.c
View file

@ -71,8 +71,8 @@ int _mi_prim_free(void* addr, size_t size) {
extern void* emmalloc_memalign(size_t alignment, size_t size);
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit);
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
MI_UNUSED(try_alignment); MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
*is_large = false;
// TODO: Track the highest address ever seen; first uses of it are zeroes.
// That assumes no one else uses sbrk but us (they could go up,

4
src/prim/osx/alloc-override-zone.c
View file

@ -418,9 +418,9 @@ static inline malloc_zone_t* mi_get_default_zone(void)
}
#if defined(__clang__)
__attribute__((constructor(0)))
__attribute__((constructor(101))) // highest priority
#else
__attribute__((constructor)) // seems not supported by g++-11 on the M1
__attribute__((constructor)) // priority level is not supported by gcc
#endif
__attribute__((used))
static void _mi_macos_override_malloc(void) {

49
src/prim/prim.c
View file

@ -25,3 +25,52 @@ terms of the MIT license. A copy of the license can be found in the file
#include "unix/prim.c" // mmap() (Linux, macOSX, BSD, Illumnos, Haiku, DragonFly, etc.)
#endif
// Generic process initialization
#ifndef MI_PRIM_HAS_PROCESS_ATTACH
#if defined(__GNUC__) || defined(__clang__)
// gcc,clang: use the constructor/destructor attribute
// which for both seem to run before regular constructors/destructors
#if defined(__clang__)
#define mi_attr_constructor __attribute__((constructor(101)))
#define mi_attr_destructor __attribute__((destructor(101)))
#else
#define mi_attr_constructor __attribute__((constructor))
#define mi_attr_destructor __attribute__((destructor))
#endif
static void mi_attr_constructor mi_process_attach(void) {
_mi_process_load();
}
static void mi_attr_destructor mi_process_detach(void) {
_mi_process_done();
}
#elif defined(__cplusplus)
// C++: use static initialization to detect process start/end
// This is not guaranteed to be first/last but the best we can generally do?
struct mi_init_done_t {
mi_init_done_t() {
_mi_process_load();
}
~mi_init_done_t() {
_mi_process_done();
}
};
static mi_init_done_t mi_init_done;
#else
#pragma message("define a way to call _mi_process_load/done on your platform")
#endif
#endif
// Generic allocator init/done callback
#ifndef MI_PRIM_HAS_ALLOCATOR_INIT
bool _mi_is_redirected(void) {
return false;
}
bool _mi_allocator_init(const char** message) {
if (message != NULL) { *message = NULL; }
return true;
}
void _mi_allocator_done(void) {
// nothing to do
}
#endif

21
src/prim/unix/prim.c
View file

@ -27,6 +27,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <sys/mman.h> // mmap
#include <unistd.h> // sysconf
#include <fcntl.h> // open, close, read, access
#include <stdlib.h> // getenv, arc4random_buf
#if defined(__linux__)
#include <features.h>
@ -139,6 +140,12 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
if (psize > 0) {
config->page_size = (size_t)psize;
config->alloc_granularity = (size_t)psize;
#if defined(_SC_PHYS_PAGES)
long pphys = sysconf(_SC_PHYS_PAGES);
if (pphys > 0 && (size_t)pphys < (SIZE_MAX/(size_t)psize)) {
config->physical_memory = (size_t)pphys * (size_t)psize;
}
#endif
}
config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this?
config->has_overcommit = unix_detect_overcommit();
@ -181,10 +188,11 @@ int _mi_prim_free(void* addr, size_t size ) {
static int unix_madvise(void* addr, size_t size, int advice) {
#if defined(__sun)
return madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)
int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520)
#else
return madvise(addr, size, advice);
int res = madvise(addr, size, advice);
#endif
return (res==0 ? 0 : errno);
}
static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
@ -331,7 +339,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
// when large OS pages are enabled for mimalloc, we call `madvise` anyways.
if (allow_large && _mi_os_use_large_page(size, try_alignment)) {
if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) {
*is_large = true; // possibly
// *is_large = true; // possibly
};
}
#elif defined(__sun)
@ -340,7 +348,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
cmd.mha_pagesize = _mi_os_large_page_size();
cmd.mha_cmd = MHA_MAPSIZE_VA;
if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
*is_large = true;
// *is_large = true; // possibly
}
}
#endif
@ -350,14 +358,14 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
}
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(commit || !allow_large);
mi_assert_internal(try_alignment > 0);
*is_zero = true;
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
*addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
*addr = unix_mmap(hint_addr, size, try_alignment, protect_flags, false, allow_large, is_large);
return (*addr != NULL ? 0 : errno);
}
@ -773,7 +781,6 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
defined(__sun) || \
(defined(__APPLE__) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_7))
#include <stdlib.h>
bool _mi_prim_random_buf(void* buf, size_t buf_len) {
arc4random_buf(buf, buf_len);
return true;

4
src/prim/wasi/prim.c
View file

@ -119,8 +119,8 @@ static void* mi_prim_mem_grow(size_t size, size_t try_alignment) {
}
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
MI_UNUSED(allow_large); MI_UNUSED(commit);
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
MI_UNUSED(allow_large); MI_UNUSED(commit); MI_UNUSED(hint_addr);
*is_large = false;
*is_zero = false;
*addr = mi_prim_mem_grow(size, try_alignment);

249
src/prim/windows/prim.c
View file

@ -118,6 +118,18 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
GetSystemInfo(&si);
if (si.dwPageSize > 0) { config->page_size = si.dwPageSize; }
if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; }
// get virtual address bits
if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {
const size_t vbits = MI_INTPTR_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
config->virtual_address_bits = vbits;
}
// get physical memory
ULONGLONG memInKiB = 0;
if (GetPhysicallyInstalledSystemMemory(&memInKiB)) {
if (memInKiB > 0 && memInKiB < (SIZE_MAX / MI_KiB)) {
config->physical_memory = memInKiB * MI_KiB;
}
}
// get the VirtualAlloc2 function
HINSTANCE hDll;
hDll = LoadLibrary(TEXT("kernelbase.dll"));
@ -191,7 +203,7 @@ static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_ali
}
#endif
// on modern Windows try use VirtualAlloc2 for aligned allocation
if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) {
MI_MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 };
reqs.Alignment = try_alignment;
MI_MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} };
@ -279,14 +291,14 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW
return p;
}
int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** addr) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(commit || !allow_large);
mi_assert_internal(try_alignment > 0);
*is_zero = true;
int flags = MEM_RESERVE;
if (commit) { flags |= MEM_COMMIT; }
*addr = win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
*addr = win_virtual_alloc(hint_addr, size, try_alignment, flags, false, allow_large, is_large);
return (*addr != NULL ? 0 : (int)GetLastError());
}
@ -499,8 +511,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo)
}
// get process info
PROCESS_MEMORY_COUNTERS info;
memset(&info, 0, sizeof(info));
PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info);
if (pGetProcessMemoryInfo != NULL) {
pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info));
}
@ -602,59 +613,205 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
#endif // MI_USE_RTLGENRANDOM
//----------------------------------------------------------------
// Thread init/done
// Process & Thread Init/Done
//----------------------------------------------------------------
#if !defined(MI_SHARED_LIB)
// use thread local storage keys to detect thread ending
// note: another design could be to use special linker sections (see issue #869)
#include <fibersapi.h>
#if (_WIN32_WINNT < 0x600) // before Windows Vista
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
#endif
static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) {
mi_heap_t* heap = (mi_heap_t*)value;
if (heap != NULL) {
_mi_thread_done(heap);
FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672
static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) {
MI_UNUSED(reserved);
MI_UNUSED(module);
#if MI_TLS_SLOT >= 2
if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) {
_mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty);
}
#endif
if (reason==DLL_PROCESS_ATTACH) {
_mi_process_load();
}
else if (reason==DLL_PROCESS_DETACH) {
_mi_process_done();
}
else if (reason==DLL_THREAD_DETACH && !_mi_is_redirected()) {
_mi_thread_done(NULL);
}
}
void _mi_prim_thread_init_auto_done(void) {
mi_fls_key = FlsAlloc(&mi_fls_done);
}
void _mi_prim_thread_done_auto_done(void) {
// call thread-done on all threads (except the main thread) to prevent
// dangling callback pointer if statically linked with a DLL; Issue #208
FlsFree(mi_fls_key);
}
#if defined(MI_SHARED_LIB)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
mi_assert_internal(mi_fls_key != (DWORD)(-1));
FlsSetValue(mi_fls_key, heap);
}
// Windows DLL: easy to hook into process_init and thread_done
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
mi_win_main((PVOID)inst,reason,reserved);
return TRUE;
}
#else
// nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
void _mi_prim_thread_init_auto_done(void) { }
void _mi_prim_thread_done_auto_done(void) { }
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
MI_UNUSED(heap);
}
// Dll; nothing to do as in that case thread_done is handled through the DLL_THREAD_DETACH event.
#elif !defined(MI_WIN_USE_FLS)
#define MI_PRIM_HAS_PROCESS_ATTACH 1
void _mi_prim_thread_init_auto_done(void) {
}
static void NTAPI mi_win_main_attach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_ATTACH || reason == DLL_THREAD_ATTACH) {
mi_win_main(module, reason, reserved);
}
}
static void NTAPI mi_win_main_detach(PVOID module, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_DETACH || reason == DLL_THREAD_DETACH) {
mi_win_main(module, reason, reserved);
}
}
void _mi_prim_thread_done_auto_done(void) {
}
// Set up TLS callbacks in a statically linked library by using special data sections.
// See <https://stackoverflow.com/questions/14538159/tls-callback-in-windows>
// We use 2 entries to ensure we call attach events before constructors
// are called, and detach events after destructors are called.
#if defined(__cplusplus)
extern "C" {
#endif
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
MI_UNUSED(heap);
}
#if defined(_WIN64)
#pragma comment(linker, "/INCLUDE:_tls_used")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:_mi_tls_callback_post")
#pragma const_seg(".CRT$XLB")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
#pragma const_seg()
#pragma const_seg(".CRT$XLY")
extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[];
const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
#pragma const_seg()
#else
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre")
#pragma comment(linker, "/INCLUDE:__mi_tls_callback_post")
#pragma data_seg(".CRT$XLB")
PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach };
#pragma data_seg()
#pragma data_seg(".CRT$XLY")
PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach };
#pragma data_seg()
#endif
#if defined(__cplusplus)
}
#endif
// nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event.
void _mi_prim_thread_init_auto_done(void) { }
void _mi_prim_thread_done_auto_done(void) { }
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
MI_UNUSED(heap);
}
#else // deprecated: statically linked, use fiber api
#if defined(_MSC_VER) // on clang/gcc use the constructor attribute (in `src/prim/prim.c`)
// MSVC: use data section magic for static libraries
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
#define MI_PRIM_HAS_PROCESS_ATTACH 1
static int mi_process_attach(void) {
mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL);
atexit(&_mi_process_done);
return 0;
}
typedef int(*mi_crt_callback_t)(void);
#if defined(_WIN64)
#pragma comment(linker, "/INCLUDE:_mi_tls_callback")
#pragma section(".CRT$XIU", long, read)
#else
#pragma comment(linker, "/INCLUDE:__mi_tls_callback")
#endif
#pragma data_seg(".CRT$XIU")
mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach };
#pragma data_seg()
#endif
// use the fiber api for calling `_mi_thread_done`.
#include <fibersapi.h>
#if (_WIN32_WINNT < 0x600) // before Windows Vista
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
#endif
static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) {
mi_heap_t* heap = (mi_heap_t*)value;
if (heap != NULL) {
_mi_thread_done(heap);
FlsSetValue(mi_fls_key, NULL); // prevent recursion as _mi_thread_done may set it back to the main heap, issue #672
}
}
void _mi_prim_thread_init_auto_done(void) {
mi_fls_key = FlsAlloc(&mi_fls_done);
}
void _mi_prim_thread_done_auto_done(void) {
// call thread-done on all threads (except the main thread) to prevent
// dangling callback pointer if statically linked with a DLL; Issue #208
FlsFree(mi_fls_key);
}
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
mi_assert_internal(mi_fls_key != (DWORD)(-1));
FlsSetValue(mi_fls_key, heap);
}
#endif
// ----------------------------------------------------
// Communicate with the redirection module on Windows
// ----------------------------------------------------
#if defined(MI_SHARED_LIB) && !defined(MI_WIN_NOREDIRECT)
#define MI_PRIM_HAS_ALLOCATOR_INIT 1
static bool mi_redirected = false; // true if malloc redirects to mi_malloc
bool _mi_is_redirected(void) {
return mi_redirected;
}
#ifdef __cplusplus
extern "C" {
#endif
mi_decl_export void _mi_redirect_entry(DWORD reason) {
// called on redirection; careful as this may be called before DllMain
#if MI_TLS_SLOT >= 2
if ((reason==DLL_PROCESS_ATTACH || reason==DLL_THREAD_ATTACH) && mi_prim_get_default_heap() == NULL) {
_mi_heap_set_default_direct((mi_heap_t*)&_mi_heap_empty);
}
#endif
if (reason == DLL_PROCESS_ATTACH) {
mi_redirected = true;
}
else if (reason == DLL_PROCESS_DETACH) {
mi_redirected = false;
}
else if (reason == DLL_THREAD_DETACH) {
_mi_thread_done(NULL);
}
}
__declspec(dllimport) bool mi_cdecl mi_allocator_init(const char** message);
__declspec(dllimport) void mi_cdecl mi_allocator_done(void);
#ifdef __cplusplus
}
#endif
bool _mi_allocator_init(const char** message) {
return mi_allocator_init(message);
}
void _mi_allocator_done(void) {
mi_allocator_done();
}
#endif

16
src/segment-map.c
View file

@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file
#elif (MI_INTPTR_SIZE > 4)
#define MI_SEGMENT_MAP_MAX_ADDRESS (48*1024ULL*MI_GiB) // 48 TiB
#else
#define MI_SEGMENT_MAP_MAX_ADDRESS (MAX_UINT32)
#define MI_SEGMENT_MAP_MAX_ADDRESS (UINT32_MAX)
#endif
#define MI_SEGMENT_MAP_PART_SIZE (MI_INTPTR_SIZE*MI_KiB - 128) // 128 > sizeof(mi_memid_t) !
@ -55,11 +55,12 @@ static mi_segmap_part_t* mi_segment_map_index_of(const mi_segment_t* segment, bo
if (part == NULL) {
if (!create_on_demand) return NULL;
mi_memid_t memid;
part = (mi_segmap_part_t*)_mi_os_alloc(sizeof(mi_segmap_part_t), &memid, NULL);
part = (mi_segmap_part_t*)_mi_os_alloc(sizeof(mi_segmap_part_t), &memid);
if (part == NULL) return NULL;
part->memid = memid;
mi_segmap_part_t* expected = NULL;
if (!mi_atomic_cas_ptr_strong_release(mi_segmap_part_t, &mi_segment_map[segindex], &expected, part)) {
_mi_os_free(part, sizeof(mi_segmap_part_t), memid, NULL);
_mi_os_free(part, sizeof(mi_segmap_part_t), memid);
part = expected;
if (part == NULL) return NULL;
}
@ -124,3 +125,12 @@ static bool mi_is_valid_pointer(const void* p) {
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
return mi_is_valid_pointer(p);
}
void _mi_segment_map_unsafe_destroy(void) {
for (size_t i = 0; i < MI_SEGMENT_MAP_MAX_PARTS; i++) {
mi_segmap_part_t* part = mi_atomic_exchange_ptr_relaxed(mi_segmap_part_t, &mi_segment_map[i], NULL);
if (part != NULL) {
_mi_os_free(part, sizeof(mi_segmap_part_t), part->memid);
}
}
}

190
src/segment.c
View file

@ -189,7 +189,7 @@ static void mi_segment_protect_range(void* p, size_t size, bool protect) {
}
}
static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t* tld) {
static void mi_segment_protect(mi_segment_t* segment, bool protect) {
// add/remove guard pages
if (MI_SECURE != 0) {
// in secure mode, we set up a protected page in between the segment info and the page data
@ -207,7 +207,7 @@ static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t*
if (protect && !segment->memid.initially_committed) {
if (protect) {
// ensure secure page is committed
if (_mi_os_commit(start, os_psize, NULL, tld->stats)) { // if this fails that is ok (as it is an unaccessible page)
if (_mi_os_commit(start, os_psize, NULL)) { // if this fails that is ok (as it is an unaccessible page)
mi_segment_protect_range(start, os_psize, protect);
}
}
@ -241,23 +241,23 @@ static void mi_page_purge(mi_segment_t* segment, mi_page_t* page, mi_segments_tl
if (!segment->allow_purge) return;
mi_assert_internal(page->used == 0);
mi_assert_internal(page->free == NULL);
mi_assert_expensive(!mi_pages_purge_contains(page, tld));
mi_assert_expensive(!mi_pages_purge_contains(page, tld)); MI_UNUSED(tld);
size_t psize;
void* start = mi_segment_raw_page_start(segment, page, &psize);
const bool needs_recommit = _mi_os_purge(start, psize, tld->stats);
const bool needs_recommit = _mi_os_purge(start, psize);
if (needs_recommit) { page->is_committed = false; }
}
static bool mi_page_ensure_committed(mi_segment_t* segment, mi_page_t* page, mi_segments_tld_t* tld) {
if (page->is_committed) return true;
mi_assert_internal(segment->allow_decommit);
mi_assert_expensive(!mi_pages_purge_contains(page, tld));
mi_assert_expensive(!mi_pages_purge_contains(page, tld)); MI_UNUSED(tld);
size_t psize;
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
bool is_zero = false;
const size_t gsize = (MI_SECURE >= 2 ? _mi_os_page_size() : 0);
bool ok = _mi_os_commit(start, psize + gsize, &is_zero, tld->stats);
bool ok = _mi_os_commit(start, psize + gsize, &is_zero);
if (!ok) return false; // failed to commit!
page->is_committed = true;
page->used = 0;
@ -436,6 +436,8 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
mi_assert_internal((uintptr_t)p % block_size == 0);
}
}
mi_assert_internal(_mi_is_aligned(p, MI_MAX_ALIGN_SIZE));
mi_assert_internal(block_size == 0 || block_size > MI_MAX_ALIGN_GUARANTEE || _mi_is_aligned(p,block_size));
if (page_size != NULL) *page_size = psize;
mi_assert_internal(_mi_ptr_page(p) == page);
@ -446,13 +448,18 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
static size_t mi_segment_calculate_sizes(size_t capacity, size_t required, size_t* pre_size, size_t* info_size)
{
const size_t minsize = sizeof(mi_segment_t) + ((capacity - 1) * sizeof(mi_page_t)) + 16 /* padding */;
const size_t minsize = sizeof(mi_segment_t) + ((capacity - 1) * sizeof(mi_page_t)) + 16 /* padding */;
size_t guardsize = 0;
size_t isize = 0;
if (MI_SECURE == 0) {
// normally no guard pages
#if MI_GUARDED
isize = _mi_align_up(minsize, _mi_os_page_size());
#else
isize = _mi_align_up(minsize, 16 * MI_MAX_ALIGN_SIZE);
#endif
}
else {
// in secure mode, we set up a protected page in between the segment info
@ -460,7 +467,7 @@ static size_t mi_segment_calculate_sizes(size_t capacity, size_t required, size_
const size_t page_size = _mi_os_page_size();
isize = _mi_align_up(minsize, page_size);
guardsize = page_size;
required = _mi_align_up(required, page_size);
//required = _mi_align_up(required, isize + guardsize);
}
if (info_size != NULL) *info_size = isize;
@ -495,7 +502,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
if (MI_SECURE != 0) {
mi_assert_internal(!segment->memid.is_pinned);
mi_segment_protect(segment, false, tld->os); // ensure no more guard pages are set
mi_segment_protect(segment, false); // ensure no more guard pages are set
}
bool fully_committed = true;
@ -509,7 +516,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
MI_UNUSED(fully_committed);
mi_assert_internal((fully_committed && committed_size == segment_size) || (!fully_committed && committed_size < segment_size));
_mi_arena_free(segment, segment_size, committed_size, segment->memid, tld->stats);
_mi_arena_free(segment, segment_size, committed_size, segment->memid);
}
// called from `heap_collect`.
@ -530,7 +537,7 @@ void _mi_segments_collect(bool force, mi_segments_tld_t* tld) {
static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignment, mi_arena_id_t req_arena_id,
size_t pre_size, size_t info_size, bool commit, size_t segment_size,
mi_segments_tld_t* tld, mi_os_tld_t* tld_os)
mi_segments_tld_t* tld)
{
mi_memid_t memid;
bool allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy
@ -542,7 +549,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme
segment_size = segment_size + (align_offset - pre_size); // adjust the segment size
}
mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os);
mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid);
if (segment == NULL) {
return NULL; // failed to allocate
}
@ -550,10 +557,10 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme
if (!memid.initially_committed) {
// ensure the initial info is committed
mi_assert_internal(!memid.is_pinned);
bool ok = _mi_os_commit(segment, pre_size, NULL, tld_os->stats);
bool ok = _mi_os_commit(segment, pre_size, NULL);
if (!ok) {
// commit failed; we cannot touch the memory: free the segment directly and return `NULL`
_mi_arena_free(segment, segment_size, 0, memid, tld_os->stats);
_mi_arena_free(segment, segment_size, 0, memid);
return NULL;
}
}
@ -571,7 +578,7 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme
// Allocate a segment from the OS aligned to `MI_SEGMENT_SIZE` .
static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_shift, size_t page_alignment,
mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
mi_arena_id_t req_arena_id, mi_segments_tld_t* tld)
{
// required is only > 0 for huge page allocations
mi_assert_internal((required > 0 && page_kind > MI_PAGE_LARGE)|| (required==0 && page_kind <= MI_PAGE_LARGE));
@ -603,7 +610,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
const bool init_commit = eager; // || (page_kind >= MI_PAGE_LARGE);
// Allocate the segment from the OS (segment_size can change due to alignment)
mi_segment_t* segment = mi_segment_os_alloc(eager_delayed, page_alignment, req_arena_id, pre_size, info_size, init_commit, init_segment_size, tld, os_tld);
mi_segment_t* segment = mi_segment_os_alloc(eager_delayed, page_alignment, req_arena_id, pre_size, info_size, init_commit, init_segment_size, tld);
if (segment == NULL) return NULL;
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
mi_assert_internal(segment->memid.is_pinned ? segment->memid.initially_committed : true);
@ -631,7 +638,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
segment->cookie = _mi_ptr_cookie(segment);
// set protection
mi_segment_protect(segment, true, tld->os);
mi_segment_protect(segment, true);
// insert in free lists for small and medium pages
if (page_kind <= MI_PAGE_MEDIUM) {
@ -645,6 +652,10 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) {
MI_UNUSED(force);
mi_assert(segment != NULL);
// in `mi_segment_force_abandon` we set this to true to ensure the segment's memory stays valid
if (segment->dont_free) return;
// don't purge as we are freeing now
mi_segment_remove_all_purges(segment, false /* don't force as we are about to free */, tld);
mi_segment_remove_from_free_queue(segment, tld);
@ -945,6 +956,9 @@ bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) {
if (mi_atomic_load_relaxed(&segment->thread_id) != 0) return false; // it is not abandoned
if (segment->subproc != heap->tld->segments.subproc) return false; // only reclaim within the same subprocess
if (!_mi_heap_memid_is_suitable(heap,segment->memid)) return false; // don't reclaim between exclusive and non-exclusive arena's
const long target = _mi_option_get_fast(mi_option_target_segments_per_thread);
if (target > 0 && (size_t)target <= heap->tld->segments.count) return false; // don't reclaim if going above the target count
// don't reclaim more from a `free` call than half the current segments
// this is to prevent a pure free-ing thread to start owning too many segments
// (but not for out-of-arena segments as that is the main way to be reclaimed for those)
@ -969,6 +983,13 @@ void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) {
_mi_arena_field_cursor_done(&current);
}
static bool segment_count_is_within_target(mi_segments_tld_t* tld, size_t* ptarget) {
const size_t target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread, 0, 1024);
if (ptarget != NULL) { *ptarget = target; }
return (target == 0 || tld->count < target);
}
static long mi_segment_get_reclaim_tries(mi_segments_tld_t* tld) {
// limit the tries to 10% (default) of the abandoned segments with at least 8 and at most 1024 tries.
const size_t perc = (size_t)mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 100);
@ -991,7 +1012,7 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size,
mi_segment_t* segment = NULL;
mi_arena_field_cursor_t current;
_mi_arena_field_cursor_init(heap, tld->subproc, false /* non-blocking */, &current);
while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(&current)) != NULL))
while (segment_count_is_within_target(tld,NULL) && (max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(&current)) != NULL))
{
mi_assert(segment->subproc == heap->tld->segments.subproc); // cursor only visits segments in our sub-process
segment->abandoned_visits++;
@ -1016,8 +1037,8 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size,
result = mi_segment_reclaim(segment, heap, block_size, reclaimed, tld);
break;
}
else if (segment->abandoned_visits >= 3 && is_suitable) {
// always reclaim on 3rd visit to limit the list length.
else if (segment->abandoned_visits > 3 && is_suitable) {
// always reclaim on 3rd visit to limit the abandoned segment count.
mi_segment_reclaim(segment, heap, 0, NULL, tld);
}
else {
@ -1031,15 +1052,104 @@ static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size,
}
/* -----------------------------------------------------------
Force abandon a segment that is in use by our thread
----------------------------------------------------------- */
// force abandon a segment
static void mi_segment_force_abandon(mi_segment_t* segment, mi_segments_tld_t* tld)
{
mi_assert_internal(segment->abandoned < segment->used);
mi_assert_internal(!segment->dont_free);
// ensure the segment does not get free'd underneath us (so we can check if a page has been freed in `mi_page_force_abandon`)
segment->dont_free = true;
// for all pages
for (size_t i = 0; i < segment->capacity; i++) {
mi_page_t* page = &segment->pages[i];
if (page->segment_in_use) {
// abandon the page if it is still in-use (this will free the page if possible as well (but not our segment))
mi_assert_internal(segment->used > 0);
if (segment->used == segment->abandoned+1) {
// the last page.. abandon and return as the segment will be abandoned after this
// and we should no longer access it.
segment->dont_free = false;
_mi_page_force_abandon(page);
return;
}
else {
// abandon and continue
_mi_page_force_abandon(page);
}
}
}
segment->dont_free = false;
mi_assert(segment->used == segment->abandoned);
mi_assert(segment->used == 0);
if (segment->used == 0) { // paranoia
// all free now
mi_segment_free(segment, false, tld);
}
else {
// perform delayed purges
mi_pages_try_purge(false /* force? */, tld);
}
}
// try abandon segments.
// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use.
static void mi_segments_try_abandon_to_target(mi_heap_t* heap, size_t target, mi_segments_tld_t* tld) {
if (target <= 1) return;
const size_t min_target = (target > 4 ? (target*3)/4 : target); // 75%
// todo: we should maintain a list of segments per thread; for now, only consider segments from the heap full pages
for (int i = 0; i < 64 && tld->count >= min_target; i++) {
mi_page_t* page = heap->pages[MI_BIN_FULL].first;
while (page != NULL && mi_page_is_huge(page)) {
page = page->next;
}
if (page==NULL) {
break;
}
mi_segment_t* segment = _mi_page_segment(page);
mi_segment_force_abandon(segment, tld);
mi_assert_internal(page != heap->pages[MI_BIN_FULL].first); // as it is just abandoned
}
}
// try abandon segments.
// this should be called from `reclaim_or_alloc` so we know all segments are (about) fully in use.
static void mi_segments_try_abandon(mi_heap_t* heap, mi_segments_tld_t* tld) {
// we call this when we are about to add a fresh segment so we should be under our target segment count.
size_t target = 0;
if (segment_count_is_within_target(tld, &target)) return;
mi_segments_try_abandon_to_target(heap, target, tld);
}
void mi_collect_reduce(size_t target_size) mi_attr_noexcept {
mi_collect(true);
mi_heap_t* heap = mi_heap_get_default();
mi_segments_tld_t* tld = &heap->tld->segments;
size_t target = target_size / MI_SEGMENT_SIZE;
if (target == 0) {
target = (size_t)mi_option_get_clamp(mi_option_target_segments_per_thread, 1, 1024);
}
mi_segments_try_abandon_to_target(heap, target, tld);
}
/* -----------------------------------------------------------
Reclaim or allocate
----------------------------------------------------------- */
static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, size_t page_shift, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, size_t page_shift, mi_segments_tld_t* tld)
{
mi_assert_internal(page_kind <= MI_PAGE_LARGE);
mi_assert_internal(block_size <= MI_LARGE_OBJ_SIZE_MAX);
// try to abandon some segments to increase reuse between threads
mi_segments_try_abandon(heap,tld);
// 1. try to reclaim an abandoned segment
bool reclaimed;
mi_segment_t* segment = mi_segment_try_reclaim(heap, block_size, page_kind, &reclaimed, tld);
@ -1054,7 +1164,7 @@ static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t block_s
return segment;
}
// 2. otherwise allocate a fresh segment
return mi_segment_alloc(0, page_kind, page_shift, 0, heap->arena_id, tld, os_tld);
return mi_segment_alloc(0, page_kind, page_shift, 0, heap->arena_id, tld);
}
@ -1093,11 +1203,11 @@ static mi_page_t* mi_segment_page_try_alloc_in_queue(mi_heap_t* heap, mi_page_ki
return NULL;
}
static mi_page_t* mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_page_kind_t kind, size_t page_shift, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
static mi_page_t* mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_page_kind_t kind, size_t page_shift, mi_segments_tld_t* tld) {
mi_page_t* page = mi_segment_page_try_alloc_in_queue(heap, kind, tld);
if (page == NULL) {
// possibly allocate or reclaim a fresh segment
mi_segment_t* const segment = mi_segment_reclaim_or_alloc(heap, block_size, kind, page_shift, tld, os_tld);
mi_segment_t* const segment = mi_segment_reclaim_or_alloc(heap, block_size, kind, page_shift, tld);
if (segment == NULL) return NULL; // return NULL if out-of-memory (or reclaimed)
mi_assert_internal(segment->page_kind==kind);
mi_assert_internal(segment->used < segment->capacity);
@ -1112,20 +1222,20 @@ static mi_page_t* mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_p
return page;
}
static mi_page_t* mi_segment_small_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
return mi_segment_page_alloc(heap, block_size, MI_PAGE_SMALL,MI_SMALL_PAGE_SHIFT,tld,os_tld);
static mi_page_t* mi_segment_small_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld) {
return mi_segment_page_alloc(heap, block_size, MI_PAGE_SMALL,MI_SMALL_PAGE_SHIFT,tld);
}
static mi_page_t* mi_segment_medium_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
return mi_segment_page_alloc(heap, block_size, MI_PAGE_MEDIUM, MI_MEDIUM_PAGE_SHIFT, tld, os_tld);
static mi_page_t* mi_segment_medium_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld) {
return mi_segment_page_alloc(heap, block_size, MI_PAGE_MEDIUM, MI_MEDIUM_PAGE_SHIFT, tld);
}
/* -----------------------------------------------------------
large page allocation
----------------------------------------------------------- */
static mi_page_t* mi_segment_large_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
mi_segment_t* segment = mi_segment_reclaim_or_alloc(heap,block_size,MI_PAGE_LARGE,MI_LARGE_PAGE_SHIFT,tld,os_tld);
static mi_page_t* mi_segment_large_page_alloc(mi_heap_t* heap, size_t block_size, mi_segments_tld_t* tld) {
mi_segment_t* segment = mi_segment_reclaim_or_alloc(heap,block_size,MI_PAGE_LARGE,MI_LARGE_PAGE_SHIFT,tld);
if (segment == NULL) return NULL;
mi_page_t* page = mi_segment_find_free(segment, tld);
mi_assert_internal(page != NULL);
@ -1135,9 +1245,9 @@ static mi_page_t* mi_segment_large_page_alloc(mi_heap_t* heap, size_t block_size
return page;
}
static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld)
{
mi_segment_t* segment = mi_segment_alloc(size, MI_PAGE_HUGE, MI_SEGMENT_SHIFT + 1, page_alignment, req_arena_id, tld, os_tld);
mi_segment_t* segment = mi_segment_alloc(size, MI_PAGE_HUGE, MI_SEGMENT_SHIFT + 1, page_alignment, req_arena_id, tld);
if (segment == NULL) return NULL;
mi_assert_internal(mi_segment_page_size(segment) - segment->segment_info_size - (2*(MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= size);
#if MI_HUGE_PAGE_ABANDON
@ -1161,7 +1271,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment,
mi_assert_internal(psize - (aligned_p - start) >= size);
uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list
ptrdiff_t decommit_size = aligned_p - decommit_start;
_mi_os_reset(decommit_start, decommit_size, os_tld->stats); // do not decommit as it may be in a region
_mi_os_reset(decommit_start, decommit_size); // do not decommit as it may be in a region
}
return page;
@ -1208,7 +1318,7 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc
if (usize > sizeof(mi_block_t)) {
usize = usize - sizeof(mi_block_t);
uint8_t* p = (uint8_t*)block + sizeof(mi_block_t);
_mi_os_reset(p, usize, &_mi_stats_main);
_mi_os_reset(p, usize);
}
}
}
@ -1218,26 +1328,26 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc
Page allocation
----------------------------------------------------------- */
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld) {
mi_page_t* page;
if mi_unlikely(page_alignment > MI_BLOCK_ALIGNMENT_MAX) {
mi_assert_internal(_mi_is_power_of_two(page_alignment));
mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE);
//mi_assert_internal((MI_SEGMENT_SIZE % page_alignment) == 0);
if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; }
page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld);
page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld);
}
else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) {
page = mi_segment_small_page_alloc(heap, block_size, tld, os_tld);
page = mi_segment_small_page_alloc(heap, block_size, tld);
}
else if (block_size <= MI_MEDIUM_OBJ_SIZE_MAX) {
page = mi_segment_medium_page_alloc(heap, block_size, tld, os_tld);
page = mi_segment_medium_page_alloc(heap, block_size, tld);
}
else if (block_size <= MI_LARGE_OBJ_SIZE_MAX /* || mi_is_good_fit(block_size, MI_LARGE_PAGE_SIZE - sizeof(mi_segment_t)) */ ) {
page = mi_segment_large_page_alloc(heap, block_size, tld, os_tld);
page = mi_segment_large_page_alloc(heap, block_size, tld);
}
else {
page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld);
page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld);
}
mi_assert_expensive(page == NULL || mi_segment_is_valid(_mi_page_segment(page),tld));
mi_assert_internal(page == NULL || (mi_segment_page_size(_mi_page_segment(page)) - (MI_SECURE == 0 ? 0 : _mi_os_page_size())) >= block_size);

33
src/stats.c
View file

@ -26,7 +26,7 @@ static bool mi_is_in_main(void* stat) {
static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
if (amount == 0) return;
if (mi_is_in_main(stat))
if mi_unlikely(mi_is_in_main(stat))
{
// add atomically (for abandoned pages)
int64_t current = mi_atomic_addi64_relaxed(&stat->current, amount);
@ -51,6 +51,27 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) {
}
}
// Adjust stats to compensate; for example before committing a range,
// first adjust downwards with parts that were already committed so
// we avoid double counting.
static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) {
if (amount == 0) return;
if mi_unlikely(mi_is_in_main(stat))
{
// adjust atomically
mi_atomic_addi64_relaxed(&stat->current, amount);
mi_atomic_addi64_relaxed(&stat->allocated, amount);
mi_atomic_addi64_relaxed(&stat->freed, amount);
}
else {
// don't affect the peak
stat->current += amount;
// add to both
stat->allocated += amount;
stat->freed += amount;
}
}
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
if (mi_is_in_main(stat)) {
mi_atomic_addi64_relaxed( &stat->count, 1 );
@ -70,6 +91,14 @@ void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) {
mi_stat_update(stat, -((int64_t)amount));
}
void _mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) {
mi_stat_adjust(stat, (int64_t)amount);
}
void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) {
mi_stat_adjust(stat, -((int64_t)amount));
}
// must be thread safe as it is called from stats_merge
static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) {
if (stat==src) return;
@ -118,6 +147,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
mi_stat_counter_add(&stats->searches, &src->searches, 1);
mi_stat_counter_add(&stats->normal_count, &src->normal_count, 1);
mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1);
mi_stat_counter_add(&stats->guarded_alloc_count, &src->guarded_alloc_count, 1);
#if MI_STAT>1
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
if (src->normal_bins[i].allocated > 0 || src->normal_bins[i].freed > 0) {
@ -342,6 +372,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
mi_stat_counter_print(&stats->commit_calls, "commits", out, arg);
mi_stat_counter_print(&stats->reset_calls, "resets", out, arg);
mi_stat_counter_print(&stats->purge_calls, "purges", out, arg);
mi_stat_counter_print(&stats->guarded_alloc_count, "guarded", out, arg);
mi_stat_print(&stats->threads, "threads", -1, out, arg);
mi_stat_counter_print_avg(&stats->searches, "searches", out, arg);
_mi_fprintf(out, arg, "%10s: %5zu\n", "numa nodes", _mi_os_numa_node_count());