kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-20 14:09:32 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Heap local deferred free fun

Browse source
This commit is contained in:
playX 2021-03-09 19:12:52 +03:00
parent 217f2e2cc7
commit f459d576bd
5 changed files with 1489 additions and 1214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
include/mimalloc-types.h
View file

@ -60,7 +60,6 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_PADDING 1
#endif
// Encoded free lists allow detection of corrupted free lists
// and can detect buffer overflows, modify after free, and double `free`s.
#if (MI_SECURE >= 3 || MI_DEBUG >= 1 || MI_PADDING > 0)
@ -98,7 +97,6 @@ terms of the MIT license. A copy of the license can be found in the file
#define MiB (KiB * KiB)
#define GiB (MiB * KiB)
// ------------------------------------------------------
// Main internal data-structures
// ------------------------------------------------------
@ -145,35 +143,39 @@ terms of the MIT license. A copy of the license can be found in the file
typedef uintptr_t mi_encoded_t;
// free lists contain blocks
typedef struct mi_block_s {
typedef struct mi_block_s
{
mi_encoded_t next;
} mi_block_t;
// The delayed flags are used for efficient multi-threaded free-ing
typedef enum mi_delayed_e {
typedef enum mi_delayed_e
{
MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list
MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap
MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list
MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim
} mi_delayed_t;
// The `in_full` and `has_aligned` page flags are put in a union to efficiently
// test if both are false (`full_aligned == 0`) in the `mi_free` routine.
#if !MI_TSAN
typedef union mi_page_flags_s {
typedef union mi_page_flags_s
{
uint8_t full_aligned;
struct {
struct
{
uint8_t in_full : 1;
uint8_t has_aligned : 1;
} x;
} mi_page_flags_t;
#else
// under thread sanitizer, use a byte for each flag to suppress warning, issue #130
typedef union mi_page_flags_s {
typedef union mi_page_flags_s
{
uint16_t full_aligned;
struct {
struct
{
uint8_t in_full;
uint8_t has_aligned;
} x;
@ -216,7 +218,8 @@ typedef uintptr_t mi_thread_free_t;
// at least one block that will be added, or as already been added, to
// the owning heap `thread_delayed_free` list. This guarantees that pages
// will be freed correctly even if only other threads free blocks.
typedef struct mi_page_s {
typedef struct mi_page_s
{
// "owned" by the segment
uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
uint8_t segment_in_use : 1; // `true` if the segment allocated this page
@ -246,9 +249,8 @@ typedef struct mi_page_s {
struct mi_page_s *prev; // previous page owned by this thread with the same `block_size`
} mi_page_t;
typedef enum mi_page_kind_e {
typedef enum mi_page_kind_e
{
MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment
MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
@ -258,7 +260,8 @@ typedef enum mi_page_kind_e {
// Segments are large allocated memory blocks (2mb on 64 bit) from
// the OS. Inside segments we allocated fixed size _pages_ that
// contain blocks.
typedef struct mi_segment_s {
typedef struct mi_segment_s
{
// memory fields
size_t memid; // id for the os-level memory manager
bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
@ -285,7 +288,6 @@ typedef struct mi_segment_s {
mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages
} mi_segment_t;
// ------------------------------------------------------
// Heaps
// Provide first-class heaps to allocate from.
@ -303,7 +305,8 @@ typedef struct mi_segment_s {
typedef struct mi_tld_s mi_tld_t;
// Pages of a certain block size are held in a queue.
typedef struct mi_page_queue_s {
typedef struct mi_page_queue_s
{
mi_page_t *first;
mi_page_t *last;
size_t block_size;
@ -312,16 +315,17 @@ typedef struct mi_page_queue_s {
#define MI_BIN_FULL (MI_BIN_HUGE + 1)
// Random context
typedef struct mi_random_cxt_s {
typedef struct mi_random_cxt_s
{
uint32_t input[16];
uint32_t output[16];
int output_available;
} mi_random_ctx_t;
// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows
#if (MI_PADDING)
typedef struct mi_padding_s {
typedef struct mi_padding_s
{
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
} mi_padding_t;
@ -334,9 +338,9 @@ typedef struct mi_padding_s {
#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
// A heap owns a set of pages.
struct mi_heap_s {
struct mi_heap_s
{
mi_tld_t *tld;
mi_page_t *pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
@ -350,10 +354,10 @@ struct mi_heap_s {
size_t page_retired_max; // largest retired index into the `pages` array.
mi_heap_t *next; // list of heaps per thread
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
void *deferred_free;
void *deferred_arg;
};
// ------------------------------------------------------
// Debug
// ------------------------------------------------------
@ -394,19 +398,22 @@ void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line
#endif
#endif
typedef struct mi_stat_count_s {
typedef struct mi_stat_count_s
{
int64_t allocated;
int64_t freed;
int64_t peak;
int64_t current;
} mi_stat_count_t;
typedef struct mi_stat_counter_s {
typedef struct mi_stat_counter_s
{
int64_t total;
int64_t count;
} mi_stat_counter_t;
typedef struct mi_stats_s {
typedef struct mi_stats_s
{
mi_stat_count_t segments;
mi_stat_count_t pages;
mi_stat_count_t reserved;
@ -434,7 +441,6 @@ typedef struct mi_stats_s {
#endif
} mi_stats_t;
void _mi_stat_increase(mi_stat_count_t *stat, size_t amount);
void _mi_stat_decrease(mi_stat_count_t *stat, size_t amount);
void _mi_stat_counter_increase(mi_stat_counter_t *stat, size_t amount);
@ -460,19 +466,22 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
typedef int64_t mi_msecs_t;
// Queue of segments
typedef struct mi_segment_queue_s {
typedef struct mi_segment_queue_s
{
mi_segment_t *first;
mi_segment_t *last;
} mi_segment_queue_t;
// OS thread local data
typedef struct mi_os_tld_s {
typedef struct mi_os_tld_s
{
size_t region_idx; // start point for next allocation
mi_stats_t *stats; // points to tld stats
} mi_os_tld_t;
// Segments thread local data
typedef struct mi_segments_tld_s {
typedef struct mi_segments_tld_s
{
mi_segment_queue_t small_free; // queue of segments with free small pages
mi_segment_queue_t medium_free; // queue of segments with free medium pages
mi_page_queue_t pages_reset; // queue of freed pages that can be reset
@ -488,7 +497,8 @@ typedef struct mi_segments_tld_s {
} mi_segments_tld_t;
// Thread local data
struct mi_tld_s {
struct mi_tld_s
{
unsigned long long heartbeat; // monotonic heartbeat count
bool recurse; // true if deferred was called; used to prevent infinite recursion.
mi_heap_t *heap_backing; // backing heap of this thread (cannot be deleted)

65
include/mimalloc.h
View file

@ -93,7 +93,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stdbool.h> // bool
#ifdef __cplusplus
extern "C" {
extern "C"
{
#endif
// ------------------------------------------------------
@ -127,7 +128,6 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)
mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void *p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
// ------------------------------------------------------
// Internals
// ------------------------------------------------------
@ -172,7 +172,6 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(siz
mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned(void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
// -------------------------------------------------------------------------------------
// Heaps: first-class, but can only allocate from the same thread that created it.
// -------------------------------------------------------------------------------------
@ -187,6 +186,8 @@ mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
mi_decl_export mi_heap_t *mi_heap_get_default(void);
mi_decl_export mi_heap_t *mi_heap_get_backing(void);
mi_decl_export void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept;
typedef void(mi_local_deferred_free_fun)(mi_heap_t *heap, bool force, unsigned long long heartbeat, void *arg);
mi_decl_export void mi_heap_register_local_deferred_free(mi_heap_t *heap, mi_local_deferred_free_fun *deferred_free, void *arg) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_zalloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
@ -211,7 +212,6 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_a
mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned(mi_heap_t *heap, void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
// --------------------------------------------------------------------------------
// Zero initialized re-allocation.
// Only valid on memory that was originally allocated with zero initialization too.
@ -235,7 +235,6 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* he
mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned_at(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
// ------------------------------------------------------
// Analysis
// ------------------------------------------------------
@ -245,7 +244,8 @@ mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
mi_decl_export bool mi_check_owned(const void *p);
// An area of heap space contains blocks of a single size.
typedef struct mi_heap_area_s {
typedef struct mi_heap_area_s
{
void *blocks; // start of the area containing heap blocks
size_t reserved; // bytes reserved for this area (virtual)
size_t committed; // current available bytes for this area
@ -267,11 +267,9 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
mi_decl_export bool mi_manage_os_memory(void *start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
// deprecated
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t *pages_reserved) mi_attr_noexcept;
// ------------------------------------------------------
// Convenience
// ------------------------------------------------------
@ -290,12 +288,12 @@ mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size
#define mi_heap_reallocn_tp(hp, p, tp, n) ((tp *)mi_heap_reallocn(hp, p, n, sizeof(tp)))
#define mi_heap_recalloc_tp(hp, p, tp, n) ((tp *)mi_heap_recalloc(hp, p, n, sizeof(tp)))
// ------------------------------------------------------
// Options, all `false` by default
// ------------------------------------------------------
typedef enum mi_option_e {
typedef enum mi_option_e
{
// stable options
mi_option_show_errors,
mi_option_show_stats,
@ -321,7 +319,6 @@ typedef enum mi_option_e {
_mi_option_last
} mi_option_t;
mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
mi_decl_export void mi_option_enable(mi_option_t option);
mi_decl_export void mi_option_disable(mi_option_t option);
@ -332,7 +329,6 @@ mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
mi_decl_export void mi_option_set(mi_option_t option, long value);
mi_decl_export void mi_option_set_default(mi_option_t option, long value);
// -------------------------------------------------------------------------------------------------------
// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
@ -389,7 +385,9 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
#include <utility> // std::forward
#endif
template<class T> struct mi_stl_allocator {
template <class T>
struct mi_stl_allocator
{
typedef T value_type;
typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type;
@ -397,19 +395,30 @@ template<class T> struct mi_stl_allocator {
typedef value_type const &const_reference;
typedef value_type *pointer;
typedef value_type const *const_pointer;
template <class U> struct rebind { typedef mi_stl_allocator<U> other; };
template <class U>
struct rebind
{
typedef mi_stl_allocator<U> other;
};
mi_stl_allocator() mi_attr_noexcept = default;
mi_stl_allocator(const mi_stl_allocator &) mi_attr_noexcept = default;
template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { }
template <class U>
mi_stl_allocator(const mi_stl_allocator<U> &) mi_attr_noexcept {}
mi_stl_allocator select_on_container_copy_construction() const { return *this; }
void deallocate(T *p, size_type) { mi_free(p); }
#if (__cplusplus >= 201703L) // C++17
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); }
mi_decl_nodiscard T *allocate(size_type count)
{
return static_cast<T *>(mi_new_n(count, sizeof(T)));
}
mi_decl_nodiscard T *allocate(size_type count, const void *) { return allocate(count); }
#else
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); }
mi_decl_nodiscard pointer allocate(size_type count, const void * = 0)
{
return static_cast<pointer>(mi_new_n(count, sizeof(value_type)));
}
#endif
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
@ -417,20 +426,30 @@ template<class T> struct mi_stl_allocator {
using propagate_on_container_move_assignment = std::true_type;
using propagate_on_container_swap = std::true_type;
using is_always_equal = std::true_type;
template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); }
template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); }
template <class U, class... Args>
void construct(U *p, Args &&...args) { ::new (p) U(std::forward<Args>(args)...); }
template <class U>
void destroy(U *p) mi_attr_noexcept { p->~U(); }
#else
void construct(pointer p, value_type const& val) { ::new(p) value_type(val); }
void construct(pointer p, value_type const &val)
{
::new (p) value_type(val);
}
void destroy(pointer p) { p->~value_type(); }
#endif
size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); }
size_type max_size() const mi_attr_noexcept
{
return (PTRDIFF_MAX / sizeof(value_type));
}
pointer address(reference x) const { return &x; }
const_pointer address(const_reference x) const { return &x; }
};
template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; }
template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; }
template <class T1, class T2>
bool operator==(const mi_stl_allocator<T1> &, const mi_stl_allocator<T2> &) mi_attr_noexcept { return true; }
template <class T1, class T2>
bool operator!=(const mi_stl_allocator<T1> &, const mi_stl_allocator<T2> &) mi_attr_noexcept { return false; }
#endif // __cplusplus
#endif

265
src/heap.c
View file

@ -25,21 +25,25 @@ typedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
// Visit all pages in a heap; returns `false` if break was called.
static bool mi_heap_visit_pages(mi_heap_t *heap, heap_page_visitor_fun *fn, void *arg1, void *arg2)
{
if (heap==NULL || heap->page_count==0) return 0;
if (heap == NULL || heap->page_count == 0)
return 0;
// visit all pages
#if MI_DEBUG > 1
size_t total = heap->page_count;
#endif
size_t count = 0;
for (size_t i = 0; i <= MI_BIN_FULL; i++) {
for (size_t i = 0; i <= MI_BIN_FULL; i++)
{
mi_page_queue_t *pq = &heap->pages[i];
mi_page_t *page = pq->first;
while(page != NULL) {
while (page != NULL)
{
mi_page_t *next = page->next; // save next in case the page gets removed from the queue
mi_assert_internal(mi_page_heap(page) == heap);
count++;
if (!fn(heap, pq, page, arg1, arg2)) return false;
if (!fn(heap, pq, page, arg1, arg2))
return false;
page = next; // and continue
}
}
@ -47,9 +51,9 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void
return true;
}
#if MI_DEBUG >= 2
static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
static bool mi_heap_page_is_valid(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1);
UNUSED(arg2);
UNUSED(pq);
@ -61,16 +65,14 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
}
#endif
#if MI_DEBUG >= 3
static bool mi_heap_is_valid(mi_heap_t* heap) {
static bool mi_heap_is_valid(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL);
mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL);
return true;
}
#endif
/* -----------------------------------------------------------
"Collect" pages by migrating `local_free` and `thread_free`
lists and freeing empty pages. This is done when a thread
@ -78,32 +80,36 @@ static bool mi_heap_is_valid(mi_heap_t* heap) {
blocks alive)
----------------------------------------------------------- */
typedef enum mi_collect_e {
typedef enum mi_collect_e
{
MI_NORMAL,
MI_FORCE,
MI_ABANDON
} mi_collect_t;
static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) {
static bool mi_heap_page_collect(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg_collect, void *arg2)
{
UNUSED(arg2);
UNUSED(heap);
mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL));
mi_collect_t collect = *((mi_collect_t *)arg_collect);
_mi_page_free_collect(page, collect >= MI_FORCE);
if (mi_page_all_free(page)) {
if (mi_page_all_free(page))
{
// no more used blocks, free the page.
// note: this will free retired pages as well.
_mi_page_free(page, pq, collect >= MI_FORCE);
}
else if (collect == MI_ABANDON) {
else if (collect == MI_ABANDON)
{
// still used blocks but the thread is done; abandon the page
_mi_page_abandon(page, pq);
}
return true; // don't break
}
static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
static bool mi_heap_page_never_delayed_free(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1);
UNUSED(arg2);
UNUSED(heap);
@ -114,7 +120,8 @@ static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq
static void mi_heap_collect_ex(mi_heap_t *heap, mi_collect_t collect)
{
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (heap == NULL || !mi_heap_is_initialized(heap))
return;
_mi_deferred_free(heap, collect >= MI_FORCE);
// note: never reclaim on collect but leave it to threads that need storage to reclaim
@ -132,7 +139,8 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
}
// if abandoning, mark all pages to no longer add to delayed_free
if (collect == MI_ABANDON) {
if (collect == MI_ABANDON)
{
mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL);
}
@ -148,39 +156,45 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
mi_assert_internal(collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t, &heap->thread_delayed_free) == NULL);
// collect segment caches
if (collect >= MI_FORCE) {
if (collect >= MI_FORCE)
{
_mi_segment_thread_collect(&heap->tld->segments);
}
// collect regions on program-exit (or shared library unload)
if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap))
{
_mi_mem_collect(&heap->tld->os);
}
}
void _mi_heap_collect_abandon(mi_heap_t* heap) {
void _mi_heap_collect_abandon(mi_heap_t *heap)
{
mi_heap_collect_ex(heap, MI_ABANDON);
}
void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept {
void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept
{
mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL));
}
void mi_collect(bool force) mi_attr_noexcept {
void mi_collect(bool force) mi_attr_noexcept
{
mi_heap_collect(mi_get_default_heap(), force);
}
/* -----------------------------------------------------------
Heap new
----------------------------------------------------------- */
mi_heap_t* mi_heap_get_default(void) {
mi_heap_t *mi_heap_get_default(void)
{
mi_thread_init();
return mi_get_default_heap();
}
mi_heap_t* mi_heap_get_backing(void) {
mi_heap_t *mi_heap_get_backing(void)
{
mi_heap_t *heap = mi_heap_get_default();
mi_assert_internal(heap != NULL);
mi_heap_t *bheap = heap->tld->heap_backing;
@ -189,10 +203,12 @@ mi_heap_t* mi_heap_get_backing(void) {
return bheap;
}
mi_heap_t* mi_heap_new(void) {
mi_heap_t *mi_heap_new(void)
{
mi_heap_t *bheap = mi_heap_get_backing();
mi_heap_t *heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
if (heap==NULL) return NULL;
if (heap == NULL)
return NULL;
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->tld = bheap->tld;
heap->thread_id = _mi_thread_id();
@ -204,15 +220,18 @@ mi_heap_t* mi_heap_new(void) {
// push on the thread local heaps list
heap->next = heap->tld->heaps;
heap->tld->heaps = heap;
heap->deferred_free = NULL;
return heap;
}
uintptr_t _mi_heap_random_next(mi_heap_t* heap) {
uintptr_t _mi_heap_random_next(mi_heap_t *heap)
{
return _mi_random_next(&heap->random);
}
// zero out the page queues
static void mi_heap_reset_pages(mi_heap_t* heap) {
static void mi_heap_reset_pages(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap));
// TODO: copy full empty heap instead?
@ -226,14 +245,18 @@ static void mi_heap_reset_pages(mi_heap_t* heap) {
}
// called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources.
static void mi_heap_free(mi_heap_t* heap) {
static void mi_heap_free(mi_heap_t *heap)
{
mi_assert(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (mi_heap_is_backing(heap)) return; // dont free the backing heap
if (heap == NULL || !mi_heap_is_initialized(heap))
return;
if (mi_heap_is_backing(heap))
return; // dont free the backing heap
// reset default
if (mi_heap_is_default(heap)) {
if (mi_heap_is_default(heap))
{
_mi_heap_set_default_direct(heap->tld->heap_backing);
}
@ -241,14 +264,22 @@ static void mi_heap_free(mi_heap_t* heap) {
// linear search but we expect the number of heaps to be relatively small
mi_heap_t *prev = NULL;
mi_heap_t *curr = heap->tld->heaps;
while (curr != heap && curr != NULL) {
while (curr != heap && curr != NULL)
{
prev = curr;
curr = curr->next;
}
mi_assert_internal(curr == heap);
if (curr == heap) {
if (prev != NULL) { prev->next = heap->next; }
else { heap->tld->heaps = heap->next; }
if (curr == heap)
{
if (prev != NULL)
{
prev->next = heap->next;
}
else
{
heap->tld->heaps = heap->next;
}
}
mi_assert_internal(heap->tld->heaps != NULL);
@ -256,12 +287,12 @@ static void mi_heap_free(mi_heap_t* heap) {
mi_free(heap);
}
/* -----------------------------------------------------------
Heap destroy
----------------------------------------------------------- */
static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) {
static bool _mi_heap_page_destroy(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1);
UNUSED(arg2);
UNUSED(heap);
@ -272,18 +303,22 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
// stats
const size_t bsize = mi_page_block_size(page);
if (bsize > MI_LARGE_OBJ_SIZE_MAX) {
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
if (bsize > MI_LARGE_OBJ_SIZE_MAX)
{
if (bsize > MI_HUGE_OBJ_SIZE_MAX)
{
mi_heap_stat_decrease(heap, giant, bsize);
}
else {
else
{
mi_heap_stat_decrease(heap, huge, bsize);
}
}
#if (MI_STAT)
_mi_page_free_collect(page, false); // update used count
const size_t inuse = page->used;
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) {
if (bsize <= MI_LARGE_OBJ_SIZE_MAX)
{
mi_heap_stat_decrease(heap, normal, bsize * inuse);
#if (MI_STAT > 1)
mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse);
@ -305,38 +340,43 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
return true; // keep going
}
void _mi_heap_destroy_pages(mi_heap_t* heap) {
void _mi_heap_destroy_pages(mi_heap_t *heap)
{
mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL);
mi_heap_reset_pages(heap);
}
void mi_heap_destroy(mi_heap_t* heap) {
void mi_heap_destroy(mi_heap_t *heap)
{
mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap));
mi_assert(heap->no_reclaim);
mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (!heap->no_reclaim) {
if (heap == NULL || !mi_heap_is_initialized(heap))
return;
if (!heap->no_reclaim)
{
// don't free in case it may contain reclaimed pages
mi_heap_delete(heap);
}
else {
else
{
// free all pages
_mi_heap_destroy_pages(heap);
mi_heap_free(heap);
}
}
/* -----------------------------------------------------------
Safe Heap delete
----------------------------------------------------------- */
// Tranfer the pages from one heap to the other
static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
static void mi_heap_absorb(mi_heap_t *heap, mi_heap_t *from)
{
mi_assert_internal(heap != NULL);
if (from==NULL || from->page_count == 0) return;
if (from == NULL || from->page_count == 0)
return;
// reduce the size of the delayed frees
_mi_heap_delayed_free(from);
@ -345,7 +385,8 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
// so threads may do delayed frees in either heap for a while.
// note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state
// so after this only the new heap will get delayed frees
for (size_t i = 0; i <= MI_BIN_FULL; i++) {
for (size_t i = 0; i <= MI_BIN_FULL; i++)
{
mi_page_queue_t *pq = &heap->pages[i];
mi_page_queue_t *append = &from->pages[i];
size_t pcount = _mi_page_queue_append(heap, pq, append);
@ -371,13 +412,16 @@ void mi_heap_delete(mi_heap_t* heap)
mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap));
mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return;
if (heap == NULL || !mi_heap_is_initialized(heap))
return;
if (!mi_heap_is_backing(heap)) {
if (!mi_heap_is_backing(heap))
{
// tranfer still used pages to the backing heap
mi_heap_absorb(heap->tld->heap_backing, heap);
}
else {
else
{
// the backing heap abandons its pages
_mi_heap_collect_abandon(heap);
}
@ -385,41 +429,45 @@ void mi_heap_delete(mi_heap_t* heap)
mi_heap_free(heap);
}
mi_heap_t* mi_heap_set_default(mi_heap_t* heap) {
mi_heap_t *mi_heap_set_default(mi_heap_t *heap)
{
mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL;
if (heap == NULL || !mi_heap_is_initialized(heap))
return NULL;
mi_assert_expensive(mi_heap_is_valid(heap));
mi_heap_t *old = mi_get_default_heap();
_mi_heap_set_default_direct(heap);
return old;
}
/* -----------------------------------------------------------
Analysis
----------------------------------------------------------- */
// static since it is not thread safe to access heaps from other threads.
static mi_heap_t* mi_heap_of_block(const void* p) {
if (p == NULL) return NULL;
static mi_heap_t *mi_heap_of_block(const void *p)
{
if (p == NULL)
return NULL;
mi_segment_t *segment = _mi_ptr_segment(p);
bool valid = (_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(valid);
if (mi_unlikely(!valid)) return NULL;
if (mi_unlikely(!valid))
return NULL;
return mi_page_heap(_mi_segment_page_of(segment, p));
}
bool mi_heap_contains_block(mi_heap_t* heap, const void* p) {
bool mi_heap_contains_block(mi_heap_t *heap, const void *p)
{
mi_assert(heap != NULL);
if (heap==NULL || !mi_heap_is_initialized(heap)) return false;
if (heap == NULL || !mi_heap_is_initialized(heap))
return false;
return (heap == mi_heap_of_block(p));
}
static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) {
static bool mi_heap_page_check_owned(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *p, void *vfound)
{
UNUSED(heap);
UNUSED(pq);
bool *found = (bool *)vfound;
@ -430,16 +478,20 @@ static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
return (!*found); // continue if not found
}
bool mi_heap_check_owned(mi_heap_t* heap, const void* p) {
bool mi_heap_check_owned(mi_heap_t *heap, const void *p)
{
mi_assert(heap != NULL);
if (heap==NULL || !mi_heap_is_initialized(heap)) return false;
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers
if (heap == NULL || !mi_heap_is_initialized(heap))
return false;
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)
return false; // only aligned pointers
bool found = false;
mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void *)p, &found);
return found;
}
bool mi_check_owned(const void* p) {
bool mi_check_owned(const void *p)
{
return mi_heap_check_owned(mi_get_default_heap(), p);
}
@ -450,28 +502,34 @@ bool mi_check_owned(const void* p) {
----------------------------------------------------------- */
// Separate struct to keep `mi_page_t` out of the public interface
typedef struct mi_heap_area_ex_s {
typedef struct mi_heap_area_ex_s
{
mi_heap_area_t area;
mi_page_t *page;
} mi_heap_area_ex_t;
static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) {
static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t *xarea, mi_block_visit_fun *visitor, void *arg)
{
mi_assert(xarea != NULL);
if (xarea==NULL) return true;
if (xarea == NULL)
return true;
const mi_heap_area_t *area = &xarea->area;
mi_page_t *page = xarea->page;
mi_assert(page != NULL);
if (page == NULL) return true;
if (page == NULL)
return true;
_mi_page_free_collect(page, true);
mi_assert_internal(page->local_free == NULL);
if (page->used == 0) return true;
if (page->used == 0)
return true;
const size_t bsize = mi_page_block_size(page);
size_t psize;
uint8_t *pstart = _mi_page_start(_mi_page_segment(page), page, &psize);
if (page->capacity == 1) {
if (page->capacity == 1)
{
// optimize page with one block
mi_assert_internal(page->used == 1 && page->free == NULL);
return visitor(mi_page_heap(page), area, pstart, bsize, arg);
@ -483,7 +541,8 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
memset(free_map, 0, sizeof(free_map));
size_t free_count = 0;
for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) {
for (mi_block_t *block = page->free; block != NULL; block = mi_block_next(page, block))
{
free_count++;
mi_assert_internal((uint8_t *)block >= pstart && (uint8_t *)block < (pstart + psize));
size_t offset = (uint8_t *)block - pstart;
@ -498,17 +557,21 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
// walk through all blocks skipping the free ones
size_t used_count = 0;
for (size_t i = 0; i < page->capacity; i++) {
for (size_t i = 0; i < page->capacity; i++)
{
size_t bitidx = (i / sizeof(uintptr_t));
size_t bit = i - (bitidx * sizeof(uintptr_t));
uintptr_t m = free_map[bitidx];
if (bit == 0 && m == UINTPTR_MAX) {
if (bit == 0 && m == UINTPTR_MAX)
{
i += (sizeof(uintptr_t) - 1); // skip a run of free blocks
}
else if ((m & ((uintptr_t)1 << bit)) == 0) {
else if ((m & ((uintptr_t)1 << bit)) == 0)
{
used_count++;
uint8_t *block = pstart + (i * bsize);
if (!visitor(mi_page_heap(page), area, block, bsize, arg)) return false;
if (!visitor(mi_page_heap(page), area, block, bsize, arg))
return false;
}
}
mi_assert_internal(page->used == used_count);
@ -517,8 +580,8 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
typedef bool(mi_heap_area_visit_fun)(const mi_heap_t *heap, const mi_heap_area_ex_t *area, void *arg);
static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) {
static bool mi_heap_visit_areas_page(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *vfun, void *arg)
{
UNUSED(heap);
UNUSED(pq);
mi_heap_area_visit_fun *fun = (mi_heap_area_visit_fun *)vfun;
@ -534,31 +597,45 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
}
// Visit all heap pages as areas
static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) {
if (visitor == NULL) return false;
static bool mi_heap_visit_areas(const mi_heap_t *heap, mi_heap_area_visit_fun *visitor, void *arg)
{
if (visitor == NULL)
return false;
return mi_heap_visit_pages((mi_heap_t *)heap, &mi_heap_visit_areas_page, (void *)(visitor), arg); // note: function pointer to void* :-{
}
// Just to pass arguments
typedef struct mi_visit_blocks_args_s {
typedef struct mi_visit_blocks_args_s
{
bool visit_blocks;
mi_block_visit_fun *visitor;
void *arg;
} mi_visit_blocks_args_t;
static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) {
static bool mi_heap_area_visitor(const mi_heap_t *heap, const mi_heap_area_ex_t *xarea, void *arg)
{
mi_visit_blocks_args_t *args = (mi_visit_blocks_args_t *)arg;
if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false;
if (args->visit_blocks) {
if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg))
return false;
if (args->visit_blocks)
{
return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg);
}
else {
else
{
return true;
}
}
// Visit all blocks in a heap
bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) {
bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
{
mi_visit_blocks_args_t args = {visit_blocks, visitor, arg};
return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);
}
void mi_heap_register_local_deferred_free(mi_heap_t *heap, mi_local_deferred_free_fun *deferred_free, void *arg)
{
heap->deferred_free = (void *)deferred_free;
heap->deferred_arg = arg;
}

265
src/init.c
View file

@ -27,24 +27,36 @@ const mi_page_t _mi_page_empty = {
NULL, // local_free
ATOMIC_VAR_INIT(0), // xthread_free
ATOMIC_VAR_INIT(0), // xheap
NULL, NULL
};
NULL,
NULL};
#define MI_PAGE_EMPTY() ((mi_page_t *)&_mi_page_empty)
#if (MI_PADDING > 0) && (MI_INTPTR_SIZE >= 8)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() \
}
#elif (MI_PADDING > 0)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() }
#define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() \
}
#else
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() }
#define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() \
}
#endif
// Empty page queues for every bin
#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) }
#define QNULL(sz) \
{ \
NULL, NULL, (sz) * sizeof(uintptr_t) \
}
#define MI_PAGE_QUEUES_EMPTY \
{ QNULL(1), \
{ \
QNULL(1), \
QNULL(1), QNULL(2), QNULL(3), QNULL(4), QNULL(5), QNULL(6), QNULL(7), QNULL(8), /* 8 */ \
QNULL(10), QNULL(12), QNULL(14), QNULL(16), QNULL(20), QNULL(24), QNULL(28), QNULL(32), /* 16 */ \
QNULL(40), QNULL(48), QNULL(56), QNULL(64), QNULL(80), QNULL(96), QNULL(112), QNULL(128), /* 24 */ \
@ -55,13 +67,18 @@ const mi_page_t _mi_page_empty = {
QNULL(40960), QNULL(49152), QNULL(57344), QNULL(65536), QNULL(81920), QNULL(98304), QNULL(114688), QNULL(131072), /* 64 */ \
QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ }
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ \
}
#define MI_STAT_COUNT_NULL() {0,0,0,0}
#define MI_STAT_COUNT_NULL() \
{ \
0, 0, 0, 0 \
}
// Empty statistics
#if MI_STAT > 1
#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) }
#define MI_STAT_COUNT_END_NULL() \
, { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) }
#else
#define MI_STAT_COUNT_END_NULL()
#endif
@ -75,8 +92,7 @@ const mi_page_t _mi_page_empty = {
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
{0, 0}, {0, 0}, {0, 0}, {0, 0}, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \
MI_STAT_COUNT_END_NULL()
{0, 0}, {0, 0}, {0, 0}, {0, 0} MI_STAT_COUNT_END_NULL()
// --------------------------------------------------------
// Statically allocate an empty heap as the initial
@ -97,10 +113,12 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
{0, 0}, // keys
{{0}, {0}, 0},
0, // page count
MI_BIN_FULL, 0, // page retired min/max
MI_BIN_FULL,
0, // page retired min/max
NULL, // next
false
};
false,
0,
0};
// the thread-local default heap for allocation
mi_decl_thread mi_heap_t *_mi_heap_default = (mi_heap_t *)&_mi_heap_empty;
@ -108,12 +126,7 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
extern mi_heap_t _mi_heap_main;
static mi_tld_t tld_main = {
0, false,
&_mi_heap_main, &_mi_heap_main,
{ { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0},
0, 0, 0, 0, 0, 0, NULL,
&tld_main.stats, &tld_main.os
}, // segments
0, false, &_mi_heap_main, &_mi_heap_main, {{NULL, NULL}, {NULL, NULL}, {NULL, NULL, 0}, 0, 0, 0, 0, 0, 0, NULL, &tld_main.stats, &tld_main.os}, // segments
{0, &tld_main.stats}, // os
{MI_STATS_NULL} // stats
};
@ -128,18 +141,21 @@ mi_heap_t _mi_heap_main = {
{0, 0}, // the key of the main heap can be fixed (unlike page keys that need to be secure!)
{{0x846ca68b}, {0}, 0}, // random
0, // page count
MI_BIN_FULL, 0, // page retired min/max
MI_BIN_FULL,
0, // page retired min/max
NULL, // next heap
false // can reclaim
};
false, // can reclaim
0,
0};
bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
mi_stats_t _mi_stats_main = {MI_STATS_NULL};
static void mi_heap_main_init(void) {
if (_mi_heap_main.cookie == 0) {
static void mi_heap_main_init(void)
{
if (_mi_heap_main.cookie == 0)
{
_mi_heap_main.thread_id = _mi_thread_id();
_mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init);
_mi_random_init(&_mi_heap_main.random);
@ -148,39 +164,46 @@ static void mi_heap_main_init(void) {
}
}
mi_heap_t* _mi_heap_main_get(void) {
mi_heap_t *_mi_heap_main_get(void)
{
mi_heap_main_init();
return &_mi_heap_main;
}
/* -----------------------------------------------------------
Initialization and freeing of the thread local heaps
----------------------------------------------------------- */
// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size).
typedef struct mi_thread_data_s {
typedef struct mi_thread_data_s
{
mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
mi_tld_t tld;
} mi_thread_data_t;
// Initialize the thread local default heap, called from `mi_thread_init`
static bool _mi_heap_init(void) {
if (mi_heap_is_initialized(mi_get_default_heap())) return true;
if (_mi_is_main_thread()) {
static bool _mi_heap_init(void)
{
if (mi_heap_is_initialized(mi_get_default_heap()))
return true;
if (_mi_is_main_thread())
{
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization
// the main heap is statically allocated
mi_heap_main_init();
_mi_heap_set_default_direct(&_mi_heap_main);
//mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap());
}
else {
else
{
// use `_mi_os_alloc` to allocate directly from the OS
mi_thread_data_t *td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); // Todo: more efficient allocation?
if (td == NULL) {
if (td == NULL)
{
// if this fails, try once more. (issue #257)
td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main);
if (td == NULL) {
if (td == NULL)
{
// really out of memory
_mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t));
return false;
@ -207,21 +230,26 @@ static bool _mi_heap_init(void) {
}
// Free the thread local default heap (called from `mi_thread_done`)
static bool _mi_heap_done(mi_heap_t* heap) {
if (!mi_heap_is_initialized(heap)) return true;
static bool _mi_heap_done(mi_heap_t *heap)
{
if (!mi_heap_is_initialized(heap))
return true;
// reset default heap
_mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t *)&_mi_heap_empty);
// switch to backing heap
heap = heap->tld->heap_backing;
if (!mi_heap_is_initialized(heap)) return false;
if (!mi_heap_is_initialized(heap))
return false;
// delete all non-backing heaps in this thread
mi_heap_t *curr = heap->tld->heaps;
while (curr != NULL) {
while (curr != NULL)
{
mi_heap_t *next = curr->next; // save `next` as `curr` will be freed
if (curr != heap) {
if (curr != heap)
{
mi_assert_internal(!mi_heap_is_backing(curr));
mi_heap_delete(curr);
}
@ -231,7 +259,8 @@ static bool _mi_heap_done(mi_heap_t* heap) {
mi_assert_internal(mi_heap_is_backing(heap));
// collect if not the main thread
if (heap != &_mi_heap_main) {
if (heap != &_mi_heap_main)
{
_mi_heap_collect_abandon(heap);
}
@ -239,7 +268,8 @@ static bool _mi_heap_done(mi_heap_t* heap) {
_mi_stats_done(&heap->tld->stats);
// free if not the main thread
if (heap != &_mi_heap_main) {
if (heap != &_mi_heap_main)
{
mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id());
_mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main);
}
@ -254,8 +284,6 @@ static bool _mi_heap_done(mi_heap_t* heap) {
return false;
}
// --------------------------------------------------------
// Try to run `mi_thread_done()` automatically so any memory
// owned by the thread but not yet released can be abandoned
@ -293,16 +321,20 @@ static void _mi_thread_done(mi_heap_t* default_heap);
WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
#endif
static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) {
if (value!=NULL) _mi_thread_done((mi_heap_t*)value);
static void NTAPI mi_fls_done(PVOID value)
{
if (value != NULL)
_mi_thread_done((mi_heap_t *)value);
}
#elif defined(MI_USE_PTHREADS)
// use pthread local storage keys to detect thread ending
// (and used with MI_TLS_PTHREADS for the default heap)
#include <pthread.h>
pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
static void mi_pthread_done(void* value) {
if (value!=NULL) _mi_thread_done((mi_heap_t*)value);
static void mi_pthread_done(void *value)
{
if (value != NULL)
_mi_thread_done((mi_heap_t *)value);
}
#elif defined(__wasi__)
// no pthreads in the WebAssembly Standard Interface
@ -311,9 +343,11 @@ static void _mi_thread_done(mi_heap_t* default_heap);
#endif
// Set up handlers so `mi_thread_done` is called automatically
static void mi_process_setup_auto_thread_done(void) {
static void mi_process_setup_auto_thread_done(void)
{
static bool tls_initialized = false; // fine if it races
if (tls_initialized) return;
if (tls_initialized)
return;
tls_initialized = true;
#if defined(_WIN32) && defined(MI_SHARED_LIB)
// nothing to do as it is done in DllMain
@ -326,8 +360,8 @@ static void mi_process_setup_auto_thread_done(void) {
_mi_heap_set_default_direct(&_mi_heap_main);
}
bool _mi_is_main_thread(void) {
bool _mi_is_main_thread(void)
{
return (_mi_heap_main.thread_id == 0 || _mi_heap_main.thread_id == _mi_thread_id());
}
@ -340,27 +374,33 @@ void mi_thread_init(void) mi_attr_noexcept
// initialize the thread local default heap
// (this will call `_mi_heap_set_default_direct` and thus set the
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
if (_mi_heap_init()) return; // returns true if already initialized
if (_mi_heap_init())
return; // returns true if already initialized
_mi_stat_increase(&_mi_stats_main.threads, 1);
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
}
void mi_thread_done(void) mi_attr_noexcept {
void mi_thread_done(void) mi_attr_noexcept
{
_mi_thread_done(mi_get_default_heap());
}
static void _mi_thread_done(mi_heap_t* heap) {
static void _mi_thread_done(mi_heap_t *heap)
{
_mi_stat_decrease(&_mi_stats_main.threads, 1);
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps...
if (heap->thread_id != _mi_thread_id()) return;
if (heap->thread_id != _mi_thread_id())
return;
// abandon the thread local heap
if (_mi_heap_done(heap)) return; // returns true if already ran
if (_mi_heap_done(heap))
return; // returns true if already ran
}
void _mi_heap_set_default_direct(mi_heap_t* heap) {
void _mi_heap_set_default_direct(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL);
#if defined(MI_TLS_SLOT)
mi_tls_slot_set(MI_TLS_SLOT, heap);
@ -380,13 +420,13 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) {
mi_assert_internal(mi_fls_key != 0);
FlsSetValue(mi_fls_key, heap);
#elif defined(MI_USE_PTHREADS)
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD
if (_mi_heap_default_key != (pthread_key_t)(-1))
{ // can happen during recursive invocation on freeBSD
pthread_setspecific(_mi_heap_default_key, heap);
}
#endif
}
// --------------------------------------------------------
// Run functions on process init/done, and thread init/done
// --------------------------------------------------------
@ -396,28 +436,35 @@ static bool os_preloading = true; // true until this module is initialized
static bool mi_redirected = false; // true if malloc redirects to mi_malloc
// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
bool _mi_preloading(void) {
bool _mi_preloading(void)
{
return os_preloading;
}
bool mi_is_redirected(void) mi_attr_noexcept {
bool mi_is_redirected(void) mi_attr_noexcept
{
return mi_redirected;
}
// Communicate with the redirection module on Windows
#if defined(_WIN32) && defined(MI_SHARED_LIB)
#ifdef __cplusplus
extern "C" {
extern "C"
{
#endif
mi_decl_export void _mi_redirect_entry(DWORD reason) {
mi_decl_export void _mi_redirect_entry(DWORD reason)
{
// called on redirection; careful as this may be called before DllMain
if (reason == DLL_PROCESS_ATTACH) {
if (reason == DLL_PROCESS_ATTACH)
{
mi_redirected = true;
}
else if (reason == DLL_PROCESS_DETACH) {
else if (reason == DLL_PROCESS_DETACH)
{
mi_redirected = false;
}
else if (reason == DLL_THREAD_DETACH) {
else if (reason == DLL_THREAD_DETACH)
{
mi_thread_done();
}
}
@ -427,17 +474,21 @@ __declspec(dllimport) void mi_allocator_done(void);
}
#endif
#else
static bool mi_allocator_init(const char** message) {
if (message != NULL) *message = NULL;
static bool mi_allocator_init(const char **message)
{
if (message != NULL)
*message = NULL;
return true;
}
static void mi_allocator_done(void) {
static void mi_allocator_done(void)
{
// nothing to do
}
#endif
// Called once by the process loader
static void mi_process_load(void) {
static void mi_process_load(void)
{
mi_heap_main_init();
#if defined(MI_TLS_RECURSE_GUARD)
volatile mi_heap_t *dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true;
@ -448,12 +499,14 @@ static void mi_process_load(void) {
_mi_options_init();
mi_process_init();
//mi_stats_reset();-
if (mi_redirected) _mi_verbose_message("malloc is redirected.\n");
if (mi_redirected)
_mi_verbose_message("malloc is redirected.\n");
// show message from the redirector (if present)
const char *msg = NULL;
mi_allocator_init(&msg);
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors)))
{
_mi_fputs(NULL, NULL, NULL, msg);
}
}
@ -462,22 +515,26 @@ static void mi_process_load(void) {
#include <intrin.h>
mi_decl_cache_align bool _mi_cpu_has_fsrm = false;
static void mi_detect_cpu_features(void) {
static void mi_detect_cpu_features(void)
{
// FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
int32_t cpu_info[4];
__cpuid(cpu_info, 7);
_mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https ://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
}
#else
static void mi_detect_cpu_features(void) {
static void mi_detect_cpu_features(void)
{
// nothing
}
#endif
// Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept {
void mi_process_init(void) mi_attr_noexcept
{
// ensure we are called once
if (_mi_process_is_initialized) return;
if (_mi_process_is_initialized)
return;
_mi_process_is_initialized = true;
mi_process_setup_auto_thread_done();
@ -492,23 +549,29 @@ void mi_process_init(void) mi_attr_noexcept {
mi_thread_init();
mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL)
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages))
{
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
mi_reserve_huge_os_pages_interleave(pages, 0, pages * 500);
}
if (mi_option_is_enabled(mi_option_reserve_os_memory)) {
if (mi_option_is_enabled(mi_option_reserve_os_memory))
{
long ksize = mi_option_get(mi_option_reserve_os_memory);
if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true);
if (ksize > 0)
mi_reserve_os_memory((size_t)ksize * KiB, true, true);
}
}
// Called when the process is done (through `at_exit`)
static void mi_process_done(void) {
static void mi_process_done(void)
{
// only shutdown if we were initialized
if (!_mi_process_is_initialized) return;
if (!_mi_process_is_initialized)
return;
// ensure we are called once
static bool process_done = false;
if (process_done) return;
if (process_done)
return;
process_done = true;
#if defined(_WIN32) && !defined(MI_SHARED_LIB)
@ -523,7 +586,8 @@ static void mi_process_done(void) {
mi_collect(true /* force */);
#endif
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) {
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose))
{
mi_stats_print(NULL);
}
mi_allocator_done();
@ -531,25 +595,28 @@ static void mi_process_done(void) {
os_preloading = true; // don't call the C runtime anymore
}
#if defined(_WIN32) && defined(MI_SHARED_LIB)
// Windows DLL: easy to hook into process_init and thread_done
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved)
{
UNUSED(reserved);
UNUSED(inst);
if (reason==DLL_PROCESS_ATTACH) {
if (reason == DLL_PROCESS_ATTACH)
{
mi_process_load();
}
else if (reason==DLL_THREAD_DETACH) {
if (!mi_is_redirected()) mi_thread_done();
else if (reason == DLL_THREAD_DETACH)
{
if (!mi_is_redirected())
mi_thread_done();
}
return TRUE;
}
#elif defined(__cplusplus)
// C++: use static initialization to detect process start
static bool _mi_process_init(void) {
static bool _mi_process_init(void)
{
mi_process_load();
return (_mi_heap_main.thread_id != 0);
}
@ -557,23 +624,27 @@ static void mi_process_done(void) {
#elif defined(__GNUC__) || defined(__clang__)
// GCC,Clang: use the constructor attribute
static void __attribute__((constructor)) _mi_process_init(void) {
static void __attribute__((constructor)) _mi_process_init(void)
{
mi_process_load();
}
#elif defined(_MSC_VER)
// MSVC: use data section magic for static libraries
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
static int _mi_process_init(void) {
static int _mi_process_init(void)
{
mi_process_load();
return 0;
}
typedef int (*_crt_cb)(void);
#ifdef _M_X64
__pragma(comment(linker, "/include:" "_mi_msvc_initu"))
__pragma(comment(linker, "/include:"
"_mi_msvc_initu"))
#pragma section(".CRT$XIU", long, read)
#else
__pragma(comment(linker, "/include:" "__mi_msvc_initu"))
__pragma(comment(linker, "/include:"
"__mi_msvc_initu"))
#endif
#pragma data_seg(".CRT$XIU")
_crt_cb _mi_msvc_initu[] = {&_mi_process_init};

324
src/page.c
View file

@ -23,13 +23,13 @@ terms of the MIT license. A copy of the license can be found in the file
#include "page-queue.c"
#undef MI_IN_PAGE_C
/* -----------------------------------------------------------
Page helpers
----------------------------------------------------------- */
// Index a block in a page
static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) {
static inline mi_block_t *mi_page_block_at(const mi_page_t *page, void *page_start, size_t block_size, size_t i)
{
UNUSED(page);
mi_assert_internal(page != NULL);
mi_assert_internal(i <= page->reserved);
@ -40,9 +40,11 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t
static void mi_page_extend_free(mi_heap_t *heap, mi_page_t *page, mi_tld_t *tld);
#if (MI_DEBUG >= 3)
static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {
static size_t mi_page_list_count(mi_page_t *page, mi_block_t *head)
{
size_t count = 0;
while (head != NULL) {
while (head != NULL)
{
mi_assert_internal(page == _mi_ptr_page(head));
count++;
head = mi_block_next(page, head);
@ -57,19 +59,23 @@ static inline uint8_t* mi_page_area(const mi_page_t* page) {
}
*/
static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {
static bool mi_page_list_is_valid(mi_page_t *page, mi_block_t *p)
{
size_t psize;
uint8_t *page_area = _mi_page_start(_mi_page_segment(page), page, &psize);
mi_block_t *start = (mi_block_t *)page_area;
mi_block_t *end = (mi_block_t *)(page_area + psize);
while(p != NULL) {
if (p < start || p >= end) return false;
while (p != NULL)
{
if (p < start || p >= end)
return false;
p = mi_block_next(page, p);
}
return true;
}
static bool mi_page_is_valid_init(mi_page_t* page) {
static bool mi_page_is_valid_init(mi_page_t *page)
{
mi_assert_internal(page->xblock_size > 0);
mi_assert_internal(page->used <= page->capacity);
mi_assert_internal(page->capacity <= page->reserved);
@ -84,8 +90,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
mi_assert_internal(mi_page_list_is_valid(page, page->local_free));
#if MI_DEBUG > 3 // generally too expensive to check this
if (page->flags.is_zero) {
for(mi_block_t* block = page->free; block != NULL; mi_block_next(page,block)) {
if (page->flags.is_zero)
{
for (mi_block_t *block = page->free; block != NULL; mi_block_next(page, block))
{
mi_assert_expensive(mi_mem_is_zero(block + 1, page->block_size - sizeof(mi_block_t)));
}
}
@ -102,15 +110,18 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
return true;
}
bool _mi_page_is_valid(mi_page_t* page) {
bool _mi_page_is_valid(mi_page_t *page)
{
mi_assert_internal(mi_page_is_valid_init(page));
#if MI_SECURE
mi_assert_internal(page->keys[0] != 0);
#endif
if (mi_page_heap(page)!=NULL) {
if (mi_page_heap(page) != NULL)
{
mi_segment_t *segment = _mi_page_segment(page);
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == mi_page_heap(page)->thread_id || segment->thread_id == 0);
if (segment->page_kind != MI_PAGE_HUGE) {
if (segment->page_kind != MI_PAGE_HUGE)
{
mi_page_queue_t *pq = mi_page_queue_of(page);
mi_assert_internal(mi_page_queue_contains(pq, page));
mi_assert_internal(pq->block_size == mi_page_block_size(page) || mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page));
@ -121,22 +132,27 @@ bool _mi_page_is_valid(mi_page_t* page) {
}
#endif
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) {
void _mi_page_use_delayed_free(mi_page_t *page, mi_delayed_t delay, bool override_never)
{
mi_thread_free_t tfreex;
mi_delayed_t old_delay;
mi_thread_free_t tfree;
do {
do
{
tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS;
tfreex = mi_tf_set_delayed(tfree, delay);
old_delay = mi_tf_delayed(tfree);
if (mi_unlikely(old_delay == MI_DELAYED_FREEING)) {
if (mi_unlikely(old_delay == MI_DELAYED_FREEING))
{
mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
// tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail
}
else if (delay == old_delay) {
else if (delay == old_delay)
{
break; // avoid atomic operation if already equal
}
else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE) {
else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE)
{
break; // leave never-delayed flag set
}
} while ((old_delay == MI_DELAYED_FREEING) ||
@ -156,25 +172,29 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
mi_block_t *head;
mi_thread_free_t tfreex;
mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);
do {
do
{
head = mi_tf_block(tfree);
tfreex = mi_tf_set_block(tfree, NULL);
} while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));
// return if the list is empty
if (head == NULL) return;
if (head == NULL)
return;
// find the tail -- also to get a proper count (without data races)
uint32_t max_count = page->capacity; // cannot collect more than capacity
uint32_t count = 1;
mi_block_t *tail = head;
mi_block_t *next;
while ((next = mi_block_next(page,tail)) != NULL && count <= max_count) {
while ((next = mi_block_next(page, tail)) != NULL && count <= max_count)
{
count++;
tail = next;
}
// if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)
if (count > max_count) {
if (count > max_count)
{
_mi_error_message(EFAULT, "corrupted thread-free list\n");
return; // the thread-free items cannot be freed
}
@ -187,27 +207,33 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
page->used -= count;
}
void _mi_page_free_collect(mi_page_t* page, bool force) {
void _mi_page_free_collect(mi_page_t *page, bool force)
{
mi_assert_internal(page != NULL);
// collect the thread free list
if (force || mi_page_thread_free(page) != NULL) { // quick test to avoid an atomic operation
if (force || mi_page_thread_free(page) != NULL)
{ // quick test to avoid an atomic operation
_mi_page_thread_free_collect(page);
}
// and the local free list
if (page->local_free != NULL) {
if (mi_likely(page->free == NULL)) {
if (page->local_free != NULL)
{
if (mi_likely(page->free == NULL))
{
// usual case
page->free = page->local_free;
page->local_free = NULL;
page->is_zero = false;
}
else if (force) {
else if (force)
{
// append -- only on shutdown (force) as this is a linear operation
mi_block_t *tail = page->local_free;
mi_block_t *next;
while ((next = mi_block_next(page, tail)) != NULL) {
while ((next = mi_block_next(page, tail)) != NULL)
{
tail = next;
}
mi_block_set_next(page, tail, page->free);
@ -220,14 +246,13 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
mi_assert_internal(!force || page->local_free == NULL);
}
/* -----------------------------------------------------------
Page fresh and retire
----------------------------------------------------------- */
// called from segments when reclaiming abandoned pages
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {
void _mi_page_reclaim(mi_heap_t *heap, mi_page_t *page)
{
mi_assert_expensive(mi_page_is_valid_init(page));
mi_assert_internal(mi_page_heap(page) == heap);
mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);
@ -240,11 +265,13 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {
}
// allocate a fresh page from a segment
static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size_t block_size) {
static mi_page_t *mi_page_fresh_alloc(mi_heap_t *heap, mi_page_queue_t *pq, size_t block_size)
{
mi_assert_internal(pq == NULL || mi_heap_contains_queue(heap, pq));
mi_assert_internal(pq == NULL || block_size == pq->block_size);
mi_page_t *page = _mi_segment_page_alloc(heap, block_size, &heap->tld->segments, &heap->tld->os);
if (page == NULL) {
if (page == NULL)
{
// this may be out-of-memory, or an abandoned page was reclaimed (and in our queue)
return NULL;
}
@ -252,16 +279,19 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
mi_assert_internal(pq == NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
mi_page_init(heap, page, block_size, heap->tld);
_mi_stat_increase(&heap->tld->stats.pages, 1);
if (pq!=NULL) mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL
if (pq != NULL)
mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL
mi_assert_expensive(_mi_page_is_valid(page));
return page;
}
// Get a fresh page to use
static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) {
static mi_page_t *mi_page_fresh(mi_heap_t *heap, mi_page_queue_t *pq)
{
mi_assert_internal(mi_heap_contains_queue(heap, pq));
mi_page_t *page = mi_page_fresh_alloc(heap, pq, pq->block_size);
if (page==NULL) return NULL;
if (page == NULL)
return NULL;
mi_assert_internal(pq->block_size == mi_page_block_size(page));
mi_assert_internal(pq == mi_page_queue(heap, mi_page_block_size(page)));
return page;
@ -271,20 +301,26 @@ static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) {
Do any delayed frees
(put there by other threads if they deallocated in a full page)
----------------------------------------------------------- */
void _mi_heap_delayed_free(mi_heap_t* heap) {
void _mi_heap_delayed_free(mi_heap_t *heap)
{
// take over the list (note: no atomic exchange since it is often NULL)
mi_block_t *block = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);
while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL)) { /* nothing */ };
while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL))
{ /* nothing */
};
// and free them all
while(block != NULL) {
while (block != NULL)
{
mi_block_t *next = mi_block_nextx(heap, block, heap->keys);
// use internal free instead of regular one to keep stats etc correct
if (!_mi_free_delayed_block(block)) {
if (!_mi_free_delayed_block(block))
{
// we might already start delayed freeing while another thread has not yet
// reset the delayed_freeing flag; in that case delay it further by reinserting.
mi_block_t *dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);
do {
do
{
mi_block_set_nextx(heap, block, dfree, heap->keys);
} while (!mi_atomic_cas_ptr_weak_release(mi_block_t, &heap->thread_delayed_free, &dfree, block));
}
@ -297,11 +333,13 @@ void _mi_heap_delayed_free(mi_heap_t* heap) {
----------------------------------------------------------- */
// Move a page from the full list back to a regular list
void _mi_page_unfull(mi_page_t* page) {
void _mi_page_unfull(mi_page_t *page)
{
mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(mi_page_is_in_full(page));
if (!mi_page_is_in_full(page)) return;
if (!mi_page_is_in_full(page))
return;
mi_heap_t *heap = mi_page_heap(page);
mi_page_queue_t *pqfull = &heap->pages[MI_BIN_FULL];
@ -311,22 +349,24 @@ void _mi_page_unfull(mi_page_t* page) {
mi_page_queue_enqueue_from(pq, pqfull, page);
}
static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) {
static void mi_page_to_full(mi_page_t *page, mi_page_queue_t *pq)
{
mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(!mi_page_immediate_available(page));
mi_assert_internal(!mi_page_is_in_full(page));
if (mi_page_is_in_full(page)) return;
if (mi_page_is_in_full(page))
return;
mi_page_queue_enqueue_from(&mi_page_heap(page)->pages[MI_BIN_FULL], pq, page);
_mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
}
// Abandon a page with used blocks at the end of a thread.
// Note: only call if it is ensured that no references exist from
// the `page->heap->thread_delayed_free` into this page.
// Currently only called through `mi_heap_collect_ex` which ensures this.
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
void _mi_page_abandon(mi_page_t *page, mi_page_queue_t *pq)
{
mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
@ -344,7 +384,8 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
#if MI_DEBUG > 1
// check there are no references left..
for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->keys)) {
for (mi_block_t *block = (mi_block_t *)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->keys))
{
mi_assert_internal(_mi_ptr_page(block) != page);
}
#endif
@ -354,9 +395,9 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
_mi_segment_page_abandon(page, segments_tld);
}
// Free a page with no more free blocks
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
void _mi_page_free(mi_page_t *page, mi_page_queue_t *pq, bool force)
{
mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page));
@ -385,7 +426,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
// Note: called from `mi_free` and benchmarks often
// trigger this due to freeing everything and then
// allocating again so careful when changing this.
void _mi_page_retire(mi_page_t* page) {
void _mi_page_retire(mi_page_t *page)
{
mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(mi_page_all_free(page));
@ -399,16 +441,20 @@ void _mi_page_retire(mi_page_t* page) {
// how to check this efficiently though...
// for now, we don't retire if it is the only page left of this size class.
mi_page_queue_t *pq = mi_page_queue_of(page);
if (mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page))) {
if (pq->last==page && pq->first==page) { // the only page in the queue?
if (mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page)))
{
if (pq->last == page && pq->first == page)
{ // the only page in the queue?
mi_stat_counter_increase(_mi_stats_main.page_no_retire, 1);
page->retire_expire = (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES / 4);
mi_heap_t *heap = mi_page_heap(page);
mi_assert_internal(pq >= heap->pages);
const size_t index = pq - heap->pages;
mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);
if (index < heap->page_retired_min) heap->page_retired_min = index;
if (index > heap->page_retired_max) heap->page_retired_max = index;
if (index < heap->page_retired_min)
heap->page_retired_min = index;
if (index > heap->page_retired_max)
heap->page_retired_max = index;
mi_assert_internal(mi_page_all_free(page));
return; // dont't free after all
}
@ -419,25 +465,34 @@ void _mi_page_retire(mi_page_t* page) {
// free retired pages: we don't need to look at the entire queues
// since we only retire pages that are at the head position in a queue.
void _mi_heap_collect_retired(mi_heap_t* heap, bool force) {
void _mi_heap_collect_retired(mi_heap_t *heap, bool force)
{
size_t min = MI_BIN_FULL;
size_t max = 0;
for(size_t bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++) {
for (size_t bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++)
{
mi_page_queue_t *pq = &heap->pages[bin];
mi_page_t *page = pq->first;
if (page != NULL && page->retire_expire != 0) {
if (mi_page_all_free(page)) {
if (page != NULL && page->retire_expire != 0)
{
if (mi_page_all_free(page))
{
page->retire_expire--;
if (force || page->retire_expire == 0) {
if (force || page->retire_expire == 0)
{
_mi_page_free(pq->first, pq, force);
}
else {
else
{
// keep retired, update min/max
if (bin < min) min = bin;
if (bin > max) max = bin;
if (bin < min)
min = bin;
if (bin > max)
max = bin;
}
}
else {
else
{
page->retire_expire = 0;
}
}
@ -446,7 +501,6 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) {
heap->page_retired_max = max;
}
/* -----------------------------------------------------------
Initialize the initial free list in a page.
In secure mode we initialize a randomized list by
@ -457,7 +511,8 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) {
#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT)
#define MI_MIN_SLICES (2)
static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) {
static void mi_page_free_list_extend_secure(mi_heap_t *const heap, mi_page_t *const page, const size_t bsize, const size_t extend, mi_stats_t *const stats)
{
UNUSED(stats);
#if (MI_SECURE <= 2)
mi_assert_internal(page->free == NULL);
@ -470,7 +525,8 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
// initialize a randomized free list
// set up `slice_count` slices to alternate between
size_t shift = MI_MAX_SLICE_SHIFT;
while ((extend >> shift) == 0) {
while ((extend >> shift) == 0)
{
shift--;
}
const size_t slice_count = (size_t)1U << shift;
@ -478,7 +534,8 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
mi_assert_internal(slice_extend >= 1);
mi_block_t *blocks[MI_MAX_SLICES]; // current start of the slice
size_t counts[MI_MAX_SLICES]; // available objects in the slice
for (size_t i = 0; i < slice_count; i++) {
for (size_t i = 0; i < slice_count; i++)
{
blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i * slice_extend);
counts[i] = slice_extend;
}
@ -492,15 +549,19 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
mi_block_t *const free_start = blocks[current];
// and iterate through the rest; use `random_shuffle` for performance
uintptr_t rnd = _mi_random_shuffle(r | 1); // ensure not 0
for (size_t i = 1; i < extend; i++) {
for (size_t i = 1; i < extend; i++)
{
// call random_shuffle only every INTPTR_SIZE rounds
const size_t round = i % MI_INTPTR_SIZE;
if (round == 0) rnd = _mi_random_shuffle(rnd);
if (round == 0)
rnd = _mi_random_shuffle(rnd);
// select a random next slice index
size_t next = ((rnd >> 8 * round) & (slice_count - 1));
while (counts[next]==0) { // ensure it still has space
while (counts[next] == 0)
{ // ensure it still has space
next++;
if (next==slice_count) next = 0;
if (next == slice_count)
next = 0;
}
// and link the current block to it
counts[next]--;
@ -530,7 +591,8 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
// initialize a sequential free list
mi_block_t *const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);
mi_block_t *block = start;
while(block <= last) {
while (block <= last)
{
mi_block_t *next = (mi_block_t *)((uint8_t *)block + bsize);
mi_block_set_next(page, block, next);
block = next;
@ -556,14 +618,17 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
// Note: we also experimented with "bump" allocation on the first
// allocations but this did not speed up any benchmark (due to an
// extra test in malloc? or cache effects?)
static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) {
static void mi_page_extend_free(mi_heap_t *heap, mi_page_t *page, mi_tld_t *tld)
{
mi_assert_expensive(mi_page_is_valid_init(page));
#if (MI_SECURE <= 2)
mi_assert(page->free == NULL);
mi_assert(page->local_free == NULL);
if (page->free != NULL) return;
if (page->free != NULL)
return;
#endif
if (page->capacity >= page->reserved) return;
if (page->capacity >= page->reserved)
return;
size_t page_size;
//uint8_t* page_start =
@ -574,9 +639,11 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
const size_t bsize = (page->xblock_size < MI_HUGE_BLOCK_SIZE ? page->xblock_size : page_size);
size_t extend = page->reserved - page->capacity;
size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE / (uint32_t)bsize);
if (max_extend < MI_MIN_EXTEND) max_extend = MI_MIN_EXTEND;
if (max_extend < MI_MIN_EXTEND)
max_extend = MI_MIN_EXTEND;
if (extend > max_extend) {
if (extend > max_extend)
{
// ensure we don't touch memory beyond the page to reduce page commit.
// the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%.
extend = (max_extend == 0 ? 1 : max_extend);
@ -586,10 +653,12 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
mi_assert_internal(extend < (1UL << 16));
// and append the extend the free list
if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) {
if (extend < MI_MIN_SLICES || MI_SECURE == 0)
{ //!mi_option_is_enabled(mi_option_secure)) {
mi_page_free_list_extend(page, bsize, extend, &tld->stats);
}
else {
else
{
mi_page_free_list_extend_secure(heap, page, bsize, extend, &tld->stats);
}
// enable the new free list
@ -597,14 +666,16 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
mi_stat_increase(tld->stats.page_committed, extend * bsize);
// extension into zero initialized memory preserves the zero'd free list
if (!page->is_zero_init) {
if (!page->is_zero_init)
{
page->is_zero = false;
}
mi_assert_expensive(mi_page_is_valid_init(page));
}
// Initialize a fresh page
static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi_tld_t* tld) {
static void mi_page_init(mi_heap_t *heap, mi_page_t *page, size_t block_size, mi_tld_t *tld)
{
mi_assert(page != NULL);
mi_segment_t *segment = _mi_page_segment(page);
mi_assert(segment != NULL);
@ -641,7 +712,6 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert(mi_page_immediate_available(page));
}
/* -----------------------------------------------------------
Find pages with free blocks
-------------------------------------------------------------*/
@ -661,12 +731,14 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
_mi_page_free_collect(page, false);
// 1. if the page contains free blocks, we are done
if (mi_page_immediate_available(page)) {
if (mi_page_immediate_available(page))
{
break; // pick this one
}
// 2. Try to extend
if (page->capacity < page->reserved) {
if (page->capacity < page->reserved)
{
mi_page_extend_free(heap, page, heap->tld);
mi_assert_internal(mi_page_immediate_available(page));
break;
@ -682,15 +754,18 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
mi_stat_counter_increase(heap->tld->stats.searches, count);
if (page == NULL) {
if (page == NULL)
{
_mi_heap_collect_retired(heap, false); // perhaps make a page available
page = mi_page_fresh(heap, pq);
if (page == NULL && first_try) {
if (page == NULL && first_try)
{
// out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again
page = mi_page_queue_find_free_ex(heap, pq, false);
}
}
else {
else
{
mi_assert(pq->first == page);
page->retire_expire = 0;
}
@ -698,15 +773,16 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
return page;
}
// Find a page with free blocks of `size`.
static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
static inline mi_page_t *mi_find_free_page(mi_heap_t *heap, size_t size)
{
mi_page_queue_t *pq = mi_page_queue(heap, size);
mi_page_t *page = pq->first;
if (page != NULL) {
if (page != NULL)
{
#if (MI_SECURE >= 3) // in secure mode, we extend half the time to increase randomness
if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) {
if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1))
{
mi_page_extend_free(heap, page, heap->tld);
mi_assert_internal(mi_page_immediate_available(page));
}
@ -716,7 +792,8 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
_mi_page_free_collect(page, false);
}
if (mi_page_immediate_available(page)) {
if (mi_page_immediate_available(page))
{
page->retire_expire = 0;
return page; // fast path
}
@ -724,7 +801,6 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
return mi_page_queue_find_free_ex(heap, pq, true);
}
/* -----------------------------------------------------------
Users can register a deferred free function called
when the `free` list is empty. Since the `local_free`
@ -735,21 +811,29 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
static mi_deferred_free_fun *volatile deferred_free = NULL;
static _Atomic(void *) deferred_arg; // = NULL
void _mi_deferred_free(mi_heap_t* heap, bool force) {
void _mi_deferred_free(mi_heap_t *heap, bool force)
{
heap->tld->heartbeat++;
if (deferred_free != NULL && !heap->tld->recurse) {
if (heap->deferred_free != NULL && !heap->tld->recurse)
{
heap->tld->recurse = true;
((mi_local_deferred_free_fun *)heap->deferred_free)(heap, force, heap->tld->heartbeat, heap->deferred_arg);
heap->tld->recurse = false;
}
if (deferred_free != NULL && !heap->tld->recurse)
{
heap->tld->recurse = true;
deferred_free(force, heap->tld->heartbeat, mi_atomic_load_ptr_relaxed(void, &deferred_arg));
heap->tld->recurse = false;
}
}
void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept {
void mi_register_deferred_free(mi_deferred_free_fun *fn, void *arg) mi_attr_noexcept
{
deferred_free = fn;
mi_atomic_store_ptr_release(void, &deferred_arg, arg);
}
/* -----------------------------------------------------------
General allocation
----------------------------------------------------------- */
@ -758,11 +842,13 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex
// Because huge pages contain just one block, and the segment contains
// just that page, we always treat them as abandoned and any thread
// that frees the block can free the whole page and segment directly.
static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
static mi_page_t *mi_huge_page_alloc(mi_heap_t *heap, size_t size)
{
size_t block_size = _mi_os_good_alloc_size(size);
mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE);
mi_page_t *page = mi_page_fresh_alloc(heap, NULL, block_size);
if (page != NULL) {
if (page != NULL)
{
const size_t bsize = mi_page_block_size(page); // note: not `mi_page_usable_block_size` as `size` includes padding already
mi_assert_internal(bsize >= size);
mi_assert_internal(mi_page_immediate_available(page));
@ -771,11 +857,13 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
mi_assert_internal(_mi_page_segment(page)->thread_id == 0); // abandoned, not in the huge queue
mi_page_set_heap(page, NULL);
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
if (bsize > MI_HUGE_OBJ_SIZE_MAX)
{
_mi_stat_increase(&heap->tld->stats.giant, bsize);
_mi_stat_counter_increase(&heap->tld->stats.giant_count, 1);
}
else {
else
{
_mi_stat_increase(&heap->tld->stats.huge, bsize);
_mi_stat_counter_increase(&heap->tld->stats.huge_count, 1);
}
@ -783,22 +871,26 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
return page;
}
// Allocate a page
// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size) mi_attr_noexcept {
static mi_page_t *mi_find_page(mi_heap_t *heap, size_t size) mi_attr_noexcept
{
// huge allocation?
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE) )) {
if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE)))
{
if (mi_unlikely(req_size > PTRDIFF_MAX))
{ // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
_mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
return NULL;
}
else {
else
{
return mi_huge_page_alloc(heap, size);
}
}
else {
else
{
// otherwise find a page with free blocks in our size segregated queues
mi_assert_internal(size >= MI_PADDING_SIZE);
return mi_find_free_page(heap, size);
@ -812,10 +904,14 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
mi_assert_internal(heap != NULL);
// initialize if necessary
if (mi_unlikely(!mi_heap_is_initialized(heap))) {
if (mi_unlikely(!mi_heap_is_initialized(heap)))
{
mi_thread_init(); // calls `_mi_heap_init` in turn
heap = mi_get_default_heap();
if (mi_unlikely(!mi_heap_is_initialized(heap))) { return NULL; }
if (mi_unlikely(!mi_heap_is_initialized(heap)))
{
return NULL;
}
}
mi_assert_internal(mi_heap_is_initialized(heap));
@ -827,12 +923,14 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
// find (or allocate) a page of the right size
mi_page_t *page = mi_find_page(heap, size);
if (mi_unlikely(page == NULL)) { // first time out of memory, try to collect and retry the allocation once more
if (mi_unlikely(page == NULL))
{ // first time out of memory, try to collect and retry the allocation once more
mi_heap_collect(heap, true /* force */);
page = mi_find_page(heap, size);
}
if (mi_unlikely(page == NULL)) { // out of memory
if (mi_unlikely(page == NULL))
{ // out of memory
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
_mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);
return NULL;