kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-20 22:19:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Heap local deferred free fun

Browse source
This commit is contained in:
playX 2021-03-09 19:12:52 +03:00
parent 217f2e2cc7
commit f459d576bd
5 changed files with 1489 additions and 1214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

222
include/mimalloc-types.h
View file

@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <mimalloc-atomic.h> // _Atomic #include <mimalloc-atomic.h> // _Atomic
#ifdef _MSC_VER #ifdef _MSC_VER
#pragma warning(disable:4214) // bitfield is not int #pragma warning(disable : 4214) // bitfield is not int
#endif #endif
// Minimal alignment necessary. On most platforms 16 bytes are needed // Minimal alignment necessary. On most platforms 16 bytes are needed
@ -56,14 +56,13 @@ terms of the MIT license. A copy of the license can be found in the file
// Reserve extra padding at the end of each block to be more resilient against heap block overflows. // Reserve extra padding at the end of each block to be more resilient against heap block overflows.
// The padding can detect byte-precise buffer overflow on free. // The padding can detect byte-precise buffer overflow on free.
#if !defined(MI_PADDING) && (MI_DEBUG>=1) #if !defined(MI_PADDING) && (MI_DEBUG >= 1)
#define MI_PADDING 1 #define MI_PADDING 1
#endif #endif
// Encoded free lists allow detection of corrupted free lists // Encoded free lists allow detection of corrupted free lists
// and can detect buffer overflows, modify after free, and double `free`s. // and can detect buffer overflows, modify after free, and double `free`s.
#if (MI_SECURE>=3 || MI_DEBUG>=1 || MI_PADDING > 0) #if (MI_SECURE >= 3 || MI_DEBUG >= 1 || MI_PADDING > 0)
#define MI_ENCODE_FREELIST 1 #define MI_ENCODE_FREELIST 1
#endif #endif
@ -84,20 +83,19 @@ terms of the MIT license. A copy of the license can be found in the file
// ------------------------------------------------------ // ------------------------------------------------------
#if INTPTR_MAX == 9223372036854775807LL #if INTPTR_MAX == 9223372036854775807LL
# define MI_INTPTR_SHIFT (3) #define MI_INTPTR_SHIFT (3)
#elif INTPTR_MAX == 2147483647LL #elif INTPTR_MAX == 2147483647LL
# define MI_INTPTR_SHIFT (2) #define MI_INTPTR_SHIFT (2)
#else #else
#error platform must be 32 or 64 bits #error platform must be 32 or 64 bits
#endif #endif
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT) #define MI_INTPTR_SIZE (1 << MI_INTPTR_SHIFT)
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8) #define MI_INTPTR_BITS (MI_INTPTR_SIZE * 8)
#define KiB ((size_t)1024) #define KiB ((size_t)1024)
#define MiB (KiB*KiB) #define MiB (KiB * KiB)
#define GiB (MiB*KiB) #define GiB (MiB * KiB)
// ------------------------------------------------------ // ------------------------------------------------------
// Main internal data-structures // Main internal data-structures
@ -106,29 +104,29 @@ terms of the MIT license. A copy of the license can be found in the file
// Main tuning parameters for segment and page sizes // Main tuning parameters for segment and page sizes
// Sizes for 64-bit, divide by two for 32-bit // Sizes for 64-bit, divide by two for 32-bit
#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb #define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb
#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb #define MI_MEDIUM_PAGE_SHIFT (3 + MI_SMALL_PAGE_SHIFT) // 512kb
#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb #define MI_LARGE_PAGE_SHIFT (3 + MI_MEDIUM_PAGE_SHIFT) // 4mb
#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb #define MI_SEGMENT_SHIFT (MI_LARGE_PAGE_SHIFT) // 4mb
// Derived constants // Derived constants
#define MI_SEGMENT_SIZE (1UL<<MI_SEGMENT_SHIFT) #define MI_SEGMENT_SIZE (1UL << MI_SEGMENT_SHIFT)
#define MI_SEGMENT_MASK ((uintptr_t)MI_SEGMENT_SIZE - 1) #define MI_SEGMENT_MASK ((uintptr_t)MI_SEGMENT_SIZE - 1)
#define MI_SMALL_PAGE_SIZE (1UL<<MI_SMALL_PAGE_SHIFT) #define MI_SMALL_PAGE_SIZE (1UL << MI_SMALL_PAGE_SHIFT)
#define MI_MEDIUM_PAGE_SIZE (1UL<<MI_MEDIUM_PAGE_SHIFT) #define MI_MEDIUM_PAGE_SIZE (1UL << MI_MEDIUM_PAGE_SHIFT)
#define MI_LARGE_PAGE_SIZE (1UL<<MI_LARGE_PAGE_SHIFT) #define MI_LARGE_PAGE_SIZE (1UL << MI_LARGE_PAGE_SHIFT)
#define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE) #define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_SMALL_PAGE_SIZE)
#define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_MEDIUM_PAGE_SIZE) #define MI_MEDIUM_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_MEDIUM_PAGE_SIZE)
#define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE) #define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE / MI_LARGE_PAGE_SIZE)
// The max object size are checked to not waste more than 12.5% internally over the page sizes. // The max object size are checked to not waste more than 12.5% internally over the page sizes.
// (Except for large pages since huge objects are allocated in 4MiB chunks) // (Except for large pages since huge objects are allocated in 4MiB chunks)
#define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE/4) // 16kb #define MI_SMALL_OBJ_SIZE_MAX (MI_SMALL_PAGE_SIZE / 4) // 16kb
#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128kb #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE / 4) // 128kb
#define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE/2) // 2mb #define MI_LARGE_OBJ_SIZE_MAX (MI_LARGE_PAGE_SIZE / 2) // 2mb
#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX / MI_INTPTR_SIZE)
#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) #define MI_HUGE_OBJ_SIZE_MAX (2 * MI_INTPTR_SIZE * MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)
// Maximum number of size classes. (spaced exponentially in 12.5% increments) // Maximum number of size classes. (spaced exponentially in 12.5% increments)
#define MI_BIN_HUGE (73U) #define MI_BIN_HUGE (73U)
@ -145,35 +143,39 @@ terms of the MIT license. A copy of the license can be found in the file
typedef uintptr_t mi_encoded_t; typedef uintptr_t mi_encoded_t;
// free lists contain blocks // free lists contain blocks
typedef struct mi_block_s { typedef struct mi_block_s
{
mi_encoded_t next; mi_encoded_t next;
} mi_block_t; } mi_block_t;
// The delayed flags are used for efficient multi-threaded free-ing // The delayed flags are used for efficient multi-threaded free-ing
typedef enum mi_delayed_e { typedef enum mi_delayed_e
{
MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list
MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap
MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list
MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim
} mi_delayed_t; } mi_delayed_t;
// The `in_full` and `has_aligned` page flags are put in a union to efficiently // The `in_full` and `has_aligned` page flags are put in a union to efficiently
// test if both are false (`full_aligned == 0`) in the `mi_free` routine. // test if both are false (`full_aligned == 0`) in the `mi_free` routine.
#if !MI_TSAN #if !MI_TSAN
typedef union mi_page_flags_s { typedef union mi_page_flags_s
{
uint8_t full_aligned; uint8_t full_aligned;
struct { struct
{
uint8_t in_full : 1; uint8_t in_full : 1;
uint8_t has_aligned : 1; uint8_t has_aligned : 1;
} x; } x;
} mi_page_flags_t; } mi_page_flags_t;
#else #else
// under thread sanitizer, use a byte for each flag to suppress warning, issue #130 // under thread sanitizer, use a byte for each flag to suppress warning, issue #130
typedef union mi_page_flags_s { typedef union mi_page_flags_s
{
uint16_t full_aligned; uint16_t full_aligned;
struct { struct
{
uint8_t in_full; uint8_t in_full;
uint8_t has_aligned; uint8_t has_aligned;
} x; } x;
@ -216,39 +218,39 @@ typedef uintptr_t mi_thread_free_t;
// at least one block that will be added, or as already been added, to // at least one block that will be added, or as already been added, to
// the owning heap `thread_delayed_free` list. This guarantees that pages // the owning heap `thread_delayed_free` list. This guarantees that pages
// will be freed correctly even if only other threads free blocks. // will be freed correctly even if only other threads free blocks.
typedef struct mi_page_s { typedef struct mi_page_s
{
// "owned" by the segment // "owned" by the segment
uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
uint8_t segment_in_use:1; // `true` if the segment allocated this page uint8_t segment_in_use : 1; // `true` if the segment allocated this page
uint8_t is_reset:1; // `true` if the page memory was reset uint8_t is_reset : 1; // `true` if the page memory was reset
uint8_t is_committed:1; // `true` if the page virtual memory is committed uint8_t is_committed : 1; // `true` if the page virtual memory is committed
uint8_t is_zero_init:1; // `true` if the page was zero initialized uint8_t is_zero_init : 1; // `true` if the page was zero initialized
// layout like this to optimize access in `mi_malloc` and `mi_free` // layout like this to optimize access in `mi_malloc` and `mi_free`
uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear`
uint16_t reserved; // number of blocks reserved in memory uint16_t reserved; // number of blocks reserved in memory
mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits) mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits)
uint8_t is_zero:1; // `true` if the blocks in the free list are zero initialized uint8_t is_zero : 1; // `true` if the blocks in the free list are zero initialized
uint8_t retire_expire:7; // expiration count for retired blocks uint8_t retire_expire : 7; // expiration count for retired blocks
mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) mi_block_t *free; // list of available free blocks (`malloc` allocates from this list)
#ifdef MI_ENCODE_FREELIST #ifdef MI_ENCODE_FREELIST
uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`) uintptr_t keys[2]; // two random keys to encode the free lists (see `_mi_block_next`)
#endif #endif
uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) uint32_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`)
uint32_t xblock_size; // size available in each block (always `>0`) uint32_t xblock_size; // size available in each block (always `>0`)
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) mi_block_t *local_free; // list of deferred free blocks by this thread (migrates to `free`)
_Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads
_Atomic(uintptr_t) xheap; _Atomic(uintptr_t) xheap;
struct mi_page_s* next; // next page owned by this thread with the same `block_size` struct mi_page_s *next; // next page owned by this thread with the same `block_size`
struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` struct mi_page_s *prev; // previous page owned by this thread with the same `block_size`
} mi_page_t; } mi_page_t;
typedef enum mi_page_kind_e
{
typedef enum mi_page_kind_e {
MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment
MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
@ -258,16 +260,17 @@ typedef enum mi_page_kind_e {
// Segments are large allocated memory blocks (2mb on 64 bit) from // Segments are large allocated memory blocks (2mb on 64 bit) from
// the OS. Inside segments we allocated fixed size _pages_ that // the OS. Inside segments we allocated fixed size _pages_ that
// contain blocks. // contain blocks.
typedef struct mi_segment_s { typedef struct mi_segment_s
{
// memory fields // memory fields
size_t memid; // id for the os-level memory manager size_t memid; // id for the os-level memory manager
bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages) bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
bool mem_is_committed; // `true` if the whole segment is eagerly committed bool mem_is_committed; // `true` if the whole segment is eagerly committed
// segment fields // segment fields
_Atomic(struct mi_segment_s*) abandoned_next; _Atomic(struct mi_segment_s *) abandoned_next;
struct mi_segment_s* next; // must be the first segment field after abandoned_next -- see `segment.c:segment_init` struct mi_segment_s *next; // must be the first segment field after abandoned_next -- see `segment.c:segment_init`
struct mi_segment_s* prev; struct mi_segment_s *prev;
size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long) size_t abandoned_visits; // count how often this segment is visited in the abandoned list (to force reclaim it it is too long)
@ -275,7 +278,7 @@ typedef struct mi_segment_s {
size_t used; // count of pages in use (`used <= capacity`) size_t used; // count of pages in use (`used <= capacity`)
size_t capacity; // count of available pages (`#free + used`) size_t capacity; // count of available pages (`#free + used`)
size_t segment_size; // for huge pages this may be different from `MI_SEGMENT_SIZE` size_t segment_size; // for huge pages this may be different from `MI_SEGMENT_SIZE`
size_t segment_info_size;// space we are using from the first page for segment meta-data and possible guard pages. size_t segment_info_size; // space we are using from the first page for segment meta-data and possible guard pages.
uintptr_t cookie; // verify addresses in secure mode: `_mi_ptr_cookie(segment) == segment->cookie` uintptr_t cookie; // verify addresses in secure mode: `_mi_ptr_cookie(segment) == segment->cookie`
// layout like this to optimize access in `mi_free` // layout like this to optimize access in `mi_free`
@ -285,7 +288,6 @@ typedef struct mi_segment_s {
mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages
} mi_segment_t; } mi_segment_t;
// ------------------------------------------------------ // ------------------------------------------------------
// Heaps // Heaps
// Provide first-class heaps to allocate from. // Provide first-class heaps to allocate from.
@ -303,25 +305,27 @@ typedef struct mi_segment_s {
typedef struct mi_tld_s mi_tld_t; typedef struct mi_tld_s mi_tld_t;
// Pages of a certain block size are held in a queue. // Pages of a certain block size are held in a queue.
typedef struct mi_page_queue_s { typedef struct mi_page_queue_s
mi_page_t* first; {
mi_page_t* last; mi_page_t *first;
mi_page_t *last;
size_t block_size; size_t block_size;
} mi_page_queue_t; } mi_page_queue_t;
#define MI_BIN_FULL (MI_BIN_HUGE+1) #define MI_BIN_FULL (MI_BIN_HUGE + 1)
// Random context // Random context
typedef struct mi_random_cxt_s { typedef struct mi_random_cxt_s
{
uint32_t input[16]; uint32_t input[16];
uint32_t output[16]; uint32_t output[16];
int output_available; int output_available;
} mi_random_ctx_t; } mi_random_ctx_t;
// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows // In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows
#if (MI_PADDING) #if (MI_PADDING)
typedef struct mi_padding_s { typedef struct mi_padding_s
{
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
} mi_padding_t; } mi_padding_t;
@ -334,13 +338,13 @@ typedef struct mi_padding_s {
#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1) #define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
// A heap owns a set of pages. // A heap owns a set of pages.
struct mi_heap_s { struct mi_heap_s
mi_tld_t* tld; {
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_tld_t *tld;
mi_page_t *pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
_Atomic(mi_block_t*) thread_delayed_free; _Atomic(mi_block_t *) thread_delayed_free;
uintptr_t thread_id; // thread this heap belongs too uintptr_t thread_id; // thread this heap belongs too
uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`)
uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list
@ -348,12 +352,12 @@ struct mi_heap_s {
size_t page_count; // total number of pages in the `pages` queues. size_t page_count; // total number of pages in the `pages` queues.
size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues)
size_t page_retired_max; // largest retired index into the `pages` array. size_t page_retired_max; // largest retired index into the `pages` array.
mi_heap_t* next; // list of heaps per thread mi_heap_t *next; // list of heaps per thread
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
void *deferred_free;
void *deferred_arg;
}; };
// ------------------------------------------------------ // ------------------------------------------------------
// Debug // Debug
// ------------------------------------------------------ // ------------------------------------------------------
@ -364,19 +368,19 @@ struct mi_heap_s {
#if (MI_DEBUG) #if (MI_DEBUG)
// use our own assertion to print without memory allocation // use our own assertion to print without memory allocation
void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); void _mi_assert_fail(const char *assertion, const char *fname, unsigned int line, const char *func);
#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr, __FILE__, __LINE__, __func__))
#else #else
#define mi_assert(x) #define mi_assert(x)
#endif #endif
#if (MI_DEBUG>1) #if (MI_DEBUG > 1)
#define mi_assert_internal mi_assert #define mi_assert_internal mi_assert
#else #else
#define mi_assert_internal(x) #define mi_assert_internal(x)
#endif #endif
#if (MI_DEBUG>2) #if (MI_DEBUG > 2)
#define mi_assert_expensive mi_assert #define mi_assert_expensive mi_assert
#else #else
#define mi_assert_expensive(x) #define mi_assert_expensive(x)
@ -387,26 +391,29 @@ void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line
// ------------------------------------------------------ // ------------------------------------------------------
#ifndef MI_STAT #ifndef MI_STAT
#if (MI_DEBUG>0) #if (MI_DEBUG > 0)
#define MI_STAT 2 #define MI_STAT 2
#else #else
#define MI_STAT 0 #define MI_STAT 0
#endif #endif
#endif #endif
typedef struct mi_stat_count_s { typedef struct mi_stat_count_s
{
int64_t allocated; int64_t allocated;
int64_t freed; int64_t freed;
int64_t peak; int64_t peak;
int64_t current; int64_t current;
} mi_stat_count_t; } mi_stat_count_t;
typedef struct mi_stat_counter_s { typedef struct mi_stat_counter_s
{
int64_t total; int64_t total;
int64_t count; int64_t count;
} mi_stat_counter_t; } mi_stat_counter_t;
typedef struct mi_stats_s { typedef struct mi_stats_s
{
mi_stat_count_t segments; mi_stat_count_t segments;
mi_stat_count_t pages; mi_stat_count_t pages;
mi_stat_count_t reserved; mi_stat_count_t reserved;
@ -429,29 +436,28 @@ typedef struct mi_stats_s {
mi_stat_counter_t normal_count; mi_stat_counter_t normal_count;
mi_stat_counter_t huge_count; mi_stat_counter_t huge_count;
mi_stat_counter_t giant_count; mi_stat_counter_t giant_count;
#if MI_STAT>1 #if MI_STAT > 1
mi_stat_count_t normal_bins[MI_BIN_HUGE+1]; mi_stat_count_t normal_bins[MI_BIN_HUGE + 1];
#endif #endif
} mi_stats_t; } mi_stats_t;
void _mi_stat_increase(mi_stat_count_t *stat, size_t amount);
void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); void _mi_stat_decrease(mi_stat_count_t *stat, size_t amount);
void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); void _mi_stat_counter_increase(mi_stat_counter_t *stat, size_t amount);
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
#if (MI_STAT) #if (MI_STAT)
#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) #define mi_stat_increase(stat, amount) _mi_stat_increase(&(stat), amount)
#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) #define mi_stat_decrease(stat, amount) _mi_stat_decrease(&(stat), amount)
#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) #define mi_stat_counter_increase(stat, amount) _mi_stat_counter_increase(&(stat), amount)
#else #else
#define mi_stat_increase(stat,amount) (void)0 #define mi_stat_increase(stat, amount) (void)0
#define mi_stat_decrease(stat,amount) (void)0 #define mi_stat_decrease(stat, amount) (void)0
#define mi_stat_counter_increase(stat,amount) (void)0 #define mi_stat_counter_increase(stat, amount) (void)0
#endif #endif
#define mi_heap_stat_counter_increase(heap,stat,amount) mi_stat_counter_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_counter_increase(heap, stat, amount) mi_stat_counter_increase((heap)->tld->stats.stat, amount)
#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_increase(heap, stat, amount) mi_stat_increase((heap)->tld->stats.stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) #define mi_heap_stat_decrease(heap, stat, amount) mi_stat_decrease((heap)->tld->stats.stat, amount)
// ------------------------------------------------------ // ------------------------------------------------------
// Thread Local data // Thread Local data
@ -460,19 +466,22 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
typedef int64_t mi_msecs_t; typedef int64_t mi_msecs_t;
// Queue of segments // Queue of segments
typedef struct mi_segment_queue_s { typedef struct mi_segment_queue_s
mi_segment_t* first; {
mi_segment_t* last; mi_segment_t *first;
mi_segment_t *last;
} mi_segment_queue_t; } mi_segment_queue_t;
// OS thread local data // OS thread local data
typedef struct mi_os_tld_s { typedef struct mi_os_tld_s
{
size_t region_idx; // start point for next allocation size_t region_idx; // start point for next allocation
mi_stats_t* stats; // points to tld stats mi_stats_t *stats; // points to tld stats
} mi_os_tld_t; } mi_os_tld_t;
// Segments thread local data // Segments thread local data
typedef struct mi_segments_tld_s { typedef struct mi_segments_tld_s
{
mi_segment_queue_t small_free; // queue of segments with free small pages mi_segment_queue_t small_free; // queue of segments with free small pages
mi_segment_queue_t medium_free; // queue of segments with free medium pages mi_segment_queue_t medium_free; // queue of segments with free medium pages
mi_page_queue_t pages_reset; // queue of freed pages that can be reset mi_page_queue_t pages_reset; // queue of freed pages that can be reset
@ -482,17 +491,18 @@ typedef struct mi_segments_tld_s {
size_t peak_size; // peak size of all segments size_t peak_size; // peak size of all segments
size_t cache_count; // number of segments in the cache size_t cache_count; // number of segments in the cache
size_t cache_size; // total size of all segments in the cache size_t cache_size; // total size of all segments in the cache
mi_segment_t* cache; // (small) cache of segments mi_segment_t *cache; // (small) cache of segments
mi_stats_t* stats; // points to tld stats mi_stats_t *stats; // points to tld stats
mi_os_tld_t* os; // points to os stats mi_os_tld_t *os; // points to os stats
} mi_segments_tld_t; } mi_segments_tld_t;
// Thread local data // Thread local data
struct mi_tld_s { struct mi_tld_s
{
unsigned long long heartbeat; // monotonic heartbeat count unsigned long long heartbeat; // monotonic heartbeat count
bool recurse; // true if deferred was called; used to prevent infinite recursion. bool recurse; // true if deferred was called; used to prevent infinite recursion.
mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) mi_heap_t *heap_backing; // backing heap of this thread (cannot be deleted)
mi_heap_t* heaps; // list of heaps in this thread (so we can abandon all when the thread terminates) mi_heap_t *heaps; // list of heaps in this thread (so we can abandon all when the thread terminates)
mi_segments_tld_t segments; // segment tld mi_segments_tld_t segments; // segment tld
mi_os_tld_t os; // os tld mi_os_tld_t os; // os tld
mi_stats_t stats; // statistics mi_stats_t stats; // statistics

573
include/mimalloc.h
View file

@ -15,74 +15,74 @@ terms of the MIT license. A copy of the license can be found in the file
// ------------------------------------------------------ // ------------------------------------------------------
#ifdef __cplusplus #ifdef __cplusplus
#if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11
#define mi_attr_noexcept noexcept #define mi_attr_noexcept noexcept
#else
#define mi_attr_noexcept throw()
#endif
#else #else
#define mi_attr_noexcept #define mi_attr_noexcept throw()
#endif
#else
#define mi_attr_noexcept
#endif #endif
#if defined(__cplusplus) && (__cplusplus >= 201703) #if defined(__cplusplus) && (__cplusplus >= 201703)
#define mi_decl_nodiscard [[nodiscard]] #define mi_decl_nodiscard [[nodiscard]]
#elif (__GNUC__ >= 4) || defined(__clang__) // includes clang, icc, and clang-cl #elif (__GNUC__ >= 4) || defined(__clang__) // includes clang, icc, and clang-cl
#define mi_decl_nodiscard __attribute__((warn_unused_result)) #define mi_decl_nodiscard __attribute__((warn_unused_result))
#elif (_MSC_VER >= 1700) #elif (_MSC_VER >= 1700)
#define mi_decl_nodiscard _Check_return_ #define mi_decl_nodiscard _Check_return_
#else #else
#define mi_decl_nodiscard #define mi_decl_nodiscard
#endif #endif
#if defined(_MSC_VER) || defined(__MINGW32__) #if defined(_MSC_VER) || defined(__MINGW32__)
#if !defined(MI_SHARED_LIB) #if !defined(MI_SHARED_LIB)
#define mi_decl_export #define mi_decl_export
#elif defined(MI_SHARED_LIB_EXPORT) #elif defined(MI_SHARED_LIB_EXPORT)
#define mi_decl_export __declspec(dllexport) #define mi_decl_export __declspec(dllexport)
#else
#define mi_decl_export __declspec(dllimport)
#endif
#if defined(__MINGW32__)
#define mi_decl_restrict
#define mi_attr_malloc __attribute__((malloc))
#else
#if (_MSC_VER >= 1900) && !defined(__EDG__)
#define mi_decl_restrict __declspec(allocator) __declspec(restrict)
#else
#define mi_decl_restrict __declspec(restrict)
#endif
#define mi_attr_malloc
#endif
#define mi_cdecl __cdecl
#define mi_attr_alloc_size(s)
#define mi_attr_alloc_size2(s1,s2)
#define mi_attr_alloc_align(p)
#elif defined(__GNUC__) // includes clang and icc
#define mi_cdecl // leads to warnings... __attribute__((cdecl))
#define mi_decl_export __attribute__((visibility("default")))
#define mi_decl_restrict
#define mi_attr_malloc __attribute__((malloc))
#if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
#define mi_attr_alloc_size(s)
#define mi_attr_alloc_size2(s1,s2)
#define mi_attr_alloc_align(p)
#elif defined(__INTEL_COMPILER)
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
#define mi_attr_alloc_align(p)
#else
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
#define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))
#endif
#else #else
#define mi_cdecl #define mi_decl_export __declspec(dllimport)
#define mi_decl_export #endif
#define mi_decl_restrict #if defined(__MINGW32__)
#define mi_attr_malloc #define mi_decl_restrict
#define mi_attr_alloc_size(s) #define mi_attr_malloc __attribute__((malloc))
#define mi_attr_alloc_size2(s1,s2) #else
#define mi_attr_alloc_align(p) #if (_MSC_VER >= 1900) && !defined(__EDG__)
#define mi_decl_restrict __declspec(allocator) __declspec(restrict)
#else
#define mi_decl_restrict __declspec(restrict)
#endif
#define mi_attr_malloc
#endif
#define mi_cdecl __cdecl
#define mi_attr_alloc_size(s)
#define mi_attr_alloc_size2(s1, s2)
#define mi_attr_alloc_align(p)
#elif defined(__GNUC__) // includes clang and icc
#define mi_cdecl // leads to warnings... __attribute__((cdecl))
#define mi_decl_export __attribute__((visibility("default")))
#define mi_decl_restrict
#define mi_attr_malloc __attribute__((malloc))
#if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5)
#define mi_attr_alloc_size(s)
#define mi_attr_alloc_size2(s1, s2)
#define mi_attr_alloc_align(p)
#elif defined(__INTEL_COMPILER)
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
#define mi_attr_alloc_size2(s1, s2) __attribute__((alloc_size(s1, s2)))
#define mi_attr_alloc_align(p)
#else
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
#define mi_attr_alloc_size2(s1, s2) __attribute__((alloc_size(s1, s2)))
#define mi_attr_alloc_align(p) __attribute__((alloc_align(p)))
#endif
#else
#define mi_cdecl
#define mi_decl_export
#define mi_decl_restrict
#define mi_attr_malloc
#define mi_attr_alloc_size(s)
#define mi_attr_alloc_size2(s1, s2)
#define mi_attr_alloc_align(p)
#endif #endif
// ------------------------------------------------------ // ------------------------------------------------------
@ -93,209 +93,207 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stdbool.h> // bool #include <stdbool.h> // bool
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C"
{
#endif #endif
// ------------------------------------------------------ // ------------------------------------------------------
// Standard malloc interface // Standard malloc interface
// ------------------------------------------------------ // ------------------------------------------------------
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void *mi_realloc(void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); mi_decl_export void *mi_expand(void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_export void mi_free(void* p) mi_attr_noexcept; mi_decl_export void mi_free(void *p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc; mi_decl_nodiscard mi_decl_export mi_decl_restrict char *mi_strdup(const char *s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; mi_decl_nodiscard mi_decl_export mi_decl_restrict char *mi_strndup(const char *s, size_t n) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; mi_decl_nodiscard mi_decl_export mi_decl_restrict char *mi_realpath(const char *fname, char *resolved_name) mi_attr_noexcept mi_attr_malloc;
// ------------------------------------------------------ // ------------------------------------------------------
// Extended functionality // Extended functionality
// ------------------------------------------------------ // ------------------------------------------------------
#define MI_SMALL_WSIZE_MAX (128) #define MI_SMALL_WSIZE_MAX (128)
#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX * sizeof(void *))
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); mi_decl_nodiscard mi_decl_export void *mi_reallocn(void *p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void *mi_reallocf(void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void *p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
// ------------------------------------------------------
// Internals
// ------------------------------------------------------
// ------------------------------------------------------ typedef void(mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void *arg);
// Internals mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun *deferred_free, void *arg) mi_attr_noexcept;
// ------------------------------------------------------
typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg); typedef void(mi_cdecl mi_output_fun)(const char *msg, void *arg);
mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept; mi_decl_export void mi_register_output(mi_output_fun *out, void *arg) mi_attr_noexcept;
typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg); typedef void(mi_cdecl mi_error_fun)(int err, void *arg);
mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept; mi_decl_export void mi_register_error(mi_error_fun *fun, void *arg);
typedef void (mi_cdecl mi_error_fun)(int err, void* arg); mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); mi_decl_export int mi_version(void) mi_attr_noexcept;
mi_decl_export void mi_stats_reset(void) mi_attr_noexcept;
mi_decl_export void mi_stats_merge(void) mi_attr_noexcept;
mi_decl_export void mi_stats_print(void *out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL
mi_decl_export void mi_stats_print_out(mi_output_fun *out, void *arg) mi_attr_noexcept;
mi_decl_export void mi_collect(bool force) mi_attr_noexcept; mi_decl_export void mi_process_init(void) mi_attr_noexcept;
mi_decl_export int mi_version(void) mi_attr_noexcept; mi_decl_export void mi_thread_init(void) mi_attr_noexcept;
mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; mi_decl_export void mi_thread_done(void) mi_attr_noexcept;
mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; mi_decl_export void mi_thread_stats_print_out(mi_output_fun *out, void *arg) mi_attr_noexcept;
mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL
mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
mi_decl_export void mi_process_init(void) mi_attr_noexcept; mi_decl_export void mi_process_info(size_t *elapsed_msecs, size_t *user_msecs, size_t *system_msecs,
mi_decl_export void mi_thread_init(void) mi_attr_noexcept; size_t *current_rss, size_t *peak_rss,
mi_decl_export void mi_thread_done(void) mi_attr_noexcept; size_t *current_commit, size_t *peak_commit, size_t *page_faults) mi_attr_noexcept;
mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept;
mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, // -------------------------------------------------------------------------------------
size_t* current_rss, size_t* peak_rss, // Aligned allocation
size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept; // Note that `alignment` always follows `size` for consistency with unaligned
// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`.
// -------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------------- mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
// Aligned allocation mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
// Note that `alignment` always follows `size` for consistency with unaligned mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
// allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
// ------------------------------------------------------------------------------------- mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1, 2);
mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned(void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); // -------------------------------------------------------------------------------------
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); // Heaps: first-class, but can only allocate from the same thread that created it.
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); // -------------------------------------------------------------------------------------
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
struct mi_heap_s;
typedef struct mi_heap_s mi_heap_t;
// ------------------------------------------------------------------------------------- mi_decl_nodiscard mi_decl_export mi_heap_t *mi_heap_new(void);
// Heaps: first-class, but can only allocate from the same thread that created it. mi_decl_export void mi_heap_delete(mi_heap_t *heap);
// ------------------------------------------------------------------------------------- mi_decl_export void mi_heap_destroy(mi_heap_t *heap);
mi_decl_export mi_heap_t *mi_heap_set_default(mi_heap_t *heap);
mi_decl_export mi_heap_t *mi_heap_get_default(void);
mi_decl_export mi_heap_t *mi_heap_get_backing(void);
mi_decl_export void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept;
typedef void(mi_local_deferred_free_fun)(mi_heap_t *heap, bool force, unsigned long long heartbeat, void *arg);
mi_decl_export void mi_heap_register_local_deferred_free(mi_heap_t *heap, mi_local_deferred_free_fun *deferred_free, void *arg) mi_attr_noexcept;
struct mi_heap_s; mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
typedef struct mi_heap_s mi_heap_t; mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_zalloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_calloc(mi_heap_t *heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_mallocn(mi_heap_t *heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc_small(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); mi_decl_nodiscard mi_decl_export void *mi_heap_realloc(mi_heap_t *heap, void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
mi_decl_export void mi_heap_delete(mi_heap_t* heap); mi_decl_nodiscard mi_decl_export void *mi_heap_reallocn(mi_heap_t *heap, void *p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
mi_decl_export void mi_heap_destroy(mi_heap_t* heap); mi_decl_nodiscard mi_decl_export void *mi_heap_reallocf(mi_heap_t *heap, void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
mi_decl_export mi_heap_t* mi_heap_get_default(void);
mi_decl_export mi_heap_t* mi_heap_get_backing(void);
mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict char *mi_heap_strdup(mi_heap_t *heap, const char *s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict char *mi_heap_strndup(mi_heap_t *heap, const char *s, size_t n) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); mi_decl_nodiscard mi_decl_export mi_decl_restrict char *mi_heap_realpath(mi_heap_t *heap, const char *fname, char *resolved_name) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc_aligned(mi_heap_t *heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc_aligned_at(mi_heap_t *heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_zalloc_aligned(mi_heap_t *heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_zalloc_aligned_at(mi_heap_t *heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_calloc_aligned(mi_heap_t *heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_calloc_aligned_at(mi_heap_t *heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned(mi_heap_t *heap, void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc; // --------------------------------------------------------------------------------
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; // Zero initialized re-allocation.
mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; // Only valid on memory that was originally allocated with zero initialization too.
// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
// --------------------------------------------------------------------------------
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); mi_decl_nodiscard mi_decl_export void *mi_rezalloc(void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void *mi_recalloc(void *p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
mi_decl_nodiscard mi_decl_export void *mi_rezalloc_aligned(void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void *mi_rezalloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void *mi_recalloc_aligned(void *p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export void *mi_recalloc_aligned_at(void *p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2, 3);
// -------------------------------------------------------------------------------- mi_decl_nodiscard mi_decl_export void *mi_heap_rezalloc(mi_heap_t *heap, void *p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3);
// Zero initialized re-allocation. mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc(mi_heap_t *heap, void *p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
// Only valid on memory that was originally allocated with zero initialization too.
// e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc.
// see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992>
// --------------------------------------------------------------------------------
mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void *mi_heap_rezalloc_aligned(mi_heap_t *heap, void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); mi_decl_nodiscard mi_decl_export void *mi_heap_rezalloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned_at(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); // ------------------------------------------------------
mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); // Analysis
mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); // ------------------------------------------------------
mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3);
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); mi_decl_export bool mi_heap_contains_block(mi_heap_t *heap, const void *p);
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); mi_decl_export bool mi_heap_check_owned(mi_heap_t *heap, const void *p);
mi_decl_export bool mi_check_owned(const void *p);
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); // An area of heap space contains blocks of a single size.
mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); typedef struct mi_heap_area_s
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); {
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4); void *blocks; // start of the area containing heap blocks
// ------------------------------------------------------
// Analysis
// ------------------------------------------------------
mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
mi_decl_export bool mi_check_owned(const void* p);
// An area of heap space contains blocks of a single size.
typedef struct mi_heap_area_s {
void* blocks; // start of the area containing heap blocks
size_t reserved; // bytes reserved for this area (virtual) size_t reserved; // bytes reserved for this area (virtual)
size_t committed; // current available bytes for this area size_t committed; // current available bytes for this area
size_t used; // bytes in use by allocated blocks size_t used; // bytes in use by allocated blocks
size_t block_size; // size in bytes of each block size_t block_size; // size in bytes of each block
} mi_heap_area_t; } mi_heap_area_t;
typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); typedef bool(mi_cdecl mi_block_visit_fun)(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg);
mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_all_blocks, mi_block_visit_fun *visitor, void *arg);
// Experimental // Experimental
mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void *p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept;
mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept;
mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept;
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void *start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
// deprecated
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t *pages_reserved) mi_attr_noexcept;
// deprecated // ------------------------------------------------------
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; // Convenience
// ------------------------------------------------------
#define mi_malloc_tp(tp) ((tp *)mi_malloc(sizeof(tp)))
#define mi_zalloc_tp(tp) ((tp *)mi_zalloc(sizeof(tp)))
#define mi_calloc_tp(tp, n) ((tp *)mi_calloc(n, sizeof(tp)))
#define mi_mallocn_tp(tp, n) ((tp *)mi_mallocn(n, sizeof(tp)))
#define mi_reallocn_tp(p, tp, n) ((tp *)mi_reallocn(p, n, sizeof(tp)))
#define mi_recalloc_tp(p, tp, n) ((tp *)mi_recalloc(p, n, sizeof(tp)))
// ------------------------------------------------------ #define mi_heap_malloc_tp(hp, tp) ((tp *)mi_heap_malloc(hp, sizeof(tp)))
// Convenience #define mi_heap_zalloc_tp(hp, tp) ((tp *)mi_heap_zalloc(hp, sizeof(tp)))
// ------------------------------------------------------ #define mi_heap_calloc_tp(hp, tp, n) ((tp *)mi_heap_calloc(hp, n, sizeof(tp)))
#define mi_heap_mallocn_tp(hp, tp, n) ((tp *)mi_heap_mallocn(hp, n, sizeof(tp)))
#define mi_heap_reallocn_tp(hp, p, tp, n) ((tp *)mi_heap_reallocn(hp, p, n, sizeof(tp)))
#define mi_heap_recalloc_tp(hp, p, tp, n) ((tp *)mi_heap_recalloc(hp, p, n, sizeof(tp)))
#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) // ------------------------------------------------------
#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) // Options, all `false` by default
#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) // ------------------------------------------------------
#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))
#define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))
#define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))
#define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) typedef enum mi_option_e
#define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) {
#define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
#define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
#define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp)))
#define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp)))
// ------------------------------------------------------
// Options, all `false` by default
// ------------------------------------------------------
typedef enum mi_option_e {
// stable options // stable options
mi_option_show_errors, mi_option_show_errors,
mi_option_show_stats, mi_option_show_stats,
@ -319,59 +317,57 @@ typedef enum mi_option_e {
mi_option_max_errors, mi_option_max_errors,
mi_option_max_warnings, mi_option_max_warnings,
_mi_option_last _mi_option_last
} mi_option_t; } mi_option_t;
mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
mi_decl_export void mi_option_enable(mi_option_t option);
mi_decl_export void mi_option_disable(mi_option_t option);
mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
mi_decl_export void mi_option_enable(mi_option_t option); mi_decl_export void mi_option_set(mi_option_t option, long value);
mi_decl_export void mi_option_disable(mi_option_t option); mi_decl_export void mi_option_set_default(mi_option_t option, long value);
mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable);
mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable);
mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); // -------------------------------------------------------------------------------------------------------
mi_decl_export void mi_option_set(mi_option_t option, long value); // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
mi_decl_export void mi_option_set_default(mi_option_t option, long value); // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing.
// -------------------------------------------------------------------------------------------------------
mi_decl_export void mi_cfree(void *p) mi_attr_noexcept;
mi_decl_export void *mi__expand(void *p, size_t newsize) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void *p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
// ------------------------------------------------------------------------------------------------------- mi_decl_export int mi_posix_memalign(void **p, size_t alignment, size_t size) mi_attr_noexcept;
// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
// note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing. mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
// ------------------------------------------------------------------------------------------------------- mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void *mi_reallocarray(void *p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2, 3);
mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void *mi_aligned_recalloc(void *p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void *mi_aligned_offset_recalloc(void *p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept;
mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short *mi_wcsdup(const unsigned short *s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char *mi_mbsdup(const unsigned char *s) mi_attr_noexcept mi_attr_malloc;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_export int mi_dupenv_s(char **buf, size_t *size, const char *name) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_export int mi_wdupenv_s(unsigned short **buf, size_t *size, const unsigned short *name) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1);
mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); mi_decl_export void mi_free_size(void *p, size_t size) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void *p, size_t size, size_t alignment) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void *p, size_t alignment) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc; // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);
mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void *mi_new_realloc(void *p, size_t newsize) mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void *mi_new_reallocn(void *p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
// The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`.
// (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception).
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2);
mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3);
#ifdef __cplusplus #ifdef __cplusplus
} }
@ -389,48 +385,71 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
#include <utility> // std::forward #include <utility> // std::forward
#endif #endif
template<class T> struct mi_stl_allocator { template <class T>
struct mi_stl_allocator
{
typedef T value_type; typedef T value_type;
typedef std::size_t size_type; typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type; typedef std::ptrdiff_t difference_type;
typedef value_type& reference; typedef value_type &reference;
typedef value_type const& const_reference; typedef value_type const &const_reference;
typedef value_type* pointer; typedef value_type *pointer;
typedef value_type const* const_pointer; typedef value_type const *const_pointer;
template <class U> struct rebind { typedef mi_stl_allocator<U> other; }; template <class U>
struct rebind
{
typedef mi_stl_allocator<U> other;
};
mi_stl_allocator() mi_attr_noexcept = default; mi_stl_allocator() mi_attr_noexcept = default;
mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default; mi_stl_allocator(const mi_stl_allocator &) mi_attr_noexcept = default;
template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { } template <class U>
mi_stl_allocator(const mi_stl_allocator<U> &) mi_attr_noexcept {}
mi_stl_allocator select_on_container_copy_construction() const { return *this; } mi_stl_allocator select_on_container_copy_construction() const { return *this; }
void deallocate(T* p, size_type) { mi_free(p); } void deallocate(T *p, size_type) { mi_free(p); }
#if (__cplusplus >= 201703L) // C++17 #if (__cplusplus >= 201703L) // C++17
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); } mi_decl_nodiscard T *allocate(size_type count)
mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } {
#else return static_cast<T *>(mi_new_n(count, sizeof(T)));
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); } }
#endif mi_decl_nodiscard T *allocate(size_type count, const void *) { return allocate(count); }
#else
mi_decl_nodiscard pointer allocate(size_type count, const void * = 0)
{
return static_cast<pointer>(mi_new_n(count, sizeof(value_type)));
}
#endif
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
using propagate_on_container_copy_assignment = std::true_type; using propagate_on_container_copy_assignment = std::true_type;
using propagate_on_container_move_assignment = std::true_type; using propagate_on_container_move_assignment = std::true_type;
using propagate_on_container_swap = std::true_type; using propagate_on_container_swap = std::true_type;
using is_always_equal = std::true_type; using is_always_equal = std::true_type;
template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); } template <class U, class... Args>
template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); } void construct(U *p, Args &&...args) { ::new (p) U(std::forward<Args>(args)...); }
#else template <class U>
void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } void destroy(U *p) mi_attr_noexcept { p->~U(); }
#else
void construct(pointer p, value_type const &val)
{
::new (p) value_type(val);
}
void destroy(pointer p) { p->~value_type(); } void destroy(pointer p) { p->~value_type(); }
#endif #endif
size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } size_type max_size() const mi_attr_noexcept
{
return (PTRDIFF_MAX / sizeof(value_type));
}
pointer address(reference x) const { return &x; } pointer address(reference x) const { return &x; }
const_pointer address(const_reference x) const { return &x; } const_pointer address(const_reference x) const { return &x; }
}; };
template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; } template <class T1, class T2>
template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; } bool operator==(const mi_stl_allocator<T1> &, const mi_stl_allocator<T2> &) mi_attr_noexcept { return true; }
template <class T1, class T2>
bool operator!=(const mi_stl_allocator<T1> &, const mi_stl_allocator<T2> &) mi_attr_noexcept { return false; }
#endif // __cplusplus #endif // __cplusplus
#endif #endif

387
src/heap.c
View file

@ -12,7 +12,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <string.h> // memset, memcpy #include <string.h> // memset, memcpy
#if defined(_MSC_VER) && (_MSC_VER < 1920) #if defined(_MSC_VER) && (_MSC_VER < 1920)
#pragma warning(disable:4204) // non-constant aggregate initializer #pragma warning(disable : 4204) // non-constant aggregate initializer
#endif #endif
/* ----------------------------------------------------------- /* -----------------------------------------------------------
@ -20,26 +20,30 @@ terms of the MIT license. A copy of the license can be found in the file
----------------------------------------------------------- */ ----------------------------------------------------------- */
// return `true` if ok, `false` to break // return `true` if ok, `false` to break
typedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2); typedef bool(heap_page_visitor_fun)(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2);
// Visit all pages in a heap; returns `false` if break was called. // Visit all pages in a heap; returns `false` if break was called.
static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void* arg1, void* arg2) static bool mi_heap_visit_pages(mi_heap_t *heap, heap_page_visitor_fun *fn, void *arg1, void *arg2)
{ {
if (heap==NULL || heap->page_count==0) return 0; if (heap == NULL || heap->page_count == 0)
return 0;
// visit all pages // visit all pages
#if MI_DEBUG>1 #if MI_DEBUG > 1
size_t total = heap->page_count; size_t total = heap->page_count;
#endif #endif
size_t count = 0; size_t count = 0;
for (size_t i = 0; i <= MI_BIN_FULL; i++) { for (size_t i = 0; i <= MI_BIN_FULL; i++)
mi_page_queue_t* pq = &heap->pages[i]; {
mi_page_t* page = pq->first; mi_page_queue_t *pq = &heap->pages[i];
while(page != NULL) { mi_page_t *page = pq->first;
mi_page_t* next = page->next; // save next in case the page gets removed from the queue while (page != NULL)
{
mi_page_t *next = page->next; // save next in case the page gets removed from the queue
mi_assert_internal(mi_page_heap(page) == heap); mi_assert_internal(mi_page_heap(page) == heap);
count++; count++;
if (!fn(heap, pq, page, arg1, arg2)) return false; if (!fn(heap, pq, page, arg1, arg2))
return false;
page = next; // and continue page = next; // and continue
} }
} }
@ -47,30 +51,28 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void
return true; return true;
} }
#if MI_DEBUG >= 2
#if MI_DEBUG>=2 static bool mi_heap_page_is_valid(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { {
UNUSED(arg1); UNUSED(arg1);
UNUSED(arg2); UNUSED(arg2);
UNUSED(pq); UNUSED(pq);
mi_assert_internal(mi_page_heap(page) == heap); mi_assert_internal(mi_page_heap(page) == heap);
mi_segment_t* segment = _mi_page_segment(page); mi_segment_t *segment = _mi_page_segment(page);
mi_assert_internal(segment->thread_id == heap->thread_id); mi_assert_internal(segment->thread_id == heap->thread_id);
mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_expensive(_mi_page_is_valid(page));
return true; return true;
} }
#endif #endif
#if MI_DEBUG>=3 #if MI_DEBUG >= 3
static bool mi_heap_is_valid(mi_heap_t* heap) { static bool mi_heap_is_valid(mi_heap_t *heap)
mi_assert_internal(heap!=NULL); {
mi_assert_internal(heap != NULL);
mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL); mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL);
return true; return true;
} }
#endif #endif
/* ----------------------------------------------------------- /* -----------------------------------------------------------
"Collect" pages by migrating `local_free` and `thread_free` "Collect" pages by migrating `local_free` and `thread_free`
lists and freeing empty pages. This is done when a thread lists and freeing empty pages. This is done when a thread
@ -78,32 +80,36 @@ static bool mi_heap_is_valid(mi_heap_t* heap) {
blocks alive) blocks alive)
----------------------------------------------------------- */ ----------------------------------------------------------- */
typedef enum mi_collect_e { typedef enum mi_collect_e
{
MI_NORMAL, MI_NORMAL,
MI_FORCE, MI_FORCE,
MI_ABANDON MI_ABANDON
} mi_collect_t; } mi_collect_t;
static bool mi_heap_page_collect(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg_collect, void *arg2)
static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { {
UNUSED(arg2); UNUSED(arg2);
UNUSED(heap); UNUSED(heap);
mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL)); mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL));
mi_collect_t collect = *((mi_collect_t*)arg_collect); mi_collect_t collect = *((mi_collect_t *)arg_collect);
_mi_page_free_collect(page, collect >= MI_FORCE); _mi_page_free_collect(page, collect >= MI_FORCE);
if (mi_page_all_free(page)) { if (mi_page_all_free(page))
{
// no more used blocks, free the page. // no more used blocks, free the page.
// note: this will free retired pages as well. // note: this will free retired pages as well.
_mi_page_free(page, pq, collect >= MI_FORCE); _mi_page_free(page, pq, collect >= MI_FORCE);
} }
else if (collect == MI_ABANDON) { else if (collect == MI_ABANDON)
{
// still used blocks but the thread is done; abandon the page // still used blocks but the thread is done; abandon the page
_mi_page_abandon(page, pq); _mi_page_abandon(page, pq);
} }
return true; // don't break return true; // don't break
} }
static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { static bool mi_heap_page_never_delayed_free(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1); UNUSED(arg1);
UNUSED(arg2); UNUSED(arg2);
UNUSED(heap); UNUSED(heap);
@ -112,18 +118,19 @@ static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq
return true; // don't break return true; // don't break
} }
static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) static void mi_heap_collect_ex(mi_heap_t *heap, mi_collect_t collect)
{ {
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
return;
_mi_deferred_free(heap, collect >= MI_FORCE); _mi_deferred_free(heap, collect >= MI_FORCE);
// note: never reclaim on collect but leave it to threads that need storage to reclaim // note: never reclaim on collect but leave it to threads that need storage to reclaim
if ( if (
#ifdef NDEBUG #ifdef NDEBUG
collect == MI_FORCE collect == MI_FORCE
#else #else
collect >= MI_FORCE collect >= MI_FORCE
#endif #endif
&& _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim) && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim)
{ {
// the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments.
@ -132,7 +139,8 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
} }
// if abandoning, mark all pages to no longer add to delayed_free // if abandoning, mark all pages to no longer add to delayed_free
if (collect == MI_ABANDON) { if (collect == MI_ABANDON)
{
mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL);
} }
@ -145,54 +153,62 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
// collect all pages owned by this thread // collect all pages owned by this thread
mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL);
mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); mi_assert_internal(collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t, &heap->thread_delayed_free) == NULL);
// collect segment caches // collect segment caches
if (collect >= MI_FORCE) { if (collect >= MI_FORCE)
{
_mi_segment_thread_collect(&heap->tld->segments); _mi_segment_thread_collect(&heap->tld->segments);
} }
// collect regions on program-exit (or shared library unload) // collect regions on program-exit (or shared library unload)
if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap)) { if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap))
{
_mi_mem_collect(&heap->tld->os); _mi_mem_collect(&heap->tld->os);
} }
} }
void _mi_heap_collect_abandon(mi_heap_t* heap) { void _mi_heap_collect_abandon(mi_heap_t *heap)
{
mi_heap_collect_ex(heap, MI_ABANDON); mi_heap_collect_ex(heap, MI_ABANDON);
} }
void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept { void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept
{
mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL)); mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL));
} }
void mi_collect(bool force) mi_attr_noexcept { void mi_collect(bool force) mi_attr_noexcept
{
mi_heap_collect(mi_get_default_heap(), force); mi_heap_collect(mi_get_default_heap(), force);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Heap new Heap new
----------------------------------------------------------- */ ----------------------------------------------------------- */
mi_heap_t* mi_heap_get_default(void) { mi_heap_t *mi_heap_get_default(void)
{
mi_thread_init(); mi_thread_init();
return mi_get_default_heap(); return mi_get_default_heap();
} }
mi_heap_t* mi_heap_get_backing(void) { mi_heap_t *mi_heap_get_backing(void)
mi_heap_t* heap = mi_heap_get_default(); {
mi_assert_internal(heap!=NULL); mi_heap_t *heap = mi_heap_get_default();
mi_heap_t* bheap = heap->tld->heap_backing; mi_assert_internal(heap != NULL);
mi_assert_internal(bheap!=NULL); mi_heap_t *bheap = heap->tld->heap_backing;
mi_assert_internal(bheap != NULL);
mi_assert_internal(bheap->thread_id == _mi_thread_id()); mi_assert_internal(bheap->thread_id == _mi_thread_id());
return bheap; return bheap;
} }
mi_heap_t* mi_heap_new(void) { mi_heap_t *mi_heap_new(void)
mi_heap_t* bheap = mi_heap_get_backing(); {
mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? mi_heap_t *bheap = mi_heap_get_backing();
if (heap==NULL) return NULL; mi_heap_t *heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
if (heap == NULL)
return NULL;
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->tld = bheap->tld; heap->tld = bheap->tld;
heap->thread_id = _mi_thread_id(); heap->thread_id = _mi_thread_id();
@ -204,15 +220,18 @@ mi_heap_t* mi_heap_new(void) {
// push on the thread local heaps list // push on the thread local heaps list
heap->next = heap->tld->heaps; heap->next = heap->tld->heaps;
heap->tld->heaps = heap; heap->tld->heaps = heap;
heap->deferred_free = NULL;
return heap; return heap;
} }
uintptr_t _mi_heap_random_next(mi_heap_t* heap) { uintptr_t _mi_heap_random_next(mi_heap_t *heap)
{
return _mi_random_next(&heap->random); return _mi_random_next(&heap->random);
} }
// zero out the page queues // zero out the page queues
static void mi_heap_reset_pages(mi_heap_t* heap) { static void mi_heap_reset_pages(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap)); mi_assert_internal(mi_heap_is_initialized(heap));
// TODO: copy full empty heap instead? // TODO: copy full empty heap instead?
@ -226,29 +245,41 @@ static void mi_heap_reset_pages(mi_heap_t* heap) {
} }
// called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources. // called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources.
static void mi_heap_free(mi_heap_t* heap) { static void mi_heap_free(mi_heap_t *heap)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap)); mi_assert_internal(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
if (mi_heap_is_backing(heap)) return; // dont free the backing heap return;
if (mi_heap_is_backing(heap))
return; // dont free the backing heap
// reset default // reset default
if (mi_heap_is_default(heap)) { if (mi_heap_is_default(heap))
{
_mi_heap_set_default_direct(heap->tld->heap_backing); _mi_heap_set_default_direct(heap->tld->heap_backing);
} }
// remove ourselves from the thread local heaps list // remove ourselves from the thread local heaps list
// linear search but we expect the number of heaps to be relatively small // linear search but we expect the number of heaps to be relatively small
mi_heap_t* prev = NULL; mi_heap_t *prev = NULL;
mi_heap_t* curr = heap->tld->heaps; mi_heap_t *curr = heap->tld->heaps;
while (curr != heap && curr != NULL) { while (curr != heap && curr != NULL)
{
prev = curr; prev = curr;
curr = curr->next; curr = curr->next;
} }
mi_assert_internal(curr == heap); mi_assert_internal(curr == heap);
if (curr == heap) { if (curr == heap)
if (prev != NULL) { prev->next = heap->next; } {
else { heap->tld->heaps = heap->next; } if (prev != NULL)
{
prev->next = heap->next;
}
else
{
heap->tld->heaps = heap->next;
}
} }
mi_assert_internal(heap->tld->heaps != NULL); mi_assert_internal(heap->tld->heaps != NULL);
@ -256,12 +287,12 @@ static void mi_heap_free(mi_heap_t* heap) {
mi_free(heap); mi_free(heap);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Heap destroy Heap destroy
----------------------------------------------------------- */ ----------------------------------------------------------- */
static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { static bool _mi_heap_page_destroy(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1); UNUSED(arg1);
UNUSED(arg2); UNUSED(arg2);
UNUSED(heap); UNUSED(heap);
@ -272,20 +303,24 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
// stats // stats
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
if (bsize > MI_LARGE_OBJ_SIZE_MAX) { if (bsize > MI_LARGE_OBJ_SIZE_MAX)
if (bsize > MI_HUGE_OBJ_SIZE_MAX) { {
if (bsize > MI_HUGE_OBJ_SIZE_MAX)
{
mi_heap_stat_decrease(heap, giant, bsize); mi_heap_stat_decrease(heap, giant, bsize);
} }
else { else
{
mi_heap_stat_decrease(heap, huge, bsize); mi_heap_stat_decrease(heap, huge, bsize);
} }
} }
#if (MI_STAT) #if (MI_STAT)
_mi_page_free_collect(page, false); // update used count _mi_page_free_collect(page, false); // update used count
const size_t inuse = page->used; const size_t inuse = page->used;
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { if (bsize <= MI_LARGE_OBJ_SIZE_MAX)
{
mi_heap_stat_decrease(heap, normal, bsize * inuse); mi_heap_stat_decrease(heap, normal, bsize * inuse);
#if (MI_STAT>1) #if (MI_STAT > 1)
mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse); mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse);
#endif #endif
} }
@ -300,43 +335,48 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
// mi_page_free(page,false); // mi_page_free(page,false);
page->next = NULL; page->next = NULL;
page->prev = NULL; page->prev = NULL;
_mi_segment_page_free(page,false /* no force? */, &heap->tld->segments); _mi_segment_page_free(page, false /* no force? */, &heap->tld->segments);
return true; // keep going return true; // keep going
} }
void _mi_heap_destroy_pages(mi_heap_t* heap) { void _mi_heap_destroy_pages(mi_heap_t *heap)
{
mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL); mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL);
mi_heap_reset_pages(heap); mi_heap_reset_pages(heap);
} }
void mi_heap_destroy(mi_heap_t* heap) { void mi_heap_destroy(mi_heap_t *heap)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap)); mi_assert(mi_heap_is_initialized(heap));
mi_assert(heap->no_reclaim); mi_assert(heap->no_reclaim);
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
if (!heap->no_reclaim) { return;
if (!heap->no_reclaim)
{
// don't free in case it may contain reclaimed pages // don't free in case it may contain reclaimed pages
mi_heap_delete(heap); mi_heap_delete(heap);
} }
else { else
{
// free all pages // free all pages
_mi_heap_destroy_pages(heap); _mi_heap_destroy_pages(heap);
mi_heap_free(heap); mi_heap_free(heap);
} }
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Safe Heap delete Safe Heap delete
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Tranfer the pages from one heap to the other // Tranfer the pages from one heap to the other
static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { static void mi_heap_absorb(mi_heap_t *heap, mi_heap_t *from)
mi_assert_internal(heap!=NULL); {
if (from==NULL || from->page_count == 0) return; mi_assert_internal(heap != NULL);
if (from == NULL || from->page_count == 0)
return;
// reduce the size of the delayed frees // reduce the size of the delayed frees
_mi_heap_delayed_free(from); _mi_heap_delayed_free(from);
@ -345,9 +385,10 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
// so threads may do delayed frees in either heap for a while. // so threads may do delayed frees in either heap for a while.
// note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state // note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state
// so after this only the new heap will get delayed frees // so after this only the new heap will get delayed frees
for (size_t i = 0; i <= MI_BIN_FULL; i++) { for (size_t i = 0; i <= MI_BIN_FULL; i++)
mi_page_queue_t* pq = &heap->pages[i]; {
mi_page_queue_t* append = &from->pages[i]; mi_page_queue_t *pq = &heap->pages[i];
mi_page_queue_t *append = &from->pages[i];
size_t pcount = _mi_page_queue_append(heap, pq, append); size_t pcount = _mi_page_queue_append(heap, pq, append);
heap->page_count += pcount; heap->page_count += pcount;
from->page_count -= pcount; from->page_count -= pcount;
@ -359,87 +400,98 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
// turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a // turns out to be ok as `_mi_heap_delayed_free` only visits the list and calls a
// the regular `_mi_free_delayed_block` which is safe. // the regular `_mi_free_delayed_block` which is safe.
_mi_heap_delayed_free(from); _mi_heap_delayed_free(from);
mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t,&from->thread_delayed_free) == NULL); mi_assert_internal(mi_atomic_load_ptr_relaxed(mi_block_t, &from->thread_delayed_free) == NULL);
// and reset the `from` heap // and reset the `from` heap
mi_heap_reset_pages(from); mi_heap_reset_pages(from);
} }
// Safe delete a heap without freeing any still allocated blocks in that heap. // Safe delete a heap without freeing any still allocated blocks in that heap.
void mi_heap_delete(mi_heap_t* heap) void mi_heap_delete(mi_heap_t *heap)
{ {
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap)); mi_assert(mi_heap_is_initialized(heap));
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
return;
if (!mi_heap_is_backing(heap)) { if (!mi_heap_is_backing(heap))
{
// tranfer still used pages to the backing heap // tranfer still used pages to the backing heap
mi_heap_absorb(heap->tld->heap_backing, heap); mi_heap_absorb(heap->tld->heap_backing, heap);
} }
else { else
{
// the backing heap abandons its pages // the backing heap abandons its pages
_mi_heap_collect_abandon(heap); _mi_heap_collect_abandon(heap);
} }
mi_assert_internal(heap->page_count==0); mi_assert_internal(heap->page_count == 0);
mi_heap_free(heap); mi_heap_free(heap);
} }
mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { mi_heap_t *mi_heap_set_default(mi_heap_t *heap)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap)); mi_assert(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL; if (heap == NULL || !mi_heap_is_initialized(heap))
return NULL;
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
mi_heap_t* old = mi_get_default_heap(); mi_heap_t *old = mi_get_default_heap();
_mi_heap_set_default_direct(heap); _mi_heap_set_default_direct(heap);
return old; return old;
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Analysis Analysis
----------------------------------------------------------- */ ----------------------------------------------------------- */
// static since it is not thread safe to access heaps from other threads. // static since it is not thread safe to access heaps from other threads.
static mi_heap_t* mi_heap_of_block(const void* p) { static mi_heap_t *mi_heap_of_block(const void *p)
if (p == NULL) return NULL; {
mi_segment_t* segment = _mi_ptr_segment(p); if (p == NULL)
return NULL;
mi_segment_t *segment = _mi_ptr_segment(p);
bool valid = (_mi_ptr_cookie(segment) == segment->cookie); bool valid = (_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(valid); mi_assert_internal(valid);
if (mi_unlikely(!valid)) return NULL; if (mi_unlikely(!valid))
return mi_page_heap(_mi_segment_page_of(segment,p)); return NULL;
return mi_page_heap(_mi_segment_page_of(segment, p));
} }
bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { bool mi_heap_contains_block(mi_heap_t *heap, const void *p)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
if (heap==NULL || !mi_heap_is_initialized(heap)) return false; if (heap == NULL || !mi_heap_is_initialized(heap))
return false;
return (heap == mi_heap_of_block(p)); return (heap == mi_heap_of_block(p));
} }
static bool mi_heap_page_check_owned(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *p, void *vfound)
static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) { {
UNUSED(heap); UNUSED(heap);
UNUSED(pq); UNUSED(pq);
bool* found = (bool*)vfound; bool *found = (bool *)vfound;
mi_segment_t* segment = _mi_page_segment(page); mi_segment_t *segment = _mi_page_segment(page);
void* start = _mi_page_start(segment, page, NULL); void *start = _mi_page_start(segment, page, NULL);
void* end = (uint8_t*)start + (page->capacity * mi_page_block_size(page)); void *end = (uint8_t *)start + (page->capacity * mi_page_block_size(page));
*found = (p >= start && p < end); *found = (p >= start && p < end);
return (!*found); // continue if not found return (!*found); // continue if not found
} }
bool mi_heap_check_owned(mi_heap_t* heap, const void* p) { bool mi_heap_check_owned(mi_heap_t *heap, const void *p)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
if (heap==NULL || !mi_heap_is_initialized(heap)) return false; if (heap == NULL || !mi_heap_is_initialized(heap))
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers return false;
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)
return false; // only aligned pointers
bool found = false; bool found = false;
mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void*)p, &found); mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void *)p, &found);
return found; return found;
} }
bool mi_check_owned(const void* p) { bool mi_check_owned(const void *p)
{
return mi_heap_check_owned(mi_get_default_heap(), p); return mi_heap_check_owned(mi_get_default_heap(), p);
} }
@ -450,46 +502,53 @@ bool mi_check_owned(const void* p) {
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Separate struct to keep `mi_page_t` out of the public interface // Separate struct to keep `mi_page_t` out of the public interface
typedef struct mi_heap_area_ex_s { typedef struct mi_heap_area_ex_s
{
mi_heap_area_t area; mi_heap_area_t area;
mi_page_t* page; mi_page_t *page;
} mi_heap_area_ex_t; } mi_heap_area_ex_t;
static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) { static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t *xarea, mi_block_visit_fun *visitor, void *arg)
{
mi_assert(xarea != NULL); mi_assert(xarea != NULL);
if (xarea==NULL) return true; if (xarea == NULL)
const mi_heap_area_t* area = &xarea->area; return true;
mi_page_t* page = xarea->page; const mi_heap_area_t *area = &xarea->area;
mi_page_t *page = xarea->page;
mi_assert(page != NULL); mi_assert(page != NULL);
if (page == NULL) return true; if (page == NULL)
return true;
_mi_page_free_collect(page,true); _mi_page_free_collect(page, true);
mi_assert_internal(page->local_free == NULL); mi_assert_internal(page->local_free == NULL);
if (page->used == 0) return true; if (page->used == 0)
return true;
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
size_t psize; size_t psize;
uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); uint8_t *pstart = _mi_page_start(_mi_page_segment(page), page, &psize);
if (page->capacity == 1) { if (page->capacity == 1)
{
// optimize page with one block // optimize page with one block
mi_assert_internal(page->used == 1 && page->free == NULL); mi_assert_internal(page->used == 1 && page->free == NULL);
return visitor(mi_page_heap(page), area, pstart, bsize, arg); return visitor(mi_page_heap(page), area, pstart, bsize, arg);
} }
// create a bitmap of free blocks. // create a bitmap of free blocks.
#define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void *))
uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)]; uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)];
memset(free_map, 0, sizeof(free_map)); memset(free_map, 0, sizeof(free_map));
size_t free_count = 0; size_t free_count = 0;
for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { for (mi_block_t *block = page->free; block != NULL; block = mi_block_next(page, block))
{
free_count++; free_count++;
mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); mi_assert_internal((uint8_t *)block >= pstart && (uint8_t *)block < (pstart + psize));
size_t offset = (uint8_t*)block - pstart; size_t offset = (uint8_t *)block - pstart;
mi_assert_internal(offset % bsize == 0); mi_assert_internal(offset % bsize == 0);
size_t blockidx = offset / bsize; // Todo: avoid division? size_t blockidx = offset / bsize; // Todo: avoid division?
mi_assert_internal( blockidx < MI_MAX_BLOCKS); mi_assert_internal(blockidx < MI_MAX_BLOCKS);
size_t bitidx = (blockidx / sizeof(uintptr_t)); size_t bitidx = (blockidx / sizeof(uintptr_t));
size_t bit = blockidx - (bitidx * sizeof(uintptr_t)); size_t bit = blockidx - (bitidx * sizeof(uintptr_t));
free_map[bitidx] |= ((uintptr_t)1 << bit); free_map[bitidx] |= ((uintptr_t)1 << bit);
@ -498,30 +557,34 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
// walk through all blocks skipping the free ones // walk through all blocks skipping the free ones
size_t used_count = 0; size_t used_count = 0;
for (size_t i = 0; i < page->capacity; i++) { for (size_t i = 0; i < page->capacity; i++)
{
size_t bitidx = (i / sizeof(uintptr_t)); size_t bitidx = (i / sizeof(uintptr_t));
size_t bit = i - (bitidx * sizeof(uintptr_t)); size_t bit = i - (bitidx * sizeof(uintptr_t));
uintptr_t m = free_map[bitidx]; uintptr_t m = free_map[bitidx];
if (bit == 0 && m == UINTPTR_MAX) { if (bit == 0 && m == UINTPTR_MAX)
{
i += (sizeof(uintptr_t) - 1); // skip a run of free blocks i += (sizeof(uintptr_t) - 1); // skip a run of free blocks
} }
else if ((m & ((uintptr_t)1 << bit)) == 0) { else if ((m & ((uintptr_t)1 << bit)) == 0)
{
used_count++; used_count++;
uint8_t* block = pstart + (i * bsize); uint8_t *block = pstart + (i * bsize);
if (!visitor(mi_page_heap(page), area, block, bsize, arg)) return false; if (!visitor(mi_page_heap(page), area, block, bsize, arg))
return false;
} }
} }
mi_assert_internal(page->used == used_count); mi_assert_internal(page->used == used_count);
return true; return true;
} }
typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); typedef bool(mi_heap_area_visit_fun)(const mi_heap_t *heap, const mi_heap_area_ex_t *area, void *arg);
static bool mi_heap_visit_areas_page(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *vfun, void *arg)
static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { {
UNUSED(heap); UNUSED(heap);
UNUSED(pq); UNUSED(pq);
mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; mi_heap_area_visit_fun *fun = (mi_heap_area_visit_fun *)vfun;
mi_heap_area_ex_t xarea; mi_heap_area_ex_t xarea;
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
xarea.page = page; xarea.page = page;
@ -534,31 +597,45 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
} }
// Visit all heap pages as areas // Visit all heap pages as areas
static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) { static bool mi_heap_visit_areas(const mi_heap_t *heap, mi_heap_area_visit_fun *visitor, void *arg)
if (visitor == NULL) return false; {
return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, (void*)(visitor), arg); // note: function pointer to void* :-{ if (visitor == NULL)
return false;
return mi_heap_visit_pages((mi_heap_t *)heap, &mi_heap_visit_areas_page, (void *)(visitor), arg); // note: function pointer to void* :-{
} }
// Just to pass arguments // Just to pass arguments
typedef struct mi_visit_blocks_args_s { typedef struct mi_visit_blocks_args_s
{
bool visit_blocks; bool visit_blocks;
mi_block_visit_fun* visitor; mi_block_visit_fun *visitor;
void* arg; void *arg;
} mi_visit_blocks_args_t; } mi_visit_blocks_args_t;
static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) { static bool mi_heap_area_visitor(const mi_heap_t *heap, const mi_heap_area_ex_t *xarea, void *arg)
mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; {
if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; mi_visit_blocks_args_t *args = (mi_visit_blocks_args_t *)arg;
if (args->visit_blocks) { if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg))
return false;
if (args->visit_blocks)
{
return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg); return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg);
} }
else { else
{
return true; return true;
} }
} }
// Visit all blocks in a heap // Visit all blocks in a heap
bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
mi_visit_blocks_args_t args = { visit_blocks, visitor, arg }; {
mi_visit_blocks_args_t args = {visit_blocks, visitor, arg};
return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args); return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);
} }
void mi_heap_register_local_deferred_free(mi_heap_t *heap, mi_local_deferred_free_fun *deferred_free, void *arg)
{
heap->deferred_free = (void *)deferred_free;
heap->deferred_arg = arg;
}

471
src/init.c
View file

@ -15,53 +15,70 @@ const mi_page_t _mi_page_empty = {
0, false, false, false, false, 0, false, false, false, false,
0, // capacity 0, // capacity
0, // reserved capacity 0, // reserved capacity
{ 0 }, // flags {0}, // flags
false, // is_zero false, // is_zero
0, // retire_expire 0, // retire_expire
NULL, // free NULL, // free
#if MI_ENCODE_FREELIST #if MI_ENCODE_FREELIST
{ 0, 0 }, {0, 0},
#endif #endif
0, // used 0, // used
0, // xblock_size 0, // xblock_size
NULL, // local_free NULL, // local_free
ATOMIC_VAR_INIT(0), // xthread_free ATOMIC_VAR_INIT(0), // xthread_free
ATOMIC_VAR_INIT(0), // xheap ATOMIC_VAR_INIT(0), // xheap
NULL, NULL NULL,
}; NULL};
#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) #define MI_PAGE_EMPTY() ((mi_page_t *)&_mi_page_empty)
#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) #if (MI_PADDING > 0) && (MI_INTPTR_SIZE >= 8)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #define MI_SMALL_PAGES_EMPTY \
#elif (MI_PADDING>0) { \
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() \
}
#elif (MI_PADDING > 0)
#define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() \
}
#else #else
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() \
}
#endif #endif
// Empty page queues for every bin // Empty page queues for every bin
#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } #define QNULL(sz) \
{ \
NULL, NULL, (sz) * sizeof(uintptr_t) \
}
#define MI_PAGE_QUEUES_EMPTY \ #define MI_PAGE_QUEUES_EMPTY \
{ QNULL(1), \ { \
QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ QNULL(1), \
QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \ QNULL(1), QNULL(2), QNULL(3), QNULL(4), QNULL(5), QNULL(6), QNULL(7), QNULL(8), /* 8 */ \
QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \ QNULL(10), QNULL(12), QNULL(14), QNULL(16), QNULL(20), QNULL(24), QNULL(28), QNULL(32), /* 16 */ \
QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \ QNULL(40), QNULL(48), QNULL(56), QNULL(64), QNULL(80), QNULL(96), QNULL(112), QNULL(128), /* 24 */ \
QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \ QNULL(160), QNULL(192), QNULL(224), QNULL(256), QNULL(320), QNULL(384), QNULL(448), QNULL(512), /* 32 */ \
QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \ QNULL(640), QNULL(768), QNULL(896), QNULL(1024), QNULL(1280), QNULL(1536), QNULL(1792), QNULL(2048), /* 40 */ \
QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ QNULL(2560), QNULL(3072), QNULL(3584), QNULL(4096), QNULL(5120), QNULL(6144), QNULL(7168), QNULL(8192), /* 48 */ \
QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ QNULL(10240), QNULL(12288), QNULL(14336), QNULL(16384), QNULL(20480), QNULL(24576), QNULL(28672), QNULL(32768), /* 56 */ \
QNULL(40960), QNULL(49152), QNULL(57344), QNULL(65536), QNULL(81920), QNULL(98304), QNULL(114688), QNULL(131072), /* 64 */ \
QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ } QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ \
}
#define MI_STAT_COUNT_NULL() {0,0,0,0} #define MI_STAT_COUNT_NULL() \
{ \
0, 0, 0, 0 \
}
// Empty statistics // Empty statistics
#if MI_STAT>1 #if MI_STAT > 1
#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } #define MI_STAT_COUNT_END_NULL() \
, { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) }
#else #else
#define MI_STAT_COUNT_END_NULL() #define MI_STAT_COUNT_END_NULL()
#endif #endif
@ -74,9 +91,8 @@ const mi_page_t _mi_page_empty = {
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ {0, 0}, {0, 0}, {0, 0}, {0, 0}, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ {0, 0}, {0, 0}, {0, 0}, {0, 0} MI_STAT_COUNT_END_NULL()
MI_STAT_COUNT_END_NULL()
// -------------------------------------------------------- // --------------------------------------------------------
// Statically allocate an empty heap as the initial // Statically allocate an empty heap as the initial
@ -94,28 +110,25 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
ATOMIC_VAR_INIT(NULL), ATOMIC_VAR_INIT(NULL),
0, // tid 0, // tid
0, // cookie 0, // cookie
{ 0, 0 }, // keys {0, 0}, // keys
{ {0}, {0}, 0 }, {{0}, {0}, 0},
0, // page count 0, // page count
MI_BIN_FULL, 0, // page retired min/max MI_BIN_FULL,
0, // page retired min/max
NULL, // next NULL, // next
false false,
}; 0,
0};
// the thread-local default heap for allocation // the thread-local default heap for allocation
mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; mi_decl_thread mi_heap_t *_mi_heap_default = (mi_heap_t *)&_mi_heap_empty;
extern mi_heap_t _mi_heap_main; extern mi_heap_t _mi_heap_main;
static mi_tld_t tld_main = { static mi_tld_t tld_main = {
0, false, 0, false, &_mi_heap_main, &_mi_heap_main, {{NULL, NULL}, {NULL, NULL}, {NULL, NULL, 0}, 0, 0, 0, 0, 0, 0, NULL, &tld_main.stats, &tld_main.os}, // segments
&_mi_heap_main, &_mi_heap_main, {0, &tld_main.stats}, // os
{ { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0}, {MI_STATS_NULL} // stats
0, 0, 0, 0, 0, 0, NULL,
&tld_main.stats, &tld_main.os
}, // segments
{ 0, &tld_main.stats }, // os
{ MI_STATS_NULL } // stats
}; };
mi_heap_t _mi_heap_main = { mi_heap_t _mi_heap_main = {
@ -125,21 +138,24 @@ mi_heap_t _mi_heap_main = {
ATOMIC_VAR_INIT(NULL), ATOMIC_VAR_INIT(NULL),
0, // thread id 0, // thread id
0, // initial cookie 0, // initial cookie
{ 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!) {0, 0}, // the key of the main heap can be fixed (unlike page keys that need to be secure!)
{ {0x846ca68b}, {0}, 0 }, // random {{0x846ca68b}, {0}, 0}, // random
0, // page count 0, // page count
MI_BIN_FULL, 0, // page retired min/max MI_BIN_FULL,
0, // page retired min/max
NULL, // next heap NULL, // next heap
false // can reclaim false, // can reclaim
}; 0,
0};
bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
mi_stats_t _mi_stats_main = { MI_STATS_NULL }; mi_stats_t _mi_stats_main = {MI_STATS_NULL};
static void mi_heap_main_init(void)
static void mi_heap_main_init(void) { {
if (_mi_heap_main.cookie == 0) { if (_mi_heap_main.cookie == 0)
{
_mi_heap_main.thread_id = _mi_thread_id(); _mi_heap_main.thread_id = _mi_thread_id();
_mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init); _mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init);
_mi_random_init(&_mi_heap_main.random); _mi_random_init(&_mi_heap_main.random);
@ -148,47 +164,54 @@ static void mi_heap_main_init(void) {
} }
} }
mi_heap_t* _mi_heap_main_get(void) { mi_heap_t *_mi_heap_main_get(void)
{
mi_heap_main_init(); mi_heap_main_init();
return &_mi_heap_main; return &_mi_heap_main;
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Initialization and freeing of the thread local heaps Initialization and freeing of the thread local heaps
----------------------------------------------------------- */ ----------------------------------------------------------- */
// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). // note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size).
typedef struct mi_thread_data_s { typedef struct mi_thread_data_s
{
mi_heap_t heap; // must come first due to cast in `_mi_heap_done` mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
mi_tld_t tld; mi_tld_t tld;
} mi_thread_data_t; } mi_thread_data_t;
// Initialize the thread local default heap, called from `mi_thread_init` // Initialize the thread local default heap, called from `mi_thread_init`
static bool _mi_heap_init(void) { static bool _mi_heap_init(void)
if (mi_heap_is_initialized(mi_get_default_heap())) return true; {
if (_mi_is_main_thread()) { if (mi_heap_is_initialized(mi_get_default_heap()))
return true;
if (_mi_is_main_thread())
{
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization
// the main heap is statically allocated // the main heap is statically allocated
mi_heap_main_init(); mi_heap_main_init();
_mi_heap_set_default_direct(&_mi_heap_main); _mi_heap_set_default_direct(&_mi_heap_main);
//mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap()); //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap());
} }
else { else
{
// use `_mi_os_alloc` to allocate directly from the OS // use `_mi_os_alloc` to allocate directly from the OS
mi_thread_data_t* td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); // Todo: more efficient allocation? mi_thread_data_t *td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); // Todo: more efficient allocation?
if (td == NULL) { if (td == NULL)
{
// if this fails, try once more. (issue #257) // if this fails, try once more. (issue #257)
td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main);
if (td == NULL) { if (td == NULL)
{
// really out of memory // really out of memory
_mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t));
return false; return false;
} }
} }
// OS allocated so already zero initialized // OS allocated so already zero initialized
mi_tld_t* tld = &td->tld; mi_tld_t *tld = &td->tld;
mi_heap_t* heap = &td->heap; mi_heap_t *heap = &td->heap;
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap));
heap->thread_id = _mi_thread_id(); heap->thread_id = _mi_thread_id();
_mi_random_init(&heap->random); _mi_random_init(&heap->random);
@ -207,21 +230,26 @@ static bool _mi_heap_init(void) {
} }
// Free the thread local default heap (called from `mi_thread_done`) // Free the thread local default heap (called from `mi_thread_done`)
static bool _mi_heap_done(mi_heap_t* heap) { static bool _mi_heap_done(mi_heap_t *heap)
if (!mi_heap_is_initialized(heap)) return true; {
if (!mi_heap_is_initialized(heap))
return true;
// reset default heap // reset default heap
_mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t *)&_mi_heap_empty);
// switch to backing heap // switch to backing heap
heap = heap->tld->heap_backing; heap = heap->tld->heap_backing;
if (!mi_heap_is_initialized(heap)) return false; if (!mi_heap_is_initialized(heap))
return false;
// delete all non-backing heaps in this thread // delete all non-backing heaps in this thread
mi_heap_t* curr = heap->tld->heaps; mi_heap_t *curr = heap->tld->heaps;
while (curr != NULL) { while (curr != NULL)
mi_heap_t* next = curr->next; // save `next` as `curr` will be freed {
if (curr != heap) { mi_heap_t *next = curr->next; // save `next` as `curr` will be freed
if (curr != heap)
{
mi_assert_internal(!mi_heap_is_backing(curr)); mi_assert_internal(!mi_heap_is_backing(curr));
mi_heap_delete(curr); mi_heap_delete(curr);
} }
@ -231,7 +259,8 @@ static bool _mi_heap_done(mi_heap_t* heap) {
mi_assert_internal(mi_heap_is_backing(heap)); mi_assert_internal(mi_heap_is_backing(heap));
// collect if not the main thread // collect if not the main thread
if (heap != &_mi_heap_main) { if (heap != &_mi_heap_main)
{
_mi_heap_collect_abandon(heap); _mi_heap_collect_abandon(heap);
} }
@ -239,7 +268,8 @@ static bool _mi_heap_done(mi_heap_t* heap) {
_mi_stats_done(&heap->tld->stats); _mi_stats_done(&heap->tld->stats);
// free if not the main thread // free if not the main thread
if (heap != &_mi_heap_main) { if (heap != &_mi_heap_main)
{
mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id());
_mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main); _mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main);
} }
@ -254,8 +284,6 @@ static bool _mi_heap_done(mi_heap_t* heap) {
return false; return false;
} }
// -------------------------------------------------------- // --------------------------------------------------------
// Try to run `mi_thread_done()` automatically so any memory // Try to run `mi_thread_done()` automatically so any memory
// owned by the thread but not yet released can be abandoned // owned by the thread but not yet released can be abandoned
@ -272,7 +300,7 @@ static bool _mi_heap_done(mi_heap_t* heap) {
// to set up the thread local keys. // to set up the thread local keys.
// -------------------------------------------------------- // --------------------------------------------------------
static void _mi_thread_done(mi_heap_t* default_heap); static void _mi_thread_done(mi_heap_t *default_heap);
#ifdef __wasi__ #ifdef __wasi__
// no pthreads in the WebAssembly Standard Interface // no pthreads in the WebAssembly Standard Interface
@ -281,54 +309,60 @@ static void _mi_thread_done(mi_heap_t* default_heap);
#endif #endif
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
// nothing to do as it is done in DllMain // nothing to do as it is done in DllMain
#elif defined(_WIN32) && !defined(MI_SHARED_LIB) #elif defined(_WIN32) && !defined(MI_SHARED_LIB)
// use thread local storage keys to detect thread ending // use thread local storage keys to detect thread ending
#include <Windows.h> #include <Windows.h>
#include <fibersapi.h> #include <fibersapi.h>
#if (_WIN32_WINNT < 0x600) // before Windows Vista #if (_WIN32_WINNT < 0x600) // before Windows Vista
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); WINBASEAPI DWORD WINAPI FlsAlloc(_In_opt_ PFLS_CALLBACK_FUNCTION lpCallback);
WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); WINBASEAPI PVOID WINAPI FlsGetValue(_In_ DWORD dwFlsIndex);
WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); WINBASEAPI BOOL WINAPI FlsSetValue(_In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData);
WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
#endif #endif
static DWORD mi_fls_key = (DWORD)(-1); static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) { static void NTAPI mi_fls_done(PVOID value)
if (value!=NULL) _mi_thread_done((mi_heap_t*)value); {
} if (value != NULL)
_mi_thread_done((mi_heap_t *)value);
}
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
// use pthread local storage keys to detect thread ending // use pthread local storage keys to detect thread ending
// (and used with MI_TLS_PTHREADS for the default heap) // (and used with MI_TLS_PTHREADS for the default heap)
#include <pthread.h> #include <pthread.h>
pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
static void mi_pthread_done(void* value) { static void mi_pthread_done(void *value)
if (value!=NULL) _mi_thread_done((mi_heap_t*)value); {
} if (value != NULL)
_mi_thread_done((mi_heap_t *)value);
}
#elif defined(__wasi__) #elif defined(__wasi__)
// no pthreads in the WebAssembly Standard Interface // no pthreads in the WebAssembly Standard Interface
#else #else
#pragma message("define a way to call mi_thread_done when a thread is done") #pragma message("define a way to call mi_thread_done when a thread is done")
#endif #endif
// Set up handlers so `mi_thread_done` is called automatically // Set up handlers so `mi_thread_done` is called automatically
static void mi_process_setup_auto_thread_done(void) { static void mi_process_setup_auto_thread_done(void)
{
static bool tls_initialized = false; // fine if it races static bool tls_initialized = false; // fine if it races
if (tls_initialized) return; if (tls_initialized)
return;
tls_initialized = true; tls_initialized = true;
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
// nothing to do as it is done in DllMain // nothing to do as it is done in DllMain
#elif defined(_WIN32) && !defined(MI_SHARED_LIB) #elif defined(_WIN32) && !defined(MI_SHARED_LIB)
mi_fls_key = FlsAlloc(&mi_fls_done); mi_fls_key = FlsAlloc(&mi_fls_done);
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1)); mi_assert_internal(_mi_heap_default_key == (pthread_key_t)(-1));
pthread_key_create(&_mi_heap_default_key, &mi_pthread_done); pthread_key_create(&_mi_heap_default_key, &mi_pthread_done);
#endif #endif
_mi_heap_set_default_direct(&_mi_heap_main); _mi_heap_set_default_direct(&_mi_heap_main);
} }
bool _mi_is_main_thread(void)
bool _mi_is_main_thread(void) { {
return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); return (_mi_heap_main.thread_id == 0 || _mi_heap_main.thread_id == _mi_thread_id());
} }
// This is called from the `mi_malloc_generic` // This is called from the `mi_malloc_generic`
@ -340,53 +374,59 @@ void mi_thread_init(void) mi_attr_noexcept
// initialize the thread local default heap // initialize the thread local default heap
// (this will call `_mi_heap_set_default_direct` and thus set the // (this will call `_mi_heap_set_default_direct` and thus set the
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
if (_mi_heap_init()) return; // returns true if already initialized if (_mi_heap_init())
return; // returns true if already initialized
_mi_stat_increase(&_mi_stats_main.threads, 1); _mi_stat_increase(&_mi_stats_main.threads, 1);
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
} }
void mi_thread_done(void) mi_attr_noexcept { void mi_thread_done(void) mi_attr_noexcept
{
_mi_thread_done(mi_get_default_heap()); _mi_thread_done(mi_get_default_heap());
} }
static void _mi_thread_done(mi_heap_t* heap) { static void _mi_thread_done(mi_heap_t *heap)
{
_mi_stat_decrease(&_mi_stats_main.threads, 1); _mi_stat_decrease(&_mi_stats_main.threads, 1);
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps...
if (heap->thread_id != _mi_thread_id()) return; if (heap->thread_id != _mi_thread_id())
return;
// abandon the thread local heap // abandon the thread local heap
if (_mi_heap_done(heap)) return; // returns true if already ran if (_mi_heap_done(heap))
return; // returns true if already ran
} }
void _mi_heap_set_default_direct(mi_heap_t* heap) { void _mi_heap_set_default_direct(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
#if defined(MI_TLS_SLOT) #if defined(MI_TLS_SLOT)
mi_tls_slot_set(MI_TLS_SLOT,heap); mi_tls_slot_set(MI_TLS_SLOT, heap);
#elif defined(MI_TLS_PTHREAD_SLOT_OFS) #elif defined(MI_TLS_PTHREAD_SLOT_OFS)
*mi_tls_pthread_heap_slot() = heap; *mi_tls_pthread_heap_slot() = heap;
#elif defined(MI_TLS_PTHREAD) #elif defined(MI_TLS_PTHREAD)
// we use _mi_heap_default_key // we use _mi_heap_default_key
#else #else
_mi_heap_default = heap; _mi_heap_default = heap;
#endif #endif
// ensure the default heap is passed to `_mi_thread_done` // ensure the default heap is passed to `_mi_thread_done`
// setting to a non-NULL value also ensures `mi_thread_done` is called. // setting to a non-NULL value also ensures `mi_thread_done` is called.
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
// nothing to do as it is done in DllMain // nothing to do as it is done in DllMain
#elif defined(_WIN32) && !defined(MI_SHARED_LIB) #elif defined(_WIN32) && !defined(MI_SHARED_LIB)
mi_assert_internal(mi_fls_key != 0); mi_assert_internal(mi_fls_key != 0);
FlsSetValue(mi_fls_key, heap); FlsSetValue(mi_fls_key, heap);
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD if (_mi_heap_default_key != (pthread_key_t)(-1))
{ // can happen during recursive invocation on freeBSD
pthread_setspecific(_mi_heap_default_key, heap); pthread_setspecific(_mi_heap_default_key, heap);
} }
#endif #endif
} }
// -------------------------------------------------------- // --------------------------------------------------------
// Run functions on process init/done, and thread init/done // Run functions on process init/done, and thread init/done
// -------------------------------------------------------- // --------------------------------------------------------
@ -396,65 +436,78 @@ static bool os_preloading = true; // true until this module is initialized
static bool mi_redirected = false; // true if malloc redirects to mi_malloc static bool mi_redirected = false; // true if malloc redirects to mi_malloc
// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
bool _mi_preloading(void) { bool _mi_preloading(void)
{
return os_preloading; return os_preloading;
} }
bool mi_is_redirected(void) mi_attr_noexcept { bool mi_is_redirected(void) mi_attr_noexcept
{
return mi_redirected; return mi_redirected;
} }
// Communicate with the redirection module on Windows // Communicate with the redirection module on Windows
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C"
{
#endif #endif
mi_decl_export void _mi_redirect_entry(DWORD reason) { mi_decl_export void _mi_redirect_entry(DWORD reason)
{
// called on redirection; careful as this may be called before DllMain // called on redirection; careful as this may be called before DllMain
if (reason == DLL_PROCESS_ATTACH) { if (reason == DLL_PROCESS_ATTACH)
{
mi_redirected = true; mi_redirected = true;
} }
else if (reason == DLL_PROCESS_DETACH) { else if (reason == DLL_PROCESS_DETACH)
{
mi_redirected = false; mi_redirected = false;
} }
else if (reason == DLL_THREAD_DETACH) { else if (reason == DLL_THREAD_DETACH)
{
mi_thread_done(); mi_thread_done();
} }
} }
__declspec(dllimport) bool mi_allocator_init(const char** message); __declspec(dllimport) bool mi_allocator_init(const char **message);
__declspec(dllimport) void mi_allocator_done(void); __declspec(dllimport) void mi_allocator_done(void);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#else #else
static bool mi_allocator_init(const char** message) { static bool mi_allocator_init(const char **message)
if (message != NULL) *message = NULL; {
if (message != NULL)
*message = NULL;
return true; return true;
} }
static void mi_allocator_done(void) { static void mi_allocator_done(void)
{
// nothing to do // nothing to do
} }
#endif #endif
// Called once by the process loader // Called once by the process loader
static void mi_process_load(void) { static void mi_process_load(void)
{
mi_heap_main_init(); mi_heap_main_init();
#if defined(MI_TLS_RECURSE_GUARD) #if defined(MI_TLS_RECURSE_GUARD)
volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; volatile mi_heap_t *dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true;
UNUSED(dummy); UNUSED(dummy);
#endif #endif
os_preloading = false; os_preloading = false;
atexit(&mi_process_done); atexit(&mi_process_done);
_mi_options_init(); _mi_options_init();
mi_process_init(); mi_process_init();
//mi_stats_reset();- //mi_stats_reset();-
if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); if (mi_redirected)
_mi_verbose_message("malloc is redirected.\n");
// show message from the redirector (if present) // show message from the redirector (if present)
const char* msg = NULL; const char *msg = NULL;
mi_allocator_init(&msg); mi_allocator_init(&msg);
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors)))
_mi_fputs(NULL,NULL,NULL,msg); {
_mi_fputs(NULL, NULL, NULL, msg);
} }
} }
@ -462,22 +515,26 @@ static void mi_process_load(void) {
#include <intrin.h> #include <intrin.h>
mi_decl_cache_align bool _mi_cpu_has_fsrm = false; mi_decl_cache_align bool _mi_cpu_has_fsrm = false;
static void mi_detect_cpu_features(void) { static void mi_detect_cpu_features(void)
{
// FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
int32_t cpu_info[4]; int32_t cpu_info[4];
__cpuid(cpu_info, 7); __cpuid(cpu_info, 7);
_mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https ://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features> _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https ://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
} }
#else #else
static void mi_detect_cpu_features(void) { static void mi_detect_cpu_features(void)
{
// nothing // nothing
} }
#endif #endif
// Initialize the process; called by thread_init or the process loader // Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept { void mi_process_init(void) mi_attr_noexcept
{
// ensure we are called once // ensure we are called once
if (_mi_process_is_initialized) return; if (_mi_process_is_initialized)
return;
_mi_process_is_initialized = true; _mi_process_is_initialized = true;
mi_process_setup_auto_thread_done(); mi_process_setup_auto_thread_done();
@ -485,45 +542,52 @@ void mi_process_init(void) mi_attr_noexcept {
mi_detect_cpu_features(); mi_detect_cpu_features();
_mi_os_init(); _mi_os_init();
mi_heap_main_init(); mi_heap_main_init();
#if (MI_DEBUG) #if (MI_DEBUG)
_mi_verbose_message("debug level : %d\n", MI_DEBUG); _mi_verbose_message("debug level : %d\n", MI_DEBUG);
#endif #endif
_mi_verbose_message("secure level: %d\n", MI_SECURE); _mi_verbose_message("secure level: %d\n", MI_SECURE);
mi_thread_init(); mi_thread_init();
mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL)
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { if (mi_option_is_enabled(mi_option_reserve_huge_os_pages))
{
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages); size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); mi_reserve_huge_os_pages_interleave(pages, 0, pages * 500);
} }
if (mi_option_is_enabled(mi_option_reserve_os_memory)) { if (mi_option_is_enabled(mi_option_reserve_os_memory))
{
long ksize = mi_option_get(mi_option_reserve_os_memory); long ksize = mi_option_get(mi_option_reserve_os_memory);
if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true); if (ksize > 0)
mi_reserve_os_memory((size_t)ksize * KiB, true, true);
} }
} }
// Called when the process is done (through `at_exit`) // Called when the process is done (through `at_exit`)
static void mi_process_done(void) { static void mi_process_done(void)
{
// only shutdown if we were initialized // only shutdown if we were initialized
if (!_mi_process_is_initialized) return; if (!_mi_process_is_initialized)
return;
// ensure we are called once // ensure we are called once
static bool process_done = false; static bool process_done = false;
if (process_done) return; if (process_done)
return;
process_done = true; process_done = true;
#if defined(_WIN32) && !defined(MI_SHARED_LIB) #if defined(_WIN32) && !defined(MI_SHARED_LIB)
FlsSetValue(mi_fls_key, NULL); // don't call main-thread callback FlsSetValue(mi_fls_key, NULL); // don't call main-thread callback
FlsFree(mi_fls_key); // call thread-done on all threads to prevent dangling callback pointer if statically linked with a DLL; Issue #208 FlsFree(mi_fls_key); // call thread-done on all threads to prevent dangling callback pointer if statically linked with a DLL; Issue #208
#endif #endif
#if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB) #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB)
// free all memory if possible on process exit. This is not needed for a stand-alone process // free all memory if possible on process exit. This is not needed for a stand-alone process
// but should be done if mimalloc is statically linked into another shared library which // but should be done if mimalloc is statically linked into another shared library which
// is repeatedly loaded/unloaded, see issue #281. // is repeatedly loaded/unloaded, see issue #281.
mi_collect(true /* force */ ); mi_collect(true /* force */);
#endif #endif
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose))
{
mi_stats_print(NULL); mi_stats_print(NULL);
} }
mi_allocator_done(); mi_allocator_done();
@ -531,53 +595,60 @@ static void mi_process_done(void) {
os_preloading = true; // don't call the C runtime anymore os_preloading = true; // don't call the C runtime anymore
} }
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
// Windows DLL: easy to hook into process_init and thread_done // Windows DLL: easy to hook into process_init and thread_done
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved)
{
UNUSED(reserved); UNUSED(reserved);
UNUSED(inst); UNUSED(inst);
if (reason==DLL_PROCESS_ATTACH) { if (reason == DLL_PROCESS_ATTACH)
{
mi_process_load(); mi_process_load();
} }
else if (reason==DLL_THREAD_DETACH) { else if (reason == DLL_THREAD_DETACH)
if (!mi_is_redirected()) mi_thread_done(); {
if (!mi_is_redirected())
mi_thread_done();
} }
return TRUE; return TRUE;
} }
#elif defined(__cplusplus) #elif defined(__cplusplus)
// C++: use static initialization to detect process start // C++: use static initialization to detect process start
static bool _mi_process_init(void) { static bool _mi_process_init(void)
{
mi_process_load(); mi_process_load();
return (_mi_heap_main.thread_id != 0); return (_mi_heap_main.thread_id != 0);
} }
static bool mi_initialized = _mi_process_init(); static bool mi_initialized = _mi_process_init();
#elif defined(__GNUC__) || defined(__clang__) #elif defined(__GNUC__) || defined(__clang__)
// GCC,Clang: use the constructor attribute // GCC,Clang: use the constructor attribute
static void __attribute__((constructor)) _mi_process_init(void) { static void __attribute__((constructor)) _mi_process_init(void)
{
mi_process_load(); mi_process_load();
} }
#elif defined(_MSC_VER) #elif defined(_MSC_VER)
// MSVC: use data section magic for static libraries // MSVC: use data section magic for static libraries
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm> // See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
static int _mi_process_init(void) { static int _mi_process_init(void)
{
mi_process_load(); mi_process_load();
return 0; return 0;
} }
typedef int(*_crt_cb)(void); typedef int (*_crt_cb)(void);
#ifdef _M_X64 #ifdef _M_X64
__pragma(comment(linker, "/include:" "_mi_msvc_initu")) __pragma(comment(linker, "/include:"
#pragma section(".CRT$XIU", long, read) "_mi_msvc_initu"))
#else #pragma section(".CRT$XIU", long, read)
__pragma(comment(linker, "/include:" "__mi_msvc_initu")) #else
#endif __pragma(comment(linker, "/include:"
#pragma data_seg(".CRT$XIU") "__mi_msvc_initu"))
_crt_cb _mi_msvc_initu[] = { &_mi_process_init }; #endif
#pragma data_seg() #pragma data_seg(".CRT$XIU")
_crt_cb _mi_msvc_initu[] = {&_mi_process_init};
#pragma data_seg()
#else #else
#pragma message("define a way to call mi_process_load on your platform") #pragma message("define a way to call mi_process_load on your platform")

558
src/page.c
View file

File diff suppressed because it is too large Load diff