kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-20 22:19:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Heap local deferred free fun

Browse source
This commit is contained in:
playX 2021-03-09 19:12:52 +03:00
parent 217f2e2cc7
commit f459d576bd
5 changed files with 1489 additions and 1214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
include/mimalloc-types.h
View file

@ -60,7 +60,6 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_PADDING 1 #define MI_PADDING 1
#endif #endif
// Encoded free lists allow detection of corrupted free lists // Encoded free lists allow detection of corrupted free lists
// and can detect buffer overflows, modify after free, and double `free`s. // and can detect buffer overflows, modify after free, and double `free`s.
#if (MI_SECURE >= 3 || MI_DEBUG >= 1 || MI_PADDING > 0) #if (MI_SECURE >= 3 || MI_DEBUG >= 1 || MI_PADDING > 0)
@ -98,7 +97,6 @@ terms of the MIT license. A copy of the license can be found in the file
#define MiB (KiB * KiB) #define MiB (KiB * KiB)
#define GiB (MiB * KiB) #define GiB (MiB * KiB)
// ------------------------------------------------------ // ------------------------------------------------------
// Main internal data-structures // Main internal data-structures
// ------------------------------------------------------ // ------------------------------------------------------
@ -145,35 +143,39 @@ terms of the MIT license. A copy of the license can be found in the file
typedef uintptr_t mi_encoded_t; typedef uintptr_t mi_encoded_t;
// free lists contain blocks // free lists contain blocks
typedef struct mi_block_s { typedef struct mi_block_s
{
mi_encoded_t next; mi_encoded_t next;
} mi_block_t; } mi_block_t;
// The delayed flags are used for efficient multi-threaded free-ing // The delayed flags are used for efficient multi-threaded free-ing
typedef enum mi_delayed_e { typedef enum mi_delayed_e
{
MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list
MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap
MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list
MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim
} mi_delayed_t; } mi_delayed_t;
// The `in_full` and `has_aligned` page flags are put in a union to efficiently // The `in_full` and `has_aligned` page flags are put in a union to efficiently
// test if both are false (`full_aligned == 0`) in the `mi_free` routine. // test if both are false (`full_aligned == 0`) in the `mi_free` routine.
#if !MI_TSAN #if !MI_TSAN
typedef union mi_page_flags_s { typedef union mi_page_flags_s
{
uint8_t full_aligned; uint8_t full_aligned;
struct { struct
{
uint8_t in_full : 1; uint8_t in_full : 1;
uint8_t has_aligned : 1; uint8_t has_aligned : 1;
} x; } x;
} mi_page_flags_t; } mi_page_flags_t;
#else #else
// under thread sanitizer, use a byte for each flag to suppress warning, issue #130 // under thread sanitizer, use a byte for each flag to suppress warning, issue #130
typedef union mi_page_flags_s { typedef union mi_page_flags_s
{
uint16_t full_aligned; uint16_t full_aligned;
struct { struct
{
uint8_t in_full; uint8_t in_full;
uint8_t has_aligned; uint8_t has_aligned;
} x; } x;
@ -216,7 +218,8 @@ typedef uintptr_t mi_thread_free_t;
// at least one block that will be added, or as already been added, to // at least one block that will be added, or as already been added, to
// the owning heap `thread_delayed_free` list. This guarantees that pages // the owning heap `thread_delayed_free` list. This guarantees that pages
// will be freed correctly even if only other threads free blocks. // will be freed correctly even if only other threads free blocks.
typedef struct mi_page_s { typedef struct mi_page_s
{
// "owned" by the segment // "owned" by the segment
uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
uint8_t segment_in_use : 1; // `true` if the segment allocated this page uint8_t segment_in_use : 1; // `true` if the segment allocated this page
@ -246,9 +249,8 @@ typedef struct mi_page_s {
struct mi_page_s *prev; // previous page owned by this thread with the same `block_size` struct mi_page_s *prev; // previous page owned by this thread with the same `block_size`
} mi_page_t; } mi_page_t;
typedef enum mi_page_kind_e
{
typedef enum mi_page_kind_e {
MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment
MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
@ -258,7 +260,8 @@ typedef enum mi_page_kind_e {
// Segments are large allocated memory blocks (2mb on 64 bit) from // Segments are large allocated memory blocks (2mb on 64 bit) from
// the OS. Inside segments we allocated fixed size _pages_ that // the OS. Inside segments we allocated fixed size _pages_ that
// contain blocks. // contain blocks.
typedef struct mi_segment_s { typedef struct mi_segment_s
{
// memory fields // memory fields
size_t memid; // id for the os-level memory manager size_t memid; // id for the os-level memory manager
bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages) bool mem_is_pinned; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
@ -285,7 +288,6 @@ typedef struct mi_segment_s {
mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages
} mi_segment_t; } mi_segment_t;
// ------------------------------------------------------ // ------------------------------------------------------
// Heaps // Heaps
// Provide first-class heaps to allocate from. // Provide first-class heaps to allocate from.
@ -303,7 +305,8 @@ typedef struct mi_segment_s {
typedef struct mi_tld_s mi_tld_t; typedef struct mi_tld_s mi_tld_t;
// Pages of a certain block size are held in a queue. // Pages of a certain block size are held in a queue.
typedef struct mi_page_queue_s { typedef struct mi_page_queue_s
{
mi_page_t *first; mi_page_t *first;
mi_page_t *last; mi_page_t *last;
size_t block_size; size_t block_size;
@ -312,16 +315,17 @@ typedef struct mi_page_queue_s {
#define MI_BIN_FULL (MI_BIN_HUGE + 1) #define MI_BIN_FULL (MI_BIN_HUGE + 1)
// Random context // Random context
typedef struct mi_random_cxt_s { typedef struct mi_random_cxt_s
{
uint32_t input[16]; uint32_t input[16];
uint32_t output[16]; uint32_t output[16];
int output_available; int output_available;
} mi_random_ctx_t; } mi_random_ctx_t;
// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows // In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows
#if (MI_PADDING) #if (MI_PADDING)
typedef struct mi_padding_s { typedef struct mi_padding_s
{
uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) uint32_t canary; // encoded block value to check validity of the padding (in case of overflow)
uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes)
} mi_padding_t; } mi_padding_t;
@ -334,9 +338,9 @@ typedef struct mi_padding_s {
#define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1) #define MI_PAGES_DIRECT (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE + 1)
// A heap owns a set of pages. // A heap owns a set of pages.
struct mi_heap_s { struct mi_heap_s
{
mi_tld_t *tld; mi_tld_t *tld;
mi_page_t *pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_t *pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
@ -350,10 +354,10 @@ struct mi_heap_s {
size_t page_retired_max; // largest retired index into the `pages` array. size_t page_retired_max; // largest retired index into the `pages` array.
mi_heap_t *next; // list of heaps per thread mi_heap_t *next; // list of heaps per thread
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
void *deferred_free;
void *deferred_arg;
}; };
// ------------------------------------------------------ // ------------------------------------------------------
// Debug // Debug
// ------------------------------------------------------ // ------------------------------------------------------
@ -394,19 +398,22 @@ void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line
#endif #endif
#endif #endif
typedef struct mi_stat_count_s { typedef struct mi_stat_count_s
{
int64_t allocated; int64_t allocated;
int64_t freed; int64_t freed;
int64_t peak; int64_t peak;
int64_t current; int64_t current;
} mi_stat_count_t; } mi_stat_count_t;
typedef struct mi_stat_counter_s { typedef struct mi_stat_counter_s
{
int64_t total; int64_t total;
int64_t count; int64_t count;
} mi_stat_counter_t; } mi_stat_counter_t;
typedef struct mi_stats_s { typedef struct mi_stats_s
{
mi_stat_count_t segments; mi_stat_count_t segments;
mi_stat_count_t pages; mi_stat_count_t pages;
mi_stat_count_t reserved; mi_stat_count_t reserved;
@ -434,7 +441,6 @@ typedef struct mi_stats_s {
#endif #endif
} mi_stats_t; } mi_stats_t;
void _mi_stat_increase(mi_stat_count_t *stat, size_t amount); void _mi_stat_increase(mi_stat_count_t *stat, size_t amount);
void _mi_stat_decrease(mi_stat_count_t *stat, size_t amount); void _mi_stat_decrease(mi_stat_count_t *stat, size_t amount);
void _mi_stat_counter_increase(mi_stat_counter_t *stat, size_t amount); void _mi_stat_counter_increase(mi_stat_counter_t *stat, size_t amount);
@ -460,19 +466,22 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
typedef int64_t mi_msecs_t; typedef int64_t mi_msecs_t;
// Queue of segments // Queue of segments
typedef struct mi_segment_queue_s { typedef struct mi_segment_queue_s
{
mi_segment_t *first; mi_segment_t *first;
mi_segment_t *last; mi_segment_t *last;
} mi_segment_queue_t; } mi_segment_queue_t;
// OS thread local data // OS thread local data
typedef struct mi_os_tld_s { typedef struct mi_os_tld_s
{
size_t region_idx; // start point for next allocation size_t region_idx; // start point for next allocation
mi_stats_t *stats; // points to tld stats mi_stats_t *stats; // points to tld stats
} mi_os_tld_t; } mi_os_tld_t;
// Segments thread local data // Segments thread local data
typedef struct mi_segments_tld_s { typedef struct mi_segments_tld_s
{
mi_segment_queue_t small_free; // queue of segments with free small pages mi_segment_queue_t small_free; // queue of segments with free small pages
mi_segment_queue_t medium_free; // queue of segments with free medium pages mi_segment_queue_t medium_free; // queue of segments with free medium pages
mi_page_queue_t pages_reset; // queue of freed pages that can be reset mi_page_queue_t pages_reset; // queue of freed pages that can be reset
@ -488,7 +497,8 @@ typedef struct mi_segments_tld_s {
} mi_segments_tld_t; } mi_segments_tld_t;
// Thread local data // Thread local data
struct mi_tld_s { struct mi_tld_s
{
unsigned long long heartbeat; // monotonic heartbeat count unsigned long long heartbeat; // monotonic heartbeat count
bool recurse; // true if deferred was called; used to prevent infinite recursion. bool recurse; // true if deferred was called; used to prevent infinite recursion.
mi_heap_t *heap_backing; // backing heap of this thread (cannot be deleted) mi_heap_t *heap_backing; // backing heap of this thread (cannot be deleted)

65
include/mimalloc.h
View file

@ -93,7 +93,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stdbool.h> // bool #include <stdbool.h> // bool
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C"
{
#endif #endif
// ------------------------------------------------------ // ------------------------------------------------------
@ -127,7 +128,6 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize)
mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void *p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void *p) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
// ------------------------------------------------------ // ------------------------------------------------------
// Internals // Internals
// ------------------------------------------------------ // ------------------------------------------------------
@ -172,7 +172,6 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(siz
mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned(void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned(void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3);
mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export void *mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2);
// ------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------
// Heaps: first-class, but can only allocate from the same thread that created it. // Heaps: first-class, but can only allocate from the same thread that created it.
// ------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------
@ -187,6 +186,8 @@ mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
mi_decl_export mi_heap_t *mi_heap_get_default(void); mi_decl_export mi_heap_t *mi_heap_get_default(void);
mi_decl_export mi_heap_t *mi_heap_get_backing(void); mi_decl_export mi_heap_t *mi_heap_get_backing(void);
mi_decl_export void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept; mi_decl_export void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept;
typedef void(mi_local_deferred_free_fun)(mi_heap_t *heap, bool force, unsigned long long heartbeat, void *arg);
mi_decl_export void mi_heap_register_local_deferred_free(mi_heap_t *heap, mi_local_deferred_free_fun *deferred_free, void *arg) mi_attr_noexcept;
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_malloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_zalloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void *mi_heap_zalloc(mi_heap_t *heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
@ -211,7 +212,6 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_a
mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned(mi_heap_t *heap, void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned(mi_heap_t *heap, void *p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4);
mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); mi_decl_nodiscard mi_decl_export void *mi_heap_realloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3);
// -------------------------------------------------------------------------------- // --------------------------------------------------------------------------------
// Zero initialized re-allocation. // Zero initialized re-allocation.
// Only valid on memory that was originally allocated with zero initialization too. // Only valid on memory that was originally allocated with zero initialization too.
@ -235,7 +235,6 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* he
mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5); mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3, 4) mi_attr_alloc_align(5);
mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned_at(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4); mi_decl_nodiscard mi_decl_export void *mi_heap_recalloc_aligned_at(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3, 4);
// ------------------------------------------------------ // ------------------------------------------------------
// Analysis // Analysis
// ------------------------------------------------------ // ------------------------------------------------------
@ -245,7 +244,8 @@ mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
mi_decl_export bool mi_check_owned(const void *p); mi_decl_export bool mi_check_owned(const void *p);
// An area of heap space contains blocks of a single size. // An area of heap space contains blocks of a single size.
typedef struct mi_heap_area_s { typedef struct mi_heap_area_s
{
void *blocks; // start of the area containing heap blocks void *blocks; // start of the area containing heap blocks
size_t reserved; // bytes reserved for this area (virtual) size_t reserved; // bytes reserved for this area (virtual)
size_t committed; // current available bytes for this area size_t committed; // current available bytes for this area
@ -267,11 +267,9 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
mi_decl_export bool mi_manage_os_memory(void *start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void *start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
// deprecated // deprecated
mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t *pages_reserved) mi_attr_noexcept; mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t *pages_reserved) mi_attr_noexcept;
// ------------------------------------------------------ // ------------------------------------------------------
// Convenience // Convenience
// ------------------------------------------------------ // ------------------------------------------------------
@ -290,12 +288,12 @@ mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size
#define mi_heap_reallocn_tp(hp, p, tp, n) ((tp *)mi_heap_reallocn(hp, p, n, sizeof(tp))) #define mi_heap_reallocn_tp(hp, p, tp, n) ((tp *)mi_heap_reallocn(hp, p, n, sizeof(tp)))
#define mi_heap_recalloc_tp(hp, p, tp, n) ((tp *)mi_heap_recalloc(hp, p, n, sizeof(tp))) #define mi_heap_recalloc_tp(hp, p, tp, n) ((tp *)mi_heap_recalloc(hp, p, n, sizeof(tp)))
// ------------------------------------------------------ // ------------------------------------------------------
// Options, all `false` by default // Options, all `false` by default
// ------------------------------------------------------ // ------------------------------------------------------
typedef enum mi_option_e { typedef enum mi_option_e
{
// stable options // stable options
mi_option_show_errors, mi_option_show_errors,
mi_option_show_stats, mi_option_show_stats,
@ -321,7 +319,6 @@ typedef enum mi_option_e {
_mi_option_last _mi_option_last
} mi_option_t; } mi_option_t;
mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option);
mi_decl_export void mi_option_enable(mi_option_t option); mi_decl_export void mi_option_enable(mi_option_t option);
mi_decl_export void mi_option_disable(mi_option_t option); mi_decl_export void mi_option_disable(mi_option_t option);
@ -332,7 +329,6 @@ mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option);
mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set(mi_option_t option, long value);
mi_decl_export void mi_option_set_default(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value);
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
// "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions.
// (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.)
@ -389,7 +385,9 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount,
#include <utility> // std::forward #include <utility> // std::forward
#endif #endif
template<class T> struct mi_stl_allocator { template <class T>
struct mi_stl_allocator
{
typedef T value_type; typedef T value_type;
typedef std::size_t size_type; typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type; typedef std::ptrdiff_t difference_type;
@ -397,19 +395,30 @@ template<class T> struct mi_stl_allocator {
typedef value_type const &const_reference; typedef value_type const &const_reference;
typedef value_type *pointer; typedef value_type *pointer;
typedef value_type const *const_pointer; typedef value_type const *const_pointer;
template <class U> struct rebind { typedef mi_stl_allocator<U> other; }; template <class U>
struct rebind
{
typedef mi_stl_allocator<U> other;
};
mi_stl_allocator() mi_attr_noexcept = default; mi_stl_allocator() mi_attr_noexcept = default;
mi_stl_allocator(const mi_stl_allocator &) mi_attr_noexcept = default; mi_stl_allocator(const mi_stl_allocator &) mi_attr_noexcept = default;
template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { } template <class U>
mi_stl_allocator(const mi_stl_allocator<U> &) mi_attr_noexcept {}
mi_stl_allocator select_on_container_copy_construction() const { return *this; } mi_stl_allocator select_on_container_copy_construction() const { return *this; }
void deallocate(T *p, size_type) { mi_free(p); } void deallocate(T *p, size_type) { mi_free(p); }
#if (__cplusplus >= 201703L) // C++17 #if (__cplusplus >= 201703L) // C++17
mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); } mi_decl_nodiscard T *allocate(size_type count)
{
return static_cast<T *>(mi_new_n(count, sizeof(T)));
}
mi_decl_nodiscard T *allocate(size_type count, const void *) { return allocate(count); } mi_decl_nodiscard T *allocate(size_type count, const void *) { return allocate(count); }
#else #else
mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); } mi_decl_nodiscard pointer allocate(size_type count, const void * = 0)
{
return static_cast<pointer>(mi_new_n(count, sizeof(value_type)));
}
#endif #endif
#if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11
@ -417,20 +426,30 @@ template<class T> struct mi_stl_allocator {
using propagate_on_container_move_assignment = std::true_type; using propagate_on_container_move_assignment = std::true_type;
using propagate_on_container_swap = std::true_type; using propagate_on_container_swap = std::true_type;
using is_always_equal = std::true_type; using is_always_equal = std::true_type;
template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); } template <class U, class... Args>
template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); } void construct(U *p, Args &&...args) { ::new (p) U(std::forward<Args>(args)...); }
template <class U>
void destroy(U *p) mi_attr_noexcept { p->~U(); }
#else #else
void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } void construct(pointer p, value_type const &val)
{
::new (p) value_type(val);
}
void destroy(pointer p) { p->~value_type(); } void destroy(pointer p) { p->~value_type(); }
#endif #endif
size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } size_type max_size() const mi_attr_noexcept
{
return (PTRDIFF_MAX / sizeof(value_type));
}
pointer address(reference x) const { return &x; } pointer address(reference x) const { return &x; }
const_pointer address(const_reference x) const { return &x; } const_pointer address(const_reference x) const { return &x; }
}; };
template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; } template <class T1, class T2>
template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; } bool operator==(const mi_stl_allocator<T1> &, const mi_stl_allocator<T2> &) mi_attr_noexcept { return true; }
template <class T1, class T2>
bool operator!=(const mi_stl_allocator<T1> &, const mi_stl_allocator<T2> &) mi_attr_noexcept { return false; }
#endif // __cplusplus #endif // __cplusplus
#endif #endif

265
src/heap.c
View file

@ -25,21 +25,25 @@ typedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
// Visit all pages in a heap; returns `false` if break was called. // Visit all pages in a heap; returns `false` if break was called.
static bool mi_heap_visit_pages(mi_heap_t *heap, heap_page_visitor_fun *fn, void *arg1, void *arg2) static bool mi_heap_visit_pages(mi_heap_t *heap, heap_page_visitor_fun *fn, void *arg1, void *arg2)
{ {
if (heap==NULL || heap->page_count==0) return 0; if (heap == NULL || heap->page_count == 0)
return 0;
// visit all pages // visit all pages
#if MI_DEBUG > 1 #if MI_DEBUG > 1
size_t total = heap->page_count; size_t total = heap->page_count;
#endif #endif
size_t count = 0; size_t count = 0;
for (size_t i = 0; i <= MI_BIN_FULL; i++) { for (size_t i = 0; i <= MI_BIN_FULL; i++)
{
mi_page_queue_t *pq = &heap->pages[i]; mi_page_queue_t *pq = &heap->pages[i];
mi_page_t *page = pq->first; mi_page_t *page = pq->first;
while(page != NULL) { while (page != NULL)
{
mi_page_t *next = page->next; // save next in case the page gets removed from the queue mi_page_t *next = page->next; // save next in case the page gets removed from the queue
mi_assert_internal(mi_page_heap(page) == heap); mi_assert_internal(mi_page_heap(page) == heap);
count++; count++;
if (!fn(heap, pq, page, arg1, arg2)) return false; if (!fn(heap, pq, page, arg1, arg2))
return false;
page = next; // and continue page = next; // and continue
} }
} }
@ -47,9 +51,9 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void
return true; return true;
} }
#if MI_DEBUG >= 2 #if MI_DEBUG >= 2
static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { static bool mi_heap_page_is_valid(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1); UNUSED(arg1);
UNUSED(arg2); UNUSED(arg2);
UNUSED(pq); UNUSED(pq);
@ -61,16 +65,14 @@ static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
} }
#endif #endif
#if MI_DEBUG >= 3 #if MI_DEBUG >= 3
static bool mi_heap_is_valid(mi_heap_t* heap) { static bool mi_heap_is_valid(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL); mi_heap_visit_pages(heap, &mi_heap_page_is_valid, NULL, NULL);
return true; return true;
} }
#endif #endif
/* ----------------------------------------------------------- /* -----------------------------------------------------------
"Collect" pages by migrating `local_free` and `thread_free` "Collect" pages by migrating `local_free` and `thread_free`
lists and freeing empty pages. This is done when a thread lists and freeing empty pages. This is done when a thread
@ -78,32 +80,36 @@ static bool mi_heap_is_valid(mi_heap_t* heap) {
blocks alive) blocks alive)
----------------------------------------------------------- */ ----------------------------------------------------------- */
typedef enum mi_collect_e { typedef enum mi_collect_e
{
MI_NORMAL, MI_NORMAL,
MI_FORCE, MI_FORCE,
MI_ABANDON MI_ABANDON
} mi_collect_t; } mi_collect_t;
static bool mi_heap_page_collect(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg_collect, void *arg2)
static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { {
UNUSED(arg2); UNUSED(arg2);
UNUSED(heap); UNUSED(heap);
mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL)); mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL));
mi_collect_t collect = *((mi_collect_t *)arg_collect); mi_collect_t collect = *((mi_collect_t *)arg_collect);
_mi_page_free_collect(page, collect >= MI_FORCE); _mi_page_free_collect(page, collect >= MI_FORCE);
if (mi_page_all_free(page)) { if (mi_page_all_free(page))
{
// no more used blocks, free the page. // no more used blocks, free the page.
// note: this will free retired pages as well. // note: this will free retired pages as well.
_mi_page_free(page, pq, collect >= MI_FORCE); _mi_page_free(page, pq, collect >= MI_FORCE);
} }
else if (collect == MI_ABANDON) { else if (collect == MI_ABANDON)
{
// still used blocks but the thread is done; abandon the page // still used blocks but the thread is done; abandon the page
_mi_page_abandon(page, pq); _mi_page_abandon(page, pq);
} }
return true; // don't break return true; // don't break
} }
static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { static bool mi_heap_page_never_delayed_free(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1); UNUSED(arg1);
UNUSED(arg2); UNUSED(arg2);
UNUSED(heap); UNUSED(heap);
@ -114,7 +120,8 @@ static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq
static void mi_heap_collect_ex(mi_heap_t *heap, mi_collect_t collect) static void mi_heap_collect_ex(mi_heap_t *heap, mi_collect_t collect)
{ {
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
return;
_mi_deferred_free(heap, collect >= MI_FORCE); _mi_deferred_free(heap, collect >= MI_FORCE);
// note: never reclaim on collect but leave it to threads that need storage to reclaim // note: never reclaim on collect but leave it to threads that need storage to reclaim
@ -132,7 +139,8 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
} }
// if abandoning, mark all pages to no longer add to delayed_free // if abandoning, mark all pages to no longer add to delayed_free
if (collect == MI_ABANDON) { if (collect == MI_ABANDON)
{
mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL);
} }
@ -148,39 +156,45 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
mi_assert_internal(collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t, &heap->thread_delayed_free) == NULL); mi_assert_internal(collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t, &heap->thread_delayed_free) == NULL);
// collect segment caches // collect segment caches
if (collect >= MI_FORCE) { if (collect >= MI_FORCE)
{
_mi_segment_thread_collect(&heap->tld->segments); _mi_segment_thread_collect(&heap->tld->segments);
} }
// collect regions on program-exit (or shared library unload) // collect regions on program-exit (or shared library unload)
if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap)) { if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap))
{
_mi_mem_collect(&heap->tld->os); _mi_mem_collect(&heap->tld->os);
} }
} }
void _mi_heap_collect_abandon(mi_heap_t* heap) { void _mi_heap_collect_abandon(mi_heap_t *heap)
{
mi_heap_collect_ex(heap, MI_ABANDON); mi_heap_collect_ex(heap, MI_ABANDON);
} }
void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept { void mi_heap_collect(mi_heap_t *heap, bool force) mi_attr_noexcept
{
mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL)); mi_heap_collect_ex(heap, (force ? MI_FORCE : MI_NORMAL));
} }
void mi_collect(bool force) mi_attr_noexcept { void mi_collect(bool force) mi_attr_noexcept
{
mi_heap_collect(mi_get_default_heap(), force); mi_heap_collect(mi_get_default_heap(), force);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Heap new Heap new
----------------------------------------------------------- */ ----------------------------------------------------------- */
mi_heap_t* mi_heap_get_default(void) { mi_heap_t *mi_heap_get_default(void)
{
mi_thread_init(); mi_thread_init();
return mi_get_default_heap(); return mi_get_default_heap();
} }
mi_heap_t* mi_heap_get_backing(void) { mi_heap_t *mi_heap_get_backing(void)
{
mi_heap_t *heap = mi_heap_get_default(); mi_heap_t *heap = mi_heap_get_default();
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
mi_heap_t *bheap = heap->tld->heap_backing; mi_heap_t *bheap = heap->tld->heap_backing;
@ -189,10 +203,12 @@ mi_heap_t* mi_heap_get_backing(void) {
return bheap; return bheap;
} }
mi_heap_t* mi_heap_new(void) { mi_heap_t *mi_heap_new(void)
{
mi_heap_t *bheap = mi_heap_get_backing(); mi_heap_t *bheap = mi_heap_get_backing();
mi_heap_t *heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? mi_heap_t *heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
if (heap==NULL) return NULL; if (heap == NULL)
return NULL;
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
heap->tld = bheap->tld; heap->tld = bheap->tld;
heap->thread_id = _mi_thread_id(); heap->thread_id = _mi_thread_id();
@ -204,15 +220,18 @@ mi_heap_t* mi_heap_new(void) {
// push on the thread local heaps list // push on the thread local heaps list
heap->next = heap->tld->heaps; heap->next = heap->tld->heaps;
heap->tld->heaps = heap; heap->tld->heaps = heap;
heap->deferred_free = NULL;
return heap; return heap;
} }
uintptr_t _mi_heap_random_next(mi_heap_t* heap) { uintptr_t _mi_heap_random_next(mi_heap_t *heap)
{
return _mi_random_next(&heap->random); return _mi_random_next(&heap->random);
} }
// zero out the page queues // zero out the page queues
static void mi_heap_reset_pages(mi_heap_t* heap) { static void mi_heap_reset_pages(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap)); mi_assert_internal(mi_heap_is_initialized(heap));
// TODO: copy full empty heap instead? // TODO: copy full empty heap instead?
@ -226,14 +245,18 @@ static void mi_heap_reset_pages(mi_heap_t* heap) {
} }
// called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources. // called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources.
static void mi_heap_free(mi_heap_t* heap) { static void mi_heap_free(mi_heap_t *heap)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert_internal(mi_heap_is_initialized(heap)); mi_assert_internal(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
if (mi_heap_is_backing(heap)) return; // dont free the backing heap return;
if (mi_heap_is_backing(heap))
return; // dont free the backing heap
// reset default // reset default
if (mi_heap_is_default(heap)) { if (mi_heap_is_default(heap))
{
_mi_heap_set_default_direct(heap->tld->heap_backing); _mi_heap_set_default_direct(heap->tld->heap_backing);
} }
@ -241,14 +264,22 @@ static void mi_heap_free(mi_heap_t* heap) {
// linear search but we expect the number of heaps to be relatively small // linear search but we expect the number of heaps to be relatively small
mi_heap_t *prev = NULL; mi_heap_t *prev = NULL;
mi_heap_t *curr = heap->tld->heaps; mi_heap_t *curr = heap->tld->heaps;
while (curr != heap && curr != NULL) { while (curr != heap && curr != NULL)
{
prev = curr; prev = curr;
curr = curr->next; curr = curr->next;
} }
mi_assert_internal(curr == heap); mi_assert_internal(curr == heap);
if (curr == heap) { if (curr == heap)
if (prev != NULL) { prev->next = heap->next; } {
else { heap->tld->heaps = heap->next; } if (prev != NULL)
{
prev->next = heap->next;
}
else
{
heap->tld->heaps = heap->next;
}
} }
mi_assert_internal(heap->tld->heaps != NULL); mi_assert_internal(heap->tld->heaps != NULL);
@ -256,12 +287,12 @@ static void mi_heap_free(mi_heap_t* heap) {
mi_free(heap); mi_free(heap);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Heap destroy Heap destroy
----------------------------------------------------------- */ ----------------------------------------------------------- */
static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { static bool _mi_heap_page_destroy(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *arg1, void *arg2)
{
UNUSED(arg1); UNUSED(arg1);
UNUSED(arg2); UNUSED(arg2);
UNUSED(heap); UNUSED(heap);
@ -272,18 +303,22 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
// stats // stats
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
if (bsize > MI_LARGE_OBJ_SIZE_MAX) { if (bsize > MI_LARGE_OBJ_SIZE_MAX)
if (bsize > MI_HUGE_OBJ_SIZE_MAX) { {
if (bsize > MI_HUGE_OBJ_SIZE_MAX)
{
mi_heap_stat_decrease(heap, giant, bsize); mi_heap_stat_decrease(heap, giant, bsize);
} }
else { else
{
mi_heap_stat_decrease(heap, huge, bsize); mi_heap_stat_decrease(heap, huge, bsize);
} }
} }
#if (MI_STAT) #if (MI_STAT)
_mi_page_free_collect(page, false); // update used count _mi_page_free_collect(page, false); // update used count
const size_t inuse = page->used; const size_t inuse = page->used;
if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { if (bsize <= MI_LARGE_OBJ_SIZE_MAX)
{
mi_heap_stat_decrease(heap, normal, bsize * inuse); mi_heap_stat_decrease(heap, normal, bsize * inuse);
#if (MI_STAT > 1) #if (MI_STAT > 1)
mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse); mi_heap_stat_decrease(heap, normal_bins[_mi_bin(bsize)], inuse);
@ -305,38 +340,43 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
return true; // keep going return true; // keep going
} }
void _mi_heap_destroy_pages(mi_heap_t* heap) { void _mi_heap_destroy_pages(mi_heap_t *heap)
{
mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL); mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL);
mi_heap_reset_pages(heap); mi_heap_reset_pages(heap);
} }
void mi_heap_destroy(mi_heap_t* heap) { void mi_heap_destroy(mi_heap_t *heap)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap)); mi_assert(mi_heap_is_initialized(heap));
mi_assert(heap->no_reclaim); mi_assert(heap->no_reclaim);
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
if (!heap->no_reclaim) { return;
if (!heap->no_reclaim)
{
// don't free in case it may contain reclaimed pages // don't free in case it may contain reclaimed pages
mi_heap_delete(heap); mi_heap_delete(heap);
} }
else { else
{
// free all pages // free all pages
_mi_heap_destroy_pages(heap); _mi_heap_destroy_pages(heap);
mi_heap_free(heap); mi_heap_free(heap);
} }
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Safe Heap delete Safe Heap delete
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Tranfer the pages from one heap to the other // Tranfer the pages from one heap to the other
static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { static void mi_heap_absorb(mi_heap_t *heap, mi_heap_t *from)
{
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
if (from==NULL || from->page_count == 0) return; if (from == NULL || from->page_count == 0)
return;
// reduce the size of the delayed frees // reduce the size of the delayed frees
_mi_heap_delayed_free(from); _mi_heap_delayed_free(from);
@ -345,7 +385,8 @@ static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) {
// so threads may do delayed frees in either heap for a while. // so threads may do delayed frees in either heap for a while.
// note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state // note: appending waits for each page to not be in the `MI_DELAYED_FREEING` state
// so after this only the new heap will get delayed frees // so after this only the new heap will get delayed frees
for (size_t i = 0; i <= MI_BIN_FULL; i++) { for (size_t i = 0; i <= MI_BIN_FULL; i++)
{
mi_page_queue_t *pq = &heap->pages[i]; mi_page_queue_t *pq = &heap->pages[i];
mi_page_queue_t *append = &from->pages[i]; mi_page_queue_t *append = &from->pages[i];
size_t pcount = _mi_page_queue_append(heap, pq, append); size_t pcount = _mi_page_queue_append(heap, pq, append);
@ -371,13 +412,16 @@ void mi_heap_delete(mi_heap_t* heap)
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap)); mi_assert(mi_heap_is_initialized(heap));
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return; if (heap == NULL || !mi_heap_is_initialized(heap))
return;
if (!mi_heap_is_backing(heap)) { if (!mi_heap_is_backing(heap))
{
// tranfer still used pages to the backing heap // tranfer still used pages to the backing heap
mi_heap_absorb(heap->tld->heap_backing, heap); mi_heap_absorb(heap->tld->heap_backing, heap);
} }
else { else
{
// the backing heap abandons its pages // the backing heap abandons its pages
_mi_heap_collect_abandon(heap); _mi_heap_collect_abandon(heap);
} }
@ -385,41 +429,45 @@ void mi_heap_delete(mi_heap_t* heap)
mi_heap_free(heap); mi_heap_free(heap);
} }
mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { mi_heap_t *mi_heap_set_default(mi_heap_t *heap)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
mi_assert(mi_heap_is_initialized(heap)); mi_assert(mi_heap_is_initialized(heap));
if (heap==NULL || !mi_heap_is_initialized(heap)) return NULL; if (heap == NULL || !mi_heap_is_initialized(heap))
return NULL;
mi_assert_expensive(mi_heap_is_valid(heap)); mi_assert_expensive(mi_heap_is_valid(heap));
mi_heap_t *old = mi_get_default_heap(); mi_heap_t *old = mi_get_default_heap();
_mi_heap_set_default_direct(heap); _mi_heap_set_default_direct(heap);
return old; return old;
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Analysis Analysis
----------------------------------------------------------- */ ----------------------------------------------------------- */
// static since it is not thread safe to access heaps from other threads. // static since it is not thread safe to access heaps from other threads.
static mi_heap_t* mi_heap_of_block(const void* p) { static mi_heap_t *mi_heap_of_block(const void *p)
if (p == NULL) return NULL; {
if (p == NULL)
return NULL;
mi_segment_t *segment = _mi_ptr_segment(p); mi_segment_t *segment = _mi_ptr_segment(p);
bool valid = (_mi_ptr_cookie(segment) == segment->cookie); bool valid = (_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(valid); mi_assert_internal(valid);
if (mi_unlikely(!valid)) return NULL; if (mi_unlikely(!valid))
return NULL;
return mi_page_heap(_mi_segment_page_of(segment, p)); return mi_page_heap(_mi_segment_page_of(segment, p));
} }
bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { bool mi_heap_contains_block(mi_heap_t *heap, const void *p)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
if (heap==NULL || !mi_heap_is_initialized(heap)) return false; if (heap == NULL || !mi_heap_is_initialized(heap))
return false;
return (heap == mi_heap_of_block(p)); return (heap == mi_heap_of_block(p));
} }
static bool mi_heap_page_check_owned(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *p, void *vfound)
static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) { {
UNUSED(heap); UNUSED(heap);
UNUSED(pq); UNUSED(pq);
bool *found = (bool *)vfound; bool *found = (bool *)vfound;
@ -430,16 +478,20 @@ static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
return (!*found); // continue if not found return (!*found); // continue if not found
} }
bool mi_heap_check_owned(mi_heap_t* heap, const void* p) { bool mi_heap_check_owned(mi_heap_t *heap, const void *p)
{
mi_assert(heap != NULL); mi_assert(heap != NULL);
if (heap==NULL || !mi_heap_is_initialized(heap)) return false; if (heap == NULL || !mi_heap_is_initialized(heap))
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers return false;
if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)
return false; // only aligned pointers
bool found = false; bool found = false;
mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void *)p, &found); mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void *)p, &found);
return found; return found;
} }
bool mi_check_owned(const void* p) { bool mi_check_owned(const void *p)
{
return mi_heap_check_owned(mi_get_default_heap(), p); return mi_heap_check_owned(mi_get_default_heap(), p);
} }
@ -450,28 +502,34 @@ bool mi_check_owned(const void* p) {
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Separate struct to keep `mi_page_t` out of the public interface // Separate struct to keep `mi_page_t` out of the public interface
typedef struct mi_heap_area_ex_s { typedef struct mi_heap_area_ex_s
{
mi_heap_area_t area; mi_heap_area_t area;
mi_page_t *page; mi_page_t *page;
} mi_heap_area_ex_t; } mi_heap_area_ex_t;
static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) { static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t *xarea, mi_block_visit_fun *visitor, void *arg)
{
mi_assert(xarea != NULL); mi_assert(xarea != NULL);
if (xarea==NULL) return true; if (xarea == NULL)
return true;
const mi_heap_area_t *area = &xarea->area; const mi_heap_area_t *area = &xarea->area;
mi_page_t *page = xarea->page; mi_page_t *page = xarea->page;
mi_assert(page != NULL); mi_assert(page != NULL);
if (page == NULL) return true; if (page == NULL)
return true;
_mi_page_free_collect(page, true); _mi_page_free_collect(page, true);
mi_assert_internal(page->local_free == NULL); mi_assert_internal(page->local_free == NULL);
if (page->used == 0) return true; if (page->used == 0)
return true;
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
size_t psize; size_t psize;
uint8_t *pstart = _mi_page_start(_mi_page_segment(page), page, &psize); uint8_t *pstart = _mi_page_start(_mi_page_segment(page), page, &psize);
if (page->capacity == 1) { if (page->capacity == 1)
{
// optimize page with one block // optimize page with one block
mi_assert_internal(page->used == 1 && page->free == NULL); mi_assert_internal(page->used == 1 && page->free == NULL);
return visitor(mi_page_heap(page), area, pstart, bsize, arg); return visitor(mi_page_heap(page), area, pstart, bsize, arg);
@ -483,7 +541,8 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
memset(free_map, 0, sizeof(free_map)); memset(free_map, 0, sizeof(free_map));
size_t free_count = 0; size_t free_count = 0;
for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { for (mi_block_t *block = page->free; block != NULL; block = mi_block_next(page, block))
{
free_count++; free_count++;
mi_assert_internal((uint8_t *)block >= pstart && (uint8_t *)block < (pstart + psize)); mi_assert_internal((uint8_t *)block >= pstart && (uint8_t *)block < (pstart + psize));
size_t offset = (uint8_t *)block - pstart; size_t offset = (uint8_t *)block - pstart;
@ -498,17 +557,21 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
// walk through all blocks skipping the free ones // walk through all blocks skipping the free ones
size_t used_count = 0; size_t used_count = 0;
for (size_t i = 0; i < page->capacity; i++) { for (size_t i = 0; i < page->capacity; i++)
{
size_t bitidx = (i / sizeof(uintptr_t)); size_t bitidx = (i / sizeof(uintptr_t));
size_t bit = i - (bitidx * sizeof(uintptr_t)); size_t bit = i - (bitidx * sizeof(uintptr_t));
uintptr_t m = free_map[bitidx]; uintptr_t m = free_map[bitidx];
if (bit == 0 && m == UINTPTR_MAX) { if (bit == 0 && m == UINTPTR_MAX)
{
i += (sizeof(uintptr_t) - 1); // skip a run of free blocks i += (sizeof(uintptr_t) - 1); // skip a run of free blocks
} }
else if ((m & ((uintptr_t)1 << bit)) == 0) { else if ((m & ((uintptr_t)1 << bit)) == 0)
{
used_count++; used_count++;
uint8_t *block = pstart + (i * bsize); uint8_t *block = pstart + (i * bsize);
if (!visitor(mi_page_heap(page), area, block, bsize, arg)) return false; if (!visitor(mi_page_heap(page), area, block, bsize, arg))
return false;
} }
} }
mi_assert_internal(page->used == used_count); mi_assert_internal(page->used == used_count);
@ -517,8 +580,8 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
typedef bool(mi_heap_area_visit_fun)(const mi_heap_t *heap, const mi_heap_area_ex_t *area, void *arg); typedef bool(mi_heap_area_visit_fun)(const mi_heap_t *heap, const mi_heap_area_ex_t *area, void *arg);
static bool mi_heap_visit_areas_page(mi_heap_t *heap, mi_page_queue_t *pq, mi_page_t *page, void *vfun, void *arg)
static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { {
UNUSED(heap); UNUSED(heap);
UNUSED(pq); UNUSED(pq);
mi_heap_area_visit_fun *fun = (mi_heap_area_visit_fun *)vfun; mi_heap_area_visit_fun *fun = (mi_heap_area_visit_fun *)vfun;
@ -534,31 +597,45 @@ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_pa
} }
// Visit all heap pages as areas // Visit all heap pages as areas
static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) { static bool mi_heap_visit_areas(const mi_heap_t *heap, mi_heap_area_visit_fun *visitor, void *arg)
if (visitor == NULL) return false; {
if (visitor == NULL)
return false;
return mi_heap_visit_pages((mi_heap_t *)heap, &mi_heap_visit_areas_page, (void *)(visitor), arg); // note: function pointer to void* :-{ return mi_heap_visit_pages((mi_heap_t *)heap, &mi_heap_visit_areas_page, (void *)(visitor), arg); // note: function pointer to void* :-{
} }
// Just to pass arguments // Just to pass arguments
typedef struct mi_visit_blocks_args_s { typedef struct mi_visit_blocks_args_s
{
bool visit_blocks; bool visit_blocks;
mi_block_visit_fun *visitor; mi_block_visit_fun *visitor;
void *arg; void *arg;
} mi_visit_blocks_args_t; } mi_visit_blocks_args_t;
static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) { static bool mi_heap_area_visitor(const mi_heap_t *heap, const mi_heap_area_ex_t *xarea, void *arg)
{
mi_visit_blocks_args_t *args = (mi_visit_blocks_args_t *)arg; mi_visit_blocks_args_t *args = (mi_visit_blocks_args_t *)arg;
if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg))
if (args->visit_blocks) { return false;
if (args->visit_blocks)
{
return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg); return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg);
} }
else { else
{
return true; return true;
} }
} }
// Visit all blocks in a heap // Visit all blocks in a heap
bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
{
mi_visit_blocks_args_t args = {visit_blocks, visitor, arg}; mi_visit_blocks_args_t args = {visit_blocks, visitor, arg};
return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args); return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args);
} }
void mi_heap_register_local_deferred_free(mi_heap_t *heap, mi_local_deferred_free_fun *deferred_free, void *arg)
{
heap->deferred_free = (void *)deferred_free;
heap->deferred_arg = arg;
}

265
src/init.c
View file

@ -27,24 +27,36 @@ const mi_page_t _mi_page_empty = {
NULL, // local_free NULL, // local_free
ATOMIC_VAR_INIT(0), // xthread_free ATOMIC_VAR_INIT(0), // xthread_free
ATOMIC_VAR_INIT(0), // xheap ATOMIC_VAR_INIT(0), // xheap
NULL, NULL NULL,
}; NULL};
#define MI_PAGE_EMPTY() ((mi_page_t *)&_mi_page_empty) #define MI_PAGE_EMPTY() ((mi_page_t *)&_mi_page_empty)
#if (MI_PADDING > 0) && (MI_INTPTR_SIZE >= 8) #if (MI_PADDING > 0) && (MI_INTPTR_SIZE >= 8)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() \
}
#elif (MI_PADDING > 0) #elif (MI_PADDING > 0)
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() \
}
#else #else
#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #define MI_SMALL_PAGES_EMPTY \
{ \
MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() \
}
#endif #endif
// Empty page queues for every bin // Empty page queues for every bin
#define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } #define QNULL(sz) \
{ \
NULL, NULL, (sz) * sizeof(uintptr_t) \
}
#define MI_PAGE_QUEUES_EMPTY \ #define MI_PAGE_QUEUES_EMPTY \
{ QNULL(1), \ { \
QNULL(1), \
QNULL(1), QNULL(2), QNULL(3), QNULL(4), QNULL(5), QNULL(6), QNULL(7), QNULL(8), /* 8 */ \ QNULL(1), QNULL(2), QNULL(3), QNULL(4), QNULL(5), QNULL(6), QNULL(7), QNULL(8), /* 8 */ \
QNULL(10), QNULL(12), QNULL(14), QNULL(16), QNULL(20), QNULL(24), QNULL(28), QNULL(32), /* 16 */ \ QNULL(10), QNULL(12), QNULL(14), QNULL(16), QNULL(20), QNULL(24), QNULL(28), QNULL(32), /* 16 */ \
QNULL(40), QNULL(48), QNULL(56), QNULL(64), QNULL(80), QNULL(96), QNULL(112), QNULL(128), /* 24 */ \ QNULL(40), QNULL(48), QNULL(56), QNULL(64), QNULL(80), QNULL(96), QNULL(112), QNULL(128), /* 24 */ \
@ -55,13 +67,18 @@ const mi_page_t _mi_page_empty = {
QNULL(40960), QNULL(49152), QNULL(57344), QNULL(65536), QNULL(81920), QNULL(98304), QNULL(114688), QNULL(131072), /* 64 */ \ QNULL(40960), QNULL(49152), QNULL(57344), QNULL(65536), QNULL(81920), QNULL(98304), QNULL(114688), QNULL(131072), /* 64 */ \
QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \
QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ } QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ \
}
#define MI_STAT_COUNT_NULL() {0,0,0,0} #define MI_STAT_COUNT_NULL() \
{ \
0, 0, 0, 0 \
}
// Empty statistics // Empty statistics
#if MI_STAT > 1 #if MI_STAT > 1
#define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } #define MI_STAT_COUNT_END_NULL() \
, { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) }
#else #else
#define MI_STAT_COUNT_END_NULL() #define MI_STAT_COUNT_END_NULL()
#endif #endif
@ -75,8 +92,7 @@ const mi_page_t _mi_page_empty = {
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
{0, 0}, {0, 0}, {0, 0}, {0, 0}, \ {0, 0}, {0, 0}, {0, 0}, {0, 0}, \
{ 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ {0, 0}, {0, 0}, {0, 0}, {0, 0} MI_STAT_COUNT_END_NULL()
MI_STAT_COUNT_END_NULL()
// -------------------------------------------------------- // --------------------------------------------------------
// Statically allocate an empty heap as the initial // Statically allocate an empty heap as the initial
@ -97,10 +113,12 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
{0, 0}, // keys {0, 0}, // keys
{{0}, {0}, 0}, {{0}, {0}, 0},
0, // page count 0, // page count
MI_BIN_FULL, 0, // page retired min/max MI_BIN_FULL,
0, // page retired min/max
NULL, // next NULL, // next
false false,
}; 0,
0};
// the thread-local default heap for allocation // the thread-local default heap for allocation
mi_decl_thread mi_heap_t *_mi_heap_default = (mi_heap_t *)&_mi_heap_empty; mi_decl_thread mi_heap_t *_mi_heap_default = (mi_heap_t *)&_mi_heap_empty;
@ -108,12 +126,7 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
extern mi_heap_t _mi_heap_main; extern mi_heap_t _mi_heap_main;
static mi_tld_t tld_main = { static mi_tld_t tld_main = {
0, false, 0, false, &_mi_heap_main, &_mi_heap_main, {{NULL, NULL}, {NULL, NULL}, {NULL, NULL, 0}, 0, 0, 0, 0, 0, 0, NULL, &tld_main.stats, &tld_main.os}, // segments
&_mi_heap_main, &_mi_heap_main,
{ { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0},
0, 0, 0, 0, 0, 0, NULL,
&tld_main.stats, &tld_main.os
}, // segments
{0, &tld_main.stats}, // os {0, &tld_main.stats}, // os
{MI_STATS_NULL} // stats {MI_STATS_NULL} // stats
}; };
@ -128,18 +141,21 @@ mi_heap_t _mi_heap_main = {
{0, 0}, // the key of the main heap can be fixed (unlike page keys that need to be secure!) {0, 0}, // the key of the main heap can be fixed (unlike page keys that need to be secure!)
{{0x846ca68b}, {0}, 0}, // random {{0x846ca68b}, {0}, 0}, // random
0, // page count 0, // page count
MI_BIN_FULL, 0, // page retired min/max MI_BIN_FULL,
0, // page retired min/max
NULL, // next heap NULL, // next heap
false // can reclaim false, // can reclaim
}; 0,
0};
bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`.
mi_stats_t _mi_stats_main = {MI_STATS_NULL}; mi_stats_t _mi_stats_main = {MI_STATS_NULL};
static void mi_heap_main_init(void)
static void mi_heap_main_init(void) { {
if (_mi_heap_main.cookie == 0) { if (_mi_heap_main.cookie == 0)
{
_mi_heap_main.thread_id = _mi_thread_id(); _mi_heap_main.thread_id = _mi_thread_id();
_mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init); _mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init);
_mi_random_init(&_mi_heap_main.random); _mi_random_init(&_mi_heap_main.random);
@ -148,39 +164,46 @@ static void mi_heap_main_init(void) {
} }
} }
mi_heap_t* _mi_heap_main_get(void) { mi_heap_t *_mi_heap_main_get(void)
{
mi_heap_main_init(); mi_heap_main_init();
return &_mi_heap_main; return &_mi_heap_main;
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Initialization and freeing of the thread local heaps Initialization and freeing of the thread local heaps
----------------------------------------------------------- */ ----------------------------------------------------------- */
// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). // note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size).
typedef struct mi_thread_data_s { typedef struct mi_thread_data_s
{
mi_heap_t heap; // must come first due to cast in `_mi_heap_done` mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
mi_tld_t tld; mi_tld_t tld;
} mi_thread_data_t; } mi_thread_data_t;
// Initialize the thread local default heap, called from `mi_thread_init` // Initialize the thread local default heap, called from `mi_thread_init`
static bool _mi_heap_init(void) { static bool _mi_heap_init(void)
if (mi_heap_is_initialized(mi_get_default_heap())) return true; {
if (_mi_is_main_thread()) { if (mi_heap_is_initialized(mi_get_default_heap()))
return true;
if (_mi_is_main_thread())
{
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization
// the main heap is statically allocated // the main heap is statically allocated
mi_heap_main_init(); mi_heap_main_init();
_mi_heap_set_default_direct(&_mi_heap_main); _mi_heap_set_default_direct(&_mi_heap_main);
//mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap()); //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap());
} }
else { else
{
// use `_mi_os_alloc` to allocate directly from the OS // use `_mi_os_alloc` to allocate directly from the OS
mi_thread_data_t *td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); // Todo: more efficient allocation? mi_thread_data_t *td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); // Todo: more efficient allocation?
if (td == NULL) { if (td == NULL)
{
// if this fails, try once more. (issue #257) // if this fails, try once more. (issue #257)
td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main); td = (mi_thread_data_t *)_mi_os_alloc(sizeof(mi_thread_data_t), &_mi_stats_main);
if (td == NULL) { if (td == NULL)
{
// really out of memory // really out of memory
_mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t));
return false; return false;
@ -207,21 +230,26 @@ static bool _mi_heap_init(void) {
} }
// Free the thread local default heap (called from `mi_thread_done`) // Free the thread local default heap (called from `mi_thread_done`)
static bool _mi_heap_done(mi_heap_t* heap) { static bool _mi_heap_done(mi_heap_t *heap)
if (!mi_heap_is_initialized(heap)) return true; {
if (!mi_heap_is_initialized(heap))
return true;
// reset default heap // reset default heap
_mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t *)&_mi_heap_empty); _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t *)&_mi_heap_empty);
// switch to backing heap // switch to backing heap
heap = heap->tld->heap_backing; heap = heap->tld->heap_backing;
if (!mi_heap_is_initialized(heap)) return false; if (!mi_heap_is_initialized(heap))
return false;
// delete all non-backing heaps in this thread // delete all non-backing heaps in this thread
mi_heap_t *curr = heap->tld->heaps; mi_heap_t *curr = heap->tld->heaps;
while (curr != NULL) { while (curr != NULL)
{
mi_heap_t *next = curr->next; // save `next` as `curr` will be freed mi_heap_t *next = curr->next; // save `next` as `curr` will be freed
if (curr != heap) { if (curr != heap)
{
mi_assert_internal(!mi_heap_is_backing(curr)); mi_assert_internal(!mi_heap_is_backing(curr));
mi_heap_delete(curr); mi_heap_delete(curr);
} }
@ -231,7 +259,8 @@ static bool _mi_heap_done(mi_heap_t* heap) {
mi_assert_internal(mi_heap_is_backing(heap)); mi_assert_internal(mi_heap_is_backing(heap));
// collect if not the main thread // collect if not the main thread
if (heap != &_mi_heap_main) { if (heap != &_mi_heap_main)
{
_mi_heap_collect_abandon(heap); _mi_heap_collect_abandon(heap);
} }
@ -239,7 +268,8 @@ static bool _mi_heap_done(mi_heap_t* heap) {
_mi_stats_done(&heap->tld->stats); _mi_stats_done(&heap->tld->stats);
// free if not the main thread // free if not the main thread
if (heap != &_mi_heap_main) { if (heap != &_mi_heap_main)
{
mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id());
_mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main); _mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main);
} }
@ -254,8 +284,6 @@ static bool _mi_heap_done(mi_heap_t* heap) {
return false; return false;
} }
// -------------------------------------------------------- // --------------------------------------------------------
// Try to run `mi_thread_done()` automatically so any memory // Try to run `mi_thread_done()` automatically so any memory
// owned by the thread but not yet released can be abandoned // owned by the thread but not yet released can be abandoned
@ -293,16 +321,20 @@ static void _mi_thread_done(mi_heap_t* default_heap);
WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
#endif #endif
static DWORD mi_fls_key = (DWORD)(-1); static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) { static void NTAPI mi_fls_done(PVOID value)
if (value!=NULL) _mi_thread_done((mi_heap_t*)value); {
if (value != NULL)
_mi_thread_done((mi_heap_t *)value);
} }
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
// use pthread local storage keys to detect thread ending // use pthread local storage keys to detect thread ending
// (and used with MI_TLS_PTHREADS for the default heap) // (and used with MI_TLS_PTHREADS for the default heap)
#include <pthread.h> #include <pthread.h>
pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1);
static void mi_pthread_done(void* value) { static void mi_pthread_done(void *value)
if (value!=NULL) _mi_thread_done((mi_heap_t*)value); {
if (value != NULL)
_mi_thread_done((mi_heap_t *)value);
} }
#elif defined(__wasi__) #elif defined(__wasi__)
// no pthreads in the WebAssembly Standard Interface // no pthreads in the WebAssembly Standard Interface
@ -311,9 +343,11 @@ static void _mi_thread_done(mi_heap_t* default_heap);
#endif #endif
// Set up handlers so `mi_thread_done` is called automatically // Set up handlers so `mi_thread_done` is called automatically
static void mi_process_setup_auto_thread_done(void) { static void mi_process_setup_auto_thread_done(void)
{
static bool tls_initialized = false; // fine if it races static bool tls_initialized = false; // fine if it races
if (tls_initialized) return; if (tls_initialized)
return;
tls_initialized = true; tls_initialized = true;
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
// nothing to do as it is done in DllMain // nothing to do as it is done in DllMain
@ -326,8 +360,8 @@ static void mi_process_setup_auto_thread_done(void) {
_mi_heap_set_default_direct(&_mi_heap_main); _mi_heap_set_default_direct(&_mi_heap_main);
} }
bool _mi_is_main_thread(void)
bool _mi_is_main_thread(void) { {
return (_mi_heap_main.thread_id == 0 || _mi_heap_main.thread_id == _mi_thread_id()); return (_mi_heap_main.thread_id == 0 || _mi_heap_main.thread_id == _mi_thread_id());
} }
@ -340,27 +374,33 @@ void mi_thread_init(void) mi_attr_noexcept
// initialize the thread local default heap // initialize the thread local default heap
// (this will call `_mi_heap_set_default_direct` and thus set the // (this will call `_mi_heap_set_default_direct` and thus set the
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
if (_mi_heap_init()) return; // returns true if already initialized if (_mi_heap_init())
return; // returns true if already initialized
_mi_stat_increase(&_mi_stats_main.threads, 1); _mi_stat_increase(&_mi_stats_main.threads, 1);
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
} }
void mi_thread_done(void) mi_attr_noexcept { void mi_thread_done(void) mi_attr_noexcept
{
_mi_thread_done(mi_get_default_heap()); _mi_thread_done(mi_get_default_heap());
} }
static void _mi_thread_done(mi_heap_t* heap) { static void _mi_thread_done(mi_heap_t *heap)
{
_mi_stat_decrease(&_mi_stats_main.threads, 1); _mi_stat_decrease(&_mi_stats_main.threads, 1);
// check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps...
if (heap->thread_id != _mi_thread_id()) return; if (heap->thread_id != _mi_thread_id())
return;
// abandon the thread local heap // abandon the thread local heap
if (_mi_heap_done(heap)) return; // returns true if already ran if (_mi_heap_done(heap))
return; // returns true if already ran
} }
void _mi_heap_set_default_direct(mi_heap_t* heap) { void _mi_heap_set_default_direct(mi_heap_t *heap)
{
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
#if defined(MI_TLS_SLOT) #if defined(MI_TLS_SLOT)
mi_tls_slot_set(MI_TLS_SLOT, heap); mi_tls_slot_set(MI_TLS_SLOT, heap);
@ -380,13 +420,13 @@ void _mi_heap_set_default_direct(mi_heap_t* heap) {
mi_assert_internal(mi_fls_key != 0); mi_assert_internal(mi_fls_key != 0);
FlsSetValue(mi_fls_key, heap); FlsSetValue(mi_fls_key, heap);
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
if (_mi_heap_default_key != (pthread_key_t)(-1)) { // can happen during recursive invocation on freeBSD if (_mi_heap_default_key != (pthread_key_t)(-1))
{ // can happen during recursive invocation on freeBSD
pthread_setspecific(_mi_heap_default_key, heap); pthread_setspecific(_mi_heap_default_key, heap);
} }
#endif #endif
} }
// -------------------------------------------------------- // --------------------------------------------------------
// Run functions on process init/done, and thread init/done // Run functions on process init/done, and thread init/done
// -------------------------------------------------------- // --------------------------------------------------------
@ -396,28 +436,35 @@ static bool os_preloading = true; // true until this module is initialized
static bool mi_redirected = false; // true if malloc redirects to mi_malloc static bool mi_redirected = false; // true if malloc redirects to mi_malloc
// Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false.
bool _mi_preloading(void) { bool _mi_preloading(void)
{
return os_preloading; return os_preloading;
} }
bool mi_is_redirected(void) mi_attr_noexcept { bool mi_is_redirected(void) mi_attr_noexcept
{
return mi_redirected; return mi_redirected;
} }
// Communicate with the redirection module on Windows // Communicate with the redirection module on Windows
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C"
{
#endif #endif
mi_decl_export void _mi_redirect_entry(DWORD reason) { mi_decl_export void _mi_redirect_entry(DWORD reason)
{
// called on redirection; careful as this may be called before DllMain // called on redirection; careful as this may be called before DllMain
if (reason == DLL_PROCESS_ATTACH) { if (reason == DLL_PROCESS_ATTACH)
{
mi_redirected = true; mi_redirected = true;
} }
else if (reason == DLL_PROCESS_DETACH) { else if (reason == DLL_PROCESS_DETACH)
{
mi_redirected = false; mi_redirected = false;
} }
else if (reason == DLL_THREAD_DETACH) { else if (reason == DLL_THREAD_DETACH)
{
mi_thread_done(); mi_thread_done();
} }
} }
@ -427,17 +474,21 @@ __declspec(dllimport) void mi_allocator_done(void);
} }
#endif #endif
#else #else
static bool mi_allocator_init(const char** message) { static bool mi_allocator_init(const char **message)
if (message != NULL) *message = NULL; {
if (message != NULL)
*message = NULL;
return true; return true;
} }
static void mi_allocator_done(void) { static void mi_allocator_done(void)
{
// nothing to do // nothing to do
} }
#endif #endif
// Called once by the process loader // Called once by the process loader
static void mi_process_load(void) { static void mi_process_load(void)
{
mi_heap_main_init(); mi_heap_main_init();
#if defined(MI_TLS_RECURSE_GUARD) #if defined(MI_TLS_RECURSE_GUARD)
volatile mi_heap_t *dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; volatile mi_heap_t *dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true;
@ -448,12 +499,14 @@ static void mi_process_load(void) {
_mi_options_init(); _mi_options_init();
mi_process_init(); mi_process_init();
//mi_stats_reset();- //mi_stats_reset();-
if (mi_redirected) _mi_verbose_message("malloc is redirected.\n"); if (mi_redirected)
_mi_verbose_message("malloc is redirected.\n");
// show message from the redirector (if present) // show message from the redirector (if present)
const char *msg = NULL; const char *msg = NULL;
mi_allocator_init(&msg); mi_allocator_init(&msg);
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors)))
{
_mi_fputs(NULL, NULL, NULL, msg); _mi_fputs(NULL, NULL, NULL, msg);
} }
} }
@ -462,22 +515,26 @@ static void mi_process_load(void) {
#include <intrin.h> #include <intrin.h>
mi_decl_cache_align bool _mi_cpu_has_fsrm = false; mi_decl_cache_align bool _mi_cpu_has_fsrm = false;
static void mi_detect_cpu_features(void) { static void mi_detect_cpu_features(void)
{
// FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017))
int32_t cpu_info[4]; int32_t cpu_info[4];
__cpuid(cpu_info, 7); __cpuid(cpu_info, 7);
_mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https ://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features> _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https ://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features>
} }
#else #else
static void mi_detect_cpu_features(void) { static void mi_detect_cpu_features(void)
{
// nothing // nothing
} }
#endif #endif
// Initialize the process; called by thread_init or the process loader // Initialize the process; called by thread_init or the process loader
void mi_process_init(void) mi_attr_noexcept { void mi_process_init(void) mi_attr_noexcept
{
// ensure we are called once // ensure we are called once
if (_mi_process_is_initialized) return; if (_mi_process_is_initialized)
return;
_mi_process_is_initialized = true; _mi_process_is_initialized = true;
mi_process_setup_auto_thread_done(); mi_process_setup_auto_thread_done();
@ -492,23 +549,29 @@ void mi_process_init(void) mi_attr_noexcept {
mi_thread_init(); mi_thread_init();
mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL)
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { if (mi_option_is_enabled(mi_option_reserve_huge_os_pages))
{
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages); size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
mi_reserve_huge_os_pages_interleave(pages, 0, pages * 500); mi_reserve_huge_os_pages_interleave(pages, 0, pages * 500);
} }
if (mi_option_is_enabled(mi_option_reserve_os_memory)) { if (mi_option_is_enabled(mi_option_reserve_os_memory))
{
long ksize = mi_option_get(mi_option_reserve_os_memory); long ksize = mi_option_get(mi_option_reserve_os_memory);
if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true); if (ksize > 0)
mi_reserve_os_memory((size_t)ksize * KiB, true, true);
} }
} }
// Called when the process is done (through `at_exit`) // Called when the process is done (through `at_exit`)
static void mi_process_done(void) { static void mi_process_done(void)
{
// only shutdown if we were initialized // only shutdown if we were initialized
if (!_mi_process_is_initialized) return; if (!_mi_process_is_initialized)
return;
// ensure we are called once // ensure we are called once
static bool process_done = false; static bool process_done = false;
if (process_done) return; if (process_done)
return;
process_done = true; process_done = true;
#if defined(_WIN32) && !defined(MI_SHARED_LIB) #if defined(_WIN32) && !defined(MI_SHARED_LIB)
@ -523,7 +586,8 @@ static void mi_process_done(void) {
mi_collect(true /* force */); mi_collect(true /* force */);
#endif #endif
if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose))
{
mi_stats_print(NULL); mi_stats_print(NULL);
} }
mi_allocator_done(); mi_allocator_done();
@ -531,25 +595,28 @@ static void mi_process_done(void) {
os_preloading = true; // don't call the C runtime anymore os_preloading = true; // don't call the C runtime anymore
} }
#if defined(_WIN32) && defined(MI_SHARED_LIB) #if defined(_WIN32) && defined(MI_SHARED_LIB)
// Windows DLL: easy to hook into process_init and thread_done // Windows DLL: easy to hook into process_init and thread_done
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved)
{
UNUSED(reserved); UNUSED(reserved);
UNUSED(inst); UNUSED(inst);
if (reason==DLL_PROCESS_ATTACH) { if (reason == DLL_PROCESS_ATTACH)
{
mi_process_load(); mi_process_load();
} }
else if (reason==DLL_THREAD_DETACH) { else if (reason == DLL_THREAD_DETACH)
if (!mi_is_redirected()) mi_thread_done(); {
if (!mi_is_redirected())
mi_thread_done();
} }
return TRUE; return TRUE;
} }
#elif defined(__cplusplus) #elif defined(__cplusplus)
// C++: use static initialization to detect process start // C++: use static initialization to detect process start
static bool _mi_process_init(void) { static bool _mi_process_init(void)
{
mi_process_load(); mi_process_load();
return (_mi_heap_main.thread_id != 0); return (_mi_heap_main.thread_id != 0);
} }
@ -557,23 +624,27 @@ static void mi_process_done(void) {
#elif defined(__GNUC__) || defined(__clang__) #elif defined(__GNUC__) || defined(__clang__)
// GCC,Clang: use the constructor attribute // GCC,Clang: use the constructor attribute
static void __attribute__((constructor)) _mi_process_init(void) { static void __attribute__((constructor)) _mi_process_init(void)
{
mi_process_load(); mi_process_load();
} }
#elif defined(_MSC_VER) #elif defined(_MSC_VER)
// MSVC: use data section magic for static libraries // MSVC: use data section magic for static libraries
// See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm> // See <https://www.codeguru.com/cpp/misc/misc/applicationcontrol/article.php/c6945/Running-Code-Before-and-After-Main.htm>
static int _mi_process_init(void) { static int _mi_process_init(void)
{
mi_process_load(); mi_process_load();
return 0; return 0;
} }
typedef int (*_crt_cb)(void); typedef int (*_crt_cb)(void);
#ifdef _M_X64 #ifdef _M_X64
__pragma(comment(linker, "/include:" "_mi_msvc_initu")) __pragma(comment(linker, "/include:"
"_mi_msvc_initu"))
#pragma section(".CRT$XIU", long, read) #pragma section(".CRT$XIU", long, read)
#else #else
__pragma(comment(linker, "/include:" "__mi_msvc_initu")) __pragma(comment(linker, "/include:"
"__mi_msvc_initu"))
#endif #endif
#pragma data_seg(".CRT$XIU") #pragma data_seg(".CRT$XIU")
_crt_cb _mi_msvc_initu[] = {&_mi_process_init}; _crt_cb _mi_msvc_initu[] = {&_mi_process_init};

324
src/page.c
View file

@ -23,13 +23,13 @@ terms of the MIT license. A copy of the license can be found in the file
#include "page-queue.c" #include "page-queue.c"
#undef MI_IN_PAGE_C #undef MI_IN_PAGE_C
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Page helpers Page helpers
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Index a block in a page // Index a block in a page
static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) { static inline mi_block_t *mi_page_block_at(const mi_page_t *page, void *page_start, size_t block_size, size_t i)
{
UNUSED(page); UNUSED(page);
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
mi_assert_internal(i <= page->reserved); mi_assert_internal(i <= page->reserved);
@ -40,9 +40,11 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t
static void mi_page_extend_free(mi_heap_t *heap, mi_page_t *page, mi_tld_t *tld); static void mi_page_extend_free(mi_heap_t *heap, mi_page_t *page, mi_tld_t *tld);
#if (MI_DEBUG >= 3) #if (MI_DEBUG >= 3)
static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) { static size_t mi_page_list_count(mi_page_t *page, mi_block_t *head)
{
size_t count = 0; size_t count = 0;
while (head != NULL) { while (head != NULL)
{
mi_assert_internal(page == _mi_ptr_page(head)); mi_assert_internal(page == _mi_ptr_page(head));
count++; count++;
head = mi_block_next(page, head); head = mi_block_next(page, head);
@ -57,19 +59,23 @@ static inline uint8_t* mi_page_area(const mi_page_t* page) {
} }
*/ */
static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { static bool mi_page_list_is_valid(mi_page_t *page, mi_block_t *p)
{
size_t psize; size_t psize;
uint8_t *page_area = _mi_page_start(_mi_page_segment(page), page, &psize); uint8_t *page_area = _mi_page_start(_mi_page_segment(page), page, &psize);
mi_block_t *start = (mi_block_t *)page_area; mi_block_t *start = (mi_block_t *)page_area;
mi_block_t *end = (mi_block_t *)(page_area + psize); mi_block_t *end = (mi_block_t *)(page_area + psize);
while(p != NULL) { while (p != NULL)
if (p < start || p >= end) return false; {
if (p < start || p >= end)
return false;
p = mi_block_next(page, p); p = mi_block_next(page, p);
} }
return true; return true;
} }
static bool mi_page_is_valid_init(mi_page_t* page) { static bool mi_page_is_valid_init(mi_page_t *page)
{
mi_assert_internal(page->xblock_size > 0); mi_assert_internal(page->xblock_size > 0);
mi_assert_internal(page->used <= page->capacity); mi_assert_internal(page->used <= page->capacity);
mi_assert_internal(page->capacity <= page->reserved); mi_assert_internal(page->capacity <= page->reserved);
@ -84,8 +90,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
mi_assert_internal(mi_page_list_is_valid(page, page->local_free)); mi_assert_internal(mi_page_list_is_valid(page, page->local_free));
#if MI_DEBUG > 3 // generally too expensive to check this #if MI_DEBUG > 3 // generally too expensive to check this
if (page->flags.is_zero) { if (page->flags.is_zero)
for(mi_block_t* block = page->free; block != NULL; mi_block_next(page,block)) { {
for (mi_block_t *block = page->free; block != NULL; mi_block_next(page, block))
{
mi_assert_expensive(mi_mem_is_zero(block + 1, page->block_size - sizeof(mi_block_t))); mi_assert_expensive(mi_mem_is_zero(block + 1, page->block_size - sizeof(mi_block_t)));
} }
} }
@ -102,15 +110,18 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
return true; return true;
} }
bool _mi_page_is_valid(mi_page_t* page) { bool _mi_page_is_valid(mi_page_t *page)
{
mi_assert_internal(mi_page_is_valid_init(page)); mi_assert_internal(mi_page_is_valid_init(page));
#if MI_SECURE #if MI_SECURE
mi_assert_internal(page->keys[0] != 0); mi_assert_internal(page->keys[0] != 0);
#endif #endif
if (mi_page_heap(page)!=NULL) { if (mi_page_heap(page) != NULL)
{
mi_segment_t *segment = _mi_page_segment(page); mi_segment_t *segment = _mi_page_segment(page);
mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == mi_page_heap(page)->thread_id || segment->thread_id == 0); mi_assert_internal(!_mi_process_is_initialized || segment->thread_id == mi_page_heap(page)->thread_id || segment->thread_id == 0);
if (segment->page_kind != MI_PAGE_HUGE) { if (segment->page_kind != MI_PAGE_HUGE)
{
mi_page_queue_t *pq = mi_page_queue_of(page); mi_page_queue_t *pq = mi_page_queue_of(page);
mi_assert_internal(mi_page_queue_contains(pq, page)); mi_assert_internal(mi_page_queue_contains(pq, page));
mi_assert_internal(pq->block_size == mi_page_block_size(page) || mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page)); mi_assert_internal(pq->block_size == mi_page_block_size(page) || mi_page_block_size(page) > MI_LARGE_OBJ_SIZE_MAX || mi_page_is_in_full(page));
@ -121,22 +132,27 @@ bool _mi_page_is_valid(mi_page_t* page) {
} }
#endif #endif
void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never) { void _mi_page_use_delayed_free(mi_page_t *page, mi_delayed_t delay, bool override_never)
{
mi_thread_free_t tfreex; mi_thread_free_t tfreex;
mi_delayed_t old_delay; mi_delayed_t old_delay;
mi_thread_free_t tfree; mi_thread_free_t tfree;
do { do
{
tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS; tfree = mi_atomic_load_acquire(&page->xthread_free); // note: must acquire as we can break/repeat this loop and not do a CAS;
tfreex = mi_tf_set_delayed(tfree, delay); tfreex = mi_tf_set_delayed(tfree, delay);
old_delay = mi_tf_delayed(tfree); old_delay = mi_tf_delayed(tfree);
if (mi_unlikely(old_delay == MI_DELAYED_FREEING)) { if (mi_unlikely(old_delay == MI_DELAYED_FREEING))
{
mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done. mi_atomic_yield(); // delay until outstanding MI_DELAYED_FREEING are done.
// tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail // tfree = mi_tf_set_delayed(tfree, MI_NO_DELAYED_FREE); // will cause CAS to busy fail
} }
else if (delay == old_delay) { else if (delay == old_delay)
{
break; // avoid atomic operation if already equal break; // avoid atomic operation if already equal
} }
else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE) { else if (!override_never && old_delay == MI_NEVER_DELAYED_FREE)
{
break; // leave never-delayed flag set break; // leave never-delayed flag set
} }
} while ((old_delay == MI_DELAYED_FREEING) || } while ((old_delay == MI_DELAYED_FREEING) ||
@ -156,25 +172,29 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
mi_block_t *head; mi_block_t *head;
mi_thread_free_t tfreex; mi_thread_free_t tfreex;
mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free); mi_thread_free_t tfree = mi_atomic_load_relaxed(&page->xthread_free);
do { do
{
head = mi_tf_block(tfree); head = mi_tf_block(tfree);
tfreex = mi_tf_set_block(tfree, NULL); tfreex = mi_tf_set_block(tfree, NULL);
} while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex)); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));
// return if the list is empty // return if the list is empty
if (head == NULL) return; if (head == NULL)
return;
// find the tail -- also to get a proper count (without data races) // find the tail -- also to get a proper count (without data races)
uint32_t max_count = page->capacity; // cannot collect more than capacity uint32_t max_count = page->capacity; // cannot collect more than capacity
uint32_t count = 1; uint32_t count = 1;
mi_block_t *tail = head; mi_block_t *tail = head;
mi_block_t *next; mi_block_t *next;
while ((next = mi_block_next(page,tail)) != NULL && count <= max_count) { while ((next = mi_block_next(page, tail)) != NULL && count <= max_count)
{
count++; count++;
tail = next; tail = next;
} }
// if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free) // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free)
if (count > max_count) { if (count > max_count)
{
_mi_error_message(EFAULT, "corrupted thread-free list\n"); _mi_error_message(EFAULT, "corrupted thread-free list\n");
return; // the thread-free items cannot be freed return; // the thread-free items cannot be freed
} }
@ -187,27 +207,33 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
page->used -= count; page->used -= count;
} }
void _mi_page_free_collect(mi_page_t* page, bool force) { void _mi_page_free_collect(mi_page_t *page, bool force)
{
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
// collect the thread free list // collect the thread free list
if (force || mi_page_thread_free(page) != NULL) { // quick test to avoid an atomic operation if (force || mi_page_thread_free(page) != NULL)
{ // quick test to avoid an atomic operation
_mi_page_thread_free_collect(page); _mi_page_thread_free_collect(page);
} }
// and the local free list // and the local free list
if (page->local_free != NULL) { if (page->local_free != NULL)
if (mi_likely(page->free == NULL)) { {
if (mi_likely(page->free == NULL))
{
// usual case // usual case
page->free = page->local_free; page->free = page->local_free;
page->local_free = NULL; page->local_free = NULL;
page->is_zero = false; page->is_zero = false;
} }
else if (force) { else if (force)
{
// append -- only on shutdown (force) as this is a linear operation // append -- only on shutdown (force) as this is a linear operation
mi_block_t *tail = page->local_free; mi_block_t *tail = page->local_free;
mi_block_t *next; mi_block_t *next;
while ((next = mi_block_next(page, tail)) != NULL) { while ((next = mi_block_next(page, tail)) != NULL)
{
tail = next; tail = next;
} }
mi_block_set_next(page, tail, page->free); mi_block_set_next(page, tail, page->free);
@ -220,14 +246,13 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
mi_assert_internal(!force || page->local_free == NULL); mi_assert_internal(!force || page->local_free == NULL);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Page fresh and retire Page fresh and retire
----------------------------------------------------------- */ ----------------------------------------------------------- */
// called from segments when reclaiming abandoned pages // called from segments when reclaiming abandoned pages
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) { void _mi_page_reclaim(mi_heap_t *heap, mi_page_t *page)
{
mi_assert_expensive(mi_page_is_valid_init(page)); mi_assert_expensive(mi_page_is_valid_init(page));
mi_assert_internal(mi_page_heap(page) == heap); mi_assert_internal(mi_page_heap(page) == heap);
mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE); mi_assert_internal(mi_page_thread_free_flag(page) != MI_NEVER_DELAYED_FREE);
@ -240,11 +265,13 @@ void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page) {
} }
// allocate a fresh page from a segment // allocate a fresh page from a segment
static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size_t block_size) { static mi_page_t *mi_page_fresh_alloc(mi_heap_t *heap, mi_page_queue_t *pq, size_t block_size)
{
mi_assert_internal(pq == NULL || mi_heap_contains_queue(heap, pq)); mi_assert_internal(pq == NULL || mi_heap_contains_queue(heap, pq));
mi_assert_internal(pq == NULL || block_size == pq->block_size); mi_assert_internal(pq == NULL || block_size == pq->block_size);
mi_page_t *page = _mi_segment_page_alloc(heap, block_size, &heap->tld->segments, &heap->tld->os); mi_page_t *page = _mi_segment_page_alloc(heap, block_size, &heap->tld->segments, &heap->tld->os);
if (page == NULL) { if (page == NULL)
{
// this may be out-of-memory, or an abandoned page was reclaimed (and in our queue) // this may be out-of-memory, or an abandoned page was reclaimed (and in our queue)
return NULL; return NULL;
} }
@ -252,16 +279,19 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
mi_assert_internal(pq == NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE); mi_assert_internal(pq == NULL || _mi_page_segment(page)->page_kind != MI_PAGE_HUGE);
mi_page_init(heap, page, block_size, heap->tld); mi_page_init(heap, page, block_size, heap->tld);
_mi_stat_increase(&heap->tld->stats.pages, 1); _mi_stat_increase(&heap->tld->stats.pages, 1);
if (pq!=NULL) mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL if (pq != NULL)
mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL
mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_expensive(_mi_page_is_valid(page));
return page; return page;
} }
// Get a fresh page to use // Get a fresh page to use
static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) { static mi_page_t *mi_page_fresh(mi_heap_t *heap, mi_page_queue_t *pq)
{
mi_assert_internal(mi_heap_contains_queue(heap, pq)); mi_assert_internal(mi_heap_contains_queue(heap, pq));
mi_page_t *page = mi_page_fresh_alloc(heap, pq, pq->block_size); mi_page_t *page = mi_page_fresh_alloc(heap, pq, pq->block_size);
if (page==NULL) return NULL; if (page == NULL)
return NULL;
mi_assert_internal(pq->block_size == mi_page_block_size(page)); mi_assert_internal(pq->block_size == mi_page_block_size(page));
mi_assert_internal(pq == mi_page_queue(heap, mi_page_block_size(page))); mi_assert_internal(pq == mi_page_queue(heap, mi_page_block_size(page)));
return page; return page;
@ -271,20 +301,26 @@ static mi_page_t* mi_page_fresh(mi_heap_t* heap, mi_page_queue_t* pq) {
Do any delayed frees Do any delayed frees
(put there by other threads if they deallocated in a full page) (put there by other threads if they deallocated in a full page)
----------------------------------------------------------- */ ----------------------------------------------------------- */
void _mi_heap_delayed_free(mi_heap_t* heap) { void _mi_heap_delayed_free(mi_heap_t *heap)
{
// take over the list (note: no atomic exchange since it is often NULL) // take over the list (note: no atomic exchange since it is often NULL)
mi_block_t *block = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); mi_block_t *block = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);
while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL)) { /* nothing */ }; while (block != NULL && !mi_atomic_cas_ptr_weak_acq_rel(mi_block_t, &heap->thread_delayed_free, &block, NULL))
{ /* nothing */
};
// and free them all // and free them all
while(block != NULL) { while (block != NULL)
{
mi_block_t *next = mi_block_nextx(heap, block, heap->keys); mi_block_t *next = mi_block_nextx(heap, block, heap->keys);
// use internal free instead of regular one to keep stats etc correct // use internal free instead of regular one to keep stats etc correct
if (!_mi_free_delayed_block(block)) { if (!_mi_free_delayed_block(block))
{
// we might already start delayed freeing while another thread has not yet // we might already start delayed freeing while another thread has not yet
// reset the delayed_freeing flag; in that case delay it further by reinserting. // reset the delayed_freeing flag; in that case delay it further by reinserting.
mi_block_t *dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free); mi_block_t *dfree = mi_atomic_load_ptr_relaxed(mi_block_t, &heap->thread_delayed_free);
do { do
{
mi_block_set_nextx(heap, block, dfree, heap->keys); mi_block_set_nextx(heap, block, dfree, heap->keys);
} while (!mi_atomic_cas_ptr_weak_release(mi_block_t, &heap->thread_delayed_free, &dfree, block)); } while (!mi_atomic_cas_ptr_weak_release(mi_block_t, &heap->thread_delayed_free, &dfree, block));
} }
@ -297,11 +333,13 @@ void _mi_heap_delayed_free(mi_heap_t* heap) {
----------------------------------------------------------- */ ----------------------------------------------------------- */
// Move a page from the full list back to a regular list // Move a page from the full list back to a regular list
void _mi_page_unfull(mi_page_t* page) { void _mi_page_unfull(mi_page_t *page)
{
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(mi_page_is_in_full(page)); mi_assert_internal(mi_page_is_in_full(page));
if (!mi_page_is_in_full(page)) return; if (!mi_page_is_in_full(page))
return;
mi_heap_t *heap = mi_page_heap(page); mi_heap_t *heap = mi_page_heap(page);
mi_page_queue_t *pqfull = &heap->pages[MI_BIN_FULL]; mi_page_queue_t *pqfull = &heap->pages[MI_BIN_FULL];
@ -311,22 +349,24 @@ void _mi_page_unfull(mi_page_t* page) {
mi_page_queue_enqueue_from(pq, pqfull, page); mi_page_queue_enqueue_from(pq, pqfull, page);
} }
static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { static void mi_page_to_full(mi_page_t *page, mi_page_queue_t *pq)
{
mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(pq == mi_page_queue_of(page));
mi_assert_internal(!mi_page_immediate_available(page)); mi_assert_internal(!mi_page_immediate_available(page));
mi_assert_internal(!mi_page_is_in_full(page)); mi_assert_internal(!mi_page_is_in_full(page));
if (mi_page_is_in_full(page)) return; if (mi_page_is_in_full(page))
return;
mi_page_queue_enqueue_from(&mi_page_heap(page)->pages[MI_BIN_FULL], pq, page); mi_page_queue_enqueue_from(&mi_page_heap(page)->pages[MI_BIN_FULL], pq, page);
_mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set
} }
// Abandon a page with used blocks at the end of a thread. // Abandon a page with used blocks at the end of a thread.
// Note: only call if it is ensured that no references exist from // Note: only call if it is ensured that no references exist from
// the `page->heap->thread_delayed_free` into this page. // the `page->heap->thread_delayed_free` into this page.
// Currently only called through `mi_heap_collect_ex` which ensures this. // Currently only called through `mi_heap_collect_ex` which ensures this.
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { void _mi_page_abandon(mi_page_t *page, mi_page_queue_t *pq)
{
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(pq == mi_page_queue_of(page));
@ -344,7 +384,8 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
#if MI_DEBUG > 1 #if MI_DEBUG > 1
// check there are no references left.. // check there are no references left..
for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->keys)) { for (mi_block_t *block = (mi_block_t *)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->keys))
{
mi_assert_internal(_mi_ptr_page(block) != page); mi_assert_internal(_mi_ptr_page(block) != page);
} }
#endif #endif
@ -354,9 +395,9 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
_mi_segment_page_abandon(page, segments_tld); _mi_segment_page_abandon(page, segments_tld);
} }
// Free a page with no more free blocks // Free a page with no more free blocks
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { void _mi_page_free(mi_page_t *page, mi_page_queue_t *pq, bool force)
{
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(pq == mi_page_queue_of(page));
@ -385,7 +426,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) {
// Note: called from `mi_free` and benchmarks often // Note: called from `mi_free` and benchmarks often
// trigger this due to freeing everything and then // trigger this due to freeing everything and then
// allocating again so careful when changing this. // allocating again so careful when changing this.
void _mi_page_retire(mi_page_t* page) { void _mi_page_retire(mi_page_t *page)
{
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_expensive(_mi_page_is_valid(page));
mi_assert_internal(mi_page_all_free(page)); mi_assert_internal(mi_page_all_free(page));
@ -399,16 +441,20 @@ void _mi_page_retire(mi_page_t* page) {
// how to check this efficiently though... // how to check this efficiently though...
// for now, we don't retire if it is the only page left of this size class. // for now, we don't retire if it is the only page left of this size class.
mi_page_queue_t *pq = mi_page_queue_of(page); mi_page_queue_t *pq = mi_page_queue_of(page);
if (mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page))) { if (mi_likely(page->xblock_size <= MI_MAX_RETIRE_SIZE && !mi_page_is_in_full(page)))
if (pq->last==page && pq->first==page) { // the only page in the queue? {
if (pq->last == page && pq->first == page)
{ // the only page in the queue?
mi_stat_counter_increase(_mi_stats_main.page_no_retire, 1); mi_stat_counter_increase(_mi_stats_main.page_no_retire, 1);
page->retire_expire = (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES / 4); page->retire_expire = (page->xblock_size <= MI_SMALL_OBJ_SIZE_MAX ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES / 4);
mi_heap_t *heap = mi_page_heap(page); mi_heap_t *heap = mi_page_heap(page);
mi_assert_internal(pq >= heap->pages); mi_assert_internal(pq >= heap->pages);
const size_t index = pq - heap->pages; const size_t index = pq - heap->pages;
mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE); mi_assert_internal(index < MI_BIN_FULL && index < MI_BIN_HUGE);
if (index < heap->page_retired_min) heap->page_retired_min = index; if (index < heap->page_retired_min)
if (index > heap->page_retired_max) heap->page_retired_max = index; heap->page_retired_min = index;
if (index > heap->page_retired_max)
heap->page_retired_max = index;
mi_assert_internal(mi_page_all_free(page)); mi_assert_internal(mi_page_all_free(page));
return; // dont't free after all return; // dont't free after all
} }
@ -419,25 +465,34 @@ void _mi_page_retire(mi_page_t* page) {
// free retired pages: we don't need to look at the entire queues // free retired pages: we don't need to look at the entire queues
// since we only retire pages that are at the head position in a queue. // since we only retire pages that are at the head position in a queue.
void _mi_heap_collect_retired(mi_heap_t* heap, bool force) { void _mi_heap_collect_retired(mi_heap_t *heap, bool force)
{
size_t min = MI_BIN_FULL; size_t min = MI_BIN_FULL;
size_t max = 0; size_t max = 0;
for(size_t bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++) { for (size_t bin = heap->page_retired_min; bin <= heap->page_retired_max; bin++)
{
mi_page_queue_t *pq = &heap->pages[bin]; mi_page_queue_t *pq = &heap->pages[bin];
mi_page_t *page = pq->first; mi_page_t *page = pq->first;
if (page != NULL && page->retire_expire != 0) { if (page != NULL && page->retire_expire != 0)
if (mi_page_all_free(page)) { {
if (mi_page_all_free(page))
{
page->retire_expire--; page->retire_expire--;
if (force || page->retire_expire == 0) { if (force || page->retire_expire == 0)
{
_mi_page_free(pq->first, pq, force); _mi_page_free(pq->first, pq, force);
} }
else { else
{
// keep retired, update min/max // keep retired, update min/max
if (bin < min) min = bin; if (bin < min)
if (bin > max) max = bin; min = bin;
if (bin > max)
max = bin;
} }
} }
else { else
{
page->retire_expire = 0; page->retire_expire = 0;
} }
} }
@ -446,7 +501,6 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) {
heap->page_retired_max = max; heap->page_retired_max = max;
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Initialize the initial free list in a page. Initialize the initial free list in a page.
In secure mode we initialize a randomized list by In secure mode we initialize a randomized list by
@ -457,7 +511,8 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) {
#define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT) #define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT)
#define MI_MIN_SLICES (2) #define MI_MIN_SLICES (2)
static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) { static void mi_page_free_list_extend_secure(mi_heap_t *const heap, mi_page_t *const page, const size_t bsize, const size_t extend, mi_stats_t *const stats)
{
UNUSED(stats); UNUSED(stats);
#if (MI_SECURE <= 2) #if (MI_SECURE <= 2)
mi_assert_internal(page->free == NULL); mi_assert_internal(page->free == NULL);
@ -470,7 +525,8 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
// initialize a randomized free list // initialize a randomized free list
// set up `slice_count` slices to alternate between // set up `slice_count` slices to alternate between
size_t shift = MI_MAX_SLICE_SHIFT; size_t shift = MI_MAX_SLICE_SHIFT;
while ((extend >> shift) == 0) { while ((extend >> shift) == 0)
{
shift--; shift--;
} }
const size_t slice_count = (size_t)1U << shift; const size_t slice_count = (size_t)1U << shift;
@ -478,7 +534,8 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
mi_assert_internal(slice_extend >= 1); mi_assert_internal(slice_extend >= 1);
mi_block_t *blocks[MI_MAX_SLICES]; // current start of the slice mi_block_t *blocks[MI_MAX_SLICES]; // current start of the slice
size_t counts[MI_MAX_SLICES]; // available objects in the slice size_t counts[MI_MAX_SLICES]; // available objects in the slice
for (size_t i = 0; i < slice_count; i++) { for (size_t i = 0; i < slice_count; i++)
{
blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i * slice_extend); blocks[i] = mi_page_block_at(page, page_area, bsize, page->capacity + i * slice_extend);
counts[i] = slice_extend; counts[i] = slice_extend;
} }
@ -492,15 +549,19 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co
mi_block_t *const free_start = blocks[current]; mi_block_t *const free_start = blocks[current];
// and iterate through the rest; use `random_shuffle` for performance // and iterate through the rest; use `random_shuffle` for performance
uintptr_t rnd = _mi_random_shuffle(r | 1); // ensure not 0 uintptr_t rnd = _mi_random_shuffle(r | 1); // ensure not 0
for (size_t i = 1; i < extend; i++) { for (size_t i = 1; i < extend; i++)
{
// call random_shuffle only every INTPTR_SIZE rounds // call random_shuffle only every INTPTR_SIZE rounds
const size_t round = i % MI_INTPTR_SIZE; const size_t round = i % MI_INTPTR_SIZE;
if (round == 0) rnd = _mi_random_shuffle(rnd); if (round == 0)
rnd = _mi_random_shuffle(rnd);
// select a random next slice index // select a random next slice index
size_t next = ((rnd >> 8 * round) & (slice_count - 1)); size_t next = ((rnd >> 8 * round) & (slice_count - 1));
while (counts[next]==0) { // ensure it still has space while (counts[next] == 0)
{ // ensure it still has space
next++; next++;
if (next==slice_count) next = 0; if (next == slice_count)
next = 0;
} }
// and link the current block to it // and link the current block to it
counts[next]--; counts[next]--;
@ -530,7 +591,8 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
// initialize a sequential free list // initialize a sequential free list
mi_block_t *const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1); mi_block_t *const last = mi_page_block_at(page, page_area, bsize, page->capacity + extend - 1);
mi_block_t *block = start; mi_block_t *block = start;
while(block <= last) { while (block <= last)
{
mi_block_t *next = (mi_block_t *)((uint8_t *)block + bsize); mi_block_t *next = (mi_block_t *)((uint8_t *)block + bsize);
mi_block_set_next(page, block, next); mi_block_set_next(page, block, next);
block = next; block = next;
@ -556,14 +618,17 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
// Note: we also experimented with "bump" allocation on the first // Note: we also experimented with "bump" allocation on the first
// allocations but this did not speed up any benchmark (due to an // allocations but this did not speed up any benchmark (due to an
// extra test in malloc? or cache effects?) // extra test in malloc? or cache effects?)
static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) { static void mi_page_extend_free(mi_heap_t *heap, mi_page_t *page, mi_tld_t *tld)
{
mi_assert_expensive(mi_page_is_valid_init(page)); mi_assert_expensive(mi_page_is_valid_init(page));
#if (MI_SECURE <= 2) #if (MI_SECURE <= 2)
mi_assert(page->free == NULL); mi_assert(page->free == NULL);
mi_assert(page->local_free == NULL); mi_assert(page->local_free == NULL);
if (page->free != NULL) return; if (page->free != NULL)
return;
#endif #endif
if (page->capacity >= page->reserved) return; if (page->capacity >= page->reserved)
return;
size_t page_size; size_t page_size;
//uint8_t* page_start = //uint8_t* page_start =
@ -574,9 +639,11 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
const size_t bsize = (page->xblock_size < MI_HUGE_BLOCK_SIZE ? page->xblock_size : page_size); const size_t bsize = (page->xblock_size < MI_HUGE_BLOCK_SIZE ? page->xblock_size : page_size);
size_t extend = page->reserved - page->capacity; size_t extend = page->reserved - page->capacity;
size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE / (uint32_t)bsize); size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE / (uint32_t)bsize);
if (max_extend < MI_MIN_EXTEND) max_extend = MI_MIN_EXTEND; if (max_extend < MI_MIN_EXTEND)
max_extend = MI_MIN_EXTEND;
if (extend > max_extend) { if (extend > max_extend)
{
// ensure we don't touch memory beyond the page to reduce page commit. // ensure we don't touch memory beyond the page to reduce page commit.
// the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%. // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%.
extend = (max_extend == 0 ? 1 : max_extend); extend = (max_extend == 0 ? 1 : max_extend);
@ -586,10 +653,12 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
mi_assert_internal(extend < (1UL << 16)); mi_assert_internal(extend < (1UL << 16));
// and append the extend the free list // and append the extend the free list
if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) { if (extend < MI_MIN_SLICES || MI_SECURE == 0)
{ //!mi_option_is_enabled(mi_option_secure)) {
mi_page_free_list_extend(page, bsize, extend, &tld->stats); mi_page_free_list_extend(page, bsize, extend, &tld->stats);
} }
else { else
{
mi_page_free_list_extend_secure(heap, page, bsize, extend, &tld->stats); mi_page_free_list_extend_secure(heap, page, bsize, extend, &tld->stats);
} }
// enable the new free list // enable the new free list
@ -597,14 +666,16 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
mi_stat_increase(tld->stats.page_committed, extend * bsize); mi_stat_increase(tld->stats.page_committed, extend * bsize);
// extension into zero initialized memory preserves the zero'd free list // extension into zero initialized memory preserves the zero'd free list
if (!page->is_zero_init) { if (!page->is_zero_init)
{
page->is_zero = false; page->is_zero = false;
} }
mi_assert_expensive(mi_page_is_valid_init(page)); mi_assert_expensive(mi_page_is_valid_init(page));
} }
// Initialize a fresh page // Initialize a fresh page
static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi_tld_t* tld) { static void mi_page_init(mi_heap_t *heap, mi_page_t *page, size_t block_size, mi_tld_t *tld)
{
mi_assert(page != NULL); mi_assert(page != NULL);
mi_segment_t *segment = _mi_page_segment(page); mi_segment_t *segment = _mi_page_segment(page);
mi_assert(segment != NULL); mi_assert(segment != NULL);
@ -641,7 +712,6 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_assert(mi_page_immediate_available(page)); mi_assert(mi_page_immediate_available(page));
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Find pages with free blocks Find pages with free blocks
-------------------------------------------------------------*/ -------------------------------------------------------------*/
@ -661,12 +731,14 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
_mi_page_free_collect(page, false); _mi_page_free_collect(page, false);
// 1. if the page contains free blocks, we are done // 1. if the page contains free blocks, we are done
if (mi_page_immediate_available(page)) { if (mi_page_immediate_available(page))
{
break; // pick this one break; // pick this one
} }
// 2. Try to extend // 2. Try to extend
if (page->capacity < page->reserved) { if (page->capacity < page->reserved)
{
mi_page_extend_free(heap, page, heap->tld); mi_page_extend_free(heap, page, heap->tld);
mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(mi_page_immediate_available(page));
break; break;
@ -682,15 +754,18 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
mi_stat_counter_increase(heap->tld->stats.searches, count); mi_stat_counter_increase(heap->tld->stats.searches, count);
if (page == NULL) { if (page == NULL)
{
_mi_heap_collect_retired(heap, false); // perhaps make a page available _mi_heap_collect_retired(heap, false); // perhaps make a page available
page = mi_page_fresh(heap, pq); page = mi_page_fresh(heap, pq);
if (page == NULL && first_try) { if (page == NULL && first_try)
{
// out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again
page = mi_page_queue_find_free_ex(heap, pq, false); page = mi_page_queue_find_free_ex(heap, pq, false);
} }
} }
else { else
{
mi_assert(pq->first == page); mi_assert(pq->first == page);
page->retire_expire = 0; page->retire_expire = 0;
} }
@ -698,15 +773,16 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
return page; return page;
} }
// Find a page with free blocks of `size`. // Find a page with free blocks of `size`.
static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) { static inline mi_page_t *mi_find_free_page(mi_heap_t *heap, size_t size)
{
mi_page_queue_t *pq = mi_page_queue(heap, size); mi_page_queue_t *pq = mi_page_queue(heap, size);
mi_page_t *page = pq->first; mi_page_t *page = pq->first;
if (page != NULL) { if (page != NULL)
{
#if (MI_SECURE >= 3) // in secure mode, we extend half the time to increase randomness #if (MI_SECURE >= 3) // in secure mode, we extend half the time to increase randomness
if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1)) { if (page->capacity < page->reserved && ((_mi_heap_random_next(heap) & 1) == 1))
{
mi_page_extend_free(heap, page, heap->tld); mi_page_extend_free(heap, page, heap->tld);
mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(mi_page_immediate_available(page));
} }
@ -716,7 +792,8 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
_mi_page_free_collect(page, false); _mi_page_free_collect(page, false);
} }
if (mi_page_immediate_available(page)) { if (mi_page_immediate_available(page))
{
page->retire_expire = 0; page->retire_expire = 0;
return page; // fast path return page; // fast path
} }
@ -724,7 +801,6 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
return mi_page_queue_find_free_ex(heap, pq, true); return mi_page_queue_find_free_ex(heap, pq, true);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Users can register a deferred free function called Users can register a deferred free function called
when the `free` list is empty. Since the `local_free` when the `free` list is empty. Since the `local_free`
@ -735,21 +811,29 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
static mi_deferred_free_fun *volatile deferred_free = NULL; static mi_deferred_free_fun *volatile deferred_free = NULL;
static _Atomic(void *) deferred_arg; // = NULL static _Atomic(void *) deferred_arg; // = NULL
void _mi_deferred_free(mi_heap_t* heap, bool force) { void _mi_deferred_free(mi_heap_t *heap, bool force)
{
heap->tld->heartbeat++; heap->tld->heartbeat++;
if (deferred_free != NULL && !heap->tld->recurse) { if (heap->deferred_free != NULL && !heap->tld->recurse)
{
heap->tld->recurse = true;
((mi_local_deferred_free_fun *)heap->deferred_free)(heap, force, heap->tld->heartbeat, heap->deferred_arg);
heap->tld->recurse = false;
}
if (deferred_free != NULL && !heap->tld->recurse)
{
heap->tld->recurse = true; heap->tld->recurse = true;
deferred_free(force, heap->tld->heartbeat, mi_atomic_load_ptr_relaxed(void, &deferred_arg)); deferred_free(force, heap->tld->heartbeat, mi_atomic_load_ptr_relaxed(void, &deferred_arg));
heap->tld->recurse = false; heap->tld->recurse = false;
} }
} }
void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { void mi_register_deferred_free(mi_deferred_free_fun *fn, void *arg) mi_attr_noexcept
{
deferred_free = fn; deferred_free = fn;
mi_atomic_store_ptr_release(void, &deferred_arg, arg); mi_atomic_store_ptr_release(void, &deferred_arg, arg);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
General allocation General allocation
----------------------------------------------------------- */ ----------------------------------------------------------- */
@ -758,11 +842,13 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex
// Because huge pages contain just one block, and the segment contains // Because huge pages contain just one block, and the segment contains
// just that page, we always treat them as abandoned and any thread // just that page, we always treat them as abandoned and any thread
// that frees the block can free the whole page and segment directly. // that frees the block can free the whole page and segment directly.
static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { static mi_page_t *mi_huge_page_alloc(mi_heap_t *heap, size_t size)
{
size_t block_size = _mi_os_good_alloc_size(size); size_t block_size = _mi_os_good_alloc_size(size);
mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE);
mi_page_t *page = mi_page_fresh_alloc(heap, NULL, block_size); mi_page_t *page = mi_page_fresh_alloc(heap, NULL, block_size);
if (page != NULL) { if (page != NULL)
{
const size_t bsize = mi_page_block_size(page); // note: not `mi_page_usable_block_size` as `size` includes padding already const size_t bsize = mi_page_block_size(page); // note: not `mi_page_usable_block_size` as `size` includes padding already
mi_assert_internal(bsize >= size); mi_assert_internal(bsize >= size);
mi_assert_internal(mi_page_immediate_available(page)); mi_assert_internal(mi_page_immediate_available(page));
@ -771,11 +857,13 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
mi_assert_internal(_mi_page_segment(page)->thread_id == 0); // abandoned, not in the huge queue mi_assert_internal(_mi_page_segment(page)->thread_id == 0); // abandoned, not in the huge queue
mi_page_set_heap(page, NULL); mi_page_set_heap(page, NULL);
if (bsize > MI_HUGE_OBJ_SIZE_MAX) { if (bsize > MI_HUGE_OBJ_SIZE_MAX)
{
_mi_stat_increase(&heap->tld->stats.giant, bsize); _mi_stat_increase(&heap->tld->stats.giant, bsize);
_mi_stat_counter_increase(&heap->tld->stats.giant_count, 1); _mi_stat_counter_increase(&heap->tld->stats.giant_count, 1);
} }
else { else
{
_mi_stat_increase(&heap->tld->stats.huge, bsize); _mi_stat_increase(&heap->tld->stats.huge, bsize);
_mi_stat_counter_increase(&heap->tld->stats.huge_count, 1); _mi_stat_counter_increase(&heap->tld->stats.huge_count, 1);
} }
@ -783,22 +871,26 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) {
return page; return page;
} }
// Allocate a page // Allocate a page
// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed.
static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size) mi_attr_noexcept { static mi_page_t *mi_find_page(mi_heap_t *heap, size_t size) mi_attr_noexcept
{
// huge allocation? // huge allocation?
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE) )) { if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_PADDING_SIZE)))
if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>) {
if (mi_unlikely(req_size > PTRDIFF_MAX))
{ // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
_mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size);
return NULL; return NULL;
} }
else { else
{
return mi_huge_page_alloc(heap, size); return mi_huge_page_alloc(heap, size);
} }
} }
else { else
{
// otherwise find a page with free blocks in our size segregated queues // otherwise find a page with free blocks in our size segregated queues
mi_assert_internal(size >= MI_PADDING_SIZE); mi_assert_internal(size >= MI_PADDING_SIZE);
return mi_find_free_page(heap, size); return mi_find_free_page(heap, size);
@ -812,10 +904,14 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
mi_assert_internal(heap != NULL); mi_assert_internal(heap != NULL);
// initialize if necessary // initialize if necessary
if (mi_unlikely(!mi_heap_is_initialized(heap))) { if (mi_unlikely(!mi_heap_is_initialized(heap)))
{
mi_thread_init(); // calls `_mi_heap_init` in turn mi_thread_init(); // calls `_mi_heap_init` in turn
heap = mi_get_default_heap(); heap = mi_get_default_heap();
if (mi_unlikely(!mi_heap_is_initialized(heap))) { return NULL; } if (mi_unlikely(!mi_heap_is_initialized(heap)))
{
return NULL;
}
} }
mi_assert_internal(mi_heap_is_initialized(heap)); mi_assert_internal(mi_heap_is_initialized(heap));
@ -827,12 +923,14 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
// find (or allocate) a page of the right size // find (or allocate) a page of the right size
mi_page_t *page = mi_find_page(heap, size); mi_page_t *page = mi_find_page(heap, size);
if (mi_unlikely(page == NULL)) { // first time out of memory, try to collect and retry the allocation once more if (mi_unlikely(page == NULL))
{ // first time out of memory, try to collect and retry the allocation once more
mi_heap_collect(heap, true /* force */); mi_heap_collect(heap, true /* force */);
page = mi_find_page(heap, size); page = mi_find_page(heap, size);
} }
if (mi_unlikely(page == NULL)) { // out of memory if (mi_unlikely(page == NULL))
{ // out of memory
const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size`
_mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size);
return NULL; return NULL;