mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-07-06 11:34:38 +03:00
initial checkin
This commit is contained in:
daan
parent
23b4e65faa
commit
26a874eb3f
41 changed files with 11897 additions and 0 deletions
190
include/mimalloc-atomic.h
Normal file
190
include/mimalloc-atomic.h
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"license.txt" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef __MIMALLOC_ATOMIC_H
|
||||
#define __MIMALLOC_ATOMIC_H
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Atomics
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Atomically increment a value; returns the incremented result.
|
||||
static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p);
|
||||
|
||||
// Atomically increment a value; returns the incremented result.
|
||||
static inline uint32_t mi_atomic_increment32(volatile uint32_t* p);
|
||||
|
||||
// Atomically decrement a value; returns the decremented result.
|
||||
static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p);
|
||||
|
||||
// Atomically add a 64-bit value; returns the added result.
|
||||
static inline int64_t mi_atomic_add(volatile int64_t* p, int64_t add);
|
||||
|
||||
// Atomically subtract a value; returns the subtracted result.
|
||||
static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub);
|
||||
|
||||
// Atomically subtract a value; returns the subtracted result.
|
||||
static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub);
|
||||
|
||||
// Atomically compare and exchange a value; returns `true` if successful.
|
||||
static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare);
|
||||
|
||||
// Atomically compare and exchange a value; returns `true` if successful.
|
||||
static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare);
|
||||
|
||||
// Atomically exchange a value.
|
||||
static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange);
|
||||
|
||||
static inline void mi_atomic_yield();
|
||||
|
||||
// Atomically compare and exchange a pointer; returns `true` if successful.
|
||||
static inline bool mi_atomic_compare_exchange_ptr(volatile void** p, void* newp, void* compare) {
|
||||
return mi_atomic_compare_exchange((volatile uintptr_t*)p, (uintptr_t)newp, (uintptr_t)compare);
|
||||
}
|
||||
|
||||
// Atomically exchange a pointer value.
|
||||
static inline void* mi_atomic_exchange_ptr(volatile void** p, void* exchange) {
|
||||
return (void*)mi_atomic_exchange((volatile uintptr_t*)p, (uintptr_t)exchange);
|
||||
}
|
||||
|
||||
|
||||
|
||||
#define mi_atomic_locked(mutex) for(bool _mheld = mi_mutex_lock(mutex); _mheld; _mheld = mi_mutex_unlock(mutex))
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#include <intrin.h>
|
||||
#if (MI_INTPTR_SIZE==8)
|
||||
#define RC64(f) f##64
|
||||
#else
|
||||
#define RC64(f) f
|
||||
#endif
|
||||
static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p) {
|
||||
return (uintptr_t)RC64(_InterlockedIncrement)((volatile intptr_t*)p);
|
||||
}
|
||||
static inline uint32_t mi_atomic_increment32(volatile uint32_t* p) {
|
||||
return (uint32_t)_InterlockedIncrement((volatile int32_t*)p);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p) {
|
||||
return (uintptr_t)RC64(_InterlockedDecrement)((volatile intptr_t*)p);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub) {
|
||||
return (uintptr_t)RC64(_InterlockedExchangeAdd)((volatile intptr_t*)p, -((intptr_t)sub)) - sub;
|
||||
}
|
||||
static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub) {
|
||||
return (uint32_t)_InterlockedExchangeAdd((volatile int32_t*)p, -((int32_t)sub)) - sub;
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare) {
|
||||
return ((int32_t)compare == _InterlockedCompareExchange((volatile int32_t*)p, (int32_t)exchange, (int32_t)compare));
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare) {
|
||||
return (compare == RC64(_InterlockedCompareExchange)((volatile intptr_t*)p, (intptr_t)exchange, (intptr_t)compare));
|
||||
}
|
||||
static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) {
|
||||
return (uintptr_t)RC64(_InterlockedExchange)((volatile intptr_t*)p, (intptr_t)exchange);
|
||||
}
|
||||
static inline void mi_atomic_yield() {
|
||||
YieldProcessor();
|
||||
}
|
||||
static inline int64_t mi_atomic_add(volatile int64_t* p, int64_t add) {
|
||||
#if (MI_INTPTR_SIZE==8)
|
||||
return _InterlockedExchangeAdd64(p, add) + add;
|
||||
#else
|
||||
int64_t current;
|
||||
int64_t sum;
|
||||
do {
|
||||
current = *p;
|
||||
sum = current + add;
|
||||
} while (_InterlockedCompareExchange64(p, sum, current) != current);
|
||||
return sum;
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
#ifdef __cplusplus
|
||||
#include <atomic>
|
||||
#define MI_USING_STD using namespace std;
|
||||
#define _Atomic(tp) atomic<tp>
|
||||
#else
|
||||
#include <stdatomic.h>
|
||||
#define MI_USING_STD
|
||||
#endif
|
||||
static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p) {
|
||||
MI_USING_STD
|
||||
return atomic_fetch_add_explicit((volatile atomic_uintptr_t*)p, (uintptr_t)1, memory_order_relaxed) + 1;
|
||||
}
|
||||
static inline uint32_t mi_atomic_increment32(volatile uint32_t* p) {
|
||||
MI_USING_STD
|
||||
return atomic_fetch_add_explicit((volatile _Atomic(uint32_t)*)p, (uint32_t)1, memory_order_relaxed) + 1;
|
||||
}
|
||||
static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p) {
|
||||
MI_USING_STD
|
||||
return atomic_fetch_sub_explicit((volatile atomic_uintptr_t*)p, (uintptr_t)1, memory_order_relaxed) - 1;
|
||||
}
|
||||
static inline int64_t mi_atomic_add(volatile int64_t* p, int64_t add) {
|
||||
MI_USING_STD
|
||||
return atomic_fetch_add_explicit((volatile _Atomic(int64_t)*)p, add, memory_order_relaxed) + add;
|
||||
}
|
||||
static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub) {
|
||||
MI_USING_STD
|
||||
return atomic_fetch_sub_explicit((volatile atomic_uintptr_t*)p, sub, memory_order_relaxed) - sub;
|
||||
}
|
||||
static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub) {
|
||||
MI_USING_STD
|
||||
return atomic_fetch_sub_explicit((volatile _Atomic(uint32_t)*)p, sub, memory_order_relaxed) - sub;
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare) {
|
||||
MI_USING_STD
|
||||
return atomic_compare_exchange_weak_explicit((volatile _Atomic(uint32_t)*)p, &compare, exchange, memory_order_relaxed, memory_order_seq_cst);
|
||||
}
|
||||
static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare) {
|
||||
MI_USING_STD
|
||||
return atomic_compare_exchange_weak_explicit((volatile atomic_uintptr_t*)p, &compare, exchange, memory_order_relaxed, memory_order_seq_cst);
|
||||
}
|
||||
static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) {
|
||||
MI_USING_STD
|
||||
return atomic_exchange_explicit((volatile atomic_uintptr_t*)p, exchange, memory_order_relaxed);
|
||||
}
|
||||
|
||||
#if defined(__cplusplus)
|
||||
#include <thread>
|
||||
static inline void mi_atomic_yield() {
|
||||
std::this_thread::yield();
|
||||
}
|
||||
#elif (defined(__GNUC__) || defined(__clang__)) && (defined(__x86_64__) || defined(__i386__))
|
||||
static inline void mi_atomic_yield() {
|
||||
asm volatile ("pause" ::: "memory");
|
||||
}
|
||||
#else
|
||||
#include <unistd.h>
|
||||
static inline void mi_atomic_yield() {
|
||||
sleep(0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
// Light weight mutex for low contention situations
|
||||
typedef struct mi_mutex_s {
|
||||
volatile uint32_t value;
|
||||
} mi_mutex_t;
|
||||
|
||||
static inline bool mi_mutex_lock(mi_mutex_t* mutex) {
|
||||
while(!mi_atomic_compare_exchange32(&mutex->value, 1, 0)) {
|
||||
mi_atomic_yield();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool mi_mutex_unlock(mi_mutex_t* mutex) {
|
||||
mutex->value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
#endif // __MIMALLOC_ATOMIC_H
|
||||
317
include/mimalloc-internal.h
Normal file
317
include/mimalloc-internal.h
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"license.txt" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef __MIMALLOC_INTERNAL_H
|
||||
#define __MIMALLOC_INTERNAL_H
|
||||
|
||||
#include "mimalloc-types.h"
|
||||
|
||||
#if defined(MI_MALLOC_OVERRIDE) && defined(MI_INTERPOSE)
|
||||
#define MI_TLS_RECURSE_GUARD
|
||||
#endif
|
||||
|
||||
|
||||
// "options.c"
|
||||
void _mi_fprintf(FILE* out, const char* fmt, ...);
|
||||
void _mi_error_message(const char* fmt, ...);
|
||||
void _mi_warning_message(const char* fmt, ...);
|
||||
void _mi_verbose_message(const char* fmt, ...);
|
||||
|
||||
// "init.c"
|
||||
extern mi_stats_t _mi_stats_main;
|
||||
extern const mi_page_t _mi_page_empty;
|
||||
bool _mi_is_main_thread();
|
||||
uintptr_t _mi_ptr_cookie(const void* p);
|
||||
uintptr_t _mi_random_shuffle(uintptr_t x);
|
||||
uintptr_t _mi_random_init(uintptr_t seed /* can be zero */);
|
||||
|
||||
// "os.c"
|
||||
bool _mi_os_reset(void* p, size_t size);
|
||||
void* _mi_os_alloc(size_t size, mi_stats_t* stats);
|
||||
void _mi_os_free(void* p, size_t size, mi_stats_t* stats);
|
||||
bool _mi_os_protect(void* addr, size_t size);
|
||||
bool _mi_os_unprotect(void* addr, size_t size);
|
||||
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, mi_os_tld_t* tld);
|
||||
size_t _mi_os_page_size();
|
||||
uintptr_t _mi_align_up(uintptr_t sz, size_t alignment);
|
||||
|
||||
// "segment.c"
|
||||
mi_page_t* _mi_segment_page_alloc(size_t block_wsize, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
|
||||
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
|
||||
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
|
||||
bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld);
|
||||
void _mi_segment_thread_collect(mi_segments_tld_t* tld);
|
||||
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page
|
||||
|
||||
// "page.c"
|
||||
void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc;
|
||||
|
||||
void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks
|
||||
void _mi_page_unfull(mi_page_t* page);
|
||||
void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page
|
||||
void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
|
||||
void _mi_heap_delayed_free(mi_heap_t* heap);
|
||||
|
||||
void _mi_page_use_delayed_free(mi_page_t* page, bool enable);
|
||||
void _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);
|
||||
void _mi_deferred_free(mi_heap_t* heap, bool force);
|
||||
|
||||
void _mi_page_free_collect(mi_page_t* page);
|
||||
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments
|
||||
|
||||
size_t _mi_bin_size(uint8_t bin); // for stats
|
||||
uint8_t _mi_bin(size_t size); // for stats
|
||||
uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD in "os.c"
|
||||
|
||||
// "heap.c"
|
||||
void _mi_heap_destroy_pages(mi_heap_t* heap);
|
||||
void _mi_heap_collect_abandon(mi_heap_t* heap);
|
||||
uintptr_t _mi_heap_random(mi_heap_t* heap);
|
||||
|
||||
// "stats.c"
|
||||
void _mi_stats_done(mi_stats_t* stats);
|
||||
|
||||
// "alloc.c"
|
||||
void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic`
|
||||
void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero);
|
||||
void* _mi_realloc_zero(void* p, size_t size, bool zero);
|
||||
mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, void* p);
|
||||
void _mi_free_delayed_block(mi_block_t* block);
|
||||
|
||||
#if MI_DEBUG>1
|
||||
bool _mi_page_is_valid(mi_page_t* page);
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Branches
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define mi_unlikely(x) __builtin_expect((x),0)
|
||||
#define mi_likely(x) __builtin_expect((x),1)
|
||||
#else
|
||||
#define mi_unlikely(x) (x)
|
||||
#define mi_likely(x) (x)
|
||||
#endif
|
||||
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define mi_decl_noinline __declspec(noinline)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#define mi_decl_noinline __attribute__((noinline))
|
||||
#else
|
||||
#define mi_decl_noinline
|
||||
#endif
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Inlined definitions
|
||||
----------------------------------------------------------- */
|
||||
#define UNUSED(x) (void)(x)
|
||||
|
||||
#define MI_INIT4(x) x(),x(),x(),x()
|
||||
#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x)
|
||||
#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x)
|
||||
#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x)
|
||||
#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x)
|
||||
#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)
|
||||
#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)
|
||||
|
||||
|
||||
// Overflow detecting multiply
|
||||
#define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
|
||||
static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) {
|
||||
*total = size * count;
|
||||
return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW)
|
||||
&& size > 0 && (SIZE_MAX / size) < count);
|
||||
}
|
||||
|
||||
// Align a byte size to a size in _machine words_,
|
||||
// i.e. byte size == `wsize*sizeof(void*)`.
|
||||
static inline size_t _mi_wsize_from_size(size_t size) {
|
||||
return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
|
||||
}
|
||||
|
||||
//extern mi_decl_thread mi_heap_t* _mi_backing_heap; // thread local heap
|
||||
extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap
|
||||
extern mi_heap_t _mi_heap_main; // statically allocated main backing heap
|
||||
extern bool _mi_process_is_initialized;
|
||||
|
||||
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
|
||||
|
||||
static inline mi_heap_t* mi_get_default_heap() {
|
||||
#ifdef MI_TLS_RECURSE_GUARD
|
||||
// on some platforms, like MacOSX, the dynamic loader calls `malloc`
|
||||
// to initialize thread local data. To avoid recursion, we need to avoid
|
||||
// accessing the thread local `_mi_default_heap` until our module is loaded
|
||||
// and use the statically allocated main heap until that time.
|
||||
// TODO: patch ourselves dynamically to avoid this check every time?
|
||||
if (!_mi_process_is_initialized) return &_mi_heap_main;
|
||||
#endif
|
||||
return _mi_heap_default;
|
||||
}
|
||||
|
||||
static inline bool mi_heap_is_default(const mi_heap_t* heap) {
|
||||
return (heap == mi_get_default_heap());
|
||||
}
|
||||
|
||||
static inline bool mi_heap_is_backing(const mi_heap_t* heap) {
|
||||
return (heap->tld->heap_backing == heap);
|
||||
}
|
||||
|
||||
static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
|
||||
mi_assert_internal(heap != NULL);
|
||||
return (heap != &_mi_heap_empty);
|
||||
}
|
||||
|
||||
static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) {
|
||||
mi_assert_internal(size <= MI_SMALL_SIZE_MAX);
|
||||
return heap->pages_free_direct[_mi_wsize_from_size(size)];
|
||||
}
|
||||
|
||||
// Get the page belonging to a certain size class
|
||||
static inline mi_page_t* _mi_get_free_small_page(size_t size) {
|
||||
return _mi_heap_get_free_small_page(mi_get_default_heap(), size);
|
||||
}
|
||||
|
||||
|
||||
// Segment that contains the pointer
|
||||
static inline mi_segment_t* _mi_ptr_segment(const void* p) {
|
||||
// mi_assert_internal(p != NULL);
|
||||
return (mi_segment_t*)((uintptr_t)p & ~MI_SEGMENT_MASK);
|
||||
}
|
||||
|
||||
// Segment belonging to a page
|
||||
static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) {
|
||||
mi_segment_t* segment = _mi_ptr_segment(page);
|
||||
mi_assert_internal(page == &segment->pages[page->segment_idx]);
|
||||
return segment;
|
||||
}
|
||||
|
||||
// Get the page containing the pointer
|
||||
static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) {
|
||||
// if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages
|
||||
ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment;
|
||||
mi_assert_internal(diff >= 0 && diff < MI_SEGMENT_SIZE);
|
||||
uintptr_t idx = (uintptr_t)diff >> segment->page_shift;
|
||||
mi_assert_internal(idx < segment->capacity);
|
||||
mi_assert_internal(segment->page_kind == MI_PAGE_SMALL || idx == 0);
|
||||
return &((mi_segment_t*)segment)->pages[idx];
|
||||
}
|
||||
|
||||
// Quick page start for initialized pages
|
||||
static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) {
|
||||
return _mi_segment_page_start(segment, page, page_size);
|
||||
}
|
||||
|
||||
// Get the page containing the pointer
|
||||
static inline mi_page_t* _mi_ptr_page(void* p) {
|
||||
return _mi_segment_page_of(_mi_ptr_segment(p), p);
|
||||
}
|
||||
|
||||
// are all blocks in a page freed?
|
||||
static inline bool mi_page_all_free(const mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
return (page->used - page->thread_freed == 0);
|
||||
}
|
||||
|
||||
// are there immediately available blocks
|
||||
static inline bool mi_page_immediate_available(const mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
return (page->free != NULL);
|
||||
}
|
||||
// are there free blocks in this page?
|
||||
static inline bool mi_page_has_free(mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
bool hasfree = (mi_page_immediate_available(page) || page->local_free != NULL || (page->thread_free.head != 0));
|
||||
mi_assert_internal(hasfree || page->used - page->thread_freed == page->capacity);
|
||||
return hasfree;
|
||||
}
|
||||
|
||||
// are all blocks in use?
|
||||
static inline bool mi_page_all_used(mi_page_t* page) {
|
||||
mi_assert_internal(page != NULL);
|
||||
return !mi_page_has_free(page);
|
||||
}
|
||||
|
||||
// is more than 7/8th of a page in use?
|
||||
static inline bool mi_page_mostly_used(const mi_page_t* page) {
|
||||
if (page==NULL) return true;
|
||||
uint16_t frac = page->reserved / 8U;
|
||||
return (page->reserved - page->used + page->thread_freed < frac);
|
||||
}
|
||||
|
||||
static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {
|
||||
return &((mi_heap_t*)heap)->pages[_mi_bin(size)];
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// Encoding/Decoding the free list next pointers
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
static inline mi_block_t* mi_block_nextx( uintptr_t cookie, mi_block_t* block ) {
|
||||
#if MI_SECURE
|
||||
return (mi_block_t*)(block->next ^ cookie);
|
||||
#else
|
||||
UNUSED(cookie);
|
||||
return (mi_block_t*)block->next;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, mi_block_t* next) {
|
||||
#if MI_SECURE
|
||||
block->next = (mi_encoded_t)next ^ cookie;
|
||||
#else
|
||||
UNUSED(cookie);
|
||||
block->next = (mi_encoded_t)next;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline mi_block_t* mi_block_next(mi_page_t* page, mi_block_t* block) {
|
||||
return mi_block_nextx(page->cookie,block);
|
||||
}
|
||||
|
||||
static inline void mi_block_set_next(mi_page_t* page, mi_block_t* block, mi_block_t* next) {
|
||||
mi_block_set_nextx(page->cookie,block,next);
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
// Getting the thread id should be performant
|
||||
// as it is called in the fast path of `_mi_free`,
|
||||
// so we specialize for various platforms.
|
||||
// -------------------------------------------------------------------
|
||||
#if defined(_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
static inline uintptr_t _mi_thread_id() mi_attr_noexcept {
|
||||
// Windows: works on Intel and ARM in both 32- and 64-bit
|
||||
return (uintptr_t)NtCurrentTeb();
|
||||
}
|
||||
#elif (defined(__GNUC__) || defined(__clang__)) && (defined(__x86_64__) || defined(__i386__))
|
||||
// TLS register on x86 is in the FS or GS register
|
||||
// see: https://akkadia.org/drepper/tls.pdf
|
||||
static inline uintptr_t _mi_thread_id() mi_attr_noexcept {
|
||||
uintptr_t tid;
|
||||
#if defined(__i386__)
|
||||
__asm__("movl %%gs:0, %0" : "=r" (tid) : : ); // 32-bit always uses GS
|
||||
#elif defined(__MACH__)
|
||||
__asm__("movq %%gs:0, %0" : "=r" (tid) : : ); // x86_64 MacOSX uses GS
|
||||
#else
|
||||
__asm__("movq %%fs:0, %0" : "=r" (tid) : : ); // x86_64 Linux, BSD uses FS
|
||||
#endif
|
||||
return tid;
|
||||
}
|
||||
#else
|
||||
// otherwise use standard C
|
||||
static inline uintptr_t _mi_thread_id() mi_attr_noexcept {
|
||||
return (uintptr_t)&_mi_backing_heap;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#endif
|
||||
398
include/mimalloc-types.h
Normal file
398
include/mimalloc-types.h
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"license.txt" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef __MIMALLOC_TYPES_H
|
||||
#define __MIMALLOC_TYPES_H
|
||||
|
||||
#include <stdlib.h> // size_t etc.
|
||||
#include <stddef.h> // ptrdiff_t
|
||||
#include <stdint.h> // uintptr_t, uint16_t, etc
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Variants
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Define NDEBUG in the release version to disable assertions.
|
||||
// #define NDEBUG
|
||||
|
||||
// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statitistics (but costs some performance).
|
||||
// #define MI_STAT 1
|
||||
|
||||
// Define MI_SECURE as 1 to encode free lists
|
||||
// #define MI_SECURE 1
|
||||
|
||||
#if !defined(MI_SECURE)
|
||||
#define MI_SECURE 0
|
||||
#endif
|
||||
|
||||
// Define MI_DEBUG as 1 for basic assert checks and statistics
|
||||
// set it to 2 to do internal asserts,
|
||||
// and to 3 to do extensive invariant checking.
|
||||
#if !defined(MI_DEBUG)
|
||||
#if !defined(NDEBUG) || defined(_DEBUG)
|
||||
#define MI_DEBUG 1
|
||||
#else
|
||||
#define MI_DEBUG 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Platform specific values
|
||||
// ------------------------------------------------------
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Size of a pointer.
|
||||
// We assume that `sizeof(void*)==sizeof(intptr_t)`
|
||||
// and it holds for all platforms we know of.
|
||||
//
|
||||
// However, the C standard only requires that:
|
||||
// p == (void*)((intptr_t)p))
|
||||
// but we also need:
|
||||
// i == (intptr_t)((void*)i)
|
||||
// or otherwise one might define an intptr_t type that is larger than a pointer...
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if INTPTR_MAX == 9223372036854775807LL
|
||||
# define MI_INTPTR_SHIFT (3)
|
||||
#elif INTPTR_MAX == 2147483647LL
|
||||
# define MI_INTPTR_SHIFT (2)
|
||||
#else
|
||||
#error platform must be 32 or 64 bits
|
||||
#endif
|
||||
|
||||
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Main internal data-structures
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Main tuning parameters for segment and page sizes
|
||||
// Sizes for 64-bit, divide by two for 32-bit
|
||||
#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb
|
||||
#define MI_LARGE_PAGE_SHIFT ( 6 + MI_SMALL_PAGE_SHIFT) // 4mb
|
||||
#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb
|
||||
|
||||
// Derived constants
|
||||
#define MI_SEGMENT_SIZE (1<<MI_SEGMENT_SHIFT)
|
||||
#define MI_SEGMENT_MASK ((uintptr_t)MI_SEGMENT_SIZE - 1)
|
||||
|
||||
#define MI_SMALL_PAGE_SIZE (1<<MI_SMALL_PAGE_SHIFT)
|
||||
#define MI_LARGE_PAGE_SIZE (1<<MI_LARGE_PAGE_SHIFT)
|
||||
|
||||
#define MI_SMALL_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_SMALL_PAGE_SIZE)
|
||||
#define MI_LARGE_PAGES_PER_SEGMENT (MI_SEGMENT_SIZE/MI_LARGE_PAGE_SIZE)
|
||||
|
||||
#define MI_LARGE_SIZE_MAX (MI_LARGE_PAGE_SIZE/8) // 512kb on 64-bit
|
||||
#define MI_LARGE_WSIZE_MAX (MI_LARGE_SIZE_MAX>>MI_INTPTR_SHIFT)
|
||||
|
||||
|
||||
// Maximum number of size classes. (spaced exponentially in 16.7% increments)
|
||||
#define MI_BIN_HUGE (64U)
|
||||
|
||||
// Minimal aligment necessary. On most platforms 16 bytes are needed
|
||||
// due to SSE registers for example. This must be at least `MI_INTPTR_SIZE`
|
||||
#define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t)
|
||||
|
||||
#if (MI_LARGE_WSIZE_MAX > 131072)
|
||||
#error "define more bins"
|
||||
#endif
|
||||
|
||||
typedef uintptr_t mi_encoded_t;
|
||||
|
||||
// free lists contain blocks
|
||||
typedef struct mi_block_s {
|
||||
mi_encoded_t next;
|
||||
} mi_block_t;
|
||||
|
||||
|
||||
typedef enum mi_delayed_e {
|
||||
MI_NO_DELAYED_FREE = 0,
|
||||
MI_USE_DELAYED_FREE,
|
||||
MI_DELAYED_FREEING
|
||||
} mi_delayed_t;
|
||||
|
||||
|
||||
typedef union mi_page_flags_u {
|
||||
uint16_t value;
|
||||
struct {
|
||||
bool has_aligned;
|
||||
bool in_full;
|
||||
};
|
||||
} mi_page_flags_t;
|
||||
|
||||
// Thread free list.
|
||||
// We use 2 bits of the pointer for the `use_delayed_free` and `delayed_freeing` flags.
|
||||
typedef union mi_thread_free_u {
|
||||
uintptr_t value;
|
||||
struct {
|
||||
mi_delayed_t delayed:2;
|
||||
#if MI_INTPTR_SIZE==8
|
||||
uintptr_t head:62; // head free block in the list (right-shifted by 2)
|
||||
#elif MI_INTPTR_SIZE==4
|
||||
uintptr_t head:30;
|
||||
#endif
|
||||
};
|
||||
} mi_thread_free_t;
|
||||
|
||||
#define MI_TF_PTR_SHIFT (2)
|
||||
|
||||
// A page contains blocks of one specific size (`block_size`).
|
||||
// Each page has three list of free blocks:
|
||||
// `free` for blocks that can be allocated,
|
||||
// `local_free` for freed blocks that are not yet available to `mi_malloc`
|
||||
// `thread_free` for freed blocks by other threads
|
||||
// The `local_free` and `thread_free` lists are migrated to the `free` list
|
||||
// when it is exhausted. The separate `local_free` list is necessary to
|
||||
// implement a monotonic heartbeat. The `thead_free` list is needed for
|
||||
// avoiding atomic operations in the common case.
|
||||
//
|
||||
// `used - thread_freed` == actual blocks that are in use (alive)
|
||||
// `used - thread_freed + |free| + |local_free| == capacity`
|
||||
//
|
||||
// note: we don't count `freed` (as |free|) instead of `used` to reduce
|
||||
// the number of memory accesses in the `mi_page_all_free` function(s).
|
||||
// note: the funny layout here is due to:
|
||||
// - access is optimized for `mi_free` and `mi_page_alloc`
|
||||
// - using `uint16_t` does not seem to slow things down
|
||||
typedef struct mi_page_s {
|
||||
// "owned" by the segment
|
||||
uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]`
|
||||
bool segment_in_use:1; // `true` if the segment allocated this page
|
||||
bool is_reset:1; // `true` if the page memory was reset
|
||||
|
||||
// layout like this to optimize access in `mi_malloc` and `mi_free`
|
||||
mi_page_flags_t flags;
|
||||
uint16_t capacity; // number of blocks committed
|
||||
uint16_t reserved; // numbes of blocks reserved in memory
|
||||
|
||||
mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)
|
||||
uintptr_t cookie; // random cookie to encode the free lists
|
||||
size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`)
|
||||
|
||||
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
|
||||
volatile uintptr_t thread_freed; // at least this number of blocks are in `thread_free`
|
||||
volatile mi_thread_free_t thread_free; // list of deferred free blocks freed by other threads
|
||||
|
||||
// less accessed info
|
||||
size_t block_size; // size available in each block (always `>0`)
|
||||
mi_heap_t* heap; // the owning heap
|
||||
struct mi_page_s* next; // next page owned by this thread with the same `block_size`
|
||||
struct mi_page_s* prev; // previous page owned by this thread with the same `block_size`
|
||||
|
||||
// improve page index calculation
|
||||
#if MI_INTPTR_SIZE==8
|
||||
//void* padding[1]; // 10 words on 64-bit
|
||||
#elif MI_INTPTR_SIZE==4
|
||||
void* padding[1]; // 12 words on 32-bit
|
||||
#endif
|
||||
} mi_page_t;
|
||||
|
||||
|
||||
|
||||
typedef enum mi_page_kind_e {
|
||||
MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment
|
||||
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
|
||||
MI_PAGE_HUGE // huge blocks (>512kb) are put into a single page in a segment of the exact size (but still 2mb aligned)
|
||||
} mi_page_kind_t;
|
||||
|
||||
// Segments are large allocated memory blocks (2mb on 64 bit) from
|
||||
// the OS. Inside segments we allocated fixed size _pages_ that
|
||||
// contain blocks.
|
||||
typedef struct mi_segment_s {
|
||||
struct mi_segment_s* next;
|
||||
struct mi_segment_s* prev;
|
||||
struct mi_segment_s* abandoned_next;
|
||||
size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
|
||||
size_t used; // count of pages in use (`used <= capacity`)
|
||||
size_t capacity; // count of available pages (`#free + used`)
|
||||
size_t segment_size;// for huge pages this may be different from `MI_SEGMENT_SIZE`
|
||||
size_t segment_info_size; // space we are using from the first page for segment meta-data and possible guard pages.
|
||||
uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie`
|
||||
|
||||
// layout like this to optimize access in `mi_free`
|
||||
size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`).
|
||||
uintptr_t thread_id; // unique id of the thread owning this segment
|
||||
mi_page_kind_t page_kind; // kind of pages: small, large, or huge
|
||||
mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages
|
||||
} mi_segment_t;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Heaps
|
||||
// Provide first-class heaps to allocate from.
|
||||
// A heap just owns a set of pages for allocation and
|
||||
// can only be allocate/reallocate from the thread that created it.
|
||||
// Freeing blocks can be done from any thread though.
|
||||
// Per thread, the segments are shared among its heaps.
|
||||
// Per thread, there is always a default heap that is
|
||||
// used for allocation; it is initialized to statically
|
||||
// point to an empty heap to avoid initialization checks
|
||||
// in the fast path.
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Thread local data
|
||||
typedef struct mi_tld_s mi_tld_t;
|
||||
|
||||
// Pages of a certain block size are held in a queue.
|
||||
typedef struct mi_page_queue_s {
|
||||
mi_page_t* first;
|
||||
mi_page_t* last;
|
||||
size_t block_size;
|
||||
} mi_page_queue_t;
|
||||
|
||||
#define MI_BIN_FULL (MI_BIN_HUGE+1)
|
||||
|
||||
// A heap owns a set of pages.
|
||||
struct mi_heap_s {
|
||||
mi_tld_t* tld;
|
||||
mi_page_t* pages_free_direct[MI_SMALL_WSIZE_MAX + 2]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
|
||||
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
|
||||
volatile mi_block_t* thread_delayed_free;
|
||||
uintptr_t thread_id; // thread this heap belongs too
|
||||
uintptr_t cookie;
|
||||
uintptr_t random; // random number used for secure allocation
|
||||
size_t page_count; // total number of pages in the `pages` queues.
|
||||
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
|
||||
};
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Debug
|
||||
// ------------------------------------------------------
|
||||
|
||||
#define MI_DEBUG_UNINIT (0xD0)
|
||||
#define MI_DEBUG_FREED (0xDF)
|
||||
|
||||
|
||||
#if (MI_DEBUG)
|
||||
// use our own assertion to print without memory allocation
|
||||
void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func );
|
||||
#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
|
||||
#else
|
||||
#define mi_assert(x)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>1)
|
||||
#define mi_assert_internal mi_assert
|
||||
#else
|
||||
#define mi_assert_internal(x)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>2)
|
||||
#define mi_assert_expensive mi_assert
|
||||
#else
|
||||
#define mi_assert_expensive(x)
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Statistics
|
||||
// ------------------------------------------------------
|
||||
|
||||
#ifndef MI_STAT
|
||||
#if (MI_DEBUG>0)
|
||||
#define MI_STAT 2
|
||||
#else
|
||||
#define MI_STAT 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
typedef struct mi_stat_count_s {
|
||||
int64_t allocated;
|
||||
int64_t freed;
|
||||
int64_t peak;
|
||||
int64_t current;
|
||||
} mi_stat_count_t;
|
||||
|
||||
typedef struct mi_stat_counter_s {
|
||||
int64_t total;
|
||||
int64_t count;
|
||||
} mi_stat_counter_t;
|
||||
|
||||
typedef struct mi_stats_s {
|
||||
mi_stat_count_t segments;
|
||||
mi_stat_count_t pages;
|
||||
mi_stat_count_t reserved;
|
||||
mi_stat_count_t committed;
|
||||
mi_stat_count_t reset;
|
||||
mi_stat_count_t segments_abandoned;
|
||||
mi_stat_count_t pages_abandoned;
|
||||
mi_stat_count_t pages_extended;
|
||||
mi_stat_count_t mmap_calls;
|
||||
mi_stat_count_t mmap_right_align;
|
||||
mi_stat_count_t mmap_ensure_aligned;
|
||||
mi_stat_count_t threads;
|
||||
mi_stat_count_t huge;
|
||||
mi_stat_count_t malloc;
|
||||
mi_stat_counter_t searches;
|
||||
#if MI_STAT>1
|
||||
mi_stat_count_t normal[MI_BIN_HUGE+1];
|
||||
#endif
|
||||
} mi_stats_t;
|
||||
|
||||
|
||||
void _mi_stat_increase(mi_stat_count_t* stat, size_t amount);
|
||||
void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount);
|
||||
void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
|
||||
|
||||
#if (MI_STAT)
|
||||
#define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount)
|
||||
#define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount)
|
||||
#define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount)
|
||||
#else
|
||||
#define mi_stat_increase(stat,amount) (void)0
|
||||
#define mi_stat_decrease(stat,amount) (void)0
|
||||
#define mi_stat_counter_increase(stat,amount) (void)0
|
||||
#endif
|
||||
|
||||
#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount)
|
||||
#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount)
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Thread Local data
|
||||
// ------------------------------------------------------
|
||||
|
||||
// Queue of segments
|
||||
typedef struct mi_segment_queue_s {
|
||||
mi_segment_t* first;
|
||||
mi_segment_t* last;
|
||||
} mi_segment_queue_t;
|
||||
|
||||
|
||||
// Segments thread local data
|
||||
typedef struct mi_segments_tld_s {
|
||||
mi_segment_queue_t small_free; // queue of segments with free small pages
|
||||
size_t count; // current number of segments
|
||||
size_t peak; // peak number of segments
|
||||
size_t cache_count; // number of segments in the cache
|
||||
mi_segment_t* cache; // small cache of segments (to avoid repeated mmap calls)
|
||||
mi_stats_t* stats; // points to tld stats
|
||||
} mi_segments_tld_t;
|
||||
|
||||
// OS thread local data
|
||||
typedef struct mi_os_tld_s {
|
||||
uintptr_t mmap_next_probable; // probable next address start allocated by mmap (to guess which path to take on alignment)
|
||||
void* mmap_previous; // previous address returned by mmap
|
||||
uint8_t* pool; // pool of segments to reduce mmap calls on some platforms
|
||||
size_t pool_available; // bytes available in the pool
|
||||
mi_stats_t* stats; // points to tld stats
|
||||
} mi_os_tld_t;
|
||||
|
||||
// Thread local data
|
||||
struct mi_tld_s {
|
||||
unsigned long long heartbeat; // monotonic heartbeat count
|
||||
mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted)
|
||||
mi_segments_tld_t segments; // segment tld
|
||||
mi_os_tld_t os; // os tld
|
||||
mi_stats_t stats; // statistics
|
||||
};
|
||||
|
||||
#endif
|
||||
237
include/mimalloc.h
Normal file
237
include/mimalloc.h
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2018, Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"license.txt" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
#pragma once
|
||||
#ifndef __MIMALLOC_H
|
||||
#define __MIMALLOC_H
|
||||
|
||||
#define MI_MALLOC_VERSION 100 // major + 2 digits minor
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Compiler specific attributes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#ifdef __cplusplus
|
||||
#if (__GNUC__ <= 5) || (_MSC_VER <= 1900)
|
||||
#define mi_attr_noexcept throw()
|
||||
#else
|
||||
#define mi_attr_noexcept noexcept
|
||||
#endif
|
||||
#else
|
||||
#define mi_attr_noexcept
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#if !defined(MI_SHARED_LIB)
|
||||
#define mi_decl_export
|
||||
#elif defined(MI_SHARED_LIB_EXPORT)
|
||||
#define mi_decl_export __declspec(dllexport)
|
||||
#else
|
||||
#define mi_decl_export __declspec(dllimport)
|
||||
#endif
|
||||
#if (_MSC_VER >= 1900) && !defined(__EDG__)
|
||||
#define mi_decl_allocator __declspec(allocator) __declspec(restrict)
|
||||
#else
|
||||
#define mi_decl_allocator __declspec(restrict)
|
||||
#endif
|
||||
#define mi_decl_thread __declspec(thread)
|
||||
#define mi_attr_malloc
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
#define mi_decl_thread __thread
|
||||
#define mi_decl_export __attribute__((visibility("default")))
|
||||
#define mi_decl_allocator
|
||||
#define mi_attr_malloc __attribute__((malloc))
|
||||
#if defined(__clang_major__) && (__clang_major__ < 4)
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#else
|
||||
#define mi_attr_alloc_size(s) __attribute__((alloc_size(s)))
|
||||
#define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2)))
|
||||
#endif
|
||||
#else
|
||||
#define mi_decl_thread __thread
|
||||
#define mi_decl_export
|
||||
#define mi_decl_allocator
|
||||
#define mi_attr_malloc
|
||||
#define mi_attr_alloc_size(s)
|
||||
#define mi_attr_alloc_size2(s1,s2)
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Includes
|
||||
// ------------------------------------------------------
|
||||
|
||||
#include <stdlib.h> // size_t, malloc etc.
|
||||
#include <stdbool.h> // bool
|
||||
#include <stdio.h> // FILE
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Standard malloc interface
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_export mi_decl_allocator void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_export mi_decl_allocator void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_export void mi_free(void* p) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_strdup(const char* s) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_strndup(const char* s, size_t n) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept;
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Extended functionality
|
||||
// ------------------------------------------------------
|
||||
#define MI_SMALL_WSIZE_MAX (128)
|
||||
#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*))
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_export mi_decl_allocator void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_export mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept;
|
||||
mi_decl_export mi_decl_allocator void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept;
|
||||
mi_decl_export mi_decl_allocator void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3);
|
||||
|
||||
mi_decl_export size_t mi_usable_size(void* p) mi_attr_noexcept;
|
||||
mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_collect(bool force) mi_attr_noexcept;
|
||||
mi_decl_export void mi_stats_print(FILE* out) mi_attr_noexcept;
|
||||
mi_decl_export void mi_stats_reset() mi_attr_noexcept;
|
||||
|
||||
mi_decl_export void mi_process_init() mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_init() mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_done() mi_attr_noexcept;
|
||||
mi_decl_export void mi_thread_stats_print(FILE* out) mi_attr_noexcept;
|
||||
|
||||
typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat);
|
||||
mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free) mi_attr_noexcept;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Aligned allocation
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_export mi_decl_allocator void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
mi_decl_export mi_decl_allocator void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1);
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
mi_decl_export mi_decl_allocator void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2);
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_recalloc_aligned(void* p, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3);
|
||||
mi_decl_export mi_decl_allocator void* mi_recalloc_aligned_at(void* p, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3);
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Heaps
|
||||
// ------------------------------------------------------
|
||||
struct mi_heap_s;
|
||||
typedef struct mi_heap_s mi_heap_t;
|
||||
|
||||
mi_decl_export mi_heap_t* mi_heap_new();
|
||||
mi_decl_export void mi_heap_delete(mi_heap_t* heap);
|
||||
mi_decl_export void mi_heap_destroy(mi_heap_t* heap);
|
||||
mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
|
||||
mi_decl_export mi_heap_t* mi_heap_get_default();
|
||||
mi_decl_export mi_heap_t* mi_heap_get_backing();
|
||||
mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept;
|
||||
|
||||
mi_decl_export mi_decl_allocator void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2);
|
||||
mi_decl_export mi_decl_allocator void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
mi_decl_export mi_decl_allocator void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3);
|
||||
|
||||
mi_decl_export char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept;
|
||||
mi_decl_export char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept;
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Analysis
|
||||
// ------------------------------------------------------
|
||||
|
||||
mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
|
||||
|
||||
mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
|
||||
mi_decl_export bool mi_check_owned(const void* p);
|
||||
|
||||
// An area of heap space contains blocks of a single size.
|
||||
typedef struct mi_heap_area_s {
|
||||
void* blocks; // start of the area containing heap blocks
|
||||
size_t reserved; // bytes reserved for this area (virtual)
|
||||
size_t committed; // current available bytes for this area
|
||||
size_t used; // bytes in use by allocated blocks
|
||||
size_t block_size; // size in bytes of each block
|
||||
} mi_heap_area_t;
|
||||
|
||||
typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
|
||||
|
||||
mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Convenience
|
||||
// ------------------------------------------------------
|
||||
|
||||
#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
|
||||
#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
|
||||
#define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp)))
|
||||
#define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp)))
|
||||
|
||||
#define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp)))
|
||||
#define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp)))
|
||||
|
||||
#define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
|
||||
#define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
|
||||
#define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp)))
|
||||
#define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp)))
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Options, all `false` by default
|
||||
// ------------------------------------------------------
|
||||
|
||||
typedef enum mi_option_e {
|
||||
mi_option_page_reset,
|
||||
mi_option_cache_reset,
|
||||
mi_option_pool_commit,
|
||||
mi_option_secure,
|
||||
mi_option_show_stats,
|
||||
mi_option_show_errors,
|
||||
mi_option_verbose,
|
||||
_mi_option_last
|
||||
} mi_option_t;
|
||||
|
||||
mi_decl_export bool mi_option_is_enabled(mi_option_t option);
|
||||
mi_decl_export void mi_option_enable(mi_option_t option, bool enable);
|
||||
mi_decl_export void mi_option_enable_default(mi_option_t option, bool enable);
|
||||
|
||||
mi_decl_export long mi_option_get(mi_option_t option);
|
||||
mi_decl_export void mi_option_set(mi_option_t option, long value);
|
||||
mi_decl_export void mi_option_set_default(mi_option_t option, long value);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Loading…
Add table
Add a link
Reference in a new issue