mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-09-16 11:14:47 +03:00
merge from dev-exp
This commit is contained in:
daan
commit
1066be1594
30 changed files with 1167 additions and 356 deletions
91
src/init.c
91
src/init.c
|
|
@ -19,7 +19,7 @@ const mi_page_t _mi_page_empty = {
|
|||
0,
|
||||
#endif
|
||||
0, // used
|
||||
NULL,
|
||||
NULL,
|
||||
ATOMIC_VAR_INIT(0), ATOMIC_VAR_INIT(0),
|
||||
0, NULL, NULL, NULL
|
||||
#ifndef MI_ENCODE_FREELIST
|
||||
|
|
@ -103,28 +103,31 @@ const mi_heap_t _mi_heap_empty = {
|
|||
};
|
||||
|
||||
#define tld_empty_stats ((mi_stats_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,stats)))
|
||||
#define tld_empty_os ((mi_os_tld_t*)((uint8_t*)&tld_empty + offsetof(mi_tld_t,os)))
|
||||
|
||||
static const mi_tld_t tld_empty = {
|
||||
0,
|
||||
false,
|
||||
NULL,
|
||||
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_empty_stats }, // segments
|
||||
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_empty_stats, tld_empty_os }, // segments
|
||||
{ 0, tld_empty_stats }, // os
|
||||
{ MI_STATS_NULL } // stats
|
||||
};
|
||||
|
||||
// the thread-local default heap for allocation
|
||||
mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
|
||||
|
||||
|
||||
|
||||
#define tld_main_stats ((mi_stats_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,stats)))
|
||||
#define tld_main_os ((mi_os_tld_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,os)))
|
||||
|
||||
static mi_tld_t tld_main = {
|
||||
0, false,
|
||||
&_mi_heap_main,
|
||||
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats }, // segments
|
||||
{ 0, tld_main_stats }, // os
|
||||
{ MI_STATS_NULL } // stats
|
||||
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats, tld_main_os }, // segments
|
||||
{ 0, tld_main_stats }, // os
|
||||
{ MI_STATS_NULL } // stats
|
||||
};
|
||||
|
||||
mi_heap_t _mi_heap_main = {
|
||||
|
|
@ -214,7 +217,7 @@ uintptr_t _mi_random_init(uintptr_t seed /* can be zero */) {
|
|||
|
||||
typedef struct mi_thread_data_s {
|
||||
mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
|
||||
mi_tld_t tld;
|
||||
mi_tld_t tld;
|
||||
} mi_thread_data_t;
|
||||
|
||||
// Initialize the thread local default heap, called from `mi_thread_init`
|
||||
|
|
@ -222,8 +225,8 @@ static bool _mi_heap_init(void) {
|
|||
if (mi_heap_is_initialized(_mi_heap_default)) return true;
|
||||
if (_mi_is_main_thread()) {
|
||||
// the main heap is statically allocated
|
||||
_mi_heap_default = &_mi_heap_main;
|
||||
mi_assert_internal(_mi_heap_default->tld->heap_backing == _mi_heap_default);
|
||||
_mi_heap_set_default_direct(&_mi_heap_main);
|
||||
mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap());
|
||||
}
|
||||
else {
|
||||
// use `_mi_os_alloc` to allocate directly from the OS
|
||||
|
|
@ -242,26 +245,26 @@ static bool _mi_heap_init(void) {
|
|||
heap->tld = tld;
|
||||
tld->heap_backing = heap;
|
||||
tld->segments.stats = &tld->stats;
|
||||
tld->segments.os = &tld->os;
|
||||
tld->os.stats = &tld->stats;
|
||||
_mi_heap_default = heap;
|
||||
_mi_heap_set_default_direct(heap);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Free the thread local default heap (called from `mi_thread_done`)
|
||||
static bool _mi_heap_done(void) {
|
||||
mi_heap_t* heap = _mi_heap_default;
|
||||
static bool _mi_heap_done(mi_heap_t* heap) {
|
||||
if (!mi_heap_is_initialized(heap)) return true;
|
||||
|
||||
// reset default heap
|
||||
_mi_heap_default = (_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty);
|
||||
_mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty);
|
||||
|
||||
// todo: delete all non-backing heaps?
|
||||
|
||||
// switch to backing heap and free it
|
||||
heap = heap->tld->heap_backing;
|
||||
if (!mi_heap_is_initialized(heap)) return false;
|
||||
|
||||
|
||||
// collect if not the main thread
|
||||
if (heap != &_mi_heap_main) {
|
||||
_mi_heap_collect_abandon(heap);
|
||||
|
|
@ -301,6 +304,8 @@ static bool _mi_heap_done(void) {
|
|||
// to set up the thread local keys.
|
||||
// --------------------------------------------------------
|
||||
|
||||
static void _mi_thread_done(mi_heap_t* default_heap);
|
||||
|
||||
#ifdef __wasi__
|
||||
// no pthreads in the WebAssembly Standard Interface
|
||||
#elif !defined(_WIN32)
|
||||
|
|
@ -315,14 +320,14 @@ static bool _mi_heap_done(void) {
|
|||
#include <fibersapi.h>
|
||||
static DWORD mi_fls_key;
|
||||
static void NTAPI mi_fls_done(PVOID value) {
|
||||
if (value!=NULL) mi_thread_done();
|
||||
if (value!=NULL) _mi_thread_done((mi_heap_t*)value);
|
||||
}
|
||||
#elif defined(MI_USE_PTHREADS)
|
||||
// use pthread locol storage keys to detect thread ending
|
||||
#include <pthread.h>
|
||||
static pthread_key_t mi_pthread_key;
|
||||
static void mi_pthread_done(void* value) {
|
||||
if (value!=NULL) mi_thread_done();
|
||||
if (value!=NULL) _mi_thread_done((mi_heap_t*)value);
|
||||
}
|
||||
#elif defined(__wasi__)
|
||||
// no pthreads in the WebAssembly Standard Interface
|
||||
|
|
@ -356,6 +361,8 @@ void mi_thread_init(void) mi_attr_noexcept
|
|||
mi_process_init();
|
||||
|
||||
// initialize the thread local default heap
|
||||
// (this will call `_mi_heap_set_default_direct` and thus set the
|
||||
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
|
||||
if (_mi_heap_init()) return; // returns true if already initialized
|
||||
|
||||
// don't further initialize for the main thread
|
||||
|
|
@ -363,33 +370,38 @@ void mi_thread_init(void) mi_attr_noexcept
|
|||
|
||||
_mi_stat_increase(&mi_get_default_heap()->tld->stats.threads, 1);
|
||||
|
||||
// set hooks so our mi_thread_done() will be called
|
||||
#if defined(_WIN32) && defined(MI_SHARED_LIB)
|
||||
// nothing to do as it is done in DllMain
|
||||
#elif defined(_WIN32) && !defined(MI_SHARED_LIB)
|
||||
FlsSetValue(mi_fls_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_fls_done` is called
|
||||
#elif defined(MI_USE_PTHREADS)
|
||||
pthread_setspecific(mi_pthread_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_pthread_done` is called
|
||||
#endif
|
||||
|
||||
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
|
||||
}
|
||||
|
||||
void mi_thread_done(void) mi_attr_noexcept {
|
||||
_mi_thread_done(mi_get_default_heap());
|
||||
}
|
||||
|
||||
static void _mi_thread_done(mi_heap_t* heap) {
|
||||
// stats
|
||||
mi_heap_t* heap = mi_get_default_heap();
|
||||
if (!_mi_is_main_thread() && mi_heap_is_initialized(heap)) {
|
||||
_mi_stat_decrease(&heap->tld->stats.threads, 1);
|
||||
}
|
||||
|
||||
// abandon the thread local heap
|
||||
if (_mi_heap_done()) return; // returns true if already ran
|
||||
|
||||
//if (!_mi_is_main_thread()) {
|
||||
// _mi_verbose_message("thread done: 0x%zx\n", _mi_thread_id());
|
||||
//}
|
||||
if (_mi_heap_done(heap)) return; // returns true if already ran
|
||||
}
|
||||
|
||||
void _mi_heap_set_default_direct(mi_heap_t* heap) {
|
||||
mi_assert_internal(heap != NULL);
|
||||
_mi_heap_default = heap;
|
||||
|
||||
// ensure the default heap is passed to `_mi_thread_done`
|
||||
// setting to a non-NULL value also ensures `mi_thread_done` is called.
|
||||
#if defined(_WIN32) && defined(MI_SHARED_LIB)
|
||||
// nothing to do as it is done in DllMain
|
||||
#elif defined(_WIN32) && !defined(MI_SHARED_LIB)
|
||||
FlsSetValue(mi_fls_key, heap);
|
||||
#elif defined(MI_USE_PTHREADS)
|
||||
pthread_setspecific(mi_pthread_key, heap);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
// --------------------------------------------------------
|
||||
// Run functions on process init/done, and thread init/done
|
||||
|
|
@ -409,7 +421,7 @@ bool mi_is_redirected() mi_attr_noexcept {
|
|||
}
|
||||
|
||||
// Communicate with the redirection module on Windows
|
||||
#if defined(_WIN32) && defined(MI_SHARED_LIB)
|
||||
#if defined(_WIN32) && defined(MI_SHARED_LIB)
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
|
@ -455,11 +467,6 @@ static void mi_process_load(void) {
|
|||
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
|
||||
_mi_fputs(NULL,NULL,msg);
|
||||
}
|
||||
|
||||
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
|
||||
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
|
||||
mi_reserve_huge_os_pages_interleave(pages, pages*500);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize the process; called by thread_init or the process loader
|
||||
|
|
@ -469,7 +476,7 @@ void mi_process_init(void) mi_attr_noexcept {
|
|||
// access _mi_heap_default before setting _mi_process_is_initialized to ensure
|
||||
// that the TLS slot is allocated without getting into recursion on macOS
|
||||
// when using dynamic linking with interpose.
|
||||
mi_heap_t* h = _mi_heap_default;
|
||||
mi_heap_t* h = mi_get_default_heap();
|
||||
_mi_process_is_initialized = true;
|
||||
|
||||
_mi_heap_main.thread_id = _mi_thread_id();
|
||||
|
|
@ -484,8 +491,14 @@ void mi_process_init(void) mi_attr_noexcept {
|
|||
#if (MI_DEBUG)
|
||||
_mi_verbose_message("debug level : %d\n", MI_DEBUG);
|
||||
#endif
|
||||
_mi_verbose_message("secure level: %d\n", MI_SECURE);
|
||||
mi_thread_init();
|
||||
mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL)
|
||||
|
||||
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
|
||||
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
|
||||
mi_reserve_huge_os_pages_interleave(pages, 0, pages*500);
|
||||
}
|
||||
}
|
||||
|
||||
// Called when the process is done (through `at_exit`)
|
||||
|
|
@ -512,7 +525,7 @@ static void mi_process_done(void) {
|
|||
|
||||
|
||||
#if defined(_WIN32) && defined(MI_SHARED_LIB)
|
||||
// Windows DLL: easy to hook into process_init and thread_done
|
||||
// Windows DLL: easy to hook into process_init and thread_done
|
||||
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
|
||||
UNUSED(reserved);
|
||||
UNUSED(inst);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue