kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-07-06 19:38:41 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge from dev

Browse source
This commit is contained in:
daan 2019-10-17 18:24:35 -07:00
commit 4b15e2ed97
70 changed files with 2002 additions and 1695 deletions
Download patch file Download diff file Expand all files Collapse all files

123
src/alloc-aligned.c
View file

@ -14,92 +14,100 @@ terms of the MIT license. A copy of the license can be found in the file
// Aligned Allocation
// ------------------------------------------------------
static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset, bool zero) mi_attr_noexcept {
static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept {
// note: we don't require `size > offset`, we just guarantee that
// the address at offset is aligned regardless of the allocated size.
mi_assert(alignment > 0 && alignment % sizeof(uintptr_t) == 0);
if (alignment <= sizeof(uintptr_t)) return _mi_heap_malloc_zero(heap,size,zero);
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
// try if there is a current small block with just the right alignment
if (size <= MI_SMALL_SIZE_MAX) {
mi_assert(alignment > 0 && alignment % sizeof(void*) == 0);
if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see <https://sourceware.org/ml/libc-announce/2019/msg00001.html>)
if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see <https://en.cppreference.com/w/c/memory/aligned_alloc>)
const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)`
// try if there is a small block available with just the right alignment
if (mi_likely(size <= MI_SMALL_SIZE_MAX)) {
mi_page_t* page = _mi_heap_get_free_small_page(heap,size);
if (page->free != NULL &&
(((uintptr_t)page->free + offset) % alignment) == 0)
const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0;
if (mi_likely(page->free != NULL && is_aligned))
{
#if MI_STAT>1
mi_heap_stat_increase( heap, malloc, size);
mi_heap_stat_increase( heap, malloc, size);
#endif
void* p = _mi_page_malloc(heap,page,size);
void* p = _mi_page_malloc(heap,page,size); // TODO: inline _mi_page_malloc
mi_assert_internal(p != NULL);
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);
if (zero) memset(p,0,size);
if (zero) _mi_block_zero_init(page,p,size);
return p;
}
}
// use regular allocation if it is guaranteed to fit the alignment constraints
if (offset==0 && alignment<=size && size<=MI_MEDIUM_OBJ_SIZE_MAX && (size&align_mask)==0) {
void* p = _mi_heap_malloc_zero(heap, size, zero);
mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);
return p;
}
// otherwise over-allocate
void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero);
if (p == NULL) return NULL;
// .. and align within the allocation
mi_page_set_has_aligned(_mi_ptr_page(p), true);
uintptr_t adjust = alignment - (((uintptr_t)p + offset) % alignment);
uintptr_t adjust = alignment - (((uintptr_t)p + offset) & align_mask);
mi_assert_internal(adjust % sizeof(uintptr_t) == 0);
void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust));
if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true);
mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);
mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) );
return aligned_p;
}
void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false);
}
void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_malloc_aligned_at(heap, size, alignment, 0);
}
void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true);
}
void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_zalloc_aligned_at(heap, size, alignment, 0);
}
void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(count, size, &total)) return NULL;
return mi_heap_zalloc_aligned_at(heap, total, alignment, offset);
}
void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_calloc_aligned_at(heap,count,size,alignment,0);
}
void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_malloc_aligned_at(mi_get_default_heap(), size, alignment, offset);
}
void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_malloc_aligned(mi_get_default_heap(), size, alignment);
}
void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_zalloc_aligned_at(mi_get_default_heap(), size, alignment, offset);
}
void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_zalloc_aligned(mi_get_default_heap(), size, alignment);
}
void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_calloc_aligned_at(mi_get_default_heap(), count, size, alignment, offset);
}
void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_calloc_aligned(mi_get_default_heap(), count, size, alignment);
}
@ -117,9 +125,16 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne
void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset);
if (newp != NULL) {
if (zero && newsize > size) {
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
memset((uint8_t*)newp + start, 0, newsize - start);
const mi_page_t* page = _mi_ptr_page(newp);
if (page->is_zero) {
// already zero initialized
mi_assert_expensive(mi_mem_is_zero(newp,newsize));
}
else {
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
memset((uint8_t*)newp + start, 0, newsize - start);
}
}
memcpy(newp, p, (newsize > size ? size : newsize));
mi_free(p); // only free if successful
@ -135,29 +150,55 @@ static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsi
return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero);
}
void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false);
}
void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false);
}
void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true);
}
mi_decl_allocator void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true);
}
mi_decl_allocator void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(newcount, size, &total)) return NULL;
return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset);
}
mi_decl_allocator void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(newcount, size, &total)) return NULL;
return mi_heap_rezalloc_aligned(heap, p, total, alignment);
}
mi_decl_allocator void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_realloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset);
}
void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
mi_decl_allocator void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment);
}
void* mi_aligned_offset_recalloc(void* p, size_t size, size_t newcount, size_t alignment, size_t offset) mi_attr_noexcept {
size_t newsize;
if (mi_mul_overflow(size,newcount,&newsize)) return NULL;
return mi_heap_realloc_zero_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset, true );
mi_decl_allocator void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_rezalloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset);
}
void* mi_aligned_recalloc(void* p, size_t size, size_t newcount, size_t alignment) mi_attr_noexcept {
size_t newsize;
if (mi_mul_overflow(size, newcount, &newsize)) return NULL;
return mi_heap_realloc_zero_aligned(mi_get_default_heap(), p, newsize, alignment, true );
mi_decl_allocator void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept {
return mi_heap_rezalloc_aligned(mi_get_default_heap(), p, newsize, alignment);
}
mi_decl_allocator void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept {
return mi_heap_recalloc_aligned_at(mi_get_default_heap(), p, newcount, size, alignment, offset);
}
mi_decl_allocator void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept {
return mi_heap_recalloc_aligned(mi_get_default_heap(), p, newcount, size, alignment);
}

714
src/alloc-override-win.c
View file

@ -1,714 +0,0 @@
/* ----------------------------------------------------------------------------
Copyright (c) 2018, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
#include "mimalloc.h"
#include "mimalloc-internal.h"
#if !defined(_WIN32)
#error "this file should only be included on Windows"
#endif
#include <windows.h>
#include <psapi.h>
#include <stdlib.h> // getenv
#include <string.h> // strstr
/*
To override the C runtime `malloc` on Windows we need to patch the allocation
functions at runtime initialization. Unfortunately we can never patch before the
runtime initializes itself, because as soon as we call `GetProcAddress` on the
runtime module (a DLL or EXE in Windows speak), it will first load and initialize
(by the OS calling `DllMain` on it).
This means that some things might be already allocated by the C runtime itself
(and possibly other DLL's) before we get to resolve runtime adresses. This is
no problem if everyone unwinds in order: when we unload, we unpatch and restore
the original crt `free` routines and crt malloc'd memory is freed correctly.
But things go wrong if such early CRT alloc'd memory is freed or re-allocated
_after_ we patch, but _before_ we unload (and unpatch), or if any memory allocated
by us is freed after we unpatched.
There are two tricky situations to deal with:
1. The Thread Local Storage (TLS): when the main thread stops it will call registered
callbacks on TLS entries (allocated by `FlsAlloc`). This is done by the OS
before any DLL's are unloaded. Unfortunately, the C runtime registers such
TLS entries with CRT allocated memory which is freed in the callback.
2. Inside the CRT:
a. Some variables might get initialized by patched allocated
blocks but freed during CRT unloading after we unpatched
(like temporary file buffers).
b. Some blocks are allocated at CRT and freed by the CRT (like the
environment storage).
c. And some blocks are allocated by the CRT and then reallocated
while patched, and finally freed after unpatching! This
happens with the `atexit` functions for example to grow the array
of registered functions.
In principle situation 2 is hopeless: since we cannot patch before CRT initialization,
we can never be sure how to free or reallocate a pointer during CRT unloading.
However, in practice there is a good solution: when terminating, we just patch
the reallocation and free routines to no-ops -- we are winding down anyway! This leaves
just the reallocation problm of CRT alloc'd memory once we are patched. Here, a study of the
CRT reveals that there seem to be just three such situations:
1. When registering `atexit` routines (to grow the exit function table),
2. When calling `_setmaxstdio` (to grow the file handle table),
3. and `_popen`/`_wpopen` (to grow handle pairs). These turn out not to be
a problem as these are NULL initialized.
We fix these by providing wrappers:
1. We first register a _global_ `atexit` routine ourselves (`mi_patches_at_exit`) before patching,
and then patch the `_crt_atexit` function to implement our own global exit list (and the
same for `_crt_at_quick_exit`). All module local lists are no problem since they are always fully
(un)patched from initialization to end. We can register in the global list by dynamically
getting the global `_crt_atexit` entry from `ucrtbase.dll`.
2. The `_setmaxstdio` is _detoured_: we patch it by a stub that unpatches first,
calls the original routine and repatches again.
That leaves us to reliably shutdown and enter "termination mode":
1. Using our trick to get the global exit list entry point, we register an exit function `mi_patches_atexit`
that first executes all our home brew list of exit functions, and then enters a _termination_
phase that patches realloc/free variants with no-ops. Patching later again with special no-ops for
`free` also improves efficiency during the program run since no flags need to be checked.
2. That is not quite good enough yet since after executing exit routines after us on the
global exit list (registered by the CRT),
the OS starts to unwind the TLS callbacks and we would like to run callbacks registered after loading
our DLL to be done in patched mode. So, we also allocate a TLS entry when our DLL is loaded and when its
callback is called, we re-enable the original patches again. Since TLS is destroyed in FIFO order
this runs any callbacks in later DLL's in patched mode.
3. Finally the DLL's get unloaded by the OS in order (still patched) until our DLL gets unloaded
and then we start a termination phase again, and patch realloc/free with no-ops for good this time.
*/
static int __cdecl mi_setmaxstdio(int newmax);
// ------------------------------------------------------
// Microsoft allocation extensions
// ------------------------------------------------------
typedef size_t mi_nothrow_t;
static void mi_free_nothrow(void* p, mi_nothrow_t tag) {
UNUSED(tag);
mi_free(p);
}
// Versions of `free`, `realloc`, `recalloc`, `expand` and `msize`
// that are used during termination and are no-ops.
static void mi_free_term(void* p) {
UNUSED(p);
}
static void mi_free_size_term(void* p, size_t size) {
UNUSED(size);
UNUSED(p);
}
static void mi_free_nothrow_term(void* p, mi_nothrow_t tag) {
UNUSED(tag);
UNUSED(p);
}
static void* mi_realloc_term(void* p, size_t newsize) {
UNUSED(p); UNUSED(newsize);
return NULL;
}
static void* mi__recalloc_term(void* p, size_t newcount, size_t newsize) {
UNUSED(p); UNUSED(newcount); UNUSED(newsize);
return NULL;
}
static void* mi__expand_term(void* p, size_t newsize) {
UNUSED(p); UNUSED(newsize);
return NULL;
}
static size_t mi__msize_term(void* p) {
UNUSED(p);
return 0;
}
static void* mi__malloc_dbg(size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return _malloc_base(size);
}
static void* mi__calloc_dbg(size_t count, size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return _calloc_base(count, size);
}
static void* mi__realloc_dbg(void* p, size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return _realloc_base(p, size);
}
static void mi__free_dbg(void* p, int block_type) {
UNUSED(block_type);
_free_base(p);
}
// the `recalloc`,`expand`, and `msize` don't have base versions and thus need a separate term version
static void* mi__recalloc_dbg(void* p, size_t count, size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return mi_recalloc(p, count, size);
}
static void* mi__expand_dbg(void* p, size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return mi__expand(p, size);
}
static size_t mi__msize_dbg(void* p, int block_type) {
UNUSED(block_type);
return mi_usable_size(p);
}
static void* mi__recalloc_dbg_term(void* p, size_t count, size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return mi__recalloc_term(p, count, size);
}
static void* mi__expand_dbg_term(void* p, size_t size, int block_type, const char* fname, int line) {
UNUSED(block_type); UNUSED(fname); UNUSED(line);
return mi__expand_term(p, size);
}
static size_t mi__msize_dbg_term(void* p, int block_type) {
UNUSED(block_type);
return mi__msize_term(p);
}
// ------------------------------------------------------
// implement our own global atexit handler
// ------------------------------------------------------
typedef void (cbfun_t)(void);
typedef int (atexit_fun_t)(cbfun_t* fn);
typedef uintptr_t encoded_t;
typedef struct exit_list_s {
encoded_t functions; // encoded pointer to array of encoded function pointers
size_t count;
size_t capacity;
} exit_list_t;
#define MI_EXIT_INC (64)
static exit_list_t atexit_list = { 0, 0, 0 };
static exit_list_t at_quick_exit_list = { 0, 0, 0 };
static CRITICAL_SECTION atexit_lock;
// encode/decode function pointers with a random canary for security
static encoded_t canary;
static inline void *decode(encoded_t x) {
return (void*)(x^canary);
}
static inline encoded_t encode(void* p) {
return ((uintptr_t)p ^ canary);
}
static void init_canary()
{
canary = _mi_random_init(0);
atexit_list.functions = at_quick_exit_list.functions = encode(NULL);
}
// initialize the list
static void mi_initialize_atexit(void) {
InitializeCriticalSection(&atexit_lock);
init_canary();
}
// register an exit function
static int mi_register_atexit(exit_list_t* list, cbfun_t* fn) {
if (fn == NULL) return EINVAL;
EnterCriticalSection(&atexit_lock);
encoded_t* functions = (encoded_t*)decode(list->functions);
if (list->count >= list->capacity) { // at first `functions == decode(0) == NULL`
encoded_t* newf = (encoded_t*)mi_recalloc(functions, list->capacity + MI_EXIT_INC, sizeof(cbfun_t*));
if (newf != NULL) {
list->capacity += MI_EXIT_INC;
list->functions = encode(newf);
functions = newf;
}
}
int result;
if (list->count < list->capacity && functions != NULL) {
functions[list->count] = encode(fn);
list->count++;
result = 0; // success
}
else {
result = ENOMEM;
}
LeaveCriticalSection(&atexit_lock);
return result;
}
// Register a global `atexit` function
static int mi_atexit(cbfun_t* fn) {
return mi_register_atexit(&atexit_list,fn);
}
static int mi_at_quick_exit(cbfun_t* fn) {
return mi_register_atexit(&at_quick_exit_list,fn);
}
static int mi_register_onexit(void* table, cbfun_t* fn) {
// TODO: how can we distinguish a quick_exit from atexit?
return mi_atexit(fn);
}
// Execute exit functions in a list
static void mi_execute_exit_list(exit_list_t* list) {
// copy and zero the list structure
EnterCriticalSection(&atexit_lock);
exit_list_t clist = *list;
memset(list,0,sizeof(*list));
LeaveCriticalSection(&atexit_lock);
// now execute the functions outside of the lock
encoded_t* functions = (encoded_t*)decode(clist.functions);
if (functions != NULL) {
for (size_t i = clist.count; i > 0; i--) { // careful with unsigned count down..
cbfun_t* fn = (cbfun_t*)decode(functions[i-1]);
if (fn==NULL) break; // corrupted!
fn();
}
mi_free(functions);
}
}
// ------------------------------------------------------
// Jump assembly instructions for patches
// ------------------------------------------------------
#if defined(_M_IX86) || defined(_M_X64)
#define MI_JUMP_SIZE 14 // at most 2+4+8 for a long jump or 1+5 for a short one
typedef struct mi_jump_s {
uint8_t opcodes[MI_JUMP_SIZE];
} mi_jump_t;
void mi_jump_restore(void* current, const mi_jump_t* saved) {
memcpy(current, &saved->opcodes, MI_JUMP_SIZE);
}
void mi_jump_write(void* current, void* target, mi_jump_t* save) {
if (save != NULL) {
memcpy(&save->opcodes, current, MI_JUMP_SIZE);
}
uint8_t* opcodes = ((mi_jump_t*)current)->opcodes;
ptrdiff_t diff = (uint8_t*)target - (uint8_t*)current;
uint32_t ofs32 = (uint32_t)diff;
#ifdef _M_X64
uint64_t ofs64 = (uint64_t)diff;
if (ofs64 != (uint64_t)ofs32) {
// use long jump
opcodes[0] = 0xFF;
opcodes[1] = 0x25;
*((uint32_t*)&opcodes[2]) = 0;
*((uint64_t*)&opcodes[6]) = (uint64_t)target;
}
else
#endif
{
// use short jump
opcodes[0] = 0xE9;
*((uint32_t*)&opcodes[1]) = ofs32 - 5 /* size of the short jump instruction */;
}
}
#elif defined(_M_ARM64)
#define MI_JUMP_SIZE 16
typedef struct mi_jump_s {
uint8_t opcodes[MI_JUMP_SIZE];
} mi_jump_t;
void mi_jump_restore(void* current, const mi_jump_t* saved) {
memcpy(current, &saved->opcodes, MI_JUMP_SIZE);
}
void mi_jump_write(void* current, void* target, mi_jump_t* save) {
if (save != NULL) {
memcpy(&save->opcodes, current, MI_JUMP_SIZE);
}
uint8_t* opcodes = ((mi_jump_t*)current)->opcodes;
uint64_t diff = (uint8_t*)target - (uint8_t*)current;
// 0x50 0x00 0x00 0x58 ldr x16, .+8 # load PC relative +8
// 0x00 0x02 0x3F 0xD6 blr x16 # and jump
// <address>
// <address>
static const uint8_t jump_opcodes[8] = { 0x50, 0x00, 0x00, 0x58, 0x00, 0x02, 0x3F, 0xD6 };
memcpy(&opcodes[0], jump_opcodes, sizeof(jump_opcodes));
*((uint64_t*)&opcodes[8]) = diff;
}
#else
#error "define jump instructions for this platform"
#endif
// ------------------------------------------------------
// Patches
// ------------------------------------------------------
typedef enum patch_apply_e {
PATCH_NONE,
PATCH_TARGET,
PATCH_TARGET_TERM
} patch_apply_t;
#define MAX_ENTRIES 4 // maximum number of patched entry points (like `malloc` in ucrtbase and msvcrt)
typedef struct mi_patch_s {
const char* name; // name of the function to patch
void* target; // the address of the new target (never NULL)
void* target_term; // the address of the target during termination (or NULL)
patch_apply_t applied; // what target has been applied?
void* originals[MAX_ENTRIES]; // the resolved addresses of the function (or NULLs)
mi_jump_t saves[MAX_ENTRIES]; // the saved instructions in case it was applied
} mi_patch_t;
#define MI_PATCH_NAME3(name,target,term) { name, &target, &term, PATCH_NONE, {NULL,NULL,NULL,NULL} }
#define MI_PATCH_NAME2(name,target) { name, &target, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} }
#define MI_PATCH3(name,target,term) MI_PATCH_NAME3(#name, target, term)
#define MI_PATCH2(name,target) MI_PATCH_NAME2(#name, target)
#define MI_PATCH1(name) MI_PATCH2(name,mi_##name)
static mi_patch_t patches[] = {
// we implement our own global exit handler (as the CRT versions do a realloc internally)
//MI_PATCH2(_crt_atexit, mi_atexit),
//MI_PATCH2(_crt_at_quick_exit, mi_at_quick_exit),
MI_PATCH2(_setmaxstdio, mi_setmaxstdio),
MI_PATCH2(_register_onexit_function, mi_register_onexit),
// override higher level atexit functions so we can implement at_quick_exit correcty
MI_PATCH2(atexit, mi_atexit),
MI_PATCH2(at_quick_exit, mi_at_quick_exit),
// regular entries
MI_PATCH2(malloc, mi_malloc),
MI_PATCH2(calloc, mi_calloc),
MI_PATCH3(realloc, mi_realloc,mi_realloc_term),
MI_PATCH3(free, mi_free,mi_free_term),
// extended api
MI_PATCH2(_strdup, mi_strdup),
MI_PATCH2(_strndup, mi_strndup),
MI_PATCH3(_expand, mi__expand,mi__expand_term),
MI_PATCH3(_recalloc, mi_recalloc,mi__recalloc_term),
MI_PATCH3(_msize, mi_usable_size,mi__msize_term),
// base versions
MI_PATCH2(_malloc_base, mi_malloc),
MI_PATCH2(_calloc_base, mi_calloc),
MI_PATCH3(_realloc_base, mi_realloc,mi_realloc_term),
MI_PATCH3(_free_base, mi_free,mi_free_term),
// these base versions are in the crt but without import records
MI_PATCH_NAME3("_recalloc_base", mi_recalloc,mi__recalloc_term),
MI_PATCH_NAME3("_msize_base", mi_usable_size,mi__msize_term),
// debug
MI_PATCH2(_malloc_dbg, mi__malloc_dbg),
MI_PATCH2(_realloc_dbg, mi__realloc_dbg),
MI_PATCH2(_calloc_dbg, mi__calloc_dbg),
MI_PATCH2(_free_dbg, mi__free_dbg),
MI_PATCH3(_expand_dbg, mi__expand_dbg, mi__expand_dbg_term),
MI_PATCH3(_recalloc_dbg, mi__recalloc_dbg, mi__recalloc_dbg_term),
MI_PATCH3(_msize_dbg, mi__msize_dbg, mi__msize_dbg_term),
#if 0
// override new/delete variants for efficiency (?)
#ifdef _WIN64
// 64 bit new/delete
MI_PATCH_NAME2("??2@YAPEAX_K@Z", mi_new),
MI_PATCH_NAME2("??_U@YAPEAX_K@Z", mi_new),
MI_PATCH_NAME3("??3@YAXPEAX@Z", mi_free, mi_free_term),
MI_PATCH_NAME3("??_V@YAXPEAX@Z", mi_free, mi_free_term),
MI_PATCH_NAME3("??3@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized
MI_PATCH_NAME3("??_V@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized
MI_PATCH_NAME2("??2@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new),
MI_PATCH_NAME2("??_U@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new),
MI_PATCH_NAME3("??3@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term),
MI_PATCH_NAME3("??_V@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term),
#else
// 32 bit new/delete
MI_PATCH_NAME2("??2@YAPAXI@Z", mi_new),
MI_PATCH_NAME2("??_U@YAPAXI@Z", mi_new),
MI_PATCH_NAME3("??3@YAXPAX@Z", mi_free, mi_free_term),
MI_PATCH_NAME3("??_V@YAXPAX@Z", mi_free, mi_free_term),
MI_PATCH_NAME3("??3@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized
MI_PATCH_NAME3("??_V@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized
MI_PATCH_NAME2("??2@YAPAXIABUnothrow_t@std@@@Z", mi_new),
MI_PATCH_NAME2("??_U@YAPAXIABUnothrow_t@std@@@Z", mi_new),
MI_PATCH_NAME3("??3@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term),
MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term),
#endif
#endif
{ NULL, NULL, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} }
};
// Apply a patch
static bool mi_patch_apply(mi_patch_t* patch, patch_apply_t apply)
{
if (patch->originals[0] == NULL) return true; // unresolved
if (apply == PATCH_TARGET_TERM && patch->target_term == NULL) apply = PATCH_TARGET; // avoid re-applying non-term variants
if (patch->applied == apply) return false;
for (int i = 0; i < MAX_ENTRIES; i++) {
void* original = patch->originals[i];
if (original == NULL) break; // no more
DWORD protect = PAGE_READWRITE;
if (!VirtualProtect(original, MI_JUMP_SIZE, PAGE_EXECUTE_READWRITE, &protect)) return false;
if (apply == PATCH_NONE) {
mi_jump_restore(original, &patch->saves[i]);
}
else {
void* target = (apply == PATCH_TARGET ? patch->target : patch->target_term);
mi_assert_internal(target != NULL);
if (target != NULL) mi_jump_write(original, target, &patch->saves[i]);
}
VirtualProtect(original, MI_JUMP_SIZE, protect, &protect);
}
patch->applied = apply;
return true;
}
// Apply all patches
static bool _mi_patches_apply(patch_apply_t apply, patch_apply_t* previous) {
static patch_apply_t current = PATCH_NONE;
if (previous != NULL) *previous = current;
if (current == apply) return true;
current = apply;
bool ok = true;
for (size_t i = 0; patches[i].name != NULL; i++) {
if (!mi_patch_apply(&patches[i], apply)) ok = false;
}
return ok;
}
// Export the following three functions just in case
// a user needs that level of control.
// Disable all patches
mi_decl_export void mi_patches_disable(void) {
_mi_patches_apply(PATCH_NONE, NULL);
}
// Enable all patches normally
mi_decl_export bool mi_patches_enable(void) {
return _mi_patches_apply( PATCH_TARGET, NULL );
}
// Enable all patches in termination phase where free is a no-op
mi_decl_export bool mi_patches_enable_term(void) {
return _mi_patches_apply(PATCH_TARGET_TERM, NULL);
}
// ------------------------------------------------------
// Stub for _setmaxstdio
// ------------------------------------------------------
static int __cdecl mi_setmaxstdio(int newmax) {
patch_apply_t previous;
_mi_patches_apply(PATCH_NONE, &previous); // disable patches
int result = _setmaxstdio(newmax); // call original function (that calls original CRT recalloc)
_mi_patches_apply(previous,NULL); // and re-enable patches
return result;
}
// ------------------------------------------------------
// Resolve addresses dynamically
// ------------------------------------------------------
// Try to resolve patches for a given module (DLL)
static void mi_module_resolve(const char* fname, HMODULE mod, int priority) {
// see if any patches apply
for (size_t i = 0; patches[i].name != NULL; i++) {
mi_patch_t* patch = &patches[i];
if (patch->applied == PATCH_NONE) {
// find an available entry
int i = 0;
while (i < MAX_ENTRIES && patch->originals[i] != NULL) i++;
if (i < MAX_ENTRIES) {
void* addr = GetProcAddress(mod, patch->name);
if (addr != NULL) {
// found it! set the address
patch->originals[i] = addr;
_mi_trace_message(" found %s at %s!%p (entry %i)\n", patch->name, fname, addr, i);
}
}
}
}
}
#define MIMALLOC_NAME "mimalloc-override.dll"
#define UCRTBASE_NAME "ucrtbase.dll"
#define UCRTBASED_NAME "ucrtbased.dll"
// Resolve addresses of all patches by inspecting the loaded modules
static atexit_fun_t* crt_atexit = NULL;
static atexit_fun_t* crt_at_quick_exit = NULL;
static bool mi_patches_resolve(void) {
// get all loaded modules
HANDLE process = GetCurrentProcess(); // always -1, no need to release
DWORD needed = 0;
HMODULE modules[400]; // try to stay under 4k to not trigger the guard page
EnumProcessModules(process, modules, sizeof(modules), &needed);
if (needed == 0) return false;
int count = needed / sizeof(HMODULE);
int ucrtbase_index = 0;
int mimalloc_index = 0;
// iterate through the loaded modules
for (int i = 0; i < count; i++) {
HMODULE mod = modules[i];
char filename[MAX_PATH] = { 0 };
DWORD slen = GetModuleFileName(mod, filename, MAX_PATH);
if (slen > 0 && slen < MAX_PATH) {
// filter out potential crt modules only
filename[slen] = 0;
const char* lastsep = strrchr(filename, '\\');
const char* basename = (lastsep==NULL ? filename : lastsep+1);
_mi_trace_message(" %i: dynamic module %s\n", i, filename);
// remember indices so we can check load order (in debug mode)
if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i;
if (_stricmp(basename, UCRTBASE_NAME) == 0) ucrtbase_index = i;
if (_stricmp(basename, UCRTBASED_NAME) == 0) ucrtbase_index = i;
// see if we potentially patch in this module
int priority = 0;
if (i == 0) priority = 2; // main module to allow static crt linking
else if (_strnicmp(basename, "ucrt", 4) == 0) priority = 3; // new ucrtbase.dll in windows 10
// NOTE: don't override msvcr -- leads to crashes in setlocale (needs more testing)
// else if (_strnicmp(basename, "msvcr", 5) == 0) priority = 1; // older runtimes
if (priority > 0) {
// probably found a crt module, try to patch it
mi_module_resolve(basename,mod,priority);
// try to find the atexit functions for the main process (in `ucrtbase.dll`)
if (crt_atexit==NULL) crt_atexit = (atexit_fun_t*)GetProcAddress(mod, "_crt_atexit");
if (crt_at_quick_exit == NULL) crt_at_quick_exit = (atexit_fun_t*)GetProcAddress(mod, "_crt_at_quick_exit");
}
}
}
int diff = mimalloc_index - ucrtbase_index;
if (diff > 1) {
_mi_warning_message("warning: the \"mimalloc-override\" DLL seems not to load before or right after the C runtime (\"ucrtbase\").\n"
" Try to fix this by changing the linking order.\n");
}
return true;
}
// ------------------------------------------------------
// Dll Entry
// ------------------------------------------------------
extern BOOL WINAPI _DllMainCRTStartup(HINSTANCE inst, DWORD reason, LPVOID reserved);
static DWORD mi_fls_unwind_entry;
static void NTAPI mi_fls_unwind(PVOID value) {
if (value != NULL) mi_patches_enable(); // and re-enable normal patches again for DLL's loaded after us
return;
}
static void mi_patches_atexit(void) {
mi_execute_exit_list(&atexit_list);
mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op
}
static void mi_patches_at_quick_exit(void) {
mi_execute_exit_list(&at_quick_exit_list);
mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op
}
BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) {
if (reason == DLL_PROCESS_ATTACH) {
__security_init_cookie();
}
else if (reason == DLL_PROCESS_DETACH) {
// enter termination phase for good now
mi_patches_enable_term();
}
// C runtime main
BOOL ok = _DllMainCRTStartup(inst, reason, reserved);
if (reason == DLL_PROCESS_ATTACH && ok) {
// initialize at exit lists
mi_initialize_atexit();
// Now resolve patches
ok = mi_patches_resolve();
if (ok) {
// check if patching is not disabled
#pragma warning(suppress:4996)
const char* s = getenv("MIMALLOC_DISABLE_OVERRIDE");
bool enabled = (s == NULL || !(strstr("1;TRUE;YES;ON", s) != NULL));
if (!enabled) {
_mi_verbose_message("override is disabled\n");
}
else {
// and register our unwind entry (this must be after resolving due to possible delayed DLL initialization from GetProcAddress)
mi_fls_unwind_entry = FlsAlloc(&mi_fls_unwind);
if (mi_fls_unwind_entry != FLS_OUT_OF_INDEXES) {
FlsSetValue(mi_fls_unwind_entry, (void*)1);
}
// register our patch disabler in the global exit list
if (crt_atexit != NULL) (*crt_atexit)(&mi_patches_atexit);
if (crt_at_quick_exit != NULL) (*crt_at_quick_exit)(&mi_patches_at_quick_exit);
// and patch ! this also redirects the `atexit` handling for the global exit list
mi_patches_enable();
_mi_verbose_message("override is enabled\n");
// hide internal allocation
mi_stats_reset();
}
}
}
return ok;
}

2
src/alloc-override.c
View file

@ -10,7 +10,7 @@ terms of the MIT license. A copy of the license can be found in the file
#endif
#if defined(MI_MALLOC_OVERRIDE) && defined(_WIN32) && !(defined(MI_SHARED_LIB) && defined(_DLL))
#error "It is only possible to override "malloc" on Windows when building as a 64-bit DLL (and linking the C runtime as a DLL)"
#error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)"
#endif
#if defined(MI_MALLOC_OVERRIDE) && !defined(_WIN32)

22
src/alloc-posix.c
View file

@ -48,17 +48,13 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept
// <http://man7.org/linux/man-pages/man3/posix_memalign.3.html>
if (p == NULL) return EINVAL;
if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment
if ((alignment & (alignment - 1)) != 0) return EINVAL; // not a power of 2
if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2
void* q = mi_malloc_aligned(size, alignment);
if (q==NULL && size != 0) return ENOMEM;
*p = q;
return 0;
}
int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept {
return mi_posix_memalign(p, alignment, size);
}
void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept {
return mi_malloc_aligned(size, alignment);
}
@ -75,6 +71,8 @@ void* mi_pvalloc(size_t size) mi_attr_noexcept {
}
void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept {
if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL;
if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see <https://en.cppreference.com/w/c/memory/aligned_alloc>
return mi_malloc_aligned(size, alignment);
}
@ -90,12 +88,6 @@ void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft
return res;
}
void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { // Microsoft
size_t total;
if (mi_mul_overflow(count, size, &total)) return NULL;
return _mi_heap_realloc_zero(mi_get_default_heap(), p, total, true);
}
unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept {
if (s==NULL) return NULL;
size_t len;
@ -149,3 +141,11 @@ int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name)
return 0;
#endif
}
void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { // Microsoft
return mi_recalloc_aligned_at(p, newcount, size, alignment, offset);
}
void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { // Microsoft
return mi_recalloc_aligned(p, newcount, size, alignment);
}

84
src/alloc.c
View file

@ -33,7 +33,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
page->used++;
mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page);
#if (MI_DEBUG)
memset(block, MI_DEBUG_UNINIT, size);
if (!page->is_zero) { memset(block, MI_DEBUG_UNINIT, size); }
#elif (MI_SECURE)
block->next = 0;
#endif
@ -47,26 +47,26 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz
}
// allocate a small block
extern inline void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
extern inline mi_decl_allocator void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept {
mi_assert(size <= MI_SMALL_SIZE_MAX);
mi_page_t* page = _mi_heap_get_free_small_page(heap,size);
return _mi_page_malloc(heap, page, size);
}
extern inline void* mi_malloc_small(size_t size) mi_attr_noexcept {
extern inline mi_decl_allocator void* mi_malloc_small(size_t size) mi_attr_noexcept {
return mi_heap_malloc_small(mi_get_default_heap(), size);
}
// zero initialized small block
void* mi_zalloc_small(size_t size) mi_attr_noexcept {
mi_decl_allocator void* mi_zalloc_small(size_t size) mi_attr_noexcept {
void* p = mi_malloc_small(size);
if (p != NULL) { memset(p, 0, size); }
return p;
}
// The main allocation function
extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
extern inline mi_decl_allocator void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
mi_assert(heap!=NULL);
mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local
void* p;
@ -85,21 +85,41 @@ extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcep
return p;
}
extern inline void* mi_malloc(size_t size) mi_attr_noexcept {
extern inline mi_decl_allocator void* mi_malloc(size_t size) mi_attr_noexcept {
return mi_heap_malloc(mi_get_default_heap(), size);
}
void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) {
// note: we need to initialize the whole block to zero, not just size
// or the recalloc/rezalloc functions cannot safely expand in place (see issue #63)
UNUSED(size);
mi_assert_internal(p != NULL);
mi_assert_internal(size > 0 && page->block_size >= size);
mi_assert_internal(_mi_ptr_page(p)==page);
if (page->is_zero) {
// already zero initialized memory?
((mi_block_t*)p)->next = 0; // clear the free list pointer
mi_assert_expensive(mi_mem_is_zero(p,page->block_size));
}
else {
// otherwise memset
memset(p, 0, page->block_size);
}
}
void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) {
void* p = mi_heap_malloc(heap,size);
if (zero && p != NULL) memset(p,0,size);
if (zero && p != NULL) {
_mi_block_zero_init(_mi_ptr_page(p),p,size); // todo: can we avoid getting the page again?
}
return p;
}
extern inline void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
extern inline mi_decl_allocator void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept {
return _mi_heap_malloc_zero(heap, size, true);
}
void* mi_zalloc(size_t size) mi_attr_noexcept {
mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept {
return mi_heap_zalloc(mi_get_default_heap(),size);
}
@ -127,6 +147,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc
mi_block_set_next(page, block, page->free);
page->free = block;
page->used--;
page->is_zero = false;
_mi_segment_page_free(page,true,&heap->tld->segments);
}
return;
@ -254,7 +275,7 @@ void mi_free(void* p) mi_attr_noexcept
// huge page stat is accounted for in `_mi_page_retire`
#endif
if (mi_likely(tid == segment->thread_id && page->flags.value == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks
if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks
// local, and not full or aligned
mi_block_t* block = (mi_block_t*)p;
mi_block_set_next(page, block, page->local_free);
@ -339,29 +360,29 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept {
mi_free(p);
}
extern inline void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
extern inline mi_decl_allocator void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(count,size,&total)) return NULL;
return mi_heap_zalloc(heap,total);
}
void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
mi_decl_allocator void* mi_calloc(size_t count, size_t size) mi_attr_noexcept {
return mi_heap_calloc(mi_get_default_heap(),count,size);
}
// Uninitialized `calloc`
extern void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
extern mi_decl_allocator void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(count,size,&total)) return NULL;
return mi_heap_malloc(heap, total);
}
void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
mi_decl_allocator void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept {
return mi_heap_mallocn(mi_get_default_heap(),count,size);
}
// Expand in place or fail
void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
if (p == NULL) return NULL;
size_t size = mi_usable_size(p);
if (newsize > size) return NULL;
@ -387,11 +408,11 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero)
return newp;
}
void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
return _mi_heap_realloc_zero(heap, p, newsize, false);
}
void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(count, size, &total)) return NULL;
return mi_heap_realloc(heap, p, total);
@ -399,25 +420,46 @@ void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_a
// Reallocate but free `p` on errors
void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
void* newp = mi_heap_realloc(heap, p, newsize);
if (newp==NULL && p!=NULL) mi_free(p);
return newp;
}
void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
mi_decl_allocator void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept {
return _mi_heap_realloc_zero(heap, p, newsize, true);
}
mi_decl_allocator void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept {
size_t total;
if (mi_mul_overflow(count, size, &total)) return NULL;
return mi_heap_rezalloc(heap, p, total);
}
mi_decl_allocator void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept {
return mi_heap_realloc(mi_get_default_heap(),p,newsize);
}
void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
mi_decl_allocator void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept {
return mi_heap_reallocn(mi_get_default_heap(),p,count,size);
}
// Reallocate but free `p` on errors
void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
mi_decl_allocator void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept {
return mi_heap_reallocf(mi_get_default_heap(),p,newsize);
}
mi_decl_allocator void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept {
return mi_heap_rezalloc(mi_get_default_heap(), p, newsize);
}
mi_decl_allocator void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept {
return mi_heap_recalloc(mi_get_default_heap(), p, count, size);
}
// ------------------------------------------------------
// strdup, strndup, and realpath
// ------------------------------------------------------

5
src/heap.c
View file

@ -108,10 +108,9 @@ static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq
static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
{
_mi_deferred_free(heap,collect > NORMAL);
if (!mi_heap_is_initialized(heap)) return;
_mi_deferred_free(heap, collect > NORMAL);
// collect (some) abandoned pages
if (collect >= NORMAL && !heap->no_reclaim) {
if (collect == NORMAL) {

51
src/init.c
View file

@ -12,8 +12,8 @@ terms of the MIT license. A copy of the license can be found in the file
// Empty page used to initialize the small free pages array
const mi_page_t _mi_page_empty = {
0, false, false, false, 0, 0,
{ 0 },
0, false, false, false, false, 0, 0,
{ 0 }, false,
NULL, // free
#if MI_SECURE
0,
@ -106,6 +106,7 @@ const mi_heap_t _mi_heap_empty = {
static const mi_tld_t tld_empty = {
0,
false,
NULL,
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_empty_stats }, // segments
{ 0, tld_empty_stats }, // os
@ -119,7 +120,7 @@ mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty;
#define tld_main_stats ((mi_stats_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,stats)))
static mi_tld_t tld_main = {
0,
0, false,
&_mi_heap_main,
{ MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats }, // segments
{ 0, tld_main_stats }, // os
@ -375,9 +376,7 @@ void mi_thread_init(void) mi_attr_noexcept
pthread_setspecific(mi_pthread_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_pthread_done` is called
#endif
#if (MI_DEBUG>0) && !defined(NDEBUG) // not in release mode as that leads to crashes on Windows dynamic override
_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
#endif
//_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id());
}
void mi_thread_done(void) mi_attr_noexcept {
@ -390,11 +389,9 @@ void mi_thread_done(void) mi_attr_noexcept {
// abandon the thread local heap
if (_mi_heap_done()) return; // returns true if already ran
#if (MI_DEBUG>0)
if (!_mi_is_main_thread()) {
_mi_verbose_message("thread done: 0x%zx\n", _mi_thread_id());
}
#endif
//if (!_mi_is_main_thread()) {
// _mi_verbose_message("thread done: 0x%zx\n", _mi_thread_id());
//}
}
@ -411,14 +408,26 @@ bool _mi_preloading() {
return os_preloading;
}
bool mi_is_redirected() mi_attr_noexcept {
return mi_redirected;
}
// Communicate with the redirection module on Windows
#if 0
#if defined(_WIN32) && defined(MI_SHARED_LIB)
#ifdef __cplusplus
extern "C" {
#endif
mi_decl_export void _mi_redirect_init() {
// called on redirection
mi_redirected = true;
mi_decl_export void _mi_redirect_entry(DWORD reason) {
// called on redirection; careful as this may be called before DllMain
if (reason == DLL_PROCESS_ATTACH) {
mi_redirected = true;
}
else if (reason == DLL_PROCESS_DETACH) {
mi_redirected = false;
}
else if (reason == DLL_THREAD_DETACH) {
mi_thread_done();
}
}
__declspec(dllimport) bool mi_allocator_init(const char** message);
__declspec(dllimport) void mi_allocator_done();
@ -447,12 +456,14 @@ static void mi_process_load(void) {
// show message from the redirector (if present)
const char* msg = NULL;
mi_allocator_init(&msg);
if (msg != NULL) _mi_verbose_message(msg);
if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) {
_mi_fputs(NULL,NULL,msg);
}
if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
size_t pages = mi_option_get(mi_option_reserve_huge_os_pages);
double max_secs = (double)pages / 2.0; // 0.5s per page (1GiB)
mi_reserve_huge_os_pages(pages, max_secs);
mi_reserve_huge_os_pages(pages, max_secs, NULL);
}
}
@ -506,9 +517,7 @@ static void mi_process_done(void) {
#if defined(_WIN32) && defined(MI_SHARED_LIB)
// Windows DLL: easy to hook into process_init and thread_done
#include <windows.h>
// Windows DLL: easy to hook into process_init and thread_done
__declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) {
UNUSED(reserved);
UNUSED(inst);
@ -516,7 +525,7 @@ static void mi_process_done(void) {
mi_process_load();
}
else if (reason==DLL_THREAD_DETACH) {
mi_thread_done();
if (!mi_is_redirected()) mi_thread_done();
}
return TRUE;
}

217
src/memory.c
View file

@ -17,7 +17,7 @@ We need this memory layer between the raw OS calls because of:
to reuse memory effectively.
2. It turns out that for large objects, between 1MiB and 32MiB (?), the cost of
an OS allocation/free is still (much) too expensive relative to the accesses in that
object :-( (`mallloc-large` tests this). This means we need a cheaper way to
object :-( (`malloc-large` tests this). This means we need a cheaper way to
reuse memory.
3. This layer can help with a NUMA aware allocation in the future.
@ -39,14 +39,16 @@ Possible issues:
// Internal raw OS interface
size_t _mi_os_large_page_size();
bool _mi_os_protect(void* addr, size_t size);
bool _mi_os_unprotect(void* addr, size_t size);
bool _mi_os_commit(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats);
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld);
bool _mi_os_protect(void* addr, size_t size);
bool _mi_os_unprotect(void* addr, size_t size);
bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats);
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld);
void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats);
void* _mi_os_try_alloc_from_huge_reserved(size_t size, size_t try_alignment);
bool _mi_os_is_huge_reserved(void* p);
// Constants
#if (MI_INTPTR_SIZE==8)
@ -66,11 +68,25 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld
#define MI_REGION_MAP_FULL UINTPTR_MAX
typedef uintptr_t mi_region_info_t;
static inline mi_region_info_t mi_region_info_create(void* start, bool is_large, bool is_committed) {
return ((uintptr_t)start | ((uintptr_t)(is_large?1:0) << 1) | (is_committed?1:0));
}
static inline void* mi_region_info_read(mi_region_info_t info, bool* is_large, bool* is_committed) {
if (is_large) *is_large = ((info&0x02) != 0);
if (is_committed) *is_committed = ((info&0x01) != 0);
return (void*)(info & ~0x03);
}
// A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with
// a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block.
typedef struct mem_region_s {
volatile _Atomic(uintptr_t) map; // in-use bit per MI_SEGMENT_SIZE block
volatile _Atomic(void*) start; // start of virtual memory area
volatile _Atomic(uintptr_t) map; // in-use bit per MI_SEGMENT_SIZE block
volatile _Atomic(mi_region_info_t) info; // start of virtual memory area, and flags
volatile _Atomic(uintptr_t) dirty_mask; // bit per block if the contents are not zero'd
} mem_region_t;
@ -108,7 +124,7 @@ bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
if (p==NULL) return false;
size_t count = mi_atomic_read_relaxed(&regions_count);
for (size_t i = 0; i < count; i++) {
uint8_t* start = (uint8_t*)mi_atomic_read_ptr_relaxed(&regions[i].start);
uint8_t* start = (uint8_t*)mi_region_info_read( mi_atomic_read_relaxed(&regions[i].info), NULL, NULL);
if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true;
}
return false;
@ -123,7 +139,8 @@ Commit from a region
// Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
// if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call.
// (not being able to claim is not considered an error so check for `p != NULL` afterwards).
static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld)
static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks,
size_t size, bool* commit, bool* allow_large, bool* is_zero, void** p, size_t* id, mi_os_tld_t* tld)
{
size_t mask = mi_region_block_mask(blocks,bitidx);
mi_assert_internal(mask != 0);
@ -131,10 +148,21 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
mi_assert_internal(&regions[idx] == region);
// ensure the region is reserved
void* start = mi_atomic_read_ptr(&region->start);
if (start == NULL)
mi_region_info_t info = mi_atomic_read(&region->info);
if (info == 0)
{
start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, mi_option_is_enabled(mi_option_eager_region_commit), tld);
bool region_commit = mi_option_is_enabled(mi_option_eager_region_commit);
bool region_large = *allow_large;
void* start = NULL;
if (region_large) {
start = _mi_os_try_alloc_from_huge_reserved(MI_REGION_SIZE, MI_SEGMENT_ALIGN);
if (start != NULL) { region_commit = true; }
}
if (start == NULL) {
start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, region_commit, &region_large, tld);
}
mi_assert_internal(!(region_large && !*allow_large));
if (start == NULL) {
// failure to allocate from the OS! unclaim the blocks and fail
size_t map;
@ -145,41 +173,61 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
}
// set the newly allocated region
if (mi_atomic_cas_ptr_strong(&region->start, start, NULL)) {
info = mi_region_info_create(start,region_large,region_commit);
if (mi_atomic_cas_strong(&region->info, info, 0)) {
// update the region count
mi_atomic_increment(&regions_count);
}
else {
// failed, another thread allocated just before us!
// we assign it to a later slot instead (up to 4 tries).
// note: we don't need to increment the region count, this will happen on another allocation
for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) {
void* s = mi_atomic_read_ptr(&regions[idx+i].start);
if (s == NULL) { // quick test
if (mi_atomic_cas_ptr_strong(&regions[idx+i].start, start, NULL)) {
start = NULL;
break;
}
if (mi_atomic_cas_strong(&regions[idx+i].info, info, 0)) {
mi_atomic_increment(&regions_count);
start = NULL;
break;
}
}
if (start != NULL) {
// free it if we didn't succeed to save it to some other region
_mi_os_free(start, MI_REGION_SIZE, tld->stats);
_mi_os_free_ex(start, MI_REGION_SIZE, region_commit, tld->stats);
}
// and continue with the memory at our index
start = mi_atomic_read_ptr(&region->start);
info = mi_atomic_read(&region->info);
}
}
mi_assert_internal(start == mi_atomic_read_ptr(&region->start));
mi_assert_internal(start != NULL);
mi_assert_internal(info == mi_atomic_read(&region->info));
mi_assert_internal(info != 0);
// Commit the blocks to memory
bool region_is_committed = false;
bool region_is_large = false;
void* start = mi_region_info_read(info,&region_is_large,&region_is_committed);
mi_assert_internal(!(region_is_large && !*allow_large));
mi_assert_internal(start!=NULL);
// set dirty bits
uintptr_t m;
do {
m = mi_atomic_read(&region->dirty_mask);
} while (!mi_atomic_cas_weak(&region->dirty_mask, m | mask, m));
*is_zero = ((m & mask) == 0); // no dirty bit set in our claimed range?
void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE);
if (commit && !mi_option_is_enabled(mi_option_eager_region_commit)) {
_mi_os_commit(blocks_start, mi_good_commit_size(size), tld->stats); // only commit needed size (unless using large OS pages)
if (*commit && !region_is_committed) {
// ensure commit
bool commit_zero = false;
_mi_os_commit(blocks_start, mi_good_commit_size(size), &commit_zero, tld->stats); // only commit needed size (unless using large OS pages)
if (commit_zero) *is_zero = true;
}
else if (!*commit && region_is_committed) {
// but even when no commit is requested, we might have committed anyway (in a huge OS page for example)
*commit = true;
}
// and return the allocation
// and return the allocation
mi_assert_internal(blocks_start != NULL);
*allow_large = region_is_large;
*p = blocks_start;
*id = (idx*MI_REGION_MAP_BITS) + bitidx;
return true;
@ -223,7 +271,8 @@ static inline size_t mi_bsr(uintptr_t x) {
// Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
// if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call.
// (not being able to claim is not considered an error so check for `p != NULL` afterwards).
static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld)
static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size,
bool* commit, bool* allow_large, bool* is_zero, void** p, size_t* id, mi_os_tld_t* tld)
{
mi_assert_internal(p != NULL && id != NULL);
mi_assert_internal(blocks < MI_REGION_MAP_BITS);
@ -231,6 +280,7 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
const uintptr_t mask = mi_region_block_mask(blocks, 0);
const size_t bitidx_max = MI_REGION_MAP_BITS - blocks;
uintptr_t map = mi_atomic_read(&region->map);
if (map==MI_REGION_MAP_FULL) return true;
#ifdef MI_HAVE_BITSCAN
size_t bitidx = mi_bsf(~map); // quickly find the first zero bit if possible
@ -245,7 +295,7 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
mi_assert_internal((m >> bitidx) == mask); // no overflow?
uintptr_t newmap = map | m;
mi_assert_internal((newmap^map) >> bitidx == mask);
if (!mi_atomic_cas_weak(&region->map, newmap, map)) {
if (!mi_atomic_cas_weak(&region->map, newmap, map)) { // TODO: use strong cas here?
// no success, another thread claimed concurrently.. keep going
map = mi_atomic_read(&region->map);
continue;
@ -253,7 +303,8 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
else {
// success, we claimed the bits
// now commit the block memory -- this can still fail
return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, p, id, tld);
return mi_region_commit_blocks(region, idx, bitidx, blocks,
size, commit, allow_large, is_zero, p, id, tld);
}
}
else {
@ -276,18 +327,32 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
// Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
// if the blocks were successfully claimed so ensure they are initialized to NULL/0 before the call.
// (not being able to claim is not considered an error so check for `p != NULL` afterwards).
static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld)
static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size,
bool* commit, bool* allow_large, bool* is_zero,
void** p, size_t* id, mi_os_tld_t* tld)
{
// check if there are available blocks in the region..
mi_assert_internal(idx < MI_REGION_MAX);
mem_region_t* region = &regions[idx];
uintptr_t m = mi_atomic_read_relaxed(&region->map);
if (m != MI_REGION_MAP_FULL) { // some bits are zero
return mi_region_alloc_blocks(region, idx, blocks, size, commit, p, id, tld);
}
else {
return true; // no error, but no success either
if (m != MI_REGION_MAP_FULL) { // some bits are zero
bool ok = (*commit || *allow_large); // committing or allow-large is always ok
if (!ok) {
// otherwise skip incompatible regions if possible.
// this is not guaranteed due to multiple threads allocating at the same time but
// that's ok. In secure mode, large is never allowed for any thread, so that works out;
// otherwise we might just not be able to reset/decommit individual pages sometimes.
mi_region_info_t info = mi_atomic_read_relaxed(&region->info);
bool is_large;
bool is_committed;
void* start = mi_region_info_read(info,&is_large,&is_committed);
ok = (start == NULL || (*commit || !is_committed) || (*allow_large || !is_large)); // Todo: test with one bitmap operation?
}
if (ok) {
return mi_region_alloc_blocks(region, idx, blocks, size, commit, allow_large, is_zero, p, id, tld);
}
}
return true; // no error, but no success either
}
/* ----------------------------------------------------------------------------
@ -296,15 +361,20 @@ static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, b
// Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`.
// (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`)
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld)
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero,
size_t* id, mi_os_tld_t* tld)
{
mi_assert_internal(id != NULL && tld != NULL);
mi_assert_internal(size > 0);
*id = SIZE_MAX;
*is_zero = false;
bool default_large = false;
if (large==NULL) large = &default_large; // ensure `large != NULL`
// use direct OS allocation for huge blocks or alignment (with `id = SIZE_MAX`)
if (size > MI_REGION_MAX_ALLOC_SIZE || alignment > MI_SEGMENT_ALIGN) {
return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, true, tld); // round up size
*is_zero = true;
return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, *commit, large, tld); // round up size
}
// always round size to OS page size multiple (so commit/decommit go over the entire range)
@ -318,27 +388,29 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t*
// find a range of free blocks
void* p = NULL;
size_t count = mi_atomic_read(&regions_count);
size_t idx = 0; // tld->region_idx; // start index is per-thread to reduce contention
size_t idx = tld->region_idx; // start at 0 to reuse low addresses? Or, use tld->region_idx to reduce contention?
for (size_t visited = 0; visited < count; visited++, idx++) {
if (idx >= count) idx = 0; // wrap around
if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error
if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, large, is_zero, &p, id, tld)) return NULL; // error
if (p != NULL) break;
}
if (p == NULL) {
// no free range in existing regions -- try to extend beyond the count.. but at most 4 regions
for (idx = count; idx < count + 4 && idx < MI_REGION_MAX; idx++) {
if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error
// no free range in existing regions -- try to extend beyond the count.. but at most 8 regions
for (idx = count; idx < mi_atomic_read_relaxed(&regions_count) + 8 && idx < MI_REGION_MAX; idx++) {
if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, large, is_zero, &p, id, tld)) return NULL; // error
if (p != NULL) break;
}
}
if (p == NULL) {
// we could not find a place to allocate, fall back to the os directly
p = _mi_os_alloc_aligned(size, alignment, commit, tld);
_mi_warning_message("unable to allocate from region: size %zu\n", size);
*is_zero = true;
p = _mi_os_alloc_aligned(size, alignment, commit, large, tld);
}
else {
tld->region_idx = idx; // next start of search
tld->region_idx = idx; // next start of search? currently not used as we use first-fit
}
mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0);
@ -346,10 +418,6 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t*
}
// Allocate `size` memory. Return non NULL on success, with a given memory `id`.
void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld) {
return _mi_mem_alloc_aligned(size,0,commit,id,tld);
}
/* ----------------------------------------------------------------------------
Free
@ -377,7 +445,10 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) {
mi_assert_internal(idx < MI_REGION_MAX); if (idx >= MI_REGION_MAX) return; // or `abort`?
mem_region_t* region = &regions[idx];
mi_assert_internal((mi_atomic_read_relaxed(&region->map) & mask) == mask ); // claimed?
void* start = mi_atomic_read_ptr(&region->start);
mi_region_info_t info = mi_atomic_read(&region->info);
bool is_large;
bool is_eager_committed;
void* start = mi_region_info_read(info,&is_large,&is_eager_committed);
mi_assert_internal(start != NULL);
void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE);
mi_assert_internal(blocks_start == p); // not a pointer in our area?
@ -388,18 +459,20 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) {
// TODO: implement delayed decommit/reset as these calls are too expensive
// if the memory is reused soon.
// reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large
if (!mi_option_is_enabled(mi_option_large_os_pages)) {
if (mi_option_is_enabled(mi_option_eager_region_commit)) {
//_mi_os_reset(p, size, stats);
}
else {
//_mi_os_decommit(p, size, stats);
if (!is_large) {
if (mi_option_is_enabled(mi_option_segment_reset)) {
_mi_os_reset(p, size, stats); //
// _mi_os_decommit(p,size,stats); // if !is_eager_committed (and clear dirty bits)
}
// else { _mi_os_reset(p,size,stats); }
}
if (!is_eager_committed) {
// adjust commit statistics as we commit again when re-using the same slot
_mi_stat_decrease(&stats->committed, mi_good_commit_size(size));
}
// TODO: should we free empty regions? currently only done _mi_mem_collect.
// this frees up virtual address space which
// might be useful on 32-bit systems?
// this frees up virtual address space which might be useful on 32-bit systems?
// and unclaim
uintptr_t map;
@ -419,17 +492,21 @@ void _mi_mem_collect(mi_stats_t* stats) {
// free every region that has no segments in use.
for (size_t i = 0; i < regions_count; i++) {
mem_region_t* region = &regions[i];
if (mi_atomic_read_relaxed(&region->map) == 0 && region->start != NULL) {
if (mi_atomic_read_relaxed(&region->map) == 0) {
// if no segments used, try to claim the whole region
uintptr_t m;
do {
m = mi_atomic_read_relaxed(&region->map);
} while(m == 0 && !mi_atomic_cas_weak(&region->map, ~((uintptr_t)0), 0 ));
if (m == 0) {
// on success, free the whole region
if (region->start != NULL) _mi_os_free((void*)region->start, MI_REGION_SIZE, stats);
// on success, free the whole region (unless it was huge reserved)
bool is_eager_committed;
void* start = mi_region_info_read(mi_atomic_read(&region->info), NULL, &is_eager_committed);
if (start != NULL && !_mi_os_is_huge_reserved(start)) {
_mi_os_free_ex(start, MI_REGION_SIZE, is_eager_committed, stats);
}
// and release
mi_atomic_write_ptr(&region->start,NULL);
mi_atomic_write(&region->info,0);
mi_atomic_write(&region->map,0);
}
}
@ -440,8 +517,8 @@ void _mi_mem_collect(mi_stats_t* stats) {
Other
-----------------------------------------------------------------------------*/
bool _mi_mem_commit(void* p, size_t size, mi_stats_t* stats) {
return _mi_os_commit(p, size, stats);
bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats) {
return _mi_os_commit(p, size, is_zero, stats);
}
bool _mi_mem_decommit(void* p, size_t size, mi_stats_t* stats) {
@ -452,8 +529,8 @@ bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats) {
return _mi_os_reset(p, size, stats);
}
bool _mi_mem_unreset(void* p, size_t size, mi_stats_t* stats) {
return _mi_os_unreset(p, size, stats);
bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats) {
return _mi_os_unreset(p, size, is_zero, stats);
}
bool _mi_mem_protect(void* p, size_t size) {

154
src/options.c
View file

@ -51,25 +51,22 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(show_stats) },
{ 0, UNINIT, MI_OPTION(verbose) },
#if MI_SECURE
{ MI_SECURE, INITIALIZED, MI_OPTION(secure) }, // in a secure build the environment setting is ignored
#else
{ 0, UNINIT, MI_OPTION(secure) },
#endif
// the following options are experimental and not all combinations make sense.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // note: needs to be on when eager_region_commit is enabled
#ifdef _WIN32 // and BSD?
{ 1, UNINIT, MI_OPTION(lazy_commit) },
{ 0, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...)
#else
{ 0, UNINIT, MI_OPTION(lazy_commit) },
{ 1, UNINIT, MI_OPTION(eager_region_commit) },
#endif
{ 0, UNINIT, MI_OPTION(decommit) },
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
{ 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread
{ 0, UNINIT, MI_OPTION(page_reset) },
{ 0, UNINIT, MI_OPTION(cache_reset) },
{ 0, UNINIT, MI_OPTION(reset_decommits) } // note: cannot enable this if secure is on
{ 0, UNINIT, MI_OPTION(reset_decommits) }, // note: cannot enable this if secure is on
{ 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
{ 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free
{ 100, UNINIT, MI_OPTION(os_tag) } // only apple specific for now but might serve more or less related purpose
};
static void mi_option_init(mi_option_desc_t* desc);
@ -77,7 +74,12 @@ static void mi_option_init(mi_option_desc_t* desc);
void _mi_options_init(void) {
// called on process load
for(int i = 0; i < _mi_option_last; i++ ) {
mi_option_get((mi_option_t)i); // initialize
mi_option_t option = (mi_option_t)i;
mi_option_get(option); // initialize
if (option != mi_option_verbose) {
mi_option_desc_t* desc = &options[option];
_mi_verbose_message("option '%s': %ld\n", desc->name, desc->value);
}
}
}
@ -86,10 +88,7 @@ long mi_option_get(mi_option_t option) {
mi_option_desc_t* desc = &options[option];
mi_assert(desc->option == option); // index should match the option
if (mi_unlikely(desc->init == UNINIT)) {
mi_option_init(desc);
if (option != mi_option_verbose) {
_mi_verbose_message("option '%s': %ld\n", desc->name, desc->value);
}
mi_option_init(desc);
}
return desc->value;
}
@ -131,8 +130,80 @@ void mi_option_disable(mi_option_t option) {
}
static void mi_out_stderr(const char* msg) {
#ifdef _WIN32
// on windows with redirection, the C runtime cannot handle locale dependent output
// after the main thread closes so we use direct console output.
_cputs(msg);
#else
fputs(msg, stderr);
#endif
}
// Since an output function can be registered earliest in the `main`
// function we also buffer output that happens earlier. When
// an output function is registered it is called immediately with
// the output up to that point.
#ifndef MI_MAX_DELAY_OUTPUT
#define MI_MAX_DELAY_OUTPUT (32*1024)
#endif
static char out_buf[MI_MAX_DELAY_OUTPUT+1];
static _Atomic(uintptr_t) out_len;
static void mi_out_buf(const char* msg) {
if (msg==NULL) return;
if (mi_atomic_read_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return;
size_t n = strlen(msg);
if (n==0) return;
// claim space
uintptr_t start = mi_atomic_addu(&out_len, n);
if (start >= MI_MAX_DELAY_OUTPUT) return;
// check bound
if (start+n >= MI_MAX_DELAY_OUTPUT) {
n = MI_MAX_DELAY_OUTPUT-start-1;
}
memcpy(&out_buf[start], msg, n);
}
static void mi_out_buf_flush(mi_output_fun* out) {
if (out==NULL) return;
// claim all (no more output will be added after this point)
size_t count = mi_atomic_addu(&out_len, MI_MAX_DELAY_OUTPUT);
// and output the current contents
if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT;
out_buf[count] = 0;
out(out_buf);
}
// The initial default output, outputs to stderr and the delayed output buffer.
static void mi_out_buf_stderr(const char* msg) {
mi_out_stderr(msg);
mi_out_buf(msg);
}
// --------------------------------------------------------
// Messages
// Default output handler
// --------------------------------------------------------
// Should be atomic but gives errors on many platforms as generally we cannot cast a function pointer to a uintptr_t.
// For now, don't register output from multiple threads.
#pragma warning(suppress:4180)
static mi_output_fun* volatile mi_out_default; // = NULL
static mi_output_fun* mi_out_get_default(void) {
mi_output_fun* out = mi_out_default;
return (out == NULL ? &mi_out_buf_stderr : out);
}
void mi_register_output(mi_output_fun* out) mi_attr_noexcept {
mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer
if (out!=NULL) mi_out_buf_flush(out); // output the delayed output now
}
// --------------------------------------------------------
// Messages, all end up calling `_mi_fputs`.
// --------------------------------------------------------
#define MAX_ERROR_COUNT (10)
static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings
@ -141,33 +212,30 @@ static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT
// inside the C runtime causes another message.
static mi_decl_thread bool recurse = false;
// Define our own limited `fprintf` that avoids memory allocation.
// We do this using `snprintf` with a limited buffer.
static void mi_vfprintf( FILE* out, const char* prefix, const char* fmt, va_list args ) {
char buf[256];
if (fmt==NULL) return;
void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message) {
if (_mi_preloading() || recurse) return;
if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) out = mi_out_get_default();
recurse = true;
if (out==NULL) out = stdout;
vsnprintf(buf,sizeof(buf)-1,fmt,args);
#ifdef _WIN32
// on windows with redirection, the C runtime cannot handle locale dependent output
// after the main thread closes so use direct console output.
if (out==stderr) {
if (prefix != NULL) _cputs(prefix);
_cputs(buf);
}
else
#endif
{
if (prefix != NULL) fputs(prefix,out);
fputs(buf,out);
}
if (prefix != NULL) out(prefix);
out(message);
recurse = false;
return;
}
void _mi_fprintf( FILE* out, const char* fmt, ... ) {
// Define our own limited `fprintf` that avoids memory allocation.
// We do this using `snprintf` with a limited buffer.
static void mi_vfprintf( mi_output_fun* out, const char* prefix, const char* fmt, va_list args ) {
char buf[512];
if (fmt==NULL) return;
if (_mi_preloading() || recurse) return;
recurse = true;
vsnprintf(buf,sizeof(buf)-1,fmt,args);
recurse = false;
_mi_fputs(out,prefix,buf);
}
void _mi_fprintf( mi_output_fun* out, const char* fmt, ... ) {
va_list args;
va_start(args,fmt);
mi_vfprintf(out,NULL,fmt,args);
@ -178,7 +246,7 @@ void _mi_trace_message(const char* fmt, ...) {
if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher
va_list args;
va_start(args, fmt);
mi_vfprintf(stderr, "mimalloc: ", fmt, args);
mi_vfprintf(NULL, "mimalloc: ", fmt, args);
va_end(args);
}
@ -186,7 +254,7 @@ void _mi_verbose_message(const char* fmt, ...) {
if (!mi_option_is_enabled(mi_option_verbose)) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(stderr, "mimalloc: ", fmt, args);
mi_vfprintf(NULL, "mimalloc: ", fmt, args);
va_end(args);
}
@ -195,7 +263,7 @@ void _mi_error_message(const char* fmt, ...) {
if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(stderr, "mimalloc: error: ", fmt, args);
mi_vfprintf(NULL, "mimalloc: error: ", fmt, args);
va_end(args);
mi_assert(false);
}
@ -205,14 +273,14 @@ void _mi_warning_message(const char* fmt, ...) {
if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return;
va_list args;
va_start(args,fmt);
mi_vfprintf(stderr, "mimalloc: warning: ", fmt, args);
mi_vfprintf(NULL, "mimalloc: warning: ", fmt, args);
va_end(args);
}
#if MI_DEBUG
void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) {
_mi_fprintf(stderr,"mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
_mi_fprintf(NULL,"mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
abort();
}
#endif
@ -278,7 +346,7 @@ static void mi_option_init(mi_option_desc_t* desc) {
size_t len = strlen(s);
if (len >= sizeof(buf)) len = sizeof(buf) - 1;
for (size_t i = 0; i < len; i++) {
buf[i] = toupper(s[i]);
buf[i] = (char)toupper(s[i]);
}
buf[len] = 0;
if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) {

374
src/os.c
View file

@ -35,10 +35,9 @@ terms of the MIT license. A copy of the license can be found in the file
On windows initializes support for aligned allocation and
large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
----------------------------------------------------------- */
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
static bool mi_os_is_huge_reserved(void* p);
static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit);
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
bool _mi_os_is_huge_reserved(void* p);
void* _mi_os_try_alloc_from_huge_reserved(size_t size, size_t try_alignment);
static void* mi_align_up_ptr(void* p, size_t alignment) {
return (void*)_mi_align_up((uintptr_t)p, alignment);
@ -73,11 +72,16 @@ static bool use_large_os_page(size_t size, size_t alignment) {
return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0);
}
// round to a good allocation size
static size_t mi_os_good_alloc_size(size_t size, size_t alignment) {
UNUSED(alignment);
if (size >= (SIZE_MAX - os_alloc_granularity)) return size; // possible overflow?
return _mi_align_up(size, os_alloc_granularity);
// round to a good OS allocation size (bounded by max 12.5% waste)
size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size;
if (size < 512*KiB) align_size = _mi_os_page_size();
else if (size < 2*MiB) align_size = 64*KiB;
else if (size < 8*MiB) align_size = 256*KiB;
else if (size < 32*MiB) align_size = 1*MiB;
else align_size = 4*MiB;
if (size >= (SIZE_MAX - align_size)) return size; // possible overflow?
return _mi_align_up(size, align_size);
}
#if defined(_WIN32)
@ -91,6 +95,41 @@ typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*
static PVirtualAlloc2 pVirtualAlloc2 = NULL;
static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
static bool mi_win_enable_large_os_pages()
{
if (large_os_page_size > 0) return true;
// Try to see if large OS pages are supported
// To use large pages on Windows, we first need access permission
// Set "Lock pages in memory" permission in the group policy editor
// <https://devblogs.microsoft.com/oldnewthing/20110128-00/?p=11643>
unsigned long err = 0;
HANDLE token = NULL;
BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
if (ok) {
TOKEN_PRIVILEGES tp;
ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);
if (ok) {
tp.PrivilegeCount = 1;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
if (ok) {
err = GetLastError();
ok = (err == ERROR_SUCCESS);
if (ok) {
large_os_page_size = GetLargePageMinimum();
}
}
}
CloseHandle(token);
}
if (!ok) {
if (err == 0) err = GetLastError();
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
}
return (ok!=0);
}
void _mi_os_init(void) {
// get the page size
SYSTEM_INFO si;
@ -102,45 +141,17 @@ void _mi_os_init(void) {
hDll = LoadLibrary(TEXT("kernelbase.dll"));
if (hDll != NULL) {
// use VirtualAlloc2FromApp if possible as it is available to Windows store apps
pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2");
pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
FreeLibrary(hDll);
}
hDll = LoadLibrary(TEXT("ntdll.dll"));
if (hDll != NULL) {
pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
FreeLibrary(hDll);
}
// Try to see if large OS pages are supported
unsigned long err = 0;
bool ok = mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages);
if (ok) {
// To use large pages on Windows, we first need access permission
// Set "Lock pages in memory" permission in the group policy editor
// <https://devblogs.microsoft.com/oldnewthing/20110128-00/?p=11643>
HANDLE token = NULL;
ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token);
if (ok) {
TOKEN_PRIVILEGES tp;
ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid);
if (ok) {
tp.PrivilegeCount = 1;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0);
if (ok) {
err = GetLastError();
ok = (err == ERROR_SUCCESS);
if (ok) {
large_os_page_size = GetLargePageMinimum();
}
}
}
CloseHandle(token);
}
if (!ok) {
if (err == 0) err = GetLastError();
_mi_warning_message("cannot enable large OS page support, error %lu\n", err);
}
}
if (mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) {
mi_win_enable_large_os_pages();
}
}
#elif defined(__wasi__)
@ -167,9 +178,9 @@ void _mi_os_init() {
Raw allocation on Windows (VirtualAlloc) and Unix's (mmap).
----------------------------------------------------------- */
static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats_t* stats)
{
if (addr == NULL || size == 0 || mi_os_is_huge_reserved(addr)) return true;
if (addr == NULL || size == 0 || _mi_os_is_huge_reserved(addr)) return true;
bool err = false;
#if defined(_WIN32)
err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
@ -178,7 +189,7 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
#else
err = (munmap(addr, size) == -1);
#endif
_mi_stat_decrease(&stats->committed, size); // TODO: what if never committed?
if (was_committed) _mi_stat_decrease(&stats->committed, size);
_mi_stat_decrease(&stats->reserved, size);
if (err) {
#pragma warning(suppress:4996)
@ -190,6 +201,8 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
}
}
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size);
#ifdef _WIN32
#define MEM_COMMIT_RESERVE (MEM_COMMIT|MEM_RESERVE)
@ -197,9 +210,8 @@ static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
if ((flags&MEM_COMMIT_RESERVE)==MEM_COMMIT_RESERVE
&& (size % ((uintptr_t)1 << 30)) == 0 /* 1GiB multiple */
&& (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0
if ((size % ((uintptr_t)1 << 30)) == 0 /* 1GiB multiple */
&& (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0 && (flags & MEM_RESERVE) != 0
&& (addr != NULL || try_alignment == 0 || try_alignment % _mi_os_page_size() == 0)
&& pNtAllocateVirtualMemoryEx != NULL)
{
@ -211,7 +223,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
param.ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
SIZE_T psize = size;
void* base = addr;
NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags | MEM_RESERVE, PAGE_READWRITE, &param, 1);
NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, &param, 1);
if (err == 0) {
return base;
}
@ -222,15 +234,10 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
}
#endif
#if (MI_INTPTR_SIZE >= 8)
// on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations
static volatile _Atomic(intptr_t) aligned_base = ATOMIC_VAR_INIT((intptr_t)4 << 40); // starting at 4TiB
if (addr == NULL && try_alignment > 0 &&
try_alignment <= MI_SEGMENT_SIZE && (size%MI_SEGMENT_SIZE) == 0)
{
intptr_t hint = mi_atomic_add(&aligned_base, size);
if (hint%try_alignment == 0) {
return VirtualAlloc((void*)hint, size, flags, PAGE_READWRITE);
}
// on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations
void* hint;
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) {
return VirtualAlloc(hint, size, flags, PAGE_READWRITE);
}
#endif
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
@ -247,12 +254,12 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
}
static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only) {
static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
mi_assert_internal(!(large_only && !allow_large));
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
void* p = NULL;
if ((flags&MEM_COMMIT_RESERVE) == MEM_COMMIT_RESERVE
&& (large_only || use_large_os_page(size, try_alignment)))
{
if ((large_only || use_large_os_page(size, try_alignment))
&& allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) {
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
if (!large_only && try_ok > 0) {
// if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
@ -261,7 +268,8 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
}
else {
// large OS pages must always reserve and commit.
p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE | flags);
*is_large = true;
p = mi_win_virtual_allocx(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
if (large_only) return p;
// fall back to non-large page allocation on error (`p == NULL`).
if (p == NULL) {
@ -270,6 +278,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
}
}
if (p == NULL) {
*is_large = ((flags&MEM_LARGE_PAGES) != 0);
p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
}
if (p == NULL) {
@ -297,14 +306,13 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr
void* p = NULL;
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
// on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations
static volatile _Atomic(intptr_t) aligned_base = ATOMIC_VAR_INIT((intptr_t)1 << 42); // starting at 4TiB
if (addr==NULL && try_alignment <= MI_SEGMENT_SIZE && (size%MI_SEGMENT_SIZE)==0) {
intptr_t hint = mi_atomic_add(&aligned_base,size);
if (hint%try_alignment == 0) {
p = mmap((void*)hint,size,protect_flags,flags,fd,0);
if (p==MAP_FAILED) p = NULL; // fall back to regular mmap
}
void* hint;
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) {
p = mmap(hint,size,protect_flags,flags,fd,0);
if (p==MAP_FAILED) p = NULL; // fall back to regular mmap
}
#else
UNUSED(try_alignment);
#endif
if (p==NULL) {
p = mmap(addr,size,protect_flags,flags,fd,0);
@ -313,7 +321,7 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr
return p;
}
static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only) {
static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
void* p = NULL;
#if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON
@ -333,9 +341,11 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
#endif
#if defined(VM_MAKE_TAG)
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
fd = VM_MAKE_TAG(100);
int os_tag = (int)mi_option_get(mi_option_os_tag);
if (os_tag < 100 || os_tag > 255) os_tag = 100;
fd = VM_MAKE_TAG(os_tag);
#endif
if (large_only || use_large_os_page(size, try_alignment)) {
if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) {
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
if (!large_only && try_ok > 0) {
@ -370,7 +380,15 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
#endif
if (large_only || lflags != flags) {
// try large OS page allocation
*is_large = true;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) {
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno);
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
}
#endif
if (large_only) return p;
if (p == NULL) {
mi_atomic_write(&large_page_try_ok, 10); // on error, don't try again for the next N allocations
@ -379,6 +397,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
}
}
if (p == NULL) {
*is_large = false;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd);
#if defined(MADV_HUGEPAGE)
// Many Linux systems don't allow MAP_HUGETLB but they support instead
@ -387,8 +406,10 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
// in that case -- in particular for our large regions (in `memory.c`).
// However, some systems only allow TPH if called with explicit `madvise`, so
// when large OS pages are enabled for mimalloc, we call `madvice` anyways.
if (use_large_os_page(size, try_alignment)) {
madvise(p, size, MADV_HUGEPAGE);
if (allow_large && use_large_os_page(size, try_alignment)) {
if (madvise(p, size, MADV_HUGEPAGE) == 0) {
*is_large = true; // possibly
};
}
#endif
}
@ -396,29 +417,69 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
}
#endif
// On 64-bit systems, we can do efficient aligned allocation by using
// the 4TiB to 30TiB area to allocate them.
#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED)))
static volatile _Atomic(intptr_t) aligned_base;
// Return a 4MiB aligned address that is probably available
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL;
if ((size%MI_SEGMENT_SIZE) != 0) return NULL;
intptr_t hint = mi_atomic_add(&aligned_base, size);
if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages)
intptr_t init = ((intptr_t)4 << 40); // start at 4TiB area
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_os_get_aligned_hint ^ hint);
init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFF)); // (randomly 0-64k)*4MiB == 0 to 256GiB
#endif
mi_atomic_cas_strong(mi_atomic_cast(uintptr_t, &aligned_base), init, hint + size);
hint = mi_atomic_add(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all
}
if (hint%try_alignment != 0) return NULL;
return (void*)hint;
}
#else
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
UNUSED(try_alignment); UNUSED(size);
return NULL;
}
#endif
// Primitive allocation from the OS.
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) {
static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
if (size == 0) return NULL;
if (!commit) allow_large = false;
void* p = mi_os_alloc_from_huge_reserved(size, try_alignment, commit);
if (p != NULL) return p;
void* p = NULL;
/*
if (commit && allow_large) {
p = _mi_os_try_alloc_from_huge_reserved(size, try_alignment);
if (p != NULL) {
*is_large = true;
return p;
}
}
*/
#if defined(_WIN32)
int flags = MEM_RESERVE;
if (commit) flags |= MEM_COMMIT;
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false);
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
#elif defined(__wasi__)
*is_large = false;
p = mi_wasm_heap_grow(size, try_alignment);
#else
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false);
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
#endif
_mi_stat_increase(&stats->mmap_calls, 1);
if (p != NULL) {
_mi_stat_increase(&stats->reserved, size);
if (commit) _mi_stat_increase(&stats->committed, size);
if (commit) { _mi_stat_increase(&stats->committed, size); }
}
return p;
}
@ -426,19 +487,20 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_
// Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned.
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) {
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) {
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = _mi_align_up(size, _mi_os_page_size());
// try first with a hint (this will be aligned directly on Win 10+ or BSD)
void* p = mi_os_mem_alloc(size, alignment, commit, stats);
void* p = mi_os_mem_alloc(size, alignment, commit, allow_large, is_large, stats);
if (p == NULL) return NULL;
// if not aligned, free it, overallocate, and unmap around it
if (((uintptr_t)p % alignment != 0)) {
mi_os_mem_free(p, size, stats);
mi_os_mem_free(p, size, commit, stats);
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
size_t over_size = size + alignment;
@ -452,7 +514,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
if (commit) flags |= MEM_COMMIT;
for (int tries = 0; tries < 3; tries++) {
// over-allocate to determine a virtual memory range
p = mi_os_mem_alloc(over_size, alignment, commit, stats);
p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats);
if (p == NULL) return NULL; // error
if (((uintptr_t)p % alignment) == 0) {
// if p happens to be aligned, just decommit the left-over area
@ -461,19 +523,19 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
}
else {
// otherwise free and allocate at an aligned address in there
mi_os_mem_free(p, over_size, stats);
mi_os_mem_free(p, over_size, commit, stats);
void* aligned_p = mi_align_up_ptr(p, alignment);
p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false);
p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large);
if (p == aligned_p) break; // success!
if (p != NULL) { // should not happen?
mi_os_mem_free(p, size, stats);
mi_os_mem_free(p, size, commit, stats);
p = NULL;
}
}
}
#else
// overallocate...
p = mi_os_mem_alloc(over_size, alignment, commit, stats);
p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area.
void* aligned_p = mi_align_up_ptr(p, alignment);
@ -481,8 +543,8 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
if (pre_size > 0) mi_os_mem_free(p, pre_size, stats);
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, stats);
if (pre_size > 0) mi_os_mem_free(p, pre_size, commit, stats);
if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats);
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
#endif
@ -498,22 +560,32 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, 0);
return mi_os_mem_alloc(size, 0, true, stats);
size = _mi_os_good_alloc_size(size);
bool is_large = false;
return mi_os_mem_alloc(size, 0, true, false, &is_large, stats);
}
void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* stats) {
if (size == 0 || p == NULL) return;
size = _mi_os_good_alloc_size(size);
mi_os_mem_free(p, size, was_committed, stats);
}
void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
if (size == 0 || p == NULL) return;
size = mi_os_good_alloc_size(size, 0);
mi_os_mem_free(p, size, stats);
_mi_os_free_ex(p, size, true, stats);
}
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld)
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld)
{
if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, alignment);
size = _mi_os_good_alloc_size(size);
alignment = _mi_align_up(alignment, _mi_os_page_size());
return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats);
bool allow_large = false;
if (large != NULL) {
allow_large = *large;
*large = false;
}
return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large!=NULL?large:&allow_large), tld->stats);
}
@ -550,11 +622,12 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
// Commit/Decommit memory.
// Usuelly commit is aligned liberal, while decommit is aligned conservative.
// (but not for the reset version where we want commit to be conservative as well)
static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, mi_stats_t* stats) {
static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, bool* is_zero, mi_stats_t* stats) {
// page align in the range, commit liberally, decommit conservative
*is_zero = false;
size_t csize;
void* start = mi_os_page_align_areax(conservative, addr, size, &csize);
if (csize == 0 || mi_os_is_huge_reserved(addr)) return true;
if (csize == 0 || _mi_os_is_huge_reserved(addr)) return true;
int err = 0;
if (commit) {
_mi_stat_increase(&stats->committed, csize);
@ -566,6 +639,8 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
#if defined(_WIN32)
if (commit) {
// if the memory was already committed, the call succeeds but it is not zero'd
// *is_zero = true;
void* p = VirtualAlloc(start, csize, MEM_COMMIT, PAGE_READWRITE);
err = (p == start ? 0 : GetLastError());
}
@ -577,6 +652,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
// WebAssembly guests can't control memory protection
#else
err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
if (err != 0) { err = errno; }
#endif
if (err != 0) {
_mi_warning_message("commit/decommit error: start: 0x%p, csize: 0x%x, err: %i\n", start, csize, err);
@ -585,16 +661,17 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
return (err == 0);
}
bool _mi_os_commit(void* addr, size_t size, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, false /* conservative? */, stats);
bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, false /* conservative? */, is_zero, stats);
}
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
return mi_os_commitx(addr, size, false, true /* conservative? */, stats);
bool is_zero;
return mi_os_commitx(addr, size, false, true /* conservative? */, &is_zero, stats);
}
bool _mi_os_commit_unreset(void* addr, size_t size, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, true /* conservative? */, stats);
bool _mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, true /* conservative? */, is_zero, stats);
}
@ -606,13 +683,13 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
// page align conservatively within the range
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0 || mi_os_is_huge_reserved(addr)) return true;
if (csize == 0 || _mi_os_is_huge_reserved(addr)) return true;
if (reset) _mi_stat_increase(&stats->reset, csize);
else _mi_stat_decrease(&stats->reset, csize);
if (!reset) return true; // nothing to do on unreset!
#if (MI_DEBUG>1)
if (!mi_option_is_enabled(mi_option_secure)) {
if (MI_SECURE==0) {
memset(start, 0, csize); // pretend it is eagerly reset
}
#endif
@ -621,6 +698,11 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == start);
#if 1
if (p == start && start != NULL) {
VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set
}
#endif
if (p != start) return false;
#else
#if defined(MADV_FREE)
@ -658,11 +740,12 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
}
}
bool _mi_os_unreset(void* addr, size_t size, mi_stats_t* stats) {
bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
if (mi_option_is_enabled(mi_option_reset_decommits)) {
return _mi_os_commit_unreset(addr, size, stats); // re-commit it (conservatively!)
return _mi_os_commit_unreset(addr, size, is_zero, stats); // re-commit it (conservatively!)
}
else {
*is_zero = false;
return mi_os_resetx(addr, size, false, stats);
}
}
@ -674,8 +757,8 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
size_t csize = 0;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return false;
if (mi_os_is_huge_reserved(addr)) {
_mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
if (_mi_os_is_huge_reserved(addr)) {
_mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
}
int err = 0;
#ifdef _WIN32
@ -686,6 +769,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
err = 0;
#else
err = mprotect(start, csize, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
if (err != 0) { err = errno; }
#endif
if (err != 0) {
_mi_warning_message("mprotect error: start: 0x%p, csize: 0x%x, err: %i\n", start, csize, err);
@ -719,36 +803,37 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
// we cannot shrink on windows, but we can decommit
return _mi_os_decommit(start, size, stats);
#else
return mi_os_mem_free(start, size, stats);
return mi_os_mem_free(start, size, true, stats);
#endif
}
/* ----------------------------------------------------------------------------
Support for huge OS pages (1Gib) that are reserved up-front and never
released. Only regions are allocated in here (see `memory.c`) so the memory
will be reused.
-----------------------------------------------------------------------------*/
#define MI_HUGE_OS_PAGE_SIZE ((size_t)1 << 30) // 1GiB
typedef struct mi_huge_info_s {
volatile _Atomic(void*) start;
volatile _Atomic(size_t) reserved;
volatile _Atomic(size_t) used;
volatile _Atomic(void*) start; // start of huge page area (32TiB)
volatile _Atomic(size_t) reserved; // total reserved size
volatile _Atomic(size_t) used; // currently allocated
} mi_huge_info_t;
static mi_huge_info_t os_huge_reserved = { NULL, 0, ATOMIC_VAR_INIT(0) };
static bool mi_os_is_huge_reserved(void* p) {
bool _mi_os_is_huge_reserved(void* p) {
return (mi_atomic_read_ptr(&os_huge_reserved.start) != NULL &&
p >= mi_atomic_read_ptr(&os_huge_reserved.start) &&
(uint8_t*)p < (uint8_t*)mi_atomic_read_ptr(&os_huge_reserved.start) + mi_atomic_read(&os_huge_reserved.reserved));
}
static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit)
void* _mi_os_try_alloc_from_huge_reserved(size_t size, size_t try_alignment)
{
// only allow large aligned allocations
// only allow large aligned allocations (e.g. regions)
if (size < MI_SEGMENT_SIZE || (size % MI_SEGMENT_SIZE) != 0) return NULL;
if (try_alignment > MI_SEGMENT_SIZE) return NULL;
if (!commit) return NULL;
if (try_alignment > MI_SEGMENT_SIZE) return NULL;
if (mi_atomic_read_ptr(&os_huge_reserved.start)==NULL) return NULL;
if (mi_atomic_read(&os_huge_reserved.used) >= mi_atomic_read(&os_huge_reserved.reserved)) return NULL; // already full
@ -783,34 +868,46 @@ static void mi_os_free_huge_reserved() {
*/
#if !(MI_INTPTR_SIZE >= 8 && (defined(_WIN32) || defined(MI_OS_USE_MMAP)))
int mi_reserve_huge_os_pages(size_t pages, size_t max_secs) {
return -2; // cannot allocate
int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept {
UNUSED(pages); UNUSED(max_secs);
if (pages_reserved != NULL) *pages_reserved = 0;
return ENOMEM;
}
#else
int mi_reserve_huge_os_pages( size_t pages, double max_secs ) mi_attr_noexcept
int mi_reserve_huge_os_pages( size_t pages, double max_secs, size_t* pages_reserved ) mi_attr_noexcept
{
if (max_secs==0) return -1; // timeout
if (pages==0) return 0; // ok
if (!mi_atomic_cas_ptr_strong(&os_huge_reserved.start,(void*)1,NULL)) return -2; // already reserved
if (pages_reserved != NULL) *pages_reserved = 0;
if (max_secs==0) return ETIMEDOUT; // timeout
if (pages==0) return 0; // ok
if (!mi_atomic_cas_ptr_strong(&os_huge_reserved.start,(void*)1,NULL)) return ETIMEDOUT; // already reserved
// Set the start address after the 32TiB area
uint8_t* start = (uint8_t*)((uintptr_t)32 << 40); // 32TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_reserve_huge_os_pages);
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB
#endif
// Allocate one page at the time but try to place them contiguously
// We allocate one page at the time to be able to abort if it takes too long
double start_t = _mi_clock_start();
uint8_t* start = (uint8_t*)((uintptr_t)16 << 40); // 16TiB virtual start address
uint8_t* addr = start; // current top of the allocations
for (size_t page = 0; page < pages; page++, addr += MI_HUGE_OS_PAGE_SIZE ) {
// allocate lorgu pages
// allocate a page
void* p = NULL;
bool is_large = true;
#ifdef _WIN32
p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true);
if (page==0) { mi_win_enable_large_os_pages(); }
p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true, true, &is_large);
#elif defined(MI_OS_USE_MMAP)
p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true);
p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true, true, &is_large);
#else
// always fail
#endif
// Did we succeed at a contiguous address?
if (p != addr) {
// no success, issue a warning and return with an error
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge page %zu at 0x%p\n", page, addr);
_mi_os_free(p, MI_HUGE_OS_PAGE_SIZE, &_mi_stats_main );
@ -823,11 +920,11 @@ int mi_reserve_huge_os_pages( size_t pages, double max_secs ) mi_attr_noexcept
#endif
_mi_warning_message("could not allocate huge page %zu at 0x%p, error: %i\n", page, addr, err);
}
return -2;
return ENOMEM;
}
// success, record it
if (page==0) {
mi_atomic_write_ptr(&os_huge_reserved.start, addr);
mi_atomic_write_ptr(&os_huge_reserved.start, addr); // don't switch the order of these writes
mi_atomic_write(&os_huge_reserved.reserved, MI_HUGE_OS_PAGE_SIZE);
}
else {
@ -835,13 +932,14 @@ int mi_reserve_huge_os_pages( size_t pages, double max_secs ) mi_attr_noexcept
}
_mi_stat_increase(&_mi_stats_main.committed, MI_HUGE_OS_PAGE_SIZE);
_mi_stat_increase(&_mi_stats_main.reserved, MI_HUGE_OS_PAGE_SIZE);
if (pages_reserved != NULL) { *pages_reserved = page + 1; }
// check for timeout
double elapsed = _mi_clock_end(start_t);
if (elapsed > max_secs) return (-1); // timeout
if (elapsed > max_secs) return ETIMEDOUT;
if (page >= 1) {
double estimate = ((elapsed / (double)(page+1)) * (double)pages);
if (estimate > 1.5*max_secs) return (-1); // seems like we are going to timeout
if (estimate > 1.5*max_secs) return ETIMEDOUT; // seems like we are going to timeout
}
}
_mi_verbose_message("reserved %zu huge pages\n", pages);

2
src/page-queue.c
View file

@ -57,7 +57,7 @@ static inline uint8_t mi_bsr32(uint32_t x);
static inline uint8_t mi_bsr32(uint32_t x) {
uint32_t idx;
_BitScanReverse((DWORD*)&idx, x);
return idx;
return (uint8_t)idx;
}
#elif defined(__GNUC__) || defined(__clang__)
static inline uint8_t mi_bsr32(uint32_t x) {

56
src/page.c
View file

@ -80,6 +80,14 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
mi_assert_internal(mi_page_list_is_valid(page,page->free));
mi_assert_internal(mi_page_list_is_valid(page,page->local_free));
#if MI_DEBUG>3 // generally too expensive to check this
if (page->flags.is_zero) {
for(mi_block_t* block = page->free; block != NULL; mi_block_next(page,block)) {
mi_assert_expensive(mi_mem_is_zero(block + 1, page->block_size - sizeof(mi_block_t)));
}
}
#endif
mi_block_t* tfree = mi_tf_block(page->thread_free);
mi_assert_internal(mi_page_list_is_valid(page, tfree));
size_t tfree_count = mi_page_list_count(page, tfree);
@ -179,10 +187,11 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
// and the local free list
if (page->local_free != NULL) {
if (mi_unlikely(page->free == NULL)) {
if (mi_likely(page->free == NULL)) {
// usual case
page->free = page->local_free;
page->local_free = NULL;
page->is_zero = false;
}
else if (force) {
// append -- only on shutdown (force) as this is a linear operation
@ -194,6 +203,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) {
mi_block_set_next(page, tail, page->free);
page->free = page->local_free;
page->local_free = NULL;
page->is_zero = false;
}
}
@ -399,7 +409,7 @@ void _mi_page_retire(mi_page_t* page) {
// if its neighbours are almost fully used.
if (mi_likely(page->block_size <= (MI_SMALL_SIZE_MAX/4))) {
if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) {
_mi_stat_counter_increase(&_mi_stats_main.page_no_retire,1);
mi_stat_counter_increase(_mi_stats_main.page_no_retire,1);
return; // dont't retire after all
}
}
@ -471,7 +481,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si
heap->random = _mi_random_shuffle(rnd);
}
static void mi_page_free_list_extend( mi_page_t* page, size_t extend, mi_stats_t* stats)
static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* page, size_t extend, mi_stats_t* stats)
{
UNUSED(stats);
mi_assert_internal(page->free == NULL);
@ -480,15 +490,15 @@ static void mi_page_free_list_extend( mi_page_t* page, size_t extend, mi_stats_t
void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL );
size_t bsize = page->block_size;
mi_block_t* start = mi_page_block_at(page, page_area, page->capacity);
// initialize a sequential free list
mi_block_t* last = mi_page_block_at(page, page_area, page->capacity + extend - 1);
mi_block_t* last = mi_page_block_at(page, page_area, page->capacity + extend - 1);
mi_block_t* block = start;
while(block <= last) {
mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize);
mi_block_set_next(page,block,next);
block = next;
}
}
mi_block_set_next(page, last, NULL);
page->free = start;
}
@ -519,11 +529,11 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st
size_t page_size;
_mi_page_start(_mi_page_segment(page), page, &page_size);
_mi_stat_increase(&stats->pages_extended, 1);
mi_stat_increase(stats->pages_extended, 1);
// calculate the extend count
size_t extend = page->reserved - page->capacity;
size_t max_extend = MI_MAX_EXTEND_SIZE/page->block_size;
size_t max_extend = (page->block_size >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/(uint32_t)page->block_size);
if (max_extend < MI_MIN_EXTEND) max_extend = MI_MIN_EXTEND;
if (extend > max_extend) {
@ -536,7 +546,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st
mi_assert_internal(extend < (1UL<<16));
// and append the extend the free list
if (extend < MI_MIN_SLICES || !mi_option_is_enabled(mi_option_secure)) {
if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) {
mi_page_free_list_extend(page, extend, stats );
}
else {
@ -544,8 +554,12 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st
}
// enable the new free list
page->capacity += (uint16_t)extend;
_mi_stat_increase(&stats->page_committed, extend * page->block_size);
mi_stat_increase(stats->page_committed, extend * page->block_size);
// extension into zero initialized memory preserves the zero'd free list
if (!page->is_zero_init) {
page->is_zero = false;
}
mi_assert_expensive(mi_page_is_valid_init(page));
}
@ -566,6 +580,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
#if MI_SECURE
page->cookie = _mi_heap_random(heap) | 1;
#endif
page->is_zero = page->is_zero_init;
mi_assert_internal(page->capacity == 0);
mi_assert_internal(page->free == NULL);
@ -639,7 +654,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
page = next;
} // for each page
_mi_stat_counter_increase(&heap->tld->stats.searches,count);
mi_stat_counter_increase(heap->tld->stats.searches,count);
if (page == NULL) {
page = rpage;
@ -665,7 +680,7 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
mi_page_queue_t* pq = mi_page_queue(heap,size);
mi_page_t* page = pq->first;
if (page != NULL) {
if (mi_option_get(mi_option_secure) >= 3 && page->capacity < page->reserved && ((_mi_heap_random(heap) & 1) == 1)) {
if ((MI_SECURE >= 3) && page->capacity < page->reserved && ((_mi_heap_random(heap) & 1) == 1)) {
// in secure mode, we extend half the time to increase randomness
mi_page_extend_free(heap, page, &heap->tld->stats);
mi_assert_internal(mi_page_immediate_available(page));
@ -688,12 +703,14 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
a certain number of allocations.
----------------------------------------------------------- */
static mi_deferred_free_fun* deferred_free = NULL;
static mi_deferred_free_fun* volatile deferred_free = NULL;
void _mi_deferred_free(mi_heap_t* heap, bool force) {
heap->tld->heartbeat++;
if (deferred_free != NULL) {
if (deferred_free != NULL && !heap->tld->recurse) {
heap->tld->recurse = true;
deferred_free(force, heap->tld->heartbeat);
heap->tld->recurse = false;
}
}
@ -711,12 +728,13 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn) mi_attr_noexcept {
// Because huge pages contain just one block, and the segment contains
// just that page, we always treat them as abandoned and any thread
// that frees the block can free the whole page and segment directly.
static mi_page_t* mi_large_page_alloc(mi_heap_t* heap, size_t size) {
static mi_page_t* mi_large_huge_page_alloc(mi_heap_t* heap, size_t size) {
size_t block_size = _mi_wsize_from_size(size) * sizeof(uintptr_t);
mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE);
mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE);
bool is_huge = (block_size > MI_LARGE_OBJ_SIZE_MAX);
mi_page_queue_t* pq = (is_huge ? NULL : mi_page_queue(heap,block_size));
mi_page_t* page = mi_page_fresh_alloc(heap,pq,block_size);
mi_page_queue_t* pq = (is_huge ? NULL : mi_page_queue(heap, block_size));
mi_page_t* page = mi_page_fresh_alloc(heap, pq, block_size);
if (page != NULL) {
mi_assert_internal(mi_page_immediate_available(page));
mi_assert_internal(page->block_size == block_size);
@ -768,7 +786,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept
page = NULL;
}
else {
page = mi_large_page_alloc(heap,size);
page = mi_large_huge_page_alloc(heap,size);
}
}
else {

58
src/segment.c
View file

@ -227,8 +227,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
}
*/
long secure = mi_option_get(mi_option_secure);
if (secure > 1 || (secure == 1 && slice == &segment->slices[segment->slice_entries - 1])) {
if (MI_SECURE > 1 || (MI_SECURE == 1 && slice == &segment->slices[segment->slice_entries - 1])) {
// secure == 1: the last page has an os guard page at the end
// secure > 1: every page has an os guard page
psize -= _mi_os_page_size();
@ -245,13 +244,13 @@ static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, siz
size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size);
size_t guardsize = 0;
if (mi_option_is_enabled(mi_option_secure)) {
if (MI_SECURE>0) {
// in secure mode, we set up a protected page in between the segment info
// and the page data (and one at the end of the segment)
guardsize = page_size;
required = _mi_align_up(required, page_size);
}
;
if (pre_size != NULL) *pre_size = isize;
isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE);
if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE;
@ -281,7 +280,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
segment->thread_id = 0;
mi_segment_map_freed_at(segment);
mi_segments_track_size(-((long)mi_segment_size(segment)),tld);
if (mi_option_is_enabled(mi_option_secure)) {
if (MI_SECURE>0) {
_mi_os_unprotect(segment, mi_segment_size(segment)); // ensure no more guard pages are set
}
_mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
@ -328,8 +327,9 @@ static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld)
if (segment->segment_slices != MI_SLICES_PER_SEGMENT || mi_segment_cache_full(tld)) {
return false;
}
mi_assert_internal(segment->segment_slices == MI_SLICES_PER_SEGMENT);
if (mi_option_is_enabled(mi_option_cache_reset)) {
if (!segment->mem_is_fixed && mi_option_is_enabled(mi_option_cache_reset)) {
_mi_os_reset((uint8_t*)segment + mi_segment_info_size(segment), mi_segment_size(segment) - mi_segment_info_size(segment), tld->stats);
}
segment->next = tld->cache;
@ -371,6 +371,7 @@ static uintptr_t mi_segment_commit_mask(mi_segment_t* segment, bool conservative
start = _mi_align_down(diff, MI_COMMIT_SIZE);
end = _mi_align_up(diff + size, MI_COMMIT_SIZE);
}
mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0);
*start_p = (uint8_t*)segment + start;
*full_size = (end > start ? end - start : 0);
@ -397,7 +398,8 @@ static void mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s
if (mask==0 || full_size==0) return;
if (commit && (segment->commit_mask & mask) != mask) {
_mi_os_commit(start,full_size,stats);
bool is_zero = false;
_mi_os_commit(start,full_size,&is_zero,stats);
segment->commit_mask |= mask;
}
else if (!commit && (segment->commit_mask & mask) != 0) {
@ -412,7 +414,7 @@ static void mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_
}
static void mi_segment_perhaps_decommit(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
if (!segment->allow_decommit || !mi_option_is_enabled(mi_option_decommit)) return;
if (!segment->allow_decommit) return; // TODO: check option_decommit?
if (segment->commit_mask == 1) return; // fully decommitted
mi_segment_commitx(segment, false, p, size, stats);
}
@ -589,28 +591,32 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_segments_tld_t* tld, m
size_t segment_size = segment_slices * MI_SEGMENT_SLICE_SIZE;
// Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little)
size_t lazy = (size_t)mi_option_get(mi_option_lazy_commit);
bool commit_lazy = (lazy > tld->count) && required == 0; // lazy, and not a huge page
bool eager_delay = (tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay));
bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit);
bool commit = eager || (required > 0);
// Try to get from our cache first
mi_segment_t* segment = mi_segment_cache_pop(segment_slices, tld);
if (segment==NULL) {
// Allocate the segment from the OS
segment = (mi_segment_t*)_mi_os_alloc_aligned(segment_size, MI_SEGMENT_SIZE, !commit_lazy, /* &memid,*/ os_tld);
bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy
segment = (mi_segment_t*)_mi_os_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &mem_large, os_tld);
if (segment == NULL) return NULL; // failed to allocate
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
if (commit_lazy) {
if (!commit) {
// at least commit the info slices
mi_assert_internal(MI_COMMIT_SIZE > info_slices*MI_SEGMENT_SLICE_SIZE);
_mi_os_commit(segment, MI_COMMIT_SIZE, tld->stats);
bool is_zero = false;
_mi_os_commit(segment, MI_COMMIT_SIZE, &is_zero, tld->stats);
}
segment->mem_is_fixed = mem_large;
segment->mem_is_committed = commit;
mi_segments_track_size((long)(segment_size), tld);
mi_segment_map_allocated_at(segment);
}
// zero the segment info? -- not needed as it is zero initialized from the OS
// memset(segment, 0, info_size);
// initialize segment info
memset(segment,0,offsetof(mi_segment_t,slices));
@ -620,13 +626,13 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_segments_tld_t* tld, m
segment->cookie = _mi_ptr_cookie(segment);
segment->slice_entries = slice_entries;
segment->kind = (required == 0 ? MI_SEGMENT_NORMAL : MI_SEGMENT_HUGE);
segment->allow_decommit = commit_lazy;
segment->commit_mask = (commit_lazy ? 0x01 : ~((uintptr_t)0)); // on lazy commit, the initial part is always committed
segment->allow_decommit = !commit;
segment->commit_mask = (!commit ? 0x01 : ~((uintptr_t)0)); // on lazy commit, the initial part is always committed
memset(segment->slices, 0, sizeof(mi_slice_t)*(info_slices+1));
_mi_stat_increase(&tld->stats->page_committed, mi_segment_info_size(segment));
// set up guard pages
if (mi_option_is_enabled(mi_option_secure)) {
if (MI_SECURE>0) {
// in secure mode, we set up a protected page in between the segment info
// and the page data
size_t os_page_size = _mi_os_page_size();
@ -694,6 +700,7 @@ static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t
Page allocation
----------------------------------------------------------- */
static mi_page_t* mi_segments_page_alloc(mi_page_kind_t page_kind, size_t required, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{
mi_assert_internal(required <= MI_LARGE_OBJ_SIZE_MAX && page_kind <= MI_PAGE_LARGE);
@ -730,21 +737,18 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld
_mi_stat_decrease(&tld->stats->pages, 1);
// reset the page memory to reduce memory pressure?
if (!page->is_reset && mi_option_is_enabled(mi_option_page_reset)) {
if (!segment->mem_is_fixed && !page->is_reset && mi_option_is_enabled(mi_option_page_reset)) {
size_t psize;
uint8_t* start = _mi_page_start(segment, page, &psize);
page->is_reset = true;
_mi_os_reset(start, psize, tld->stats);
}
// zero the page data
uint32_t slice_count = page->slice_count; // don't clear the slice_count
bool is_reset = page->is_reset; // don't clear the reset flag
bool is_committed = page->is_committed; // don't clear the commit flag
memset(page, 0, sizeof(*page));
page->slice_count = slice_count;
page->is_reset = is_reset;
page->is_committed = is_committed;
// zero the page data, but not the segment fields
page->is_zero_init = false;
ptrdiff_t ofs = offsetof(mi_page_t, capacity);
memset((uint8_t*)page + ofs, 0, sizeof(*page) - ofs);
page->block_size = 1;
// and free it

61
src/stats.c
View file

@ -8,22 +8,10 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc-internal.h"
#include "mimalloc-atomic.h"
#include <stdio.h> // fputs, stderr
#include <string.h> // memset
/* -----------------------------------------------------------
Merge thread statistics with the main one.
----------------------------------------------------------- */
static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src);
void _mi_stats_done(mi_stats_t* stats) {
if (stats == &_mi_stats_main) return;
mi_stats_add(&_mi_stats_main, stats);
memset(stats,0,sizeof(*stats));
}
/* -----------------------------------------------------------
Statistics operations
----------------------------------------------------------- */
@ -85,6 +73,7 @@ static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64
mi_atomic_add64( &stat->allocated, src->allocated * unit);
mi_atomic_add64( &stat->current, src->current * unit);
mi_atomic_add64( &stat->freed, src->freed * unit);
// peak scores do not work across threads..
mi_atomic_add64( &stat->peak, src->peak * unit);
}
@ -132,7 +121,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
Display statistics
----------------------------------------------------------- */
static void mi_printf_amount(int64_t n, int64_t unit, FILE* out, const char* fmt) {
static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, const char* fmt) {
char buf[32];
int len = 32;
const char* suffix = (unit <= 0 ? " " : "b");
@ -153,16 +142,16 @@ static void mi_printf_amount(int64_t n, int64_t unit, FILE* out, const char* fmt
}
static void mi_print_amount(int64_t n, int64_t unit, FILE* out) {
static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out) {
mi_printf_amount(n,unit,out,NULL);
}
static void mi_print_count(int64_t n, int64_t unit, FILE* out) {
static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out) {
if (unit==1) _mi_fprintf(out,"%11s"," ");
else mi_print_amount(n,0,out);
}
static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, FILE* out ) {
static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out ) {
_mi_fprintf(out,"%10s:", msg);
if (unit>0) {
mi_print_amount(stat->peak, unit, out);
@ -191,24 +180,24 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
}
}
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, FILE* out ) {
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out ) {
_mi_fprintf(out, "%10s:", msg);
mi_print_amount(stat->total, -1, out);
_mi_fprintf(out, "\n");
}
static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, FILE* out) {
static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out) {
double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count);
_mi_fprintf(out, "%10s: %7.1f avg\n", msg, avg);
}
static void mi_print_header( FILE* out ) {
static void mi_print_header(mi_output_fun* out ) {
_mi_fprintf(out,"%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count ");
}
#if MI_STAT>1
static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, FILE* out) {
static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, mi_output_fun* out) {
bool found = false;
char buf[64];
for (size_t i = 0; i <= max; i++) {
@ -232,8 +221,7 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin
static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit);
static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_noexcept {
if (out == NULL) out = stderr;
static void _mi_stats_print(mi_stats_t* stats, double secs, mi_output_fun* out) mi_attr_noexcept {
mi_print_header(out);
#if MI_STAT>1
mi_stat_count_t normal = { 0,0,0,0 };
@ -293,6 +281,13 @@ static mi_stats_t* mi_stats_get_default(void) {
return &heap->tld->stats;
}
static void mi_stats_merge_from(mi_stats_t* stats) {
if (stats != &_mi_stats_main) {
mi_stats_add(&_mi_stats_main, stats);
memset(stats, 0, sizeof(mi_stats_t));
}
}
void mi_stats_reset(void) mi_attr_noexcept {
mi_stats_t* stats = mi_stats_get_default();
if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); }
@ -300,19 +295,25 @@ void mi_stats_reset(void) mi_attr_noexcept {
mi_time_start = _mi_clock_start();
}
static void mi_stats_print_ex(mi_stats_t* stats, double secs, FILE* out) {
if (stats != &_mi_stats_main) {
mi_stats_add(&_mi_stats_main,stats);
memset(stats,0,sizeof(mi_stats_t));
}
void mi_stats_merge(void) mi_attr_noexcept {
mi_stats_merge_from( mi_stats_get_default() );
}
void _mi_stats_done(mi_stats_t* stats) { // called from `mi_thread_done`
mi_stats_merge_from(stats);
}
static void mi_stats_print_ex(mi_stats_t* stats, double secs, mi_output_fun* out) {
mi_stats_merge_from(stats);
_mi_stats_print(&_mi_stats_main, secs, out);
}
void mi_stats_print(FILE* out) mi_attr_noexcept {
void mi_stats_print(mi_output_fun* out) mi_attr_noexcept {
mi_stats_print_ex(mi_stats_get_default(),_mi_clock_end(mi_time_start),out);
}
void mi_thread_stats_print(FILE* out) mi_attr_noexcept {
void mi_thread_stats_print(mi_output_fun* out) mi_attr_noexcept {
_mi_stats_print(mi_stats_get_default(), _mi_clock_end(mi_time_start), out);
}