kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-08 08:19:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge from dev-slice

Browse source
This commit is contained in:
daan 2021-11-10 16:49:43 -08:00
commit 998c2de633
7 changed files with 162 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
include/mimalloc-internal.h
View file

@ -85,8 +85,8 @@ void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_pinn
void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, mi_os_tld_t* tld); void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, mi_os_tld_t* tld);
// "segment-cache.c" // "segment-cache.c"
void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld); bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld);
void _mi_segment_map_allocated_at(const mi_segment_t* segment); void _mi_segment_map_allocated_at(const mi_segment_t* segment);
void _mi_segment_map_freed_at(const mi_segment_t* segment); void _mi_segment_map_freed_at(const mi_segment_t* segment);

4
include/mimalloc-types.h
View file

@ -118,7 +118,7 @@ terms of the MIT license. A copy of the license can be found in the file
// Main tuning parameters for segment and page sizes // Main tuning parameters for segment and page sizes
// Sizes for 64-bit, divide by two for 32-bit // Sizes for 64-bit, divide by two for 32-bit
#define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB #define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB
#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 8MiB #define MI_SEGMENT_SHIFT (10 + MI_SEGMENT_SLICE_SHIFT) // 64MiB
#define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB #define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64KiB
#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB #define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB
@ -139,7 +139,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit #define MI_MEDIUM_OBJ_SIZE_MAX (MI_MEDIUM_PAGE_SIZE/4) // 128KiB on 64-bit
#define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE) #define MI_MEDIUM_OBJ_WSIZE_MAX (MI_MEDIUM_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
#define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 4MiB on 64-bit #define MI_LARGE_OBJ_SIZE_MAX (MI_SEGMENT_SIZE/2) // 32MiB on 64-bit
#define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE) #define MI_LARGE_OBJ_WSIZE_MAX (MI_LARGE_OBJ_SIZE_MAX/MI_INTPTR_SIZE)
#define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c) #define MI_HUGE_OBJ_SIZE_MAX (2*MI_INTPTR_SIZE*MI_SEGMENT_SIZE) // (must match MI_REGION_MAX_ALLOC_SIZE in memory.c)

141
src/os.c
View file

@ -295,20 +295,21 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size);
static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
#if (MI_INTPTR_SIZE >= 8) #if (MI_INTPTR_SIZE >= 8)
// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
void* hint; if (addr == NULL) {
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { void* hint = mi_os_get_aligned_hint(try_alignment,size);
void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); if (hint != NULL) {
if (p != NULL) return p; void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE);
// for robustness always fall through in case of an error if (p != NULL) return p;
/* // for robustness always fall through in case of an error
DWORD err = GetLastError(); /*
if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) DWORD err = GetLastError();
err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210)
return NULL; err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230)
return NULL;
}
*/
_mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags);
} }
*/
_mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags);
// fall through on error
} }
#endif #endif
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
@ -405,23 +406,40 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
#else #else
#define MI_OS_USE_MMAP #define MI_OS_USE_MMAP
static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
void* p = NULL; UNUSED(try_alignment);
#if defined(MAP_ALIGNED) // BSD
if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) {
size_t n = mi_bsr(try_alignment);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
flags |= MAP_ALIGNED(n);
void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0);
if (p!=MAP_FAILED) return p;
// fall back to regular mmap
}
}
#elif defined(MAP_ALIGN) // Solaris
if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) {
void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0);
if (p!=MAP_FAILED) return p;
// fall back to regular mmap
}
#endif
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
// on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations
void* hint; if (addr == NULL) {
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { void* hint = mi_os_get_aligned_hint(try_alignment, size);
p = mmap(hint,size,protect_flags,flags,fd,0); if (hint != NULL) {
if (p==MAP_FAILED) p = NULL; // fall back to regular mmap void* p = mmap(hint, size, protect_flags, flags, fd, 0);
if (p!=MAP_FAILED) return p;
// fall back to regular mmap
}
} }
#else
UNUSED(try_alignment);
UNUSED(mi_os_get_aligned_hint);
#endif #endif
if (p==NULL) { // regular mmap
p = mmap(addr,size,protect_flags,flags,fd,0); void* p = mmap(addr, size, protect_flags, flags, fd, 0);
if (p==MAP_FAILED) p = NULL; if (p!=MAP_FAILED) return p;
} // failed to allocate
return p; return NULL;
} }
static int mi_unix_mmap_fd(void) { static int mi_unix_mmap_fd(void) {
@ -447,18 +465,17 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
int flags = MAP_PRIVATE | MAP_ANONYMOUS; int flags = MAP_PRIVATE | MAP_ANONYMOUS;
if (os_overcommit) { if (os_overcommit) {
flags |= MAP_NORESERVE; flags |= MAP_NORESERVE;
} }
#if defined(MAP_ALIGNED) // BSD
if (try_alignment > 0) {
size_t n = mi_bsr(try_alignment);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
flags |= MAP_ALIGNED(n);
}
}
#endif
#if defined(PROT_MAX) #if defined(PROT_MAX)
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
#endif #endif
#if defined(VM_MAKE_TAG)
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
int os_tag = (int)mi_option_get(mi_option_os_tag);
if (os_tag < 100 || os_tag > 255) { os_tag = 100; }
fd = VM_MAKE_TAG(os_tag);
#endif
// huge page allocation
if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) {
static _Atomic(uintptr_t) large_page_try_ok; // = 0; static _Atomic(uintptr_t) large_page_try_ok; // = 0;
uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok);
@ -507,37 +524,39 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
#endif #endif
if (large_only) return p; if (large_only) return p;
if (p == NULL) { if (p == NULL) {
mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations mi_atomic_store_release(&large_page_try_ok, (uintptr_t)8); // on error, don't try again for the next N allocations
} }
} }
} }
} }
// regular allocation
if (p == NULL) { if (p == NULL) {
*is_large = false; *is_large = false;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd);
#if defined(MADV_HUGEPAGE) if (p != NULL) {
// Many Linux systems don't allow MAP_HUGETLB but they support instead #if defined(MADV_HUGEPAGE)
// transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE // Many Linux systems don't allow MAP_HUGETLB but they support instead
// though since properly aligned allocations will already use large pages if available // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE
// in that case -- in particular for our large regions (in `memory.c`). // though since properly aligned allocations will already use large pages if available
// However, some systems only allow THP if called with explicit `madvise`, so // in that case -- in particular for our large regions (in `memory.c`).
// when large OS pages are enabled for mimalloc, we call `madvise` anyways. // However, some systems only allow THP if called with explicit `madvise`, so
if (allow_large && use_large_os_page(size, try_alignment)) { // when large OS pages are enabled for mimalloc, we call `madvise` anyways.
if (madvise(p, size, MADV_HUGEPAGE) == 0) { if (allow_large && use_large_os_page(size, try_alignment)) {
*is_large = true; // possibly if (madvise(p, size, MADV_HUGEPAGE) == 0) {
}; *is_large = true; // possibly
} };
#endif
#if defined(__sun)
if (allow_large && use_large_os_page(size, try_alignment)) {
struct memcntl_mha cmd = {0};
cmd.mha_pagesize = large_os_page_size;
cmd.mha_cmd = MHA_MAPSIZE_VA;
if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
*is_large = true;
} }
#elif defined(__sun)
if (allow_large && use_large_os_page(size, try_alignment)) {
struct memcntl_mha cmd = {0};
cmd.mha_pagesize = large_os_page_size;
cmd.mha_cmd = MHA_MAPSIZE_VA;
if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) {
*is_large = true;
}
}
#endif
} }
#endif
} }
if (p == NULL) { if (p == NULL) {
_mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large);
@ -548,7 +567,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
// On 64-bit systems, we can do efficient aligned allocation by using // On 64-bit systems, we can do efficient aligned allocation by using
// the 2TiB to 30TiB area to allocate them. // the 2TiB to 30TiB area to allocate them.
#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) #if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || defined(MI_OS_USE_MMAP))
static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; static mi_decl_cache_align _Atomic(uintptr_t) aligned_base;
// Return a 4MiB aligned address that is probably available. // Return a 4MiB aligned address that is probably available.
@ -856,7 +875,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
} }
#endif #endif
if (err != 0) { if (err != 0) {
_mi_warning_message("%s error: start: %p, csize: 0x%x, err: %i\n", commit ? "commit" : "decommit", start, csize, err); _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err);
mi_mprotect_hint(err); mi_mprotect_hint(err);
} }
mi_assert_internal(err == 0); mi_assert_internal(err == 0);
@ -926,7 +945,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
int err = madvise(start, csize, MADV_DONTNEED); int err = madvise(start, csize, MADV_DONTNEED);
#endif #endif
if (err != 0) { if (err != 0) {
_mi_warning_message("madvise reset error: start: %p, csize: 0x%x, errno: %i\n", start, csize, errno); _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, errno);
} }
//mi_assert(err == 0); //mi_assert(err == 0);
if (err != 0) return false; if (err != 0) return false;
@ -985,7 +1004,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
if (err != 0) { err = errno; } if (err != 0) { err = errno; }
#endif #endif
if (err != 0) { if (err != 0) {
_mi_warning_message("mprotect error: start: %p, csize: 0x%x, err: %i\n", start, csize, err); _mi_warning_message("mprotect error: start: %p, csize: 0x%zx, err: %i\n", start, csize, err);
mi_mprotect_hint(err); mi_mprotect_hint(err);
} }
return (err == 0); return (err == 0);

15
src/segment-cache.c
View file

@ -16,19 +16,20 @@ terms of the MIT license. A copy of the license can be found in the file
#include "bitmap.h" // atomic bitmap #include "bitmap.h" // atomic bitmap
//#define MI_CACHE_DISABLE 1 //#define MI_CACHE_DISABLE 1 // define to completely disable the segment cache
#define MI_CACHE_FIELDS (16) #define MI_CACHE_FIELDS (16)
#define MI_CACHE_MAX (MI_BITMAP_FIELD_BITS*MI_CACHE_FIELDS) // 1024 on 64-bit #define MI_CACHE_MAX (MI_BITMAP_FIELD_BITS*MI_CACHE_FIELDS) // 1024 on 64-bit
#define BITS_SET() ATOMIC_VAR_INIT(UINTPTR_MAX) #define BITS_SET() ATOMIC_VAR_INIT(UINTPTR_MAX)
#define MI_CACHE_BITS_SET MI_INIT16(BITS_SET) #define MI_CACHE_BITS_SET MI_INIT16(BITS_SET) // note: update if MI_CACHE_FIELDS changes
typedef struct mi_cache_slot_s { typedef struct mi_cache_slot_s {
void* p; void* p;
size_t memid; size_t memid;
bool is_pinned; bool is_pinned;
mi_commit_mask_t commit_mask; mi_commit_mask_t commit_mask;
mi_commit_mask_t decommit_mask;
_Atomic(mi_msecs_t) expire; _Atomic(mi_msecs_t) expire;
} mi_cache_slot_t; } mi_cache_slot_t;
@ -39,7 +40,7 @@ static mi_decl_cache_align mi_bitmap_field_t cache_available_large[MI_CACHE_FIEL
static mi_decl_cache_align mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free static mi_decl_cache_align mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free
mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld) mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
{ {
#ifdef MI_CACHE_DISABLE #ifdef MI_CACHE_DISABLE
return NULL; return NULL;
@ -76,7 +77,8 @@ mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* comm
*memid = slot->memid; *memid = slot->memid;
*is_pinned = slot->is_pinned; *is_pinned = slot->is_pinned;
*is_zero = false; *is_zero = false;
*commit_mask = slot->commit_mask; *commit_mask = slot->commit_mask;
*decommit_mask = slot->decommit_mask;
slot->p = NULL; slot->p = NULL;
mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0);
@ -138,6 +140,7 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld)
// decommit committed parts // decommit committed parts
// TODO: instead of decommit, we could also free to the OS? // TODO: instead of decommit, we could also free to the OS?
mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats); mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats);
mi_commit_mask_create_empty(&slot->decommit_mask);
} }
_mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop _mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop
} }
@ -146,7 +149,7 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld)
} }
} }
mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld) mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld)
{ {
#ifdef MI_CACHE_DISABLE #ifdef MI_CACHE_DISABLE
return false; return false;
@ -186,11 +189,13 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me
slot->is_pinned = is_pinned; slot->is_pinned = is_pinned;
mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
slot->commit_mask = *commit_mask; slot->commit_mask = *commit_mask;
slot->decommit_mask = *decommit_mask;
if (!mi_commit_mask_is_empty(commit_mask) && !is_large && !is_pinned && mi_option_is_enabled(mi_option_allow_decommit)) { if (!mi_commit_mask_is_empty(commit_mask) && !is_large && !is_pinned && mi_option_is_enabled(mi_option_allow_decommit)) {
long delay = mi_option_get(mi_option_segment_decommit_delay); long delay = mi_option_get(mi_option_segment_decommit_delay);
if (delay == 0) { if (delay == 0) {
_mi_abandoned_await_readers(); // wait until safe to decommit _mi_abandoned_await_readers(); // wait until safe to decommit
mi_commit_mask_decommit(&slot->commit_mask, start, MI_SEGMENT_SIZE, tld->stats); mi_commit_mask_decommit(&slot->commit_mask, start, MI_SEGMENT_SIZE, tld->stats);
mi_commit_mask_create_empty(&slot->decommit_mask);
} }
else { else {
mi_atomic_storei64_release(&slot->expire, _mi_clock_now() + delay); mi_atomic_storei64_release(&slot->expire, _mi_clock_now() + delay);

27
src/segment.c
View file

@ -256,7 +256,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
// _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats); // _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
const size_t size = mi_segment_size(segment); const size_t size = mi_segment_size(segment);
if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) { if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, &segment->decommit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) {
const size_t csize = mi_commit_mask_committed_size(&segment->commit_mask, size); const size_t csize = mi_commit_mask_committed_size(&segment->commit_mask, size);
if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize); if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize);
_mi_abandoned_await_readers(); // wait until safe to free _mi_abandoned_await_readers(); // wait until safe to free
@ -663,19 +663,21 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
bool is_zero = false; bool is_zero = false;
const bool commit_info_still_good = (segment != NULL); const bool commit_info_still_good = (segment != NULL);
mi_commit_mask_t commit_mask; mi_commit_mask_t commit_mask;
mi_commit_mask_t decommit_mask;
if (segment != NULL) { if (segment != NULL) {
commit_mask = segment->commit_mask; commit_mask = segment->commit_mask;
decommit_mask = segment->decommit_mask;
} }
else { else {
mi_commit_mask_create_empty(&commit_mask); mi_commit_mask_create_empty(&commit_mask);
mi_commit_mask_create_empty(&decommit_mask);
} }
if (segment==NULL) { if (segment==NULL) {
// Allocate the segment from the OS // Allocate the segment from the OS
bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy
bool is_pinned = false; bool is_pinned = false;
size_t memid = 0; size_t memid = 0;
segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, &commit_mask, &mem_large, &is_pinned, &is_zero, &memid, os_tld); segment = (mi_segment_t*)_mi_segment_cache_pop(segment_size, &commit_mask, &decommit_mask, &mem_large, &is_pinned, &is_zero, &memid, os_tld);
if (segment==NULL) { if (segment==NULL) {
segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld); segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld);
if (segment == NULL) return NULL; // failed to allocate if (segment == NULL) return NULL; // failed to allocate
@ -718,9 +720,24 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
if (!commit_info_still_good) { if (!commit_info_still_good) {
segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed
segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_pinned && !segment->mem_is_large); segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_pinned && !segment->mem_is_large);
segment->decommit_expire = 0; if (segment->allow_decommit) {
mi_commit_mask_create_empty( &segment->decommit_mask ); segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay);
segment->decommit_mask = decommit_mask;
mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask));
#if MI_DEBUG>2
const size_t commit_needed = _mi_divide_up(info_slices*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE);
mi_commit_mask_t commit_needed_mask;
mi_commit_mask_create(0, commit_needed, &commit_needed_mask);
mi_assert_internal(!mi_commit_mask_any_set(&segment->decommit_mask, &commit_needed_mask));
#endif
}
else {
mi_assert_internal(mi_commit_mask_is_empty(&decommit_mask));
segment->decommit_expire = 0;
mi_commit_mask_create_empty( &segment->decommit_mask );
}
} }
// initialize segment info // initialize segment info
segment->segment_slices = segment_slices; segment->segment_slices = segment_slices;

2
src/stats.c
View file

@ -327,7 +327,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_counter_print(&stats->commit_calls, "commits", out, arg);
mi_stat_print(&stats->threads, "threads", -1, out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg);
mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); mi_stat_counter_print_avg(&stats->searches, "searches", out, arg);
_mi_fprintf(out, arg, "%10s: %7i\n", "numa nodes", _mi_os_numa_node_count()); _mi_fprintf(out, arg, "%10s: %7zu\n", "numa nodes", _mi_os_numa_node_count());
mi_msecs_t elapsed; mi_msecs_t elapsed;
mi_msecs_t user_time; mi_msecs_t user_time;

57
test/main-override.cpp
View file

@ -35,22 +35,24 @@ static void test_mt_shutdown();
static void large_alloc(void); // issue #363 static void large_alloc(void); // issue #363
static void fail_aslr(); // issue #372 static void fail_aslr(); // issue #372
static void tsan_numa_test(); // issue #414 static void tsan_numa_test(); // issue #414
static void strdup_test(); // issue #445 static void strdup_test(); // issue #445
static void bench_alloc_large(void); // issue #xxx
int main() { int main() {
mi_stats_reset(); // ignore earlier allocations mi_stats_reset(); // ignore earlier allocations
heap_thread_free_large(); heap_thread_free_large();
heap_no_delete(); heap_no_delete();
heap_late_free(); heap_late_free();
padding_shrink(); padding_shrink();
various_tests(); various_tests();
large_alloc(); large_alloc();
tsan_numa_test(); tsan_numa_test();
strdup_test(); strdup_test();
//test_mt_shutdown(); //test_mt_shutdown();
//fail_aslr(); //fail_aslr();
//bench_alloc_large();
mi_stats_print(NULL); mi_stats_print(NULL);
return 0; return 0;
} }
@ -246,11 +248,42 @@ static void fail_aslr() {
// issues #414 // issues #414
static void dummy_worker() { static void dummy_worker() {
void* p = mi_malloc(0); void* p = mi_malloc(0);
mi_free(p); mi_free(p);
} }
static void tsan_numa_test() { static void tsan_numa_test() {
auto t1 = std::thread(dummy_worker); auto t1 = std::thread(dummy_worker);
dummy_worker(); dummy_worker();
t1.join(); t1.join();
} }
// issue #?
#include <chrono>
#include <random>
#include <iostream>
static void bench_alloc_large(void) {
static constexpr int kNumBuffers = 20;
static constexpr size_t kMinBufferSize = 5 * 1024 * 1024;
static constexpr size_t kMaxBufferSize = 25 * 1024 * 1024;
std::unique_ptr<char[]> buffers[kNumBuffers];
std::random_device rd;
std::mt19937 gen(42); //rd());
std::uniform_int_distribution<> size_distribution(kMinBufferSize, kMaxBufferSize);
std::uniform_int_distribution<> buf_number_distribution(0, kNumBuffers - 1);
static constexpr int kNumIterations = 2000;
const auto start = std::chrono::steady_clock::now();
for (int i = 0; i < kNumIterations; ++i) {
int buffer_idx = buf_number_distribution(gen);
size_t new_size = size_distribution(gen);
buffers[buffer_idx] = std::make_unique<char[]>(new_size);
}
const auto end = std::chrono::steady_clock::now();
const auto num_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
const auto us_per_allocation = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / kNumIterations;
std::cout << kNumIterations << " allocations Done in " << num_ms << "ms." << std::endl;
std::cout << "Avg " << us_per_allocation << " us per allocation" << std::endl;
}