kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

keep commit_mask live in the cache for better reuse

Browse source
This commit is contained in:
daan 2020-09-05 21:58:32 -07:00
parent 63a9f45ba6
commit f7dc4847f2
4 changed files with 126 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
include/mimalloc-internal.h
View file

@ -71,9 +71,9 @@ bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats)
size_t _mi_os_good_alloc_size(size_t size); size_t _mi_os_good_alloc_size(size_t size);
// arena.c // arena.c
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool commit, mi_commit_mask_t* commit_mask, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc(size_t size, bool commit, mi_commit_mask_t* commit_mask, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, bool is_large, mi_os_tld_t* tld); void _mi_arena_free(void* p, size_t size, size_t memid, mi_commit_mask_t commit_mask, bool is_large, mi_os_tld_t* tld);
// "segment.c" // "segment.c"
@ -655,6 +655,8 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c
// commit mask // commit mask
// ------------------------------------------------------------------- // -------------------------------------------------------------------
#define MI_COMMIT_MASK_BITS (sizeof(mi_commit_mask_t)*8)
static inline mi_commit_mask_t mi_commit_mask_empty(void) { static inline mi_commit_mask_t mi_commit_mask_empty(void) {
return 0; return 0;
} }
@ -664,9 +666,9 @@ static inline mi_commit_mask_t mi_commit_mask_full(void) {
} }
static inline mi_commit_mask_t mi_commit_mask_create(uintptr_t bitidx, uintptr_t bitcount) { static inline mi_commit_mask_t mi_commit_mask_create(uintptr_t bitidx, uintptr_t bitcount) {
mi_assert_internal(bitidx < MI_INTPTR_BITS); mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
mi_assert_internal((bitidx + bitcount) <= MI_INTPTR_BITS); mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
if (bitcount == MI_INTPTR_BITS) { if (bitcount == MI_COMMIT_MASK_BITS) {
mi_assert_internal(bitidx==0); mi_assert_internal(bitidx==0);
return mi_commit_mask_full(); return mi_commit_mask_full();
} }
@ -683,7 +685,7 @@ static inline bool mi_commit_mask_is_empty(mi_commit_mask_t mask) {
} }
static inline bool mi_commit_mask_is_full(mi_commit_mask_t mask) { static inline bool mi_commit_mask_is_full(mi_commit_mask_t mask) {
return (~mask == 0); return ((~mask) == 0);
} }
static inline bool mi_commit_mask_all_set(mi_commit_mask_t commit, mi_commit_mask_t mask) { static inline bool mi_commit_mask_all_set(mi_commit_mask_t commit, mi_commit_mask_t mask) {
@ -694,18 +696,35 @@ static inline bool mi_commit_mask_any_set(mi_commit_mask_t commit, mi_commit_mas
return ((commit & mask) != 0); return ((commit & mask) != 0);
} }
static mi_decl_nodiscard inline mi_commit_mask_t mi_commit_mask_intersect(mi_commit_mask_t commit, mi_commit_mask_t mask) { mi_decl_nodiscard static inline mi_commit_mask_t mi_commit_mask_intersect(mi_commit_mask_t commit, mi_commit_mask_t mask) {
return (commit & mask); return (commit & mask);
} }
static inline void mi_commit_mask_clear(mi_commit_mask_t* commit, mi_commit_mask_t mask) { static inline void mi_commit_mask_clear(mi_commit_mask_t* commit, mi_commit_mask_t mask) {
*commit = *commit & ~mask; *commit = (*commit) & (~mask);
} }
static inline void mi_commit_mask_set(mi_commit_mask_t* commit, mi_commit_mask_t mask) { static inline void mi_commit_mask_set(mi_commit_mask_t* commit, mi_commit_mask_t mask) {
*commit = *commit | mask; *commit = (*commit) | mask;
} }
static inline size_t mi_commit_mask_committed_size(mi_commit_mask_t mask, size_t total) {
if (mi_commit_mask_is_full(mask)) {
return total;
}
else if (mi_commit_mask_is_empty(mask)) {
return 0;
}
else {
size_t count = 0;
for (; mask != 0; mask >>= 1) { // todo: use popcount
if ((mask&1)!=0) count++;
}
return (total/MI_COMMIT_MASK_BITS)*count;
}
}
#define mi_commit_mask_foreach(mask,idx,count) \ #define mi_commit_mask_foreach(mask,idx,count) \
idx = 0; \ idx = 0; \
while (mask != 0) { \ while (mask != 0) { \

107
src/arena.c
View file

@ -135,8 +135,8 @@ static bool mi_arena_alloc(mi_arena_t* arena, size_t blocks, mi_bitmap_index_t*
typedef struct mi_cache_slot_s { typedef struct mi_cache_slot_s {
void* p; void* p;
size_t memid; size_t memid;
mi_commit_mask_t commit_mask;
_Atomic(mi_msecs_t) expire; _Atomic(mi_msecs_t) expire;
bool is_committed; // TODO: use bit from p to reduce size?
} mi_cache_slot_t; } mi_cache_slot_t;
static mi_cache_slot_t cache[MI_CACHE_MAX]; // = 0 static mi_cache_slot_t cache[MI_CACHE_MAX]; // = 0
@ -147,7 +147,10 @@ static mi_bitmap_field_t cache_available_large[MI_CACHE_FIELDS] = { MI_CACHE_BIT
static mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free static mi_bitmap_field_t cache_inuse[MI_CACHE_FIELDS]; // zero bit = free
static void* mi_cache_pop(int numa_node, size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) { static void* mi_cache_pop(int numa_node, size_t size, size_t alignment, bool commit, mi_commit_mask_t* commit_mask, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) {
UNUSED(tld);
UNUSED(commit);
// only segment blocks // only segment blocks
if (size != MI_SEGMENT_SIZE || alignment > MI_SEGMENT_ALIGN) return NULL; if (size != MI_SEGMENT_SIZE || alignment > MI_SEGMENT_ALIGN) return NULL;
@ -177,17 +180,24 @@ static void* mi_cache_pop(int numa_node, size_t size, size_t alignment, bool* co
void* p = slot->p; void* p = slot->p;
*memid = slot->memid; *memid = slot->memid;
*is_zero = false; *is_zero = false;
bool committed = slot->is_committed; mi_commit_mask_t cmask = slot->commit_mask; // copy
slot->p = NULL; slot->p = NULL;
mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0);
if (*commit && !committed) { // ignore commit request
/*
if (commit && !mi_commit_mask_is_full(cmask)) {
bool commit_zero; bool commit_zero;
_mi_os_commit(p, MI_SEGMENT_SIZE, &commit_zero, tld->stats); bool ok = _mi_os_commit(p, MI_SEGMENT_SIZE, &commit_zero, tld->stats); // todo: only commit needed parts?
*commit = true; if (!ok) {
*commit_mask = cmask;
}
else {
*commit_mask = mi_commit_mask_full();
}
} }
else { else {
*commit = committed; */
} *commit_mask = cmask;
// mark the slot as free again // mark the slot as free again
mi_assert_internal(mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx));
@ -195,6 +205,30 @@ static void* mi_cache_pop(int numa_node, size_t size, size_t alignment, bool* co
return p; return p;
} }
static void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats) {
if (mi_commit_mask_is_empty(*cmask)) {
// nothing
}
else if (mi_commit_mask_is_full(*cmask)) {
_mi_os_decommit(p, total, stats);
}
else {
// todo: one call to decommit the whole at once?
mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
size_t part = total/MI_COMMIT_MASK_BITS;
uintptr_t idx;
uintptr_t count;
mi_commit_mask_t mask = *cmask;
mi_commit_mask_foreach(mask, idx, count) {
void* start = (uint8_t*)p + (idx*part);
size_t size = count*part;
_mi_os_decommit(start, size, stats);
}
mi_commit_mask_foreach_end()
}
*cmask = mi_commit_mask_empty();
}
static void mi_cache_purge(mi_os_tld_t* tld) { static void mi_cache_purge(mi_os_tld_t* tld) {
UNUSED(tld); UNUSED(tld);
mi_msecs_t now = _mi_clock_now(); mi_msecs_t now = _mi_clock_now();
@ -214,10 +248,11 @@ static void mi_cache_purge(mi_os_tld_t* tld) {
if (expire != 0 && now >= expire) { // safe read if (expire != 0 && now >= expire) { // safe read
// still expired, decommit it // still expired, decommit it
mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
mi_assert_internal(slot->is_committed && mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(!mi_commit_mask_is_empty(slot->commit_mask) && mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx));
_mi_abandoned_await_readers(); // wait until safe to decommit _mi_abandoned_await_readers(); // wait until safe to decommit
_mi_os_decommit(slot->p, MI_SEGMENT_SIZE, tld->stats); // decommit committed parts
slot->is_committed = false; mi_commit_mask_decommit(&slot->commit_mask, slot->p, MI_SEGMENT_SIZE, tld->stats);
//_mi_os_decommit(slot->p, MI_SEGMENT_SIZE, tld->stats);
} }
mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop
} }
@ -226,7 +261,7 @@ static void mi_cache_purge(mi_os_tld_t* tld) {
} }
} }
static bool mi_cache_push(void* start, size_t size, size_t memid, bool is_committed, bool is_large, mi_os_tld_t* tld) static bool mi_cache_push(void* start, size_t size, size_t memid, mi_commit_mask_t commit_mask, bool is_large, mi_os_tld_t* tld)
{ {
// only for segment blocks // only for segment blocks
if (size != MI_SEGMENT_SIZE || ((uintptr_t)start % MI_SEGMENT_ALIGN) != 0) return false; if (size != MI_SEGMENT_SIZE || ((uintptr_t)start % MI_SEGMENT_ALIGN) != 0) return false;
@ -255,13 +290,12 @@ static bool mi_cache_push(void* start, size_t size, size_t memid, bool is_commit
slot->p = start; slot->p = start;
slot->memid = memid; slot->memid = memid;
mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
slot->is_committed = is_committed; slot->commit_mask = commit_mask;
if (is_committed && !is_large) { if (!mi_commit_mask_is_empty(commit_mask) && !is_large) {
long delay = mi_option_get(mi_option_arena_reset_delay); long delay = mi_option_get(mi_option_arena_reset_delay);
if (delay == 0) { if (delay == 0) {
_mi_abandoned_await_readers(); // wait until safe to decommit _mi_abandoned_await_readers(); // wait until safe to decommit
_mi_os_decommit(start, size, tld->stats); mi_commit_mask_decommit(&slot->commit_mask, start, MI_SEGMENT_SIZE, tld->stats);
slot->is_committed = false;
} }
else { else {
mi_atomic_storei64_release(&slot->expire, _mi_clock_now() + delay); mi_atomic_storei64_release(&slot->expire, _mi_clock_now() + delay);
@ -311,10 +345,10 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n
} }
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
bool* commit, bool* large, bool* is_zero, bool commit, mi_commit_mask_t* commit_mask, bool* large, bool* is_zero,
size_t* memid, mi_os_tld_t* tld) size_t* memid, mi_os_tld_t* tld)
{ {
mi_assert_internal(commit != NULL && large != NULL && is_zero != NULL && memid != NULL && tld != NULL); mi_assert_internal(commit_mask != NULL && large != NULL && is_zero != NULL && memid != NULL && tld != NULL);
mi_assert_internal(size > 0); mi_assert_internal(size > 0);
*memid = MI_MEMID_OS; *memid = MI_MEMID_OS;
*is_zero = false; *is_zero = false;
@ -339,9 +373,13 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
if ((arena->numa_node<0 || arena->numa_node==numa_node) && // numa local? if ((arena->numa_node<0 || arena->numa_node==numa_node) && // numa local?
(*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages
{ {
void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_zero, memid, tld); bool acommit = commit;
void* p = mi_arena_alloc_from(arena, i, bcount, &acommit, large, is_zero, memid, tld);
mi_assert_internal((uintptr_t)p % alignment == 0); mi_assert_internal((uintptr_t)p % alignment == 0);
if (p != NULL) return p; if (p != NULL) {
*commit_mask = (acommit ? mi_commit_mask_full() : mi_commit_mask_empty());
return p;
}
} }
} }
// try from another numa node instead.. // try from another numa node instead..
@ -351,43 +389,52 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
if ((arena->numa_node>=0 && arena->numa_node!=numa_node) && // not numa local! if ((arena->numa_node>=0 && arena->numa_node!=numa_node) && // not numa local!
(*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages (*large || !arena->is_large)) // large OS pages allowed, or arena is not large OS pages
{ {
void* p = mi_arena_alloc_from(arena, i, bcount, commit, large, is_zero, memid, tld); bool acommit = commit;
void* p = mi_arena_alloc_from(arena, i, bcount, &acommit, large, is_zero, memid, tld);
mi_assert_internal((uintptr_t)p % alignment == 0); mi_assert_internal((uintptr_t)p % alignment == 0);
if (p != NULL) return p; if (p != NULL) {
*commit_mask = (acommit ? mi_commit_mask_full() : mi_commit_mask_empty());
return p;
}
} }
} }
} }
// try to get from the cache // try to get from the cache
void* p = mi_cache_pop(numa_node, size, alignment, commit, large, is_zero, memid, tld); void* p = mi_cache_pop(numa_node, size, alignment, commit, commit_mask, large, is_zero, memid, tld);
if (p != NULL) return p; if (p != NULL) return p;
// finally, fall back to the OS // finally, fall back to the OS
*is_zero = true; *is_zero = true;
*memid = MI_MEMID_OS; *memid = MI_MEMID_OS;
return _mi_os_alloc_aligned(size, alignment, *commit, large, tld); p = _mi_os_alloc_aligned(size, alignment, commit, large, tld);
*commit_mask = ((p!=NULL && commit) ? mi_commit_mask_full() : mi_commit_mask_empty());
return p;
} }
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld) void* _mi_arena_alloc(size_t size, bool commit, mi_commit_mask_t* commit_mask, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
{ {
return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, large, is_zero, memid, tld); return _mi_arena_alloc_aligned(size, MI_ARENA_BLOCK_SIZE, commit, commit_mask, large, is_zero, memid, tld);
} }
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Arena free Arena free
----------------------------------------------------------- */ ----------------------------------------------------------- */
void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, bool is_large, mi_os_tld_t* tld) { void _mi_arena_free(void* p, size_t size, size_t memid, mi_commit_mask_t commit_mask, bool is_large, mi_os_tld_t* tld) {
mi_assert_internal(size > 0 && tld->stats != NULL); mi_assert_internal(size > 0 && tld->stats != NULL);
if (p==NULL) return; if (p==NULL) return;
if (size==0) return; if (size==0) return;
if (memid == MI_MEMID_OS) { if (memid == MI_MEMID_OS) {
// was a direct OS allocation, pass through // was a direct OS allocation, pass through
if (!mi_cache_push(p, size, memid, is_committed, is_large, tld)) { if (!mi_cache_push(p, size, memid, commit_mask, is_large, tld)) {
_mi_abandoned_await_readers(); // wait unti safe to free _mi_abandoned_await_readers(); // wait until safe to free
_mi_os_free_ex(p, size, is_committed, tld->stats); // TODO: is it safe on all platforms to free even it contains decommitted parts? (eg. macOS)
const size_t csize = mi_commit_mask_committed_size(commit_mask, size);
_mi_stat_decrease(&_mi_stats_main.committed, csize);
_mi_os_free_ex(p, size, false /*pretend decommitted to not double count stats*/, tld->stats);
} }
} }
else { else {

4
src/options.c
View file

@ -65,7 +65,7 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(verbose) }, { 0, UNINIT, MI_OPTION(verbose) },
// the following options are experimental and not all combinations make sense. // the following options are experimental and not all combinations make sense.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`) { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (8MiB) (but see also `eager_commit_delay`)
#if defined(_WIN32) || (MI_INTPTR_SIZE <= 4) // and other OS's without overcommit? #if defined(_WIN32) || (MI_INTPTR_SIZE <= 4) // and other OS's without overcommit?
{ 0, UNINIT, MI_OPTION(eager_region_commit) }, { 0, UNINIT, MI_OPTION(eager_region_commit) },
{ 0, UNINIT, MI_OPTION(reset_decommits) }, // reset decommits memory { 0, UNINIT, MI_OPTION(reset_decommits) }, // reset decommits memory
@ -82,7 +82,7 @@ static mi_option_desc_t options[_mi_option_last] =
#if defined(__NetBSD__) #if defined(__NetBSD__)
{ 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
#else #else
{ 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) { 4, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)
#endif #endif
{ 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit pages when not eager committed { 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit pages when not eager committed
{ 250, UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds { 250, UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds

25
src/segment.c
View file

@ -137,7 +137,7 @@ static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(segment->abandoned <= segment->used); mi_assert_internal(segment->abandoned <= segment->used);
mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id()); mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id());
mi_assert_internal((segment->commit_mask & segment->decommit_mask) == segment->decommit_mask); // can only decommit committed blocks mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask)); // can only decommit committed blocks
//mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0); //mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0);
mi_slice_t* slice = &segment->slices[0]; mi_slice_t* slice = &segment->slices[0];
const mi_slice_t* end = mi_segment_slices_end(segment); const mi_slice_t* end = mi_segment_slices_end(segment);
@ -268,8 +268,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
// mi_segment_delayed_decommit(segment,true,tld->stats); // mi_segment_delayed_decommit(segment,true,tld->stats);
// _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats); // _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
bool fully_committed = (mi_commit_mask_is_full(segment->commit_mask) && mi_commit_mask_is_empty(segment->decommit_mask)); _mi_arena_free(segment, mi_segment_size(segment), segment->memid, segment->commit_mask, segment->mem_is_fixed, tld->os);
_mi_arena_free(segment, mi_segment_size(segment), segment->memid, fully_committed, segment->mem_is_fixed, tld->os);
} }
@ -382,11 +381,15 @@ static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s
if (commit && !mi_commit_mask_all_set(segment->commit_mask, mask)) { if (commit && !mi_commit_mask_all_set(segment->commit_mask, mask)) {
bool is_zero = false; bool is_zero = false;
mi_commit_mask_t cmask = mi_commit_mask_intersect(segment->commit_mask, mask);
_mi_stat_decrease(&_mi_stats_main.committed, mi_commit_mask_committed_size(cmask, MI_SEGMENT_SIZE)); // adjust for overlap
if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false; if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false;
mi_commit_mask_set(&segment->commit_mask,mask); mi_commit_mask_set(&segment->commit_mask,mask);
} }
else if (!commit && mi_commit_mask_any_set(segment->commit_mask,mask)) { else if (!commit && mi_commit_mask_any_set(segment->commit_mask,mask)) {
mi_assert_internal((void*)start != (void*)segment); mi_assert_internal((void*)start != (void*)segment);
mi_commit_mask_t cmask = mi_commit_mask_intersect(segment->commit_mask, mask);
_mi_stat_increase(&_mi_stats_main.committed, full_size - mi_commit_mask_committed_size(cmask, MI_SEGMENT_SIZE)); // adjust for overlap
_mi_os_decommit(start, full_size, stats); // ok if this fails _mi_os_decommit(start, full_size, stats); // ok if this fails
mi_commit_mask_clear(&segment->commit_mask, mask); mi_commit_mask_clear(&segment->commit_mask, mask);
} }
@ -401,6 +404,7 @@ static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s
} }
static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask));
if (mi_commit_mask_is_full(segment->commit_mask) && mi_commit_mask_is_empty(segment->decommit_mask)) return true; // fully committed if (mi_commit_mask_is_full(segment->commit_mask) && mi_commit_mask_is_empty(segment->decommit_mask)) return true; // fully committed
return mi_segment_commitx(segment,true,p,size,stats); return mi_segment_commitx(segment,true,p,size,stats);
} }
@ -648,29 +652,30 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
// Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little)
const bool eager_delay = (tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager_delay = (tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay));
const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit); const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit);
bool commit = eager || (required > 0); const bool commit = eager || (required > 0);
// Try to get from our cache first // Try to get from our cache first
bool is_zero = false; bool is_zero = false;
const bool commit_info_still_good = (segment != NULL); const bool commit_info_still_good = (segment != NULL);
mi_commit_mask_t commit_mask = (segment != NULL ? segment->commit_mask : mi_commit_mask_empty());
if (segment==NULL) { if (segment==NULL) {
// Allocate the segment from the OS // Allocate the segment from the OS
bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy
size_t memid = 0; size_t memid = 0;
// segment = (mi_segment_t*)_mi_os_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &mem_large, os_tld); segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &commit_mask, &mem_large, &is_zero, &memid, os_tld);
segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_zero, &memid, os_tld);
if (segment == NULL) return NULL; // failed to allocate if (segment == NULL) return NULL; // failed to allocate
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
if (!commit) {
if (!mi_commit_mask_all_set(commit_mask,mi_commit_mask_create(0, 1))) {
// at least commit the info slices // at least commit the info slices
mi_assert_internal(MI_COMMIT_SIZE > info_slices*MI_SEGMENT_SLICE_SIZE); mi_assert_internal(MI_COMMIT_SIZE > info_slices*MI_SEGMENT_SLICE_SIZE);
bool ok = _mi_os_commit(segment, MI_COMMIT_SIZE, &is_zero, tld->stats); bool ok = _mi_os_commit(segment, MI_COMMIT_SIZE, &is_zero, tld->stats);
if (!ok) return NULL; // failed to commit if (!ok) return NULL; // failed to commit
mi_commit_mask_set(&commit_mask,mi_commit_mask_create(0, 1));
} }
segment->memid = memid; segment->memid = memid;
segment->mem_is_fixed = mem_large; segment->mem_is_fixed = mem_large;
segment->mem_is_committed = commit; segment->mem_is_committed = mi_commit_mask_is_full(commit_mask);
mi_segments_track_size((long)(segment_size), tld); mi_segments_track_size((long)(segment_size), tld);
mi_segment_map_allocated_at(segment); mi_segment_map_allocated_at(segment);
} }
@ -684,7 +689,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
} }
if (!commit_info_still_good) { if (!commit_info_still_good) {
segment->commit_mask = (!commit ? 0x01 : mi_commit_mask_full()); // on lazy commit, the initial part is always committed segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed
segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_fixed); segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_fixed);
segment->decommit_expire = 0; segment->decommit_expire = 0;
segment->decommit_mask = mi_commit_mask_empty(); segment->decommit_mask = mi_commit_mask_empty();