kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-10 09:19:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

limit purgeing to one purge cycle per purge delay

Browse source
This commit is contained in:
daanx 2024-12-21 15:24:46 -08:00
parent e3ebebb990
commit 476d4699ff
2 changed files with 56 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
include/mimalloc/atomic.h
View file

@ -421,9 +421,8 @@ static inline void mi_atomic_yield(void) {
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return TryAcquireSRWLockExclusive(lock); return TryAcquireSRWLockExclusive(lock);
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
AcquireSRWLockExclusive(lock); AcquireSRWLockExclusive(lock);
return true;
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
ReleaseSRWLockExclusive(lock); ReleaseSRWLockExclusive(lock);
@ -432,7 +431,7 @@ static inline void mi_lock_init(mi_lock_t* lock) {
InitializeSRWLock(lock); InitializeSRWLock(lock);
} }
static inline void mi_lock_done(mi_lock_t* lock) { static inline void mi_lock_done(mi_lock_t* lock) {
// nothing (void)(lock);
} }
#else #else
@ -440,24 +439,20 @@ static inline void mi_lock_done(mi_lock_t* lock) {
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return TryEnterCriticalSection(lock); return TryEnterCriticalSection(lock);
} }
static inline void mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
EnterCriticalSection(lock); EnterCriticalSection(lock);
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
LeaveCriticalSection(lock); LeaveCriticalSection(lock);
} }
static inline void mi_lock_init(mi_lock_t* lock) { static inline void mi_lock_init(mi_lock_t* lock) {
InitializeCriticalSection(lock); InitializeCriticalSection(lock);
} }
static inline void mi_lock_done(mi_lock_t* lock) { static inline void mi_lock_done(mi_lock_t* lock) {
DeleteCriticalSection(lock); DeleteCriticalSection(lock);
} }
#endif #endif
#elif defined(MI_USE_PTHREADS) #elif defined(MI_USE_PTHREADS)
@ -467,8 +462,11 @@ static inline void mi_lock_done(mi_lock_t* lock) {
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return (pthread_mutex_trylock(lock) == 0); return (pthread_mutex_trylock(lock) == 0);
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
return (pthread_mutex_lock(lock) == 0); const int err = pthread_mutex_lock(lock);
if (err != 0) {
mi_error_message(EFAULT, "internal error: lock cannot be acquired\n");
}
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
pthread_mutex_unlock(lock); pthread_mutex_unlock(lock);
@ -488,9 +486,8 @@ static inline void mi_lock_done(mi_lock_t* lock) {
static inline bool mi_lock_try_acquire(mi_lock_t* lock) { static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
return lock->try_lock(); return lock->try_lock();
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
lock->lock(); lock->lock();
return true;
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
lock->unlock(); lock->unlock();
@ -513,12 +510,11 @@ static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
uintptr_t expected = 0; uintptr_t expected = 0;
return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1); return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1);
} }
static inline bool mi_lock_acquire(mi_lock_t* lock) { static inline void mi_lock_acquire(mi_lock_t* lock) {
for (int i = 0; i < 1000; i++) { // for at most 1000 tries? for (int i = 0; i < 1000; i++) { // for at most 1000 tries?
if (mi_lock_try_acquire(lock)) return true; if (mi_lock_try_acquire(lock)) return;
mi_atomic_yield(); mi_atomic_yield();
} }
return true;
} }
static inline void mi_lock_release(mi_lock_t* lock) { static inline void mi_lock_release(mi_lock_t* lock) {
mi_atomic_store_release(lock, (uintptr_t)0); mi_atomic_store_release(lock, (uintptr_t)0);

69
src/arena.c
View file

@ -33,7 +33,7 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
typedef struct mi_arena_s { typedef struct mi_arena_s {
mi_arena_id_t id; // arena id; 0 for non-specific mi_arena_id_t id; // arena id; 0 for non-specific
mi_memid_t memid; // memid of the memory area mi_memid_t memid; // memid of the memory area
_Atomic(uint8_t*)start; // the start of the memory area _Atomic(uint8_t*) start; // the start of the memory area
size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`) size_t block_count; // size of the area in arena blocks (of `MI_ARENA_BLOCK_SIZE`)
size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`)
size_t meta_size; // size of the arena structure itself (including its bitmaps) size_t meta_size; // size of the arena structure itself (including its bitmaps)
@ -42,12 +42,13 @@ typedef struct mi_arena_s {
bool exclusive; // only allow allocations if specifically for this arena bool exclusive; // only allow allocations if specifically for this arena
bool is_large; // memory area consists of large- or huge OS pages (always committed) bool is_large; // memory area consists of large- or huge OS pages (always committed)
mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited
_Atomic(size_t)search_idx; // optimization to start the search for free blocks _Atomic(size_t) search_idx; // optimization to start the search for free blocks
_Atomic(mi_msecs_t)purge_expire; // expiration time when blocks should be decommitted from `blocks_decommit`. _Atomic(mi_msecs_t) purge_expire; // expiration time when blocks should be purged from `blocks_purge`.
mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero?
mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero?
mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted) mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted)
mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here) mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted)
mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here)
mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)
// do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields. // do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields.
} mi_arena_t; } mi_arena_t;
@ -60,6 +61,7 @@ typedef struct mi_arena_s {
// The available arenas // The available arenas
static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS]; static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS];
static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0
static mi_decl_cache_align _Atomic(int64_t) mi_arenas_purge_expire; // set if there exist purgeable arenas
#define MI_IN_ARENA_C #define MI_IN_ARENA_C
#include "arena-abandon.c" #include "arena-abandon.c"
@ -349,11 +351,10 @@ static mi_decl_noinline void* mi_arena_try_alloc(int numa_node, size_t size, siz
} }
// try to reserve a fresh arena space // try to reserve a fresh arena space
static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t req_arena_id, mi_arena_id_t *arena_id) static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t *arena_id)
{ {
if (_mi_preloading()) return false; // use OS only while pre loading if (_mi_preloading()) return false; // use OS only while pre loading
if (req_arena_id != _mi_arena_id_none()) return false;
const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count); const size_t arena_count = mi_atomic_load_acquire(&mi_arena_count);
if (arena_count > (MI_MAX_ARENAS - 4)) return false; if (arena_count > (MI_MAX_ARENAS - 4)) return false;
@ -403,7 +404,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset
// otherwise, try to first eagerly reserve a new arena // otherwise, try to first eagerly reserve a new arena
if (req_arena_id == _mi_arena_id_none()) { if (req_arena_id == _mi_arena_id_none()) {
mi_arena_id_t arena_id = 0; mi_arena_id_t arena_id = 0;
if (mi_arena_reserve(size, allow_large, req_arena_id, &arena_id)) { if (mi_arena_reserve(size, allow_large, &arena_id)) {
// and try allocate in there // and try allocate in there
mi_assert_internal(req_arena_id == _mi_arena_id_none()); mi_assert_internal(req_arena_id == _mi_arena_id_none());
p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid); p = mi_arena_try_alloc_at_id(arena_id, true, numa_node, size, alignment, commit, allow_large, req_arena_id, memid);
@ -497,13 +498,16 @@ static void mi_arena_schedule_purge(mi_arena_t* arena, size_t bitmap_idx, size_t
mi_arena_purge(arena, bitmap_idx, blocks); mi_arena_purge(arena, bitmap_idx, blocks);
} }
else { else {
// schedule decommit // schedule purge
mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); const mi_msecs_t expire = _mi_clock_now() + delay;
if (expire != 0) { mi_msecs_t expire0 = 0;
mi_atomic_addi64_acq_rel(&arena->purge_expire, (mi_msecs_t)(delay/10)); // add smallish extra delay if (mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire0, expire)) {
// expiration was not yet set
// maybe set the global arenas expire as well (if it wasn't set already)
mi_atomic_casi64_strong_acq_rel(&mi_arenas_purge_expire, &expire0, expire);
} }
else { else {
mi_atomic_storei64_release(&arena->purge_expire, _mi_clock_now() + delay); // already an expiration was set
} }
_mi_bitmap_claim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx, NULL); _mi_bitmap_claim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx, NULL);
} }
@ -538,14 +542,16 @@ static bool mi_arena_purge_range(mi_arena_t* arena, size_t idx, size_t startidx,
// returns true if anything was purged // returns true if anything was purged
static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
{ {
if (arena->memid.is_pinned || arena->blocks_purge == NULL) return false; // check pre-conditions
if (arena->memid.is_pinned) return false;
// expired yet?
mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
if (expire == 0) return false; if (!force && (expire == 0 || expire > now)) return false;
if (!force && expire > now) return false;
// reset expire (if not already set concurrently) // reset expire (if not already set concurrently)
mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0); mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &expire, (mi_msecs_t)0);
// potential purges scheduled, walk through the bitmap // potential purges scheduled, walk through the bitmap
bool any_purged = false; bool any_purged = false;
bool full_purge = true; bool full_purge = true;
@ -592,9 +598,15 @@ static bool mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force)
return any_purged; return any_purged;
} }
static void mi_arenas_try_purge( bool force, bool visit_all ) { static void mi_arenas_try_purge( bool force, bool visit_all )
{
if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled if (_mi_preloading() || mi_arena_purge_delay() <= 0) return; // nothing will be scheduled
// check if any arena needs purging?
const mi_msecs_t now = _mi_clock_now();
mi_msecs_t arenas_expire = mi_atomic_load_acquire(&mi_arenas_purge_expire);
if (!force && (arenas_expire == 0 || arenas_expire < now)) return;
const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count); const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count);
if (max_arena == 0) return; if (max_arena == 0) return;
@ -602,17 +614,26 @@ static void mi_arenas_try_purge( bool force, bool visit_all ) {
static mi_atomic_guard_t purge_guard; static mi_atomic_guard_t purge_guard;
mi_atomic_guard(&purge_guard) mi_atomic_guard(&purge_guard)
{ {
mi_msecs_t now = _mi_clock_now(); // increase global expire: at most one purge per delay cycle
size_t max_purge_count = (visit_all ? max_arena : 1); mi_atomic_store_release(&mi_arenas_purge_expire, now + mi_arena_purge_delay());
size_t max_purge_count = (visit_all ? max_arena : 2);
bool all_visited = true;
for (size_t i = 0; i < max_arena; i++) { for (size_t i = 0; i < max_arena; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]); mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[i]);
if (arena != NULL) { if (arena != NULL) {
if (mi_arena_try_purge(arena, now, force)) { if (mi_arena_try_purge(arena, now, force)) {
if (max_purge_count <= 1) break; if (max_purge_count <= 1) {
all_visited = false;
break;
}
max_purge_count--; max_purge_count--;
} }
} }
} }
if (all_visited) {
// all arena's were visited and purged: reset global expire
mi_atomic_store_release(&mi_arenas_purge_expire, 0);
}
} }
} }