kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-04 22:49:32 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

push os abandoned blocks at the tail end

Browse source
This commit is contained in:
daanx 2024-06-03 15:37:05 -07:00
parent a04905c88b
commit 3333f776f9
2 changed files with 76 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
include/mimalloc/types.h
View file

@ -611,10 +611,12 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
// ------------------------------------------------------ // ------------------------------------------------------
struct mi_subproc_s { struct mi_subproc_s {
_Atomic(size_t) abandoned_count; // count of abandoned segments for this sup-process _Atomic(size_t) abandoned_count; // count of abandoned segments for this sub-process
_Atomic(size_t) abandoned_os_list_count; // count of abandoned segments in the os-list
mi_lock_t abandoned_os_lock; // lock for the abandoned segments outside of arena's mi_lock_t abandoned_os_lock; // lock for the abandoned segments outside of arena's
mi_segment_t* abandoned_os_list; // doubly-linked list of abandoned segments outside of arena's (in OS allocated memory) mi_segment_t* abandoned_os_list; // doubly-linked list of abandoned segments outside of arena's (in OS allocated memory)
mi_memid_t memid; // provenance mi_segment_t* abandoned_os_list_tail; // the tail-end of the list
mi_memid_t memid; // provenance of this memory block
}; };
// ------------------------------------------------------ // ------------------------------------------------------

65
src/arena.c
View file

@ -753,11 +753,10 @@ bool _mi_arena_contains(const void* p) {
the arena bitmaps. the arena bitmaps.
----------------------------------------------------------- */ ----------------------------------------------------------- */
// reclaim a specific abandoned segment; `true` on success. // reclaim a specific OS abandoned segment; `true` on success.
// sets the thread_id. // sets the thread_id.
bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment ) static bool mi_arena_segment_os_clear_abandoned(mi_segment_t* segment) {
{ mi_assert(segment->memid.memkind != MI_MEM_ARENA);
if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) {
// not in an arena, remove from list of abandoned os segments // not in an arena, remove from list of abandoned os segments
mi_subproc_t* const subproc = segment->subproc; mi_subproc_t* const subproc = segment->subproc;
if (!mi_lock_try_acquire(&subproc->abandoned_os_lock)) { if (!mi_lock_try_acquire(&subproc->abandoned_os_lock)) {
@ -780,6 +779,7 @@ bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment )
if (prev != NULL) { prev->abandoned_os_next = next; } if (prev != NULL) { prev->abandoned_os_next = next; }
else { subproc->abandoned_os_list = next; } else { subproc->abandoned_os_list = next; }
if (next != NULL) { next->abandoned_os_prev = prev; } if (next != NULL) { next->abandoned_os_prev = prev; }
else { subproc->abandoned_os_list_tail = prev; }
segment->abandoned_os_next = NULL; segment->abandoned_os_next = NULL;
segment->abandoned_os_prev = NULL; segment->abandoned_os_prev = NULL;
mi_atomic_decrement_relaxed(&segment->subproc->abandoned_count); mi_atomic_decrement_relaxed(&segment->subproc->abandoned_count);
@ -790,6 +790,12 @@ bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment )
return reclaimed; return reclaimed;
} }
// reclaim a specific abandoned segment; `true` on success.
// sets the thread_id.
bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment ) {
if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) {
return mi_arena_segment_os_clear_abandoned(segment);
}
// arena segment: use the blocks_abandoned bitmap. // arena segment: use the blocks_abandoned bitmap.
size_t arena_idx; size_t arena_idx;
size_t bitmap_idx; size_t bitmap_idx;
@ -810,6 +816,34 @@ bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment )
return was_marked; return was_marked;
} }
// mark a specific OS segment as abandoned
static void mi_arena_segment_os_mark_abandoned(mi_segment_t* segment) {
mi_assert(segment->memid.memkind != MI_MEM_ARENA);
// not in an arena; we use a list of abandoned segments
mi_subproc_t* const subproc = segment->subproc;
if (!mi_lock_acquire(&subproc->abandoned_os_lock)) {
_mi_error_message(EFAULT, "internal error: failed to acquire the abandoned (os) segment lock to mark abandonment");
// we can continue but cannot visit/reclaim such blocks..
}
else {
mi_atomic_increment_relaxed(&subproc->abandoned_count);
// push on the tail of the list (important for the visitor)
mi_segment_t* prev = subproc->abandoned_os_list_tail;
mi_assert_internal(prev == NULL || prev->abandoned_os_next == NULL);
mi_assert_internal(segment->abandoned_os_prev == NULL);
mi_assert_internal(segment->abandoned_os_next == NULL);
if (prev != NULL) { prev->abandoned_os_next = segment; }
else { subproc->abandoned_os_list = segment; }
subproc->abandoned_os_list_tail = segment;
segment->abandoned_os_prev = prev;
segment->abandoned_os_next = NULL;
// and release the lock
mi_lock_release(&subproc->abandoned_os_lock);
}
return;
}
// mark a specific segment as abandoned // mark a specific segment as abandoned
// clears the thread_id. // clears the thread_id.
void _mi_arena_segment_mark_abandoned(mi_segment_t* segment) void _mi_arena_segment_mark_abandoned(mi_segment_t* segment)
@ -817,28 +851,9 @@ void _mi_arena_segment_mark_abandoned(mi_segment_t* segment)
mi_assert_internal(segment->used == segment->abandoned); mi_assert_internal(segment->used == segment->abandoned);
mi_atomic_store_release(&segment->thread_id, 0); // mark as abandoned for multi-thread free's mi_atomic_store_release(&segment->thread_id, 0); // mark as abandoned for multi-thread free's
if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) { if mi_unlikely(segment->memid.memkind != MI_MEM_ARENA) {
// not in an arena; we use a list of abandoned segments mi_arena_segment_os_mark_abandoned(segment);
if (!mi_lock_acquire(&segment->subproc->abandoned_os_lock)) {
_mi_error_message(EFAULT, "internal error: failed to acquire the abandoned (os) segment lock to mark abandonment");
// we can continue but cannot visit/reclaim such blocks..
}
else {
mi_atomic_increment_relaxed(&segment->subproc->abandoned_count);
// push on the front of the list
mi_segment_t* next = segment->subproc->abandoned_os_list;
mi_assert_internal(next == NULL || next->abandoned_os_prev == NULL);
mi_assert_internal(segment->abandoned_os_prev == NULL);
mi_assert_internal(segment->abandoned_os_next == NULL);
if (next != NULL) { next->abandoned_os_prev = segment; }
segment->abandoned_os_prev = NULL;
segment->abandoned_os_next = next;
segment->subproc->abandoned_os_list = segment;
// and release the lock
mi_lock_release(&segment->subproc->abandoned_os_lock);
}
return; return;
} }
// segment is in an arena, mark it in the arena `blocks_abandoned` bitmap // segment is in an arena, mark it in the arena `blocks_abandoned` bitmap
size_t arena_idx; size_t arena_idx;
size_t bitmap_idx; size_t bitmap_idx;
@ -853,6 +868,8 @@ void _mi_arena_segment_mark_abandoned(mi_segment_t* segment)
mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx));
} }
// start a cursor at a randomized arena // start a cursor at a randomized arena
void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, mi_arena_field_cursor_t* current) { void _mi_arena_field_cursor_init(mi_heap_t* heap, mi_subproc_t* subproc, mi_arena_field_cursor_t* current) {
mi_assert_internal(heap == NULL || heap->tld->segments.subproc == subproc); mi_assert_internal(heap == NULL || heap->tld->segments.subproc == subproc);