kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-07-01 17:24:38 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip: initial large bitmaps

Browse source
This commit is contained in:
daanx 2024-12-03 22:43:14 -08:00
parent 8d9c725482
commit e5fdd6e110
5 changed files with 501 additions and 440 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

162
src/arena.c
View file

@ -37,18 +37,20 @@ typedef struct mi_arena_s {
mi_arena_id_t id; // arena id; 0 for non-specific mi_arena_id_t id; // arena id; 0 for non-specific
size_t slice_count; // size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`) size_t slice_count; // size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`)
size_t info_slices; // initial slices reserved for the arena bitmaps
int numa_node; // associated NUMA node int numa_node; // associated NUMA node
bool exclusive; // only allow allocations if specifically for this arena bool exclusive; // only allow allocations if specifically for this arena
bool is_large; // memory area consists of large- or huge OS pages (always committed) bool is_large; // memory area consists of large- or huge OS pages (always committed)
mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited mi_lock_t abandoned_visit_lock; // lock is only used when abandoned segments are being visited
_Atomic(mi_msecs_t) purge_expire; // expiration time when slices should be decommitted from `slices_decommit`. _Atomic(mi_msecs_t) purge_expire; // expiration time when slices should be decommitted from `slices_decommit`.
mi_bitmap_t slices_free; // is the slice free? mi_bitmap_t* slices_free; // is the slice free?
mi_bitmap_t slices_committed; // is the slice committed? (i.e. accessible) mi_bitmap_t* slices_committed; // is the slice committed? (i.e. accessible)
mi_bitmap_t slices_purge; // can the slice be purged? (slice in purge => slice in free) mi_bitmap_t* slices_purge; // can the slice be purged? (slice in purge => slice in free)
mi_bitmap_t slices_dirty; // is the slice potentially non-zero? mi_bitmap_t* slices_dirty; // is the slice potentially non-zero?
mi_pairmap_t pages_abandoned[MI_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page) mi_pairmap_t pages_abandoned[MI_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page)
// the full queue contains abandoned full pages // the full queue contains abandoned full pages
// followed by the bitmaps (whose size depends on the arena size)
} mi_arena_t; } mi_arena_t;
#define MI_MAX_ARENAS (1024) // Limited for now (and takes up .bss) #define MI_MAX_ARENAS (1024) // Limited for now (and takes up .bss)
@ -58,6 +60,7 @@ static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS];
static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Arena id's Arena id's
id = arena_index + 1 id = arena_index + 1
@ -103,6 +106,11 @@ mi_arena_t* mi_arena_from_id(mi_arena_id_t id) {
return mi_arena_from_index(mi_arena_id_index(id)); return mi_arena_from_index(mi_arena_id_index(id));
} }
static size_t mi_arena_info_slices(mi_arena_t* arena) {
return arena->info_slices;
}
/* ----------------------------------------------------------- /* -----------------------------------------------------------
Util Util
@ -114,14 +122,6 @@ static size_t mi_arena_size(mi_arena_t* arena) {
return mi_size_of_slices(arena->slice_count); return mi_size_of_slices(arena->slice_count);
} }
static size_t mi_arena_info_slices(void) {
const size_t os_page_size = _mi_os_page_size();
const size_t info_size = _mi_align_up(sizeof(mi_arena_t), os_page_size) + os_page_size; // + guard page
const size_t info_slices = mi_slice_count_of_size(info_size);
return info_slices;
}
// Start of the arena memory area // Start of the arena memory area
static uint8_t* mi_arena_start(mi_arena_t* arena) { static uint8_t* mi_arena_start(mi_arena_t* arena) {
return ((uint8_t*)arena); return ((uint8_t*)arena);
@ -187,7 +187,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid) mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)
{ {
size_t slice_index; size_t slice_index;
if (!mi_bitmap_try_find_and_clearN(&arena->slices_free, slice_count, tseq, &slice_index)) return NULL; if (!mi_bitmap_try_find_and_clearN(arena->slices_free, slice_count, tseq, &slice_index)) return NULL;
// claimed it! // claimed it!
void* p = mi_arena_slice_start(arena, slice_index); void* p = mi_arena_slice_start(arena, slice_index);
@ -197,7 +197,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
// set the dirty bits // set the dirty bits
if (arena->memid.initially_zero) { if (arena->memid.initially_zero) {
// size_t dirty_count = 0; // size_t dirty_count = 0;
memid->initially_zero = mi_bitmap_setN(&arena->slices_dirty, slice_index, slice_count, NULL); memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, NULL);
//if (dirty_count>0) { //if (dirty_count>0) {
// if (memid->initially_zero) { // if (memid->initially_zero) {
// _mi_error_message(EFAULT, "ouch1\n"); // _mi_error_message(EFAULT, "ouch1\n");
@ -217,7 +217,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
memid->initially_committed = true; memid->initially_committed = true;
// commit requested, but the range may not be committed as a whole: ensure it is committed now // commit requested, but the range may not be committed as a whole: ensure it is committed now
if (!mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)) { if (!mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)) {
// not fully committed: commit the full range and set the commit bits // not fully committed: commit the full range and set the commit bits
// (this may race and we may double-commit which is fine) // (this may race and we may double-commit which is fine)
bool commit_zero = false; bool commit_zero = false;
@ -235,7 +235,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
} }
#endif #endif
size_t already_committed_count = 0; size_t already_committed_count = 0;
mi_bitmap_setN(&arena->slices_committed, slice_index, slice_count, &already_committed_count); mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
if (already_committed_count < slice_count) { if (already_committed_count < slice_count) {
// todo: also decrease total // todo: also decrease total
mi_stat_decrease(_mi_stats_main.committed, mi_size_of_slices(already_committed_count)); mi_stat_decrease(_mi_stats_main.committed, mi_size_of_slices(already_committed_count));
@ -245,13 +245,13 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
} }
else { else {
// no need to commit, but check if already fully committed // no need to commit, but check if already fully committed
memid->initially_committed = mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count); memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
} }
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
if (commit) { mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); } if (commit) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); }
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_dirty, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
// mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_purge, slice_index, slice_count));
return p; return p;
} }
@ -285,8 +285,8 @@ static bool mi_arena_reserve(size_t req_size, bool allow_large, mi_arena_id_t re
} }
// check arena bounds // check arena bounds
const size_t min_reserve = mi_size_of_slices(mi_arena_info_slices() + 1); const size_t min_reserve = 8; // hope that fits minimal bitmaps?
const size_t max_reserve = MI_BITMAP_MAX_BITS * MI_ARENA_SLICE_SIZE; const size_t max_reserve = MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE; // 16 GiB
if (arena_reserve < min_reserve) { if (arena_reserve < min_reserve) {
arena_reserve = min_reserve; arena_reserve = min_reserve;
} }
@ -494,10 +494,10 @@ static mi_page_t* mi_arena_page_try_find_abandoned(size_t slice_count, size_t bl
_mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_alloc, 1); _mi_stat_counter_increase(&_mi_stats_main.pages_reclaim_on_alloc, 1);
_mi_page_free_collect(page, false); // update `used` count _mi_page_free_collect(page, false); // update `used` count
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_dirty, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_purge, slice_index, slice_count));
mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN));
mi_assert_internal(_mi_ptr_page(page)==page); mi_assert_internal(_mi_ptr_page(page)==page);
mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page);
@ -670,9 +670,9 @@ void _mi_arena_page_free(mi_page_t* page) {
size_t slice_count; size_t slice_count;
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_purge, slice_index, slice_count));
mi_assert_internal(mi_pairmap_is_clear(&arena->pages_abandoned[bin], slice_index)); mi_assert_internal(mi_pairmap_is_clear(&arena->pages_abandoned[bin], slice_index));
} }
#endif #endif
@ -701,10 +701,10 @@ static void mi_arena_page_abandon_no_stat(mi_page_t* page) {
size_t slice_count; size_t slice_count;
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(!mi_page_is_singleton(page)); mi_assert_internal(!mi_page_is_singleton(page));
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_purge, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_dirty, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
mi_page_set_abandoned_mapped(page); mi_page_set_abandoned_mapped(page);
bool were_zero = mi_pairmap_set(&arena->pages_abandoned[bin], slice_index); bool were_zero = mi_pairmap_set(&arena->pages_abandoned[bin], slice_index);
@ -757,9 +757,9 @@ void _mi_arena_page_unabandon(mi_page_t* page) {
size_t slice_count; size_t slice_count;
mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count); mi_arena_t* arena = mi_page_arena(page, &slice_index, &slice_count);
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_free, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_setN(&arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
mi_assert_internal(mi_bitmap_is_clearN(&arena->slices_purge, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_clearN(arena->slices_purge, slice_index, slice_count));
// this busy waits until a concurrent reader (from alloc_abandoned) is done // this busy waits until a concurrent reader (from alloc_abandoned) is done
mi_pairmap_clear_while_not_busy(&arena->pages_abandoned[bin], slice_index); mi_pairmap_clear_while_not_busy(&arena->pages_abandoned[bin], slice_index);
@ -876,8 +876,8 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi
return; return;
} }
mi_assert_internal(slice_index < arena->slice_count); mi_assert_internal(slice_index < arena->slice_count);
mi_assert_internal(slice_index >= mi_arena_info_slices()); mi_assert_internal(slice_index >= mi_arena_info_slices(arena));
if (slice_index < mi_arena_info_slices() || slice_index > arena->slice_count) { if (slice_index < mi_arena_info_slices(arena) || slice_index > arena->slice_count) {
_mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid); _mi_error_message(EINVAL, "trying to free from an invalid arena block: %p, size %zu, memid: 0x%zx\n", p, size, memid);
return; return;
} }
@ -907,7 +907,7 @@ void _mi_arena_free(void* p, size_t size, size_t committed_size, mi_memid_t memi
} }
// and make it available to others again // and make it available to others again
bool all_inuse = mi_bitmap_setN(&arena->slices_free, slice_index, slice_count, NULL); bool all_inuse = mi_bitmap_setN(arena->slices_free, slice_index, slice_count, NULL);
if (!all_inuse) { if (!all_inuse) {
_mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count)); _mi_error_message(EAGAIN, "trying to free an already freed arena block: %p, size %zu\n", mi_arena_slice_start(arena,slice_index), mi_size_of_slices(slice_count));
return; return;
@ -989,6 +989,29 @@ static bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id, mi_stats_t*
return true; return true;
} }
static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) {
if (slice_count == 0) slice_count = MI_BITMAP_CHUNK_BITS;
mi_assert_internal((slice_count % MI_BITMAP_CHUNK_BITS) == 0);
const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BITMAP_CHUNK_SIZE);
const size_t bitmaps_size = 4 * mi_bitmap_size(slice_count,NULL);
const size_t pairmaps_size = MI_BIN_COUNT * 2 * mi_bitmap_size(slice_count,NULL);
const size_t size = base_size + bitmaps_size + pairmaps_size;
const size_t os_page_size = _mi_os_page_size();
const size_t info_size = _mi_align_up(size, os_page_size) + os_page_size; // + guard page
const size_t info_slices = mi_slice_count_of_size(info_size);
if (bitmap_base != NULL) *bitmap_base = base_size;
return info_slices;
}
static mi_bitmap_t* mi_arena_bitmap_init(size_t slice_count, uint8_t** base) {
mi_bitmap_t* bitmap = (mi_bitmap_t*)(*base);
*base = (*base) + mi_bitmap_init(bitmap, slice_count, true /* already zero */);
return bitmap;
}
static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept
{ {
mi_assert(!is_large || (memid.initially_committed && memid.is_pinned)); mi_assert(!is_large || (memid.initially_committed && memid.is_pinned));
@ -1003,23 +1026,25 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int
if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); }
const size_t info_slices = mi_arena_info_slices(); const size_t slice_count = _mi_align_down(size / MI_ARENA_SLICE_SIZE, MI_BITMAP_CHUNK_BITS);
const size_t bcount = size / MI_ARENA_SLICE_SIZE; // divide down if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now
if (bcount < info_slices+1) { // todo: allow larger areas (either by splitting it up in arena's or having larger arena's)
_mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", size/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BIT_COUNT)/MI_MiB);
return false;
}
size_t bitmap_base;
const size_t info_slices = mi_arena_info_slices_needed(slice_count, &bitmap_base);
if (slice_count < info_slices+1) {
_mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB); _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB);
return false; return false;
} }
if (bcount > MI_BITMAP_MAX_BITS) {
// todo: allow larger areas (either by splitting it up in arena's or having larger arena's)
_mi_warning_message("cannot use OS memory since it is too large (size %zu MiB, maximum is %zu MiB)", size/MI_MiB, mi_size_of_slices(MI_BITMAP_MAX_BITS)/MI_MiB);
return false;
}
mi_arena_t* arena = (mi_arena_t*)start; mi_arena_t* arena = (mi_arena_t*)start;
// commit & zero if needed // commit & zero if needed
bool is_zero = memid.initially_zero; bool is_zero = memid.initially_zero;
if (!memid.initially_committed) { if (!memid.initially_committed) {
_mi_os_commit(arena, mi_size_of_slices(info_slices), &is_zero, &_mi_stats_main); _mi_os_commit(arena, mi_size_of_slices(info_slices), NULL, &_mi_stats_main);
} }
if (!is_zero) { if (!is_zero) {
_mi_memzero(arena, mi_size_of_slices(info_slices)); _mi_memzero(arena, mi_size_of_slices(info_slices));
@ -1029,34 +1054,37 @@ static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int
arena->id = _mi_arena_id_none(); arena->id = _mi_arena_id_none();
arena->memid = memid; arena->memid = memid;
arena->exclusive = exclusive; arena->exclusive = exclusive;
arena->slice_count = bcount; arena->slice_count = slice_count;
arena->info_slices = info_slices;
arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1)
arena->is_large = is_large; arena->is_large = is_large;
arena->purge_expire = 0; arena->purge_expire = 0;
mi_lock_init(&arena->abandoned_visit_lock); mi_lock_init(&arena->abandoned_visit_lock);
// init bitmaps // init bitmaps
mi_bitmap_init(&arena->slices_free,true); uint8_t* base = mi_arena_start(arena) + bitmap_base;
mi_bitmap_init(&arena->slices_committed,true); arena->slices_free = mi_arena_bitmap_init(slice_count,&base);
mi_bitmap_init(&arena->slices_dirty,true); arena->slices_committed = mi_arena_bitmap_init(slice_count,&base);
mi_bitmap_init(&arena->slices_purge,true); arena->slices_dirty = mi_arena_bitmap_init(slice_count,&base);
arena->slices_purge = mi_arena_bitmap_init(slice_count,&base);
for( size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { for( size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
mi_pairmap_init(&arena->pages_abandoned[i],true); mi_pairmap_init(&arena->pages_abandoned[i], mi_arena_bitmap_init(slice_count, &base), mi_arena_bitmap_init(slice_count, &base));
} }
mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena)));
// reserve our meta info (and reserve slices outside the memory area) // reserve our meta info (and reserve slices outside the memory area)
mi_bitmap_unsafe_setN(&arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); mi_bitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices);
if (memid.initially_committed) { if (memid.initially_committed) {
mi_bitmap_unsafe_setN(&arena->slices_committed, 0, arena->slice_count); mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count);
} }
else { else {
mi_bitmap_setN(&arena->slices_committed, 0, info_slices, NULL); mi_bitmap_setN(arena->slices_committed, 0, info_slices, NULL);
} }
if (!memid.initially_zero) { if (!memid.initially_zero) {
mi_bitmap_unsafe_setN(&arena->slices_dirty, 0, arena->slice_count); mi_bitmap_unsafe_setN(arena->slices_dirty, 0, arena->slice_count);
} }
else { else {
mi_bitmap_setN(&arena->slices_dirty, 0, info_slices, NULL); mi_bitmap_setN(arena->slices_dirty, 0, info_slices, NULL);
} }
return mi_arena_add(arena, arena_id, &_mi_stats_main); return mi_arena_add(arena, arena_id, &_mi_stats_main);
@ -1117,7 +1145,7 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_
_mi_output_message("%s%s:\n", prefix, header); _mi_output_message("%s%s:\n", prefix, header);
size_t bit_count = 0; size_t bit_count = 0;
size_t bit_set_count = 0; size_t bit_set_count = 0;
for (int i = 0; i < MI_BITMAP_CHUNK_COUNT && bit_count < slice_count; i++) { for (int i = 0; i < mi_bitmap_chunk_count(bitmap) && bit_count < slice_count; i++) {
char buf[MI_BITMAP_CHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf)); char buf[MI_BITMAP_CHUNK_BITS + 64]; _mi_memzero(buf, sizeof(buf));
mi_bitmap_chunk_t* chunk = &bitmap->chunks[i]; mi_bitmap_chunk_t* chunk = &bitmap->chunks[i];
for (size_t j = 0, k = 0; j < MI_BITMAP_CHUNK_FIELDS; j++) { for (size_t j = 0, k = 0; j < MI_BITMAP_CHUNK_FIELDS; j++) {
@ -1161,12 +1189,12 @@ void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge)
slice_total += arena->slice_count; slice_total += arena->slice_count;
_mi_output_message("arena %zu: %zu slices (%zu MiB)%s\n", i, arena->slice_count, mi_size_of_slices(arena->slice_count)/MI_MiB, (arena->memid.is_pinned ? ", pinned" : "")); _mi_output_message("arena %zu: %zu slices (%zu MiB)%s\n", i, arena->slice_count, mi_size_of_slices(arena->slice_count)/MI_MiB, (arena->memid.is_pinned ? ", pinned" : ""));
if (show_inuse) { if (show_inuse) {
free_total += mi_debug_show_bitmap(" ", "in-use slices", arena->slice_count, &arena->slices_free, true); free_total += mi_debug_show_bitmap(" ", "in-use slices", arena->slice_count, arena->slices_free, true);
} }
mi_debug_show_bitmap(" ", "committed slices", arena->slice_count, &arena->slices_committed, false); mi_debug_show_bitmap(" ", "committed slices", arena->slice_count, arena->slices_committed, false);
// todo: abandoned slices // todo: abandoned slices
if (show_purge) { if (show_purge) {
purge_total += mi_debug_show_bitmap(" ", "purgeable slices", arena->slice_count, &arena->slices_purge, false); purge_total += mi_debug_show_bitmap(" ", "purgeable slices", arena->slice_count, arena->slices_purge, false);
} }
} }
if (show_inuse) _mi_output_message("total inuse slices : %zu\n", slice_total - free_total); if (show_inuse) _mi_output_message("total inuse slices : %zu\n", slice_total - free_total);
@ -1262,7 +1290,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slices,
const size_t size = mi_size_of_slices(slices); const size_t size = mi_size_of_slices(slices);
void* const p = mi_arena_slice_start(arena, slice_index); void* const p = mi_arena_slice_start(arena, slice_index);
bool needs_recommit; bool needs_recommit;
if (mi_bitmap_is_setN(&arena->slices_committed, slice_index, slices)) { if (mi_bitmap_is_setN(arena->slices_committed, slice_index, slices)) {
// all slices are committed, we can purge freely // all slices are committed, we can purge freely
needs_recommit = _mi_os_purge(p, size, stats); needs_recommit = _mi_os_purge(p, size, stats);
} }
@ -1277,11 +1305,11 @@ static void mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slices,
} }
// clear the purged slices // clear the purged slices
mi_bitmap_clearN(&arena->slices_purge, slices, slice_index); mi_bitmap_clearN(arena->slices_purge, slices, slice_index);
// update committed bitmap // update committed bitmap
if (needs_recommit) { if (needs_recommit) {
mi_bitmap_clearN(&arena->slices_committed, slices, slice_index); mi_bitmap_clearN(arena->slices_committed, slices, slice_index);
} }
} }

666
src/bitmap.c
View file

File diff suppressed because it is too large Load diff

108
src/bitmap.h
View file

@ -34,30 +34,56 @@ typedef mi_decl_align(MI_BITMAP_CHUNK_SIZE) struct mi_bitmap_chunk_s {
_Atomic(mi_bfield_t) bfields[MI_BITMAP_CHUNK_FIELDS]; _Atomic(mi_bfield_t) bfields[MI_BITMAP_CHUNK_FIELDS];
} mi_bitmap_chunk_t; } mi_bitmap_chunk_t;
// for now 32 (note: with ABA instructions we can make this 64) // for now 32-bit epoch + 32-bit bit-set (note: with ABA instructions we can double this)
#define MI_EPOCHSET_BITS (32) typedef uint64_t mi_chunkmap_t;
#define MI_BITMAP_CHUNK_COUNT MI_EPOCHSET_BITS typedef uint32_t mi_epoch_t;
typedef uint64_t mi_epochset_t; typedef uint32_t mi_cmap_t;
#define MI_CHUNKMAP_BITS (32) // 1 chunkmap tracks 32 chunks
#define MI_BITMAP_MAX_CHUNKMAPS (16)
#define MI_BITMAP_MAX_CHUNK_COUNT (MI_BITMAP_MAX_CHUNKMAPS * MI_CHUNKMAP_BITS)
#define MI_BITMAP_MIN_CHUNK_COUNT (1 * MI_CHUNKMAP_BITS) // 1 GiB arena
#define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BITMAP_CHUNK_BITS) // 16 GiB arena
#define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BITMAP_CHUNK_BITS) // 1 GiB arena
typedef mi_decl_align(MI_BITMAP_CHUNK_SIZE) struct mi_bitmap_s { typedef mi_decl_align(MI_BITMAP_CHUNK_SIZE) struct mi_bitmap_s {
mi_bitmap_chunk_t chunks[MI_BITMAP_CHUNK_COUNT]; _Atomic(size_t) chunk_map_count;
_Atomic(mi_epochset_t) any_set; _Atomic(size_t) chunk_count;
_Atomic(mi_chunkmap_t) chunk_maps[MI_BITMAP_MAX_CHUNKMAPS];
// padding
mi_bitmap_chunk_t chunks[MI_BITMAP_MIN_BIT_COUNT]; // or more, up to MI_BITMAP_MAX_CHUNK_COUNT
} mi_bitmap_t; } mi_bitmap_t;
// 16k bits on 64bit, 8k bits on 32bit static inline size_t mi_bitmap_chunk_map_count(const mi_bitmap_t* bitmap) {
// with 64KiB slices, this can address a 1GiB arena return mi_atomic_load_relaxed(&bitmap->chunk_map_count);
#define MI_BITMAP_MAX_BITS (MI_BITMAP_CHUNK_COUNT * MI_BITMAP_CHUNK_BITS) }
static inline size_t mi_bitmap_chunk_count(const mi_bitmap_t* bitmap) {
return mi_atomic_load_relaxed(&bitmap->chunk_count);
}
static inline size_t mi_bitmap_max_bits(const mi_bitmap_t* bitmap) {
return (mi_bitmap_chunk_count(bitmap) * MI_BITMAP_CHUNK_BITS);
}
/* -------------------------------------------------------------------------------- /* --------------------------------------------------------------------------------
Atomic bitmap Atomic bitmap
-------------------------------------------------------------------------------- */ -------------------------------------------------------------------------------- */
typedef bool mi_bit_t; typedef bool mi_xset_t;
#define MI_BIT_SET (true) #define MI_BIT_SET (true)
#define MI_BIT_CLEAR (false) #define MI_BIT_CLEAR (false)
size_t mi_bitmap_size(size_t bit_count, size_t* chunk_count);
// initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true // initialize a bitmap to all unset; avoid a mem_zero if `already_zero` is true
void mi_bitmap_init(mi_bitmap_t* bitmap, bool already_zero); // returns the size of the bitmap.
size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero);
// Set/clear a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. // Set/clear a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread.
void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n); void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n);
@ -65,7 +91,7 @@ void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n);
// Set/clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's (or all 1's to 0's). // Set/clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's (or all 1's to 0's).
// `n` cannot cross chunk boundaries (and `n <= MI_BITMAP_CHUNK_BITS`)! // `n` cannot cross chunk boundaries (and `n <= MI_BITMAP_CHUNK_BITS`)!
// If `already_xset` is not NULL, it is set to true if all the bits were already all set/cleared. // If `already_xset` is not NULL, it is set to true if all the bits were already all set/cleared.
bool mi_bitmap_xsetN(mi_bit_t set, mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_xset); bool mi_bitmap_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_xset);
static inline bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set) { static inline bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set) {
return mi_bitmap_xsetN(MI_BIT_SET, bitmap, idx, n, already_set); return mi_bitmap_xsetN(MI_BIT_SET, bitmap, idx, n, already_set);
@ -77,7 +103,7 @@ static inline bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
// Is a sequence of n bits already all set/cleared? // Is a sequence of n bits already all set/cleared?
bool mi_bitmap_is_xsetN(mi_bit_t set, mi_bitmap_t* bitmap, size_t idx, size_t n); bool mi_bitmap_is_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);
static inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) { static inline bool mi_bitmap_is_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n); return mi_bitmap_is_xsetN(MI_BIT_SET, bitmap, idx, n);
@ -88,9 +114,29 @@ static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n
} }
// Try to set/clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's)
// and false otherwise leaving the bitmask as is.
// `n` cannot cross chunk boundaries (and `n <= MI_BITMAP_CHUNK_BITS`)!
mi_decl_nodiscard bool mi_bitmap_try_xsetN(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);
static inline bool mi_bitmap_try_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
return mi_bitmap_try_xsetN(MI_BIT_SET, bitmap, idx, n);
}
static inline bool mi_bitmap_try_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
return mi_bitmap_try_xsetN(MI_BIT_CLEAR, bitmap, idx, n);
}
// Find a sequence of `n` bits in the bitmap with all bits set, and atomically unset all.
// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
mi_decl_nodiscard bool mi_bitmap_try_find_and_clearN(mi_bitmap_t* bitmap, size_t n, size_t tseq, size_t* pidx);
// Try to set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) // Try to set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0)
// and false otherwise leaving the bitmask as is. // and false otherwise leaving the bitmask as is.
//mi_decl_nodiscard bool mi_bitmap_try_xset(mi_bit_t set, mi_bitmap_t* bitmap, size_t idx); //mi_decl_nodiscard bool mi_bitmap_try_xset(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx);
// //
//static inline bool mi_bitmap_try_set(mi_bitmap_t* bitmap, size_t idx) { //static inline bool mi_bitmap_try_set(mi_bitmap_t* bitmap, size_t idx) {
// return mi_bitmap_try_xset(MI_BIT_SET, bitmap, idx); // return mi_bitmap_try_xset(MI_BIT_SET, bitmap, idx);
@ -103,7 +149,7 @@ static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n
// Try to set/clear a byte in the bitmap; returns `true` if atomically transitioned from 0 to 0xFF (or 0xFF to 0) // Try to set/clear a byte in the bitmap; returns `true` if atomically transitioned from 0 to 0xFF (or 0xFF to 0)
// and false otherwise leaving the bitmask as is. // and false otherwise leaving the bitmask as is.
//mi_decl_nodiscard bool mi_bitmap_try_xset8(mi_bit_t set, mi_bitmap_t* bitmap, size_t idx); //mi_decl_nodiscard bool mi_bitmap_try_xset8(mi_xset_t set, mi_bitmap_t* bitmap, size_t idx);
// //
//static inline bool mi_bitmap_try_set8(mi_bitmap_t* bitmap, size_t idx) { //static inline bool mi_bitmap_try_set8(mi_bitmap_t* bitmap, size_t idx) {
// return mi_bitmap_try_xset8(MI_BIT_SET, bitmap, idx); // return mi_bitmap_try_xset8(MI_BIT_SET, bitmap, idx);
@ -113,48 +159,28 @@ static inline bool mi_bitmap_is_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n
// return mi_bitmap_try_xset8(MI_BIT_CLEAR, bitmap, idx); // return mi_bitmap_try_xset8(MI_BIT_CLEAR, bitmap, idx);
//} //}
// Try to set/clear a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's)
// and false otherwise leaving the bitmask as is.
// `n` cannot cross chunk boundaries (and `n <= MI_BITMAP_CHUNK_BITS`)!
mi_decl_nodiscard bool mi_bitmap_try_xsetN(mi_bit_t set, mi_bitmap_t* bitmap, size_t idx, size_t n);
static inline bool mi_bitmap_try_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
return mi_bitmap_try_xsetN(MI_BIT_SET, bitmap, idx, n);
}
static inline bool mi_bitmap_try_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) {
return mi_bitmap_try_xsetN(MI_BIT_CLEAR, bitmap, idx, n);
}
// Find a sequence of `n` bits in the bitmap with all bits set, and atomically unset all.
// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
mi_decl_nodiscard bool mi_bitmap_try_find_and_clearN(mi_bitmap_t* bitmap, size_t n, size_t tseq, size_t* pidx );
/* -------------------------------------------------------------------------------- /* --------------------------------------------------------------------------------
Atomic bitmap for a pair of bits Atomic bitmap for a pair of bits
-------------------------------------------------------------------------------- */ -------------------------------------------------------------------------------- */
typedef mi_bfield_t mi_pair_t;
#define MI_PAIR_CLEAR (0) #define MI_PAIR_CLEAR (0)
#define MI_PAIR_BUSY (1) #define MI_PAIR_BUSY (1)
#define MI_PAIR_UNUSED (2) // should never occur #define MI_PAIR_UNUSED (2) // should never occur
#define MI_PAIR_SET (3) #define MI_PAIR_SET (3)
typedef mi_decl_align(MI_BITMAP_CHUNK_SIZE) struct mi_pairmap_s { typedef struct mi_pairmap_s {
mi_bitmap_chunk_t chunks[2*MI_BITMAP_CHUNK_COUNT]; mi_bitmap_t* bitmap1;
_Atomic(mi_epochset_t) any_set; mi_bitmap_t* bitmap2;
} mi_pairmap_t; } mi_pairmap_t;
#define MI_PAIRMAP_MAX_PAIRS (MI_BITMAP_MAX_BITS) // 16k pairs on 64bit, 8k pairs on 32bit
#define MI_PAIRMAP_MAX_BITS (2*MI_PAIRMAP_MAX_PAIRS)
// initialize a pairmap to all unset; avoid a mem_zero if `already_zero` is true // initialize a pairmap to all unset; avoid a mem_zero if `already_zero` is true
void mi_pairmap_init(mi_pairmap_t* pairmap, bool already_zero); void mi_pairmap_init(mi_pairmap_t* pairmap, mi_bitmap_t* bm1, mi_bitmap_t* bm2);
bool mi_pairmap_set(mi_pairmap_t* pairmap, size_t pair_idx); bool mi_pairmap_set(mi_pairmap_t* pairmap, size_t pair_idx);
bool mi_pairmap_clear(mi_pairmap_t* pairmap, size_t pair_idx);
bool mi_pairmap_is_clear(mi_pairmap_t* pairmap, size_t pair_idx); bool mi_pairmap_is_clear(mi_pairmap_t* pairmap, size_t pair_idx);
void mi_pairmap_clear(mi_pairmap_t* pairmap, size_t pair_idx);
void mi_pairmap_clear_while_not_busy(mi_pairmap_t* pairmap, size_t pair_idx); void mi_pairmap_clear_while_not_busy(mi_pairmap_t* pairmap, size_t pair_idx);
mi_decl_nodiscard bool mi_pairmap_try_find_and_set_busy(mi_pairmap_t* pairmap, size_t tseq, size_t* pidx); mi_decl_nodiscard bool mi_pairmap_try_find_and_set_busy(mi_pairmap_t* pairmap, size_t tseq, size_t* pidx);

3
src/page-map.c
View file

@ -22,7 +22,8 @@ static bool mi_page_map_init(void) {
// 64 KiB for 4 GiB address space (on 32-bit) // 64 KiB for 4 GiB address space (on 32-bit)
const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT)); const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_SLICE_SHIFT));
mi_page_map_entries_per_commit_bit = _mi_divide_up(page_map_size,MI_BITMAP_MAX_BITS); mi_page_map_entries_per_commit_bit = _mi_divide_up(page_map_size, MI_BITMAP_MIN_BIT_COUNT);
mi_bitmap_init(&mi_page_map_commit, MI_BITMAP_MIN_BIT_COUNT, true);
mi_page_map_all_committed = false; // _mi_os_has_overcommit(); // commit on-access on Linux systems? mi_page_map_all_committed = false; // _mi_os_has_overcommit(); // commit on-access on Linux systems?
_mi_page_map = (uint8_t*)_mi_os_alloc_aligned(page_map_size, 1, mi_page_map_all_committed, true, &mi_page_map_memid, NULL); _mi_page_map = (uint8_t*)_mi_os_alloc_aligned(page_map_size, 1, mi_page_map_all_committed, true, &mi_page_map_memid, NULL);

2
test/test-stress.c
View file

@ -41,7 +41,7 @@ static int THREADS = 8;
static int SCALE = 10; static int SCALE = 10;
static int ITER = 10; static int ITER = 10;
#elif 0 #elif 0
static int THREADS = 4; static int THREADS = 1;
static int SCALE = 100; static int SCALE = 100;
static int ITER = 10; static int ITER = 10;
#define ALLOW_LARGE false #define ALLOW_LARGE false