From e24217e69cb37d3b2087b7489ca4b2c6bc40f7d7 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 15 Dec 2024 18:35:12 -0800 Subject: [PATCH] more bbin size classes, bug fixes --- src/arena-meta.c | 12 ++--- src/arena.c | 27 ++++++++-- src/bitmap.c | 119 +++++++-------------------------------------- src/bitmap.h | 48 +++++------------- test/test-stress.c | 2 +- 5 files changed, 59 insertions(+), 149 deletions(-) diff --git a/src/arena-meta.c b/src/arena-meta.c index ceda06ba..86a89755 100644 --- a/src/arena-meta.c +++ b/src/arena-meta.c @@ -33,7 +33,7 @@ terms of the MIT license. A copy of the license can be found in the file typedef struct mi_meta_page_s { _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released) mi_memid_t memid; // provenance of the meta-page memory itself - mi_bitmap_t blocks_free; // a small bitmap with 1 bit per block. + mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block. } mi_meta_page_t; static mi_decl_cache_align _Atomic(mi_meta_page_t*) mi_meta_pages = MI_ATOMIC_VAR_INIT(NULL); @@ -76,11 +76,11 @@ static mi_meta_page_t* mi_meta_page_zalloc(void) { // initialize the page mpage->memid = memid; - mi_bitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */); + mi_bbitmap_init(&mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */); const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bitmap_size(MI_META_BLOCKS_PER_PAGE, NULL); const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE); mi_assert_internal(info_blocks < MI_META_BLOCKS_PER_PAGE); - mi_bitmap_unsafe_setN(&mpage->blocks_free, info_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks); + mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks); // push atomically in front of the meta page list // (note: there is no ABA issue since we never free meta-pages) @@ -104,7 +104,7 @@ mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid ) mi_meta_page_t* mpage = mpage0; while (mpage != NULL) { size_t block_idx; - if (mi_bitmap_try_find_and_clearN(&mpage->blocks_free, block_count, 0, &block_idx)) { + if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, block_count, 0, &block_idx)) { // found and claimed `block_count` blocks *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count); return mi_meta_block_start(mpage,block_idx); @@ -122,7 +122,7 @@ mi_decl_noinline void* _mi_meta_zalloc( size_t size, mi_memid_t* pmemid ) mpage = mi_meta_page_zalloc(); if (mpage != NULL) { size_t block_idx; - if (mi_bitmap_try_find_and_clearN(&mpage->blocks_free, block_count, 0, &block_idx)) { + if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, block_count, 0, &block_idx)) { // found and claimed `block_count` blocks *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count); return mi_meta_block_start(mpage,block_idx); @@ -145,7 +145,7 @@ mi_decl_noinline void _mi_meta_free(void* p, size_t size, mi_memid_t memid) { mi_assert_internal(mi_bitmap_is_clearN(&mpage->blocks_free, block_idx, block_count)); // we zero on free (and on the initial page allocation) so we don't need a "dirty" map _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE); - mi_bitmap_setN(&mpage->blocks_free, block_idx, block_count,NULL); + mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count); } else if (mi_memid_is_os(memid)) { _mi_os_free(p, size, memid); diff --git a/src/arena.c b/src/arena.c index 84db2fb0..1547c9b2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1243,7 +1243,7 @@ static size_t mi_debug_show_page_bfield(mi_bfield_t field, char* buf, mi_arena_t return bit_set_count; } -static size_t mi_debug_show_chunks(const char* header, size_t slice_count, size_t chunk_count, mi_bchunk_t* chunks, bool invert, mi_arena_t* arena) { +static size_t mi_debug_show_chunks(const char* header, size_t slice_count, size_t chunk_count, mi_bchunk_t* chunks, _Atomic(uint8_t)* chunk_bins, bool invert, mi_arena_t* arena) { _mi_output_message("%s:\n", header); size_t bit_count = 0; size_t bit_set_count = 0; @@ -1256,9 +1256,22 @@ static size_t mi_debug_show_chunks(const char* header, size_t slice_count, size_ else if (i<100) { buf[k++] = ('0' + (char)(i/10)); buf[k++] = ('0' + (char)(i%10)); buf[k++] = ' '; } else if (i<1000) { buf[k++] = ('0' + (char)(i/100)); buf[k++] = ('0' + (char)((i%100)/10)); buf[k++] = ('0' + (char)(i%10)); } + char chunk_kind = ' '; + if (chunk_bins != NULL) { + switch (chunk_bins[i]) { + // case MI_BBIN_SMALL: chunk_kind = 'S'; break; + case MI_BBIN_MEDIUM: chunk_kind = 'M'; break; + case MI_BBIN_LARGE: chunk_kind = 'L'; break; + case MI_BBIN_OTHER: chunk_kind = 'O'; break; + // case MI_BBIN_NONE: chunk_kind = 'N'; break; + } + } + buf[k++] = chunk_kind; + buf[k++] = ' '; + for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { if (j > 0 && (j % 4) == 0) { - buf[k++] = '\n'; _mi_memset(buf+k,' ',5); k += 5; + buf[k++] = '\n'; _mi_memset(buf+k,' ',7); k += 7; } if (bit_count < slice_count) { mi_bfield_t bfield = chunk->bfields[j]; @@ -1283,11 +1296,15 @@ static size_t mi_debug_show_chunks(const char* header, size_t slice_count, size_ } static size_t mi_debug_show_bitmap(const char* header, size_t slice_count, mi_bitmap_t* bitmap, bool invert, mi_arena_t* arena) { - return mi_debug_show_chunks(header, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], invert, arena); + return mi_debug_show_chunks(header, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], NULL, invert, arena); +} + +static size_t mi_debug_show_bitmap_binned(const char* header, size_t slice_count, mi_bitmap_t* bitmap, _Atomic(uint8_t)* chunk_bins, bool invert, mi_arena_t* arena) { + return mi_debug_show_chunks(header, slice_count, mi_bitmap_chunk_count(bitmap), &bitmap->chunks[0], chunk_bins, invert, arena); } static size_t mi_debug_show_bbitmap(const char* header, size_t slice_count, mi_bbitmap_t* bbitmap, bool invert, mi_arena_t* arena) { - return mi_debug_show_chunks(header, slice_count, mi_bbitmap_chunk_count(bbitmap), &bbitmap->chunks[0], invert, arena); + return mi_debug_show_chunks(header, slice_count, mi_bbitmap_chunk_count(bbitmap), &bbitmap->chunks[0], &bbitmap->chunk_bins[0], invert, arena); } @@ -1313,7 +1330,7 @@ void mi_debug_show_arenas(bool show_pages, bool show_inuse, bool show_committed) // purge_total += mi_debug_show_bitmap("purgeable slices", arena->slice_count, arena->slices_purge, false, NULL); //} if (show_pages) { - page_total += mi_debug_show_bitmap("pages (p:page, a:abandoned, f:full-abandoned, s:singleton-abandoned, i:arena-info, m:heap-meta-data, ~:free-purgable, _:free-committed, .:free-reserved)", arena->slice_count, arena->pages, false, arena); + page_total += mi_debug_show_bitmap_binned("pages (p:page, a:abandoned, f:full-abandoned, s:singleton-abandoned, i:arena-info, m:heap-meta-data, ~:free-purgable, _:free-committed, .:free-reserved)", arena->slice_count, arena->pages, arena->slices_free->chunk_bins, false, arena); } } if (show_inuse) _mi_output_message("total inuse slices : %zu\n", slice_total - free_total); diff --git a/src/bitmap.c b/src/bitmap.c index 4a0c4a60..a847740b 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -938,9 +938,7 @@ size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero) // Set a sequence of `n` bits in the bitmap (and can cross chunks). Not atomic so only use if local to a thread. static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, size_t idx, size_t n) { mi_assert_internal(n>0); - const size_t total = n; - - + // start chunk and index size_t chunk_idx = idx / MI_BCHUNK_BITS; const size_t cidx = idx % MI_BCHUNK_BITS; @@ -984,29 +982,6 @@ void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n) { // ------- mi_bitmap_xset --------------------------------------- -// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) -bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) { - mi_assert_internal(idx < mi_bitmap_max_bits(bitmap)); - const size_t chunk_idx = idx / MI_BCHUNK_BITS; - const size_t cidx = idx % MI_BCHUNK_BITS; - mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); - const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx); - mi_bitmap_chunkmap_set(bitmap, chunk_idx); // set afterwards - return wasclear; -} - -bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) { - mi_assert_internal(idx < mi_bitmap_max_bits(bitmap)); - const size_t chunk_idx = idx / MI_BCHUNK_BITS; - const size_t cidx = idx % MI_BCHUNK_BITS; - mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); - bool maybe_all_clear; - const bool wasset = mi_bchunk_clear(&bitmap->chunks[chunk_idx], cidx, &maybe_all_clear); - if (maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); } - return wasset; -} - - // Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from 0's to 1's (or 1's to 0's). // `n` cannot cross chunk boundaries (and `n <= MI_BCHUNK_BITS`)! bool mi_bitmap_setN(mi_bitmap_t* bitmap, size_t idx, size_t n, size_t* already_set) { @@ -1043,24 +1018,17 @@ bool mi_bitmap_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { } -// ------- mi_bitmap_try_clearN --------------------------------------- - -bool mi_bitmap_try_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n) { - mi_assert_internal(n>0); - mi_assert_internal(n<=MI_BCHUNK_BITS); - mi_assert_internal(idx + n <= mi_bitmap_max_bits(bitmap)); - - const size_t chunk_idx = idx / MI_BCHUNK_BITS; - const size_t cidx = idx % MI_BCHUNK_BITS; - mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); // don't cross chunks (for now) - mi_assert_internal(chunk_idx < mi_bitmap_chunk_count(bitmap)); - if (cidx + n > MI_BCHUNK_BITS) return false; - bool maybe_all_clear; - const bool cleared = mi_bchunk_try_clearN(&bitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear); - if (cleared && maybe_all_clear) { mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); } - return cleared; +// Set/clear a bit in the bitmap; returns `true` if atomically transitioned from 0 to 1 (or 1 to 0) +bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_setN(bitmap, idx, 1, NULL); } +bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx) { + return mi_bitmap_clearN(bitmap, idx, 1); +} + + + // ------- mi_bitmap_is_xset --------------------------------------- // Is a sequence of n bits already all set/cleared? @@ -1170,58 +1138,6 @@ static inline bool mi_bitmap_find(mi_bitmap_t* bitmap, size_t tseq, size_t n, si } -/* -------------------------------------------------------------------------------- - mi_bitmap_try_find_and_clear -- used to find free pages - note: the compiler will fully inline the indirect function calls --------------------------------------------------------------------------------- */ - - -typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx); - -static bool mi_bitmap_try_find_and_clear_visit(mi_bitmap_t* bitmap, size_t chunk_idx, size_t n, size_t* pidx, void* arg1, void* arg2) { - MI_UNUSED(arg2); - mi_bchunk_try_find_and_clear_fun_t* try_find_and_clear = (mi_bchunk_try_find_and_clear_fun_t*)arg1; - size_t cidx; - // if we find a spot in the chunk we are done - if ((*try_find_and_clear)(&bitmap->chunks[chunk_idx], n, &cidx)) { - *pidx = (chunk_idx * MI_BCHUNK_BITS) + cidx; - mi_assert_internal(*pidx + n <= mi_bitmap_max_bits(bitmap)); - return true; - } - else { - /* we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation. */ - mi_bitmap_chunkmap_try_clear(bitmap, chunk_idx); - return false; - } -} - -static inline bool mi_bitmap_try_find_and_clear_generic(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* try_find_and_clear) { - return mi_bitmap_find(bitmap, tseq, n, pidx, &mi_bitmap_try_find_and_clear_visit, (void*)try_find_and_clear, NULL); -} - -bool mi_bitmap_try_find_and_clear(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) { - return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, 1, pidx, &mi_bchunk_try_find_and_clear_1); -} - -bool mi_bitmap_try_find_and_clear8(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) { - return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, 8, pidx, &mi_bchunk_try_find_and_clear_8); -} - -bool mi_bitmap_try_find_and_clearX(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx) { - return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, MI_BFIELD_BITS, pidx, &mi_bchunk_try_find_and_clear_X); -} - -bool mi_bitmap_try_find_and_clearNX(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx) { - mi_assert_internal(n<=MI_BFIELD_BITS); - return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearNX); -} - -bool mi_bitmap_try_find_and_clearN_(mi_bitmap_t* bitmap, size_t tseq, size_t n, size_t* pidx) { - mi_assert_internal(n<=MI_BCHUNK_BITS); - return mi_bitmap_try_find_and_clear_generic(bitmap, tseq, n, pidx, &mi_bchunk_try_find_and_clearN_); -} - - /* -------------------------------------------------------------------------------- Bitmap: try_find_and_claim -- used to allocate abandoned pages note: the compiler will fully inline the indirect function call @@ -1267,7 +1183,7 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk // Find a set bit in the bitmap and try to atomically clear it and claim it. // (Used to find pages in the pages_abandoned bitmaps.) -bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, +mi_decl_nodiscard bool mi_bitmap_try_find_and_claim(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx, mi_claim_fun_t* claim, mi_arena_t* arena, mi_subproc_t* subproc, mi_heaptag_t heap_tag) { mi_claim_fun_data_t claim_data = { arena, subproc, heap_tag }; @@ -1541,15 +1457,15 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, const size_t cmap_cycle = cmap_acc+1; const mi_bbin_t bbin = mi_bbin_of(n); // visit bins from largest size bin up to the NONE bin - // for(int bin = bbin; bin >= MI_BBIN_SMALL; bin--) // no need to traverse for MI_BBIN_NONE as anyone can allocate in MI_BBIN_SMALL - const mi_bbin_t bin = bbin; + for(int bin = bbin; bin >= MI_BBIN_SMALL; bin--) // no need to traverse for MI_BBIN_NONE as anyone can allocate in MI_BBIN_SMALL + // const mi_bbin_t bin = bbin; { mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X) { // don't search into non-accessed memory until we tried other size bins as well - //if (bin > MI_BBIN_SMALL && cmap_idx > cmap_acc) { - // break; - //} + if (bin > MI_BBIN_SMALL && cmap_idx > cmap_acc) { + break; + } // and for each chunkmap entry we iterate over its bits to find the chunks const mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[cmap_idx]); @@ -1561,7 +1477,8 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); // only in the current size class! const mi_bbin_t chunk_bin = (mi_bbin_t)mi_atomic_load_acquire(&bbitmap->chunk_bins[chunk_idx]); - if (bin >= chunk_bin) { // || (bin <= MI_BBIN_SMALL && chunk_bin <= MI_BBIN_SMALL)) { + if // (bin >= chunk_bin) { + (bin == chunk_bin || (bin <= MI_BBIN_SMALL && chunk_bin <= MI_BBIN_SMALL)) { mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx]; size_t cidx; if ((*on_find)(chunk, n, &cidx)) { diff --git a/src/bitmap.h b/src/bitmap.h index b28a09e4..f221f2cd 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -136,13 +136,13 @@ size_t mi_bitmap_init(mi_bitmap_t* bitmap, size_t bit_count, bool already_zero); // Not atomic so only use if still local to a thread. void mi_bitmap_unsafe_setN(mi_bitmap_t* bitmap, size_t idx, size_t n); + // Set a bit in the bitmap; returns `true` if it atomically transitioned from 0 to 1 bool mi_bitmap_set(mi_bitmap_t* bitmap, size_t idx); // Clear a bit in the bitmap; returns `true` if it atomically transitioned from 1 to 0 bool mi_bitmap_clear(mi_bitmap_t* bitmap, size_t idx); - // Set a sequence of `n` bits in the bitmap; returns `true` if atomically transitioned from all 0's to 1's // `n` cannot cross chunk boundaries (and `n <= MI_BCHUNK_BITS`)! // If `already_set` is not NULL, it is set to count of bits were already all set. @@ -177,36 +177,6 @@ static inline bool mi_bitmap_is_clear(mi_bitmap_t* bitmap, size_t idx) { } -// Try to atomically transition `n` bits from all set to all clear. Returns `true` on succes. -// `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`. -bool mi_bitmap_try_clearN(mi_bitmap_t* bitmap, size_t idx, size_t n); - -// Try to atomically transition a bit from set to clear. Returns `true` on succes. -static inline bool mi_bitmap_try_clear(mi_bitmap_t* bitmap, size_t idx) { - return mi_bitmap_try_clearN(bitmap, idx, 1); -} - - - -// Specialized versions for common bit sequence sizes -mi_decl_nodiscard bool mi_bitmap_try_find_and_clear(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx); // 1-bit -mi_decl_nodiscard bool mi_bitmap_try_find_and_clear8(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx); // 8-bits -mi_decl_nodiscard bool mi_bitmap_try_find_and_clearX(mi_bitmap_t* bitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS -mi_decl_nodiscard bool mi_bitmap_try_find_and_clearNX(mi_bitmap_t* bitmap, size_t n, size_t tseq, size_t* pidx); // < MI_BFIELD_BITS -mi_decl_nodiscard bool mi_bitmap_try_find_and_clearN_(mi_bitmap_t* bitmap, size_t n, size_t tseq, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS - -// Find a sequence of `n` bits in the bitmap with all bits set, and try to atomically clear all. -// Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`. -mi_decl_nodiscard static inline bool mi_bitmap_try_find_and_clearN(mi_bitmap_t* bitmap, size_t n, size_t tseq, size_t* pidx) { - if (n==1) return mi_bitmap_try_find_and_clear(bitmap, tseq, pidx); // small pages - if (n==8) return mi_bitmap_try_find_and_clear8(bitmap, tseq, pidx); // medium pages - if (n==MI_BFIELD_BITS) return mi_bitmap_try_find_and_clearX(bitmap, tseq, pidx); // large pages - if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk - if (n < MI_BFIELD_BITS) return mi_bitmap_try_find_and_clearNX(bitmap, tseq, n, pidx); - return mi_bitmap_try_find_and_clearN_(bitmap, tseq, n, pidx); -} - - // Called once a bit is cleared to see if the memory slice can be claimed. typedef bool (mi_claim_fun_t)(size_t slice_index, mi_arena_t* arena, mi_subproc_t* subproc, mi_heaptag_t heap_tag, bool* keep_set); @@ -225,6 +195,7 @@ void mi_bitmap_clear_once_set(mi_bitmap_t* bitmap, size_t idx); // If a bit is set in the bitmap, return `true` and set `idx` to the index of the highest bit. // Otherwise return `false` (and `*idx` is undefined). +// Used for unloading arena's bool mi_bitmap_bsr(mi_bitmap_t* bitmap, size_t* idx); @@ -236,17 +207,24 @@ bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_a // Visit all set bits in a bitmap with larger ranges if possible (`slice_count >= 1`) bool _mi_bitmap_forall_set_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg); -// + +/* ---------------------------------------------------------------------------- + Binned concurrent bitmap + Assigns a size class to each chunk such that small blocks don't cause too + much fragmentation by keeping chunks for larger blocks separate. +---------------------------------------------------------------------------- */ + typedef enum mi_bbin_e { MI_BBIN_NONE, // no bin assigned yet (the chunk is completely free) MI_BBIN_SMALL, // slice_count == 1 MI_BBIN_MEDIUM, // slice_count == 8 - MI_BBIN_OTHER, // slice_count > 1, and not 8 + MI_BBIN_LARGE, // slice_count == MI_BFIELD_BITS + MI_BBIN_OTHER, // slice_count > 1, and not 8 or MI_BFIELD_BITS MI_BBIN_COUNT } mi_bbin_t; static inline mi_bbin_t mi_bbin_of(size_t n) { - return (n==1 ? MI_BBIN_SMALL : (n==8 ? MI_BBIN_MEDIUM : MI_BBIN_OTHER)); + return (n==1 ? MI_BBIN_SMALL : (n==8 ? MI_BBIN_MEDIUM : (n==64 ? MI_BBIN_LARGE : MI_BBIN_OTHER))); } // An atomic "binned" bitmap for the free slices where we keep chunks reserved for particalar size classes @@ -308,8 +286,6 @@ static inline bool mi_bbitmap_is_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_ // `n` cannot cross chunk boundaries, where `n <= MI_CHUNK_BITS`. bool mi_bbitmap_try_clearN(mi_bbitmap_t* bbitmap, size_t idx, size_t n); - - // Specialized versions for common bit sequence sizes bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 1-bit bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits diff --git a/test/test-stress.c b/test/test-stress.c index 277f9e6e..1996e52e 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -353,7 +353,7 @@ int main(int argc, char** argv) { mi_debug_show_arenas(true,false,false); #else //mi_collect(true); - //mi_debug_show_arenas(true,false,false); + mi_debug_show_arenas(true,false,false); // mi_stats_print(NULL); #endif #else