diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 2debaf25..1814853a 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -285,7 +285,6 @@ static inline size_t mi_clz(size_t x) { // return false if `x==0` (with `*idx` undefined) and true otherwise, // with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`). static inline bool mi_bsf(size_t x, size_t* idx) { - // we don't optimize anymore to lzcnt so we run correctly on older cpu's as well #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) && (!defined(__clang_major__) || __clang_major__ >= 9) // on x64 the carry flag is set on zero which gives better codegen bool is_zero; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 1e4914e7..fbf4ea03 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -112,16 +112,26 @@ terms of the MIT license. A copy of the license can be found in the file // (comments specify sizes on 64-bit, usually 32-bit is halved) // -------------------------------------------------------------- -// Sizes are for 64-bit +// Main size parameter; determines max arena sizes and max arena object sizes etc. #ifndef MI_ARENA_SLICE_SHIFT -#ifdef MI_SMALL_PAGE_SHIFT // backward compatibility -#define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT -#else -#define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) + #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility + #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT + #else + #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) + #endif #endif +#if MI_ARENA_SLICE_SHIFT < 12 +#error Arena slices should be at least 4KiB #endif + #ifndef MI_BCHUNK_BITS_SHIFT -#define MI_BCHUNK_BITS_SHIFT (6 + MI_SIZE_SHIFT) // optimized for 512 bits per chunk (avx512) + #if MI_ARENA_SLICE_SHIFT <= 13 // <= 8KiB + #define MI_BCHUNK_BITS_SHIFT (7) // 128 bits + #elif MI_ARENA_SLICE_SHIFT < 16 // <= 32KiB + #define MI_BCHUNK_BITS_SHIFT (8) // 256 bits + #else + #define MI_BCHUNK_BITS_SHIFT (6 + MI_SIZE_SHIFT) // 512 bits (or 256 on 32-bit) + #endif #endif #define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps are "bchunks" of 512 bits @@ -134,6 +144,10 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ARENA_MIN_OBJ_SIZE (MI_ARENA_MIN_OBJ_SLICES * MI_ARENA_SLICE_SIZE) #define MI_ARENA_MAX_OBJ_SIZE (MI_ARENA_MAX_OBJ_SLICES * MI_ARENA_SLICE_SIZE) +#if MI_ARENA_MAX_OBJ_SIZE < MI_SIZE_SIZE*1024 +#error maximum object size may be too small to hold local thread data +#endif + #define MI_SMALL_PAGE_SIZE MI_ARENA_MIN_OBJ_SIZE // 64 KiB #define MI_MEDIUM_PAGE_SIZE (8*MI_SMALL_PAGE_SIZE) // 512 KiB (=byte in the bchunk bitmap) #define MI_LARGE_PAGE_SIZE (MI_SIZE_SIZE*MI_MEDIUM_PAGE_SIZE) // 4 MiB (=word in the bchunk bitmap) diff --git a/src/arena-meta.c b/src/arena-meta.c index a6cc965d..56a7505c 100644 --- a/src/arena-meta.c +++ b/src/arena-meta.c @@ -25,11 +25,16 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE #define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN -#define MI_META_BLOCK_SIZE (128) // large enough such that META_MAX_SIZE >= 4k (even on 32-bit) +// large enough such that META_MAX_SIZE > 4k (even on 32-bit) +#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit #define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE #define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512 #define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE) +#if MI_META_MAX_SIZE <= 4096 +#error "max meta object size should be at least 4KiB" +#endif + typedef struct mi_meta_page_s { _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released) mi_memid_t memid; // provenance of the meta-page memory itself diff --git a/src/arena.c b/src/arena.c index 5f24c986..1496975a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1190,6 +1190,10 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s _mi_warning_message("cannot use OS memory since it is not large enough (size %zu KiB, minimum required is %zu KiB)", size/MI_KiB, mi_size_of_slices(info_slices+1)/MI_KiB); return false; } + else if (info_slices >= MI_ARENA_MAX_OBJ_SLICES) { + _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", size/MI_MiB, info_slices, MI_ARENA_MAX_OBJ_SLICES); + return false; + } mi_arena_t* arena = (mi_arena_t*)start; diff --git a/src/bitmap.c b/src/bitmap.c index f7f94ddb..f7e17e38 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1373,7 +1373,7 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis size_t mi_bbitmap_size(size_t bit_count, size_t* pchunk_count) { - mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); + // mi_assert_internal((bit_count % MI_BCHUNK_BITS) == 0); bit_count = _mi_align_up(bit_count, MI_BCHUNK_BITS); mi_assert_internal(bit_count <= MI_BITMAP_MAX_BIT_COUNT); mi_assert_internal(bit_count > 0); diff --git a/src/bitmap.h b/src/bitmap.h index ecca6e6f..31b57a4e 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -256,7 +256,9 @@ static inline mi_bbin_t mi_bbin_of(size_t slice_count) { typedef mi_decl_bchunk_align struct mi_bbitmap_s { _Atomic(size_t) chunk_count; // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS) _Atomic(size_t) chunk_max_accessed; // max chunk index that was once cleared or set - size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc + #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 2 + size_t _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 2]; // suppress warning on msvc by aligning manually + #endif mi_bchunkmap_t chunkmap; mi_bchunkmap_t chunkmap_bins[MI_BBIN_COUNT - 1]; // chunkmaps with bit set if the chunk is in that size class (excluding MI_BBIN_NONE) mi_bchunk_t chunks[MI_BITMAP_DEFAULT_CHUNK_COUNT]; // usually dynamic MI_BITMAP_MAX_CHUNK_COUNT