diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 32b9d528..fb6c2e8c 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -68,6 +68,14 @@ typedef int32_t mi_ssize_t; #define MI_MiB (MI_KiB*MI_KiB) #define MI_GiB (MI_MiB*MI_KiB) +#if MI_INTPTR_SIZE > 4 +#define MI_MAX_VABITS (48) +#define MI_PAGE_MAP_FLAT 0 +#else +#define MI_MAX_VABITS (32) +#define MI_PAGE_MAP_FLAT 1 +#endif + /* -------------------------------------------------------------------------------- Architecture diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 208989e3..dbc45133 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -422,6 +422,14 @@ static inline bool mi_heap_is_initialized(mi_heap_t* heap) { return (heap != &_mi_heap_empty); } +static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { + mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = _mi_wsize_from_size(size); + mi_assert_internal(idx < MI_PAGES_DIRECT); + return heap->pages_free_direct[idx]; +} + + //static inline uintptr_t _mi_ptr_cookie(const void* p) { // extern mi_heap_t _mi_heap_main; // mi_assert_internal(_mi_heap_main.cookie != 0); @@ -433,14 +441,9 @@ static inline bool mi_heap_is_initialized(mi_heap_t* heap) { Pages ----------------------------------------------------------- */ -static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { - mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - const size_t idx = _mi_wsize_from_size(size); - mi_assert_internal(idx < MI_PAGES_DIRECT); - return heap->pages_free_direct[idx]; -} - +#if MI_PAGE_MAP_FLAT +// flat page-map committed on demand extern uint8_t* _mi_page_map; static inline uintptr_t _mi_page_map_index(const void* p) { @@ -465,16 +468,59 @@ static inline mi_page_t* _mi_ptr_page_ex(const void* p, bool* valid) { static inline mi_page_t* _mi_checked_ptr_page(const void* p) { bool valid; - mi_page_t* const page = _mi_ptr_page_ex(p,&valid); + mi_page_t* const page = _mi_ptr_page_ex(p, &valid); return (valid ? page : NULL); } +static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { + return _mi_ptr_page_ex(p, NULL); +} + +#else + +// 2-level page map + +// one page-map directory = 64 KiB => covers 2^16 * 2^16 = 2^32 = 4 GiB address space +// the page-map needs 48-16-16 = 16 bits => 2^16 map directories = 2^16 * 2^3 = 2^19 = 512 KiB size. +// we commit the page-map directories on-demand. (2^16 * 2^16 = 2^32 ~= 4 GiB needed to cover 256 TeB) + +#define MI_PAGE_MAP_SUB_SHIFT (16) // 64 KiB +#define MI_PAGE_MAP_SUB_SIZE (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) +#define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) +#define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT) + +extern uint8_t** _mi_page_map; + +static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { + const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE; + if (sub_idx != NULL) { *sub_idx = (uint32_t)u % MI_PAGE_MAP_SUB_SIZE; } + return (size_t)(u / MI_PAGE_MAP_COUNT); +} + +static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { + const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE; + const uint8_t* const sub = _mi_page_map[u / MI_PAGE_MAP_COUNT]; + const uint8_t ofs = sub[(uint32_t)u % MI_PAGE_MAP_SUB_SIZE]; + return (mi_page_t*)((u - ofs + 1) * MI_ARENA_SLICE_SIZE); +} + +static inline mi_page_t* _mi_checked_ptr_page(const void* p) { + const uintptr_t u = (uintptr_t)p / MI_ARENA_SLICE_SIZE; + const uint8_t* const sub = _mi_page_map[u / MI_PAGE_MAP_COUNT]; + //if mi_unlikely(sub == NULL) { return NULL; } + const uint8_t ofs = sub[(uint32_t)u % MI_PAGE_MAP_SUB_SIZE]; + //if mi_unlikely(ofs == 0) { return NULL; } + return (mi_page_t*)((u - ofs + 1) * MI_ARENA_SLICE_SIZE); +} + +#endif + static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); #if MI_DEBUG || defined(__APPLE__) return _mi_checked_ptr_page(p); #else - return _mi_ptr_page_ex(p,NULL); + return _mi_unchecked_ptr_page(p); #endif } diff --git a/src/page-map.c b/src/page-map.c index d6517f72..a814610f 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -9,6 +9,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "bitmap.h" +#if MI_PAGE_MAP_FLAT + // The page-map contains a byte for each 64kb slice in the address space. // For an address `a` where `n = _mi_page_map[a >> 16]`: // 0 = unused @@ -17,6 +19,9 @@ terms of the MIT license. A copy of the license can be found in the file // // 1 byte per slice => 1 GiB page map = 2^30 slices of 2^16 = 2^46 = 64 TiB address space. // 4 GiB virtual for 256 TiB address space (48 bit) (and 64 KiB for 4 GiB address space (on 32-bit)). + +// 1MiB = 2^20*2^16 = 2^36 = 64GiB address space +// 2^12 pointers = 2^15 k = 32k mi_decl_cache_align uint8_t* _mi_page_map = NULL; static bool mi_page_map_all_committed = false; static size_t mi_page_map_entries_per_commit_bit = MI_ARENA_SLICE_SIZE; @@ -25,7 +30,7 @@ static mi_memid_t mi_page_map_memid; // (note: we need to initialize statically or otherwise C++ may run a default constructors after process initialization) -static mi_bitmap_t mi_page_map_commit = { MI_ATOMIC_VAR_INIT(MI_BITMAP_DEFAULT_CHUNK_COUNT), MI_ATOMIC_VAR_INIT(0), +sstatic mi_bitmap_t mi_page_map_commit = { MI_ATOMIC_VAR_INIT(MI_BITMAP_DEFAULT_CHUNK_COUNT), MI_ATOMIC_VAR_INIT(0), { 0 }, { {MI_ATOMIC_VAR_INIT(0)} }, {{{ MI_ATOMIC_VAR_INIT(0) }}} }; bool _mi_page_map_init(void) { @@ -101,7 +106,7 @@ static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* void _mi_page_map_register(mi_page_t* page) { mi_assert_internal(page != NULL); - mi_assert_internal(_mi_is_aligned(page,MI_PAGE_ALIGN)); + mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access! if mi_unlikely(_mi_page_map == NULL) { if (!_mi_page_map_init()) return; @@ -151,3 +156,137 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att return false; } } + +#else + +mi_decl_cache_align uint8_t** _mi_page_map = NULL; + +static void* mi_page_map_max_address = NULL; +static mi_memid_t mi_page_map_memid; + +bool _mi_page_map_init(void) { + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS); + if (vbits == 0) { + vbits = _mi_os_virtual_address_bits(); + mi_assert_internal(vbits <= MI_MAX_VABITS); + } + + mi_page_map_max_address = (void*)(MI_PU(1) << vbits); + const size_t os_page_size = _mi_os_page_size(); + const size_t page_map_size = _mi_align_up(MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT + MI_INTPTR_SHIFT), os_page_size); + const size_t reserve_size = page_map_size + (2 * MI_PAGE_MAP_SUB_SIZE); + _mi_page_map = (uint8_t**)_mi_os_alloc_aligned(reserve_size, 1, true /* commit */, true, &mi_page_map_memid); + if (_mi_page_map==NULL) { + _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", reserve_size / MI_KiB); + return false; + } + if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) { + _mi_warning_message("the page map was committed but not zero initialized!\n"); + _mi_memzero_aligned(_mi_page_map, reserve_size); + } + + uint8_t* sub0 = (uint8_t*)_mi_page_map + page_map_size; + uint8_t* sub1 = sub0 + MI_PAGE_MAP_SUB_SIZE; + // initialize the first part so NULL pointers get resolved without an access violation + _mi_page_map[0] = sub0; + sub0[0] = 1; // so _mi_ptr_page(NULL) == NULL + // and initialize the 4GiB range where we were allocated + _mi_page_map[_mi_page_map_index(_mi_page_map,NULL)] = sub1; + + mi_assert_internal(_mi_ptr_page(NULL)==NULL); + return true; +} + +static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* sub_idx, size_t* slice_count) { + size_t page_size; + *page_start = mi_page_area(page, &page_size); + if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE - MI_ARENA_SLICE_SIZE; } // furthest interior pointer + *slice_count = mi_slice_count_of_size(page_size) + (((uint8_t*)*page_start - (uint8_t*)page)/MI_ARENA_SLICE_SIZE); // add for large aligned blocks + return _mi_page_map_index(page,sub_idx); +} + + +static inline void mi_page_map_set_range(size_t idx, size_t sub_idx, size_t slice_count, uint8_t (*set)(uint8_t ofs)) { + // is the page map area that contains the page address committed? + uint8_t ofs = 1; + while (slice_count > 0) { + uint8_t* sub = _mi_page_map[idx]; + if (sub == NULL) { + mi_memid_t memid; + sub = (uint8_t*)_mi_os_alloc(MI_PAGE_MAP_SUB_SIZE, &memid); + if (sub == NULL) { + _mi_error_message(EFAULT, "internal error: unable to extend the page map\n"); + return; // abort? + } + } + // set the offsets for the page + while (sub_idx < MI_PAGE_MAP_SUB_SIZE && slice_count > 0) { + sub[sub_idx] = set(ofs); + sub_idx++; + ofs++; + slice_count--; + } + sub_idx = 0; // potentially wrap around to the next idx + } +} + +static uint8_t set_ofs(uint8_t ofs) { + return ofs; +} + +void _mi_page_map_register(mi_page_t* page) { + mi_assert_internal(page != NULL); + mi_assert_internal(_mi_is_aligned(page, MI_PAGE_ALIGN)); + mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access! + if mi_unlikely(_mi_page_map == NULL) { + if (!_mi_page_map_init()) return; + } + mi_assert(_mi_page_map!=NULL); + uint8_t* page_start; + size_t slice_count; + size_t sub_idx; + const size_t idx = mi_page_map_get_idx(page, &page_start, &sub_idx, &slice_count); + mi_page_map_set_range(idx, sub_idx, slice_count, &set_ofs); +} + +static uint8_t set_zero(uint8_t ofs) { + MI_UNUSED(ofs); + return 0; +} + + +void _mi_page_map_unregister(mi_page_t* page) { + mi_assert_internal(_mi_page_map != NULL); + // get index and count + uint8_t* page_start; + size_t slice_count; + size_t sub_idx; + const size_t idx = mi_page_map_get_idx(page, &page_start, &sub_idx, &slice_count); + // unset the offsets + mi_page_map_set_range(idx, sub_idx, slice_count, &set_zero); +} + +void _mi_page_map_unregister_range(void* start, size_t size) { + const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); + size_t sub_idx; + const size_t idx = _mi_page_map_index(start, &sub_idx); + mi_page_map_set_range(idx, sub_idx, slice_count, &set_zero); +} + +mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { + + if mi_unlikely(p >= mi_page_map_max_address) return false; + size_t sub_idx; + const size_t idx = _mi_page_map_index(p, &sub_idx); + uint8_t* sub = _mi_page_map[idx]; + if (sub != NULL) { + return (sub[sub_idx] != 0); + } + else { + return false; + } +} + + +#endif + diff --git a/test/test-stress.c b/test/test-stress.c index 0920a02e..bbcded65 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -302,8 +302,8 @@ int main(int argc, char** argv) { mi_option_enable(mi_option_visit_abandoned); #endif #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) - // mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */); - mi_option_set(mi_option_purge_delay,10); + mi_option_set(mi_option_arena_reserve, 32 * 1024 /* in kib = 32MiB */); + //mi_option_set(mi_option_purge_delay,10); #endif #ifndef USE_STD_MALLOC mi_stats_reset();