From a42a2a926b5fd68a40bd7b75d1362d5c1f4e7d1b Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 22 Dec 2024 14:18:33 -0800 Subject: [PATCH] improving level 2 page-map --- include/mimalloc/internal.h | 11 ++++++----- src/page-map.c | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 8955db5e..5dc2074d 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -470,11 +470,12 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { #else -// 2-level page map - -// one sub page-map = 64 KiB => covers 2^13 * 2^16 = 2^32 = 512 MiB address space -// the page-map needs 48-16-13 = 19 bits => 2^19 sub map pointers = 4 MiB size. -// we commit the page-map and the sub maps on-demand. +// 2-level page map: +// The page-map is usually 4 MiB and points to sub maps of 64 KiB. +// The page-map is committed on-demand (in 64 KiB) parts (and sub-maps are committed on-demand as well) +// One sub page-map = 64 KiB => covers 2^13 * 2^16 = 2^32 = 512 MiB address space +// The page-map needs 48-16-13 = 19 bits => 2^19 sub map pointers = 4 MiB size. +// (Choosing a MI_PAGE_MAP_SUB_SHIFT of 16 gives slightly better code but will commit the initial sub-map at 512 KiB) #define MI_PAGE_MAP_SUB_SHIFT (13) #define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) diff --git a/src/page-map.c b/src/page-map.c index 99a9b60a..5a25b839 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -159,11 +159,13 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att #else +// A 2-level page map + mi_decl_cache_align mi_page_t*** _mi_page_map; static void* mi_page_map_max_address; static mi_memid_t mi_page_map_memid; -static _Atomic(mi_bfield_t) mi_page_map_commit; // one bit per committed 64 KiB entries +static _Atomic(mi_bfield_t) mi_page_map_commit; static mi_page_t** mi_page_map_ensure_at(size_t idx); static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count); @@ -178,8 +180,10 @@ bool _mi_page_map_init(void) { } // Allocate the page map and commit bits + mi_assert(MI_MAX_VABITS >= vbits); mi_page_map_max_address = (void*)(MI_PU(1) << vbits); const size_t page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + mi_assert(page_map_count <= MI_PAGE_MAP_COUNT); const size_t os_page_size = _mi_os_page_size(); const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size); const size_t reserve_size = page_map_size + os_page_size; @@ -193,7 +197,7 @@ bool _mi_page_map_init(void) { _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); _mi_memzero_aligned(_mi_page_map, page_map_size); } - mi_atomic_store_release(&mi_page_map_commit, (commit ? ~0 : (mi_bfield_t)0)); + mi_atomic_store_release(&mi_page_map_commit, (commit ? ~MI_ZU(0) : MI_ZU(0))); // commit the first part so NULL pointers get resolved without an access violation mi_page_map_ensure_at(0); @@ -210,9 +214,12 @@ bool _mi_page_map_init(void) { return true; } + +#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS) + static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) { mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit); - const size_t bit_idx = (idx*MI_INTPTR_SIZE)/MI_ARENA_SLICE_SIZE; // we commit a slice of entries at a time + const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT; mi_assert_internal(bit_idx < MI_BFIELD_BITS); if (pbit_idx != NULL) { *pbit_idx = bit_idx; } return ((commit & (MI_ZU(1) << bit_idx)) != 0); @@ -221,8 +228,8 @@ static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) { static mi_page_t** mi_page_map_ensure_committed(size_t idx) { size_t bit_idx; if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) { - uint8_t* start = (uint8_t*)_mi_page_map + (bit_idx * MI_ARENA_SLICE_SIZE); - _mi_os_commit(start, MI_ARENA_SLICE_SIZE, NULL); + uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT]; + _mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_page_t**), NULL); mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx); } return _mi_page_map[idx];