kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 23:39:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

improving level 2 page-map

Browse source
This commit is contained in:
daanx 2024-12-22 14:18:33 -08:00
parent 3c7d7e1f11
commit a42a2a926b
2 changed files with 18 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
include/mimalloc/internal.h
View file

@ -470,11 +470,12 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) {
#else #else
// 2-level page map // 2-level page map:
// The page-map is usually 4 MiB and points to sub maps of 64 KiB.
// one sub page-map = 64 KiB => covers 2^13 * 2^16 = 2^32 = 512 MiB address space // The page-map is committed on-demand (in 64 KiB) parts (and sub-maps are committed on-demand as well)
// the page-map needs 48-16-13 = 19 bits => 2^19 sub map pointers = 4 MiB size. // One sub page-map = 64 KiB => covers 2^13 * 2^16 = 2^32 = 512 MiB address space
// we commit the page-map and the sub maps on-demand. // The page-map needs 48-16-13 = 19 bits => 2^19 sub map pointers = 4 MiB size.
// (Choosing a MI_PAGE_MAP_SUB_SHIFT of 16 gives slightly better code but will commit the initial sub-map at 512 KiB)
#define MI_PAGE_MAP_SUB_SHIFT (13) #define MI_PAGE_MAP_SUB_SHIFT (13)
#define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) #define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT)

17
src/page-map.c
View file

@ -159,11 +159,13 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att
#else #else
// A 2-level page map
mi_decl_cache_align mi_page_t*** _mi_page_map; mi_decl_cache_align mi_page_t*** _mi_page_map;
static void* mi_page_map_max_address; static void* mi_page_map_max_address;
static mi_memid_t mi_page_map_memid; static mi_memid_t mi_page_map_memid;
static _Atomic(mi_bfield_t) mi_page_map_commit; // one bit per committed 64 KiB entries static _Atomic(mi_bfield_t) mi_page_map_commit;
static mi_page_t** mi_page_map_ensure_at(size_t idx); static mi_page_t** mi_page_map_ensure_at(size_t idx);
static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count); static inline void mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count);
@ -178,8 +180,10 @@ bool _mi_page_map_init(void) {
} }
// Allocate the page map and commit bits // Allocate the page map and commit bits
mi_assert(MI_MAX_VABITS >= vbits);
mi_page_map_max_address = (void*)(MI_PU(1) << vbits); mi_page_map_max_address = (void*)(MI_PU(1) << vbits);
const size_t page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); const size_t page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT));
mi_assert(page_map_count <= MI_PAGE_MAP_COUNT);
const size_t os_page_size = _mi_os_page_size(); const size_t os_page_size = _mi_os_page_size();
const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size); const size_t page_map_size = _mi_align_up( page_map_count * sizeof(mi_page_t**), os_page_size);
const size_t reserve_size = page_map_size + os_page_size; const size_t reserve_size = page_map_size + os_page_size;
@ -193,7 +197,7 @@ bool _mi_page_map_init(void) {
_mi_warning_message("internal: the page map was committed but not zero initialized!\n"); _mi_warning_message("internal: the page map was committed but not zero initialized!\n");
_mi_memzero_aligned(_mi_page_map, page_map_size); _mi_memzero_aligned(_mi_page_map, page_map_size);
} }
mi_atomic_store_release(&mi_page_map_commit, (commit ? ~0 : (mi_bfield_t)0)); mi_atomic_store_release(&mi_page_map_commit, (commit ? ~MI_ZU(0) : MI_ZU(0)));
// commit the first part so NULL pointers get resolved without an access violation // commit the first part so NULL pointers get resolved without an access violation
mi_page_map_ensure_at(0); mi_page_map_ensure_at(0);
@ -210,9 +214,12 @@ bool _mi_page_map_init(void) {
return true; return true;
} }
#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS)
static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) { static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {
mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit); mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit);
const size_t bit_idx = (idx*MI_INTPTR_SIZE)/MI_ARENA_SLICE_SIZE; // we commit a slice of entries at a time const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT;
mi_assert_internal(bit_idx < MI_BFIELD_BITS); mi_assert_internal(bit_idx < MI_BFIELD_BITS);
if (pbit_idx != NULL) { *pbit_idx = bit_idx; } if (pbit_idx != NULL) { *pbit_idx = bit_idx; }
return ((commit & (MI_ZU(1) << bit_idx)) != 0); return ((commit & (MI_ZU(1) << bit_idx)) != 0);
@ -221,8 +228,8 @@ static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) {
static mi_page_t** mi_page_map_ensure_committed(size_t idx) { static mi_page_t** mi_page_map_ensure_committed(size_t idx) {
size_t bit_idx; size_t bit_idx;
if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) { if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) {
uint8_t* start = (uint8_t*)_mi_page_map + (bit_idx * MI_ARENA_SLICE_SIZE); uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT];
_mi_os_commit(start, MI_ARENA_SLICE_SIZE, NULL); _mi_os_commit(start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_page_t**), NULL);
mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx); mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx);
} }
return _mi_page_map[idx]; return _mi_page_map[idx];