From 0f635413d678608bd04600faf4e6d558507f7284 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 29 Nov 2024 17:50:37 -0800 Subject: [PATCH] wip: can run initial test --- include/mimalloc/internal.h | 5 +++-- src/arena.c | 9 ++++++--- src/bitmap.c | 2 +- src/init.c | 2 +- src/page-map.c | 20 ++++++++++---------- test/test-stress.c | 4 ++++ 6 files changed, 25 insertions(+), 17 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 3c8216ec..47301e79 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -447,7 +447,7 @@ static inline mi_page_t* _mi_ptr_page(const void* p) { #if MI_DEBUG if mi_unlikely(ofs==0) return MI_PAGE_PTR_INVALID; #endif - return (mi_page_t*)((up + ofs - 1) << MI_ARENA_BLOCK_SHIFT); + return (mi_page_t*)((up + ofs + 1) << MI_ARENA_BLOCK_SHIFT); } @@ -663,7 +663,8 @@ We also pass a separate `null` value to be used as `NULL` or otherwise ------------------------------------------------------------------- */ static inline bool mi_is_in_same_page(const void* p, const void* q) { - return (_mi_ptr_page(p) == _mi_ptr_page(q)); + // return (_mi_ptr_page(p) == _mi_ptr_page(q)); + return ((uintptr_t)p / MI_LARGE_PAGE_SIZE) == ((uintptr_t)q / MI_LARGE_PAGE_SIZE); } static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { diff --git a/src/arena.c b/src/arena.c index 9dbf73d6..a8dff8a5 100644 --- a/src/arena.c +++ b/src/arena.c @@ -415,7 +415,8 @@ void* _mi_arena_alloc_aligned( } // fall back to the OS - return mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena_id, memid, tld); + void* p = mi_arena_os_alloc_aligned(size, alignment, align_offset, commit, allow_large, req_arena_id, memid, tld); + return p; } void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_tld_t* tld) @@ -498,6 +499,7 @@ static mi_page_t* mi_arena_page_alloc_fresh(size_t block_count, size_t block_siz else { page->block_size_shift = 0; } + _mi_page_map_register(page); mi_assert_internal(mi_page_block_size(page) == block_size); mi_assert_internal(mi_page_is_abandoned(page)); @@ -564,12 +566,13 @@ static uint8_t* mi_arena_page_allocated_area(mi_page_t* page, size_t* psize) { const size_t diff = pstart - (uint8_t*)page; const size_t size = _mi_align_up(page_size + diff, MI_ARENA_BLOCK_SIZE); if (psize != NULL) { *psize = size; } - return pstart; + return (uint8_t*)page; } void _mi_arena_page_free(mi_page_t* page, mi_tld_t* tld) { size_t size; uint8_t* pstart = mi_arena_page_allocated_area(page, &size); + _mi_page_map_unregister(page); _mi_arena_free(pstart, size, size, page->memid, &tld->stats); } @@ -1110,7 +1113,7 @@ static void mi_arenas_try_purge(bool force, bool visit_all, mi_stats_t* stats) { const size_t max_arena = mi_atomic_load_acquire(&mi_arena_count); if (max_arena == 0) return; - _mi_error_message(EFAULT, "purging not yet implemented\n"); + // _mi_error_message(EFAULT, "purging not yet implemented\n"); MI_UNUSED(stats); MI_UNUSED(visit_all); MI_UNUSED(force); diff --git a/src/bitmap.c b/src/bitmap.c index 5ac4ca08..175bc0ec 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -144,7 +144,7 @@ static bool mi_bitmap_chunk_try_xset8(mi_bit_t set, mi_bitmap_chunk_t* chunk, si // Set/clear a sequence of `n` bits within a chunk. Returns true if all bits transitioned from 0 to 1 (or 1 to 0) static bool mi_bitmap_chunk_xsetN(mi_bit_t set, mi_bitmap_chunk_t* chunk, size_t cidx, size_t n, bool* palready_xset) { - mi_assert_internal(cidx + n < MI_BITMAP_CHUNK_BITS); + mi_assert_internal(cidx + n <= MI_BITMAP_CHUNK_BITS); mi_assert_internal(n>0); bool all_transition = true; bool all_already_xset = true; diff --git a/src/init.c b/src/init.c index 215d6be8..d11f5b5a 100644 --- a/src/init.c +++ b/src/init.c @@ -130,7 +130,7 @@ static mi_decl_cache_align mi_subproc_t mi_subproc_default; static mi_decl_cache_align mi_tld_t tld_main = { 0, false, &_mi_heap_main, &_mi_heap_main, - NULL, // subproc + &mi_subproc_default, // subproc 0, // tseq { 0, &tld_main.stats }, // os { MI_STATS_NULL } // stats diff --git a/src/page-map.c b/src/page-map.c index dc0145f2..8dfd2f26 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file mi_decl_cache_align signed char* _mi_page_map = NULL; static bool mi_page_map_all_committed = false; -static size_t mi_blocks_per_commit_bit = 1; +static size_t mi_size_per_commit_bit = MI_ARENA_BLOCK_SIZE; static mi_memid_t mi_page_map_memid; static mi_bitmap_t mi_page_map_commit; @@ -20,13 +20,12 @@ static bool mi_page_map_init(void) { if (vbits >= 48) vbits = 47; // 1 byte per block = 2 GiB for 128 TiB address space (48 bit = 256 TiB address space) // 64 KiB for 4 GiB address space (on 32-bit) - const size_t page_map_size = (MI_ZU(1) << (vbits >> MI_ARENA_BLOCK_SHIFT)); + const size_t page_map_size = (MI_ZU(1) << (vbits - MI_ARENA_BLOCK_SHIFT)); - const size_t min_commit_size = _mi_divide_up(page_map_size,MI_BITMAP_MAX_BITS); - mi_blocks_per_commit_bit = mi_block_count_of_size(min_commit_size); + mi_size_per_commit_bit = _mi_divide_up(page_map_size,MI_BITMAP_MAX_BITS); mi_page_map_all_committed = _mi_os_has_overcommit(); // commit on-access on Linux systems - _mi_page_map = (int8_t*)_mi_os_alloc_aligned(page_map_size, 0, mi_page_map_all_committed, true, &mi_page_map_memid, NULL); + _mi_page_map = (int8_t*)_mi_os_alloc_aligned(page_map_size, 1, mi_page_map_all_committed, true, &mi_page_map_memid, NULL); if (_mi_page_map==NULL) { _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB); return false; @@ -38,6 +37,7 @@ static bool mi_page_map_init(void) { // commit the first part so NULL pointers get resolved without an access violation if (!mi_page_map_all_committed) { _mi_os_commit(_mi_page_map, _mi_os_page_size(), NULL, NULL); + _mi_page_map[0] = -1; // so _mi_ptr_page(NULL) == NULL } return true; } @@ -45,12 +45,12 @@ static bool mi_page_map_init(void) { static void mi_page_map_ensure_committed(void* p, size_t idx, size_t block_count) { // is the page map area that contains the page address committed? if (!mi_page_map_all_committed) { - const size_t commit_bit_count = _mi_divide_up(block_count, mi_blocks_per_commit_bit); - const size_t commit_bit_idx = idx / mi_blocks_per_commit_bit; + const size_t commit_bit_count = _mi_divide_up(block_count, mi_size_per_commit_bit); + const size_t commit_bit_idx = idx / mi_size_per_commit_bit; for (size_t i = 0; i < commit_bit_count; i++) { // per bit to avoid crossing over bitmap chunks if (mi_bitmap_is_xsetN(MI_BIT_CLEAR, &mi_page_map_commit, commit_bit_idx + i, 1)) { // this may race, in which case we do multiple commits (which is ok) - _mi_os_commit((uint8_t*)p + (i*mi_blocks_per_commit_bit*MI_ARENA_BLOCK_SIZE), mi_blocks_per_commit_bit* MI_ARENA_BLOCK_SIZE, NULL, NULL); + _mi_os_commit(_mi_page_map + ((commit_bit_idx + i)*mi_size_per_commit_bit), mi_size_per_commit_bit, NULL, NULL); mi_bitmap_xsetN(MI_BIT_SET, &mi_page_map_commit, commit_bit_idx + i, 1, NULL); } } @@ -75,7 +75,7 @@ void _mi_page_map_register(mi_page_t* page) { size_t block_count; const size_t idx = mi_page_map_get_idx(page, &page_start, &block_count); - mi_page_map_ensure_committed(page, idx, block_count); + mi_page_map_ensure_committed(page_start, idx, block_count); // set the offsets for (int i = 0; i < (int)block_count; i++) { @@ -100,7 +100,7 @@ void _mi_page_map_unregister(mi_page_t* page) { mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { uintptr_t idx = ((uintptr_t)p >> MI_ARENA_BLOCK_SHIFT); - if (!mi_page_map_all_committed || mi_bitmap_is_xsetN(MI_BIT_SET, &mi_page_map_commit, idx/mi_blocks_per_commit_bit, 1)) { + if (!mi_page_map_all_committed || mi_bitmap_is_xsetN(MI_BIT_SET, &mi_page_map_commit, idx/mi_size_per_commit_bit, 1)) { return (_mi_page_map[idx] != 0); } else { diff --git a/test/test-stress.c b/test/test-stress.c index 1e70e699..c7288b1a 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -40,6 +40,10 @@ static int ITER = 20; static int THREADS = 8; static int SCALE = 10; static int ITER = 10; +#elif 1 +static int THREADS = 1; +static int SCALE = 10; +static int ITER = 10; #else static int THREADS = 32; // more repeatable if THREADS <= #processors static int SCALE = 25; // scaling factor