kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-03 22:19:32 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' of d:\dev\mimalloc3 into dev

Browse source
This commit is contained in:
Daan Leijen 2024-12-25 14:15:13 -08:00
commit b48ffaf230
6 changed files with 70 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/mimalloc.h
View file

@ -276,7 +276,7 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size
mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept;
mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept;
mi_decl_export void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept;
mi_decl_export void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept;
// Experimental: heaps associated with specific memory arena's
typedef int mi_arena_id_t;

2
include/mimalloc/internal.h
View file

@ -121,7 +121,7 @@ bool _mi_os_decommit(void* addr, size_t size);
bool _mi_os_protect(void* addr, size_t size);
bool _mi_os_unprotect(void* addr, size_t size);
bool _mi_os_purge(void* p, size_t size);
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset);
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size);
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);

2
include/mimalloc/types.h
View file

@ -377,7 +377,7 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
typedef struct mi_memid_os_info {
void* base; // actual base address of the block (used for offset aligned allocations)
size_t alignment; // alignment at allocation
size_t size; // full allocation size
} mi_memid_os_info_t;
typedef struct mi_memid_arena_info {

25
src/arena.c
View file

@ -474,8 +474,7 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks)
// we need to ensure we do not try to reset (as that may be invalid for uncommitted memory),
// and also undo the decommit stats (as it was already adjusted)
mi_assert_internal(mi_option_is_enabled(mi_option_purge_decommits));
needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */);
if (needs_recommit) { _mi_stat_increase(&_mi_stats_main.committed, size); }
needs_recommit = _mi_os_purge_ex(p, size, false /* allow reset? */, 0);
}
// clear the purged blocks
@ -912,11 +911,11 @@ static size_t mi_debug_show_bitmap(const char* prefix, const char* header, size_
return inuse_count;
}
void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept {
void mi_debug_show_arenas(bool show_inuse) mi_attr_noexcept {
size_t max_arenas = mi_atomic_load_relaxed(&mi_arena_count);
size_t inuse_total = 0;
size_t abandoned_total = 0;
size_t purge_total = 0;
//size_t abandoned_total = 0;
//size_t purge_total = 0;
for (size_t i = 0; i < max_arenas; i++) {
mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]);
if (arena == NULL) break;
@ -927,16 +926,16 @@ void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge)
if (arena->blocks_committed != NULL) {
mi_debug_show_bitmap(" ", "committed blocks", arena->block_count, arena->blocks_committed, arena->field_count);
}
if (show_abandoned) {
abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count);
}
if (show_purge && arena->blocks_purge != NULL) {
purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count);
}
//if (show_abandoned) {
// abandoned_total += mi_debug_show_bitmap(" ", "abandoned blocks", arena->block_count, arena->blocks_abandoned, arena->field_count);
//}
//if (show_purge && arena->blocks_purge != NULL) {
// purge_total += mi_debug_show_bitmap(" ", "purgeable blocks", arena->block_count, arena->blocks_purge, arena->field_count);
//}
}
if (show_inuse) _mi_verbose_message("total inuse blocks : %zu\n", inuse_total);
if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total);
if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total);
//if (show_abandoned) _mi_verbose_message("total abandoned blocks: %zu\n", abandoned_total);
//if (show_purge) _mi_verbose_message("total purgeable blocks: %zu\n", purge_total);
}

86
src/os.c
View file

@ -9,7 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/atomic.h"
#include "mimalloc/prim.h"
#define os_stats (&_mi_stats_main)
#define mi_os_stat_increase(stat,amount) _mi_stat_increase(&_mi_stats_main.stat, amount)
#define mi_os_stat_decrease(stat,amount) _mi_stat_decrease(&_mi_stats_main.stat, amount)
#define mi_os_stat_counter_increase(stat,inc) _mi_stat_counter_increase(&_mi_stats_main.stat, inc)
/* -----------------------------------------------------------
Initialization.
@ -157,34 +159,42 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
}
#endif
/* -----------------------------------------------------------
Free memory
-------------------------------------------------------------- */
static void mi_os_free_huge_os_pages(void* p, size_t size);
static void mi_os_prim_free(void* addr, size_t size, bool still_committed) {
static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) {
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free(addr, size);
if (err != 0) {
_mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
if (still_committed) { _mi_stat_decrease(&os_stats->committed, size); }
_mi_stat_decrease(&os_stats->reserved, size);
if (commit_size > 0) {
mi_os_stat_decrease(committed, commit_size);
}
mi_os_stat_decrease(reserved, size);
}
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) {
if (mi_memkind_is_os(memid.memkind)) {
size_t csize = _mi_os_good_alloc_size(size);
size_t csize = memid.mem.os.size;
if (csize==0) { _mi_os_good_alloc_size(size); }
size_t commit_size = (still_committed ? csize : 0);
void* base = addr;
// different base? (due to alignment)
if (memid.mem.os.base != NULL) {
mi_assert(memid.mem.os.base <= addr);
mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr);
if (memid.mem.os.base != base) {
mi_assert(memid.mem.os.base <= addr);
base = memid.mem.os.base;
csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base);
const size_t diff = (uint8_t*)addr - (uint8_t*)memid.mem.os.base;
if (memid.mem.os.size==0) {
csize += diff;
}
if (still_committed) {
commit_size -= diff; // the (addr-base) part was already un-committed
}
}
// free it
if (memid.memkind == MI_MEM_OS_HUGE) {
@ -192,7 +202,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me
mi_os_free_huge_os_pages(base, csize);
}
else {
mi_os_prim_free(base, csize, still_committed);
mi_os_prim_free(base, csize, (still_committed ? commit_size : 0));
}
}
else {
@ -228,11 +238,11 @@ static void* mi_os_prim_alloc_at(void* hint_addr, size_t size, size_t try_alignm
mi_stat_counter_increase(os_stats->mmap_calls, 1);
mi_os_stat_counter_increase(mmap_calls, 1);
if (p != NULL) {
_mi_stat_increase(&os_stats->reserved, size);
mi_os_stat_increase(reserved, size);
if (commit) {
_mi_stat_increase(&os_stats->committed, size);
mi_os_stat_increase(committed, size);
// seems needed for asan (or `mimalloc-test-api` fails)
#ifdef MI_TRACK_ASAN
if (*is_zero) { mi_track_mem_defined(p,size); }
@ -273,7 +283,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
#if !MI_TRACK_ASAN
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
#endif
mi_os_prim_free(p, size, commit);
if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); }
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment;
@ -304,8 +314,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit); }
if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); }
// we can return the aligned pointer on `mmap` systems
p = aligned_p;
*base = aligned_p; // since we freed the pre part, `*base == p`.
@ -349,7 +359,8 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
if (p != NULL) {
*memid = _mi_memid_create_os(commit, os_is_zero, os_is_large);
memid->mem.os.base = os_base;
memid->mem.os.alignment = alignment;
// memid->mem.os.alignment = alignment;
memid->mem.os.size += ((uint8_t*)p - (uint8_t*)os_base); // todo: return from prim_alloc_aligned
}
return p;
}
@ -417,10 +428,10 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
return mi_os_page_align_areax(true, addr, size, newsize);
}
bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) {
if (is_zero != NULL) { *is_zero = false; }
_mi_stat_increase(&os_stats->committed, size); // use size for precise commit vs. decommit
_mi_stat_counter_increase(&os_stats->commit_calls, 1);
mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit
mi_os_stat_counter_increase(commit_calls, 1);
// page align range
size_t csize;
@ -446,8 +457,13 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
return true;
}
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) { mi_assert_internal(needs_recommit!=NULL);
_mi_stat_decrease(&os_stats->committed, size);
bool _mi_os_commit(void* addr, size_t size, bool* is_zero) {
return _mi_os_commit_ex(addr, size, is_zero, size);
}
static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, size_t stat_size) {
mi_assert_internal(needs_recommit!=NULL);
mi_os_stat_decrease(committed, stat_size);
// page align
size_t csize;
@ -466,7 +482,7 @@ static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit) {
bool _mi_os_decommit(void* addr, size_t size) {
bool needs_recommit;
return mi_os_decommit_ex(addr, size, &needs_recommit);
return mi_os_decommit_ex(addr, size, &needs_recommit, size);
}
@ -479,8 +495,8 @@ bool _mi_os_reset(void* addr, size_t size) {
size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr)
_mi_stat_increase(&os_stats->reset, csize);
_mi_stat_counter_increase(&os_stats->reset_calls, 1);
mi_os_stat_increase(reset, csize);
mi_os_stat_counter_increase(reset_calls, 1);
#if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
memset(start, 0, csize); // pretend it is eagerly reset
@ -496,17 +512,17 @@ bool _mi_os_reset(void* addr, size_t size) {
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset)
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size)
{
if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed?
_mi_stat_counter_increase(&os_stats->purge_calls, 1);
_mi_stat_increase(&os_stats->purged, size);
mi_os_stat_counter_increase(purge_calls, 1);
mi_os_stat_increase(purged, size);
if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit?
!_mi_preloading()) // don't decommit during preloading (unsafe)
{
bool needs_recommit = true;
mi_os_decommit_ex(p, size, &needs_recommit);
mi_os_decommit_ex(p, size, &needs_recommit, stat_size);
return needs_recommit;
}
else {
@ -520,7 +536,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset)
// either resets or decommits memory, returns true if the memory needs
// to be recommitted if it is to be re-used later on.
bool _mi_os_purge(void* p, size_t size) {
return _mi_os_purge_ex(p, size, true);
return _mi_os_purge_ex(p, size, true, size);
}
@ -628,15 +644,15 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// no success, issue a warning and break
if (p != NULL) {
_mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr);
mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, true);
mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);
}
break;
}
// success, record it
page++; // increase before timeout check (see issue #711)
_mi_stat_increase(&os_stats->committed, MI_HUGE_OS_PAGE_SIZE);
_mi_stat_increase(&os_stats->reserved, MI_HUGE_OS_PAGE_SIZE);
mi_os_stat_increase(committed, MI_HUGE_OS_PAGE_SIZE);
mi_os_stat_increase(reserved, MI_HUGE_OS_PAGE_SIZE);
// check for timeout
if (max_msecs > 0) {
@ -674,7 +690,7 @@ static void mi_os_free_huge_os_pages(void* p, size_t size) {
if (p==NULL || size==0) return;
uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) {
mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, true);
mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE);
size -= MI_HUGE_OS_PAGE_SIZE;
base += MI_HUGE_OS_PAGE_SIZE;
}

7
test/test-stress.c
View file

@ -64,6 +64,7 @@ static bool main_participates = false; // main thread participates as a
#define custom_calloc(n,s) mi_calloc(n,s)
#define custom_realloc(p,s) mi_realloc(p,s)
#define custom_free(p) mi_free(p)
#ifndef NDEBUG
#define HEAP_WALK // walk the heap objects?
#endif
@ -223,7 +224,7 @@ static void test_stress(void) {
run_os_threads(THREADS, &stress);
#if !defined(NDEBUG) && !defined(USE_STD_MALLOC)
// switch between arena and OS allocation for testing
mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1);
// mi_option_set_enabled(mi_option_disallow_arena_alloc, (n%2)==1);
#endif
#ifdef HEAP_WALK
size_t total = 0;
@ -237,7 +238,7 @@ static void test_stress(void) {
}
#ifndef NDEBUG
//mi_collect(false);
//mi_debug_show_arenas();
//mi_debug_show_arenas(true);
#endif
#if !defined(NDEBUG) || defined(MI_TSAN)
if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); }
@ -315,7 +316,7 @@ int main(int argc, char** argv) {
#ifndef USE_STD_MALLOC
#ifndef NDEBUG
mi_debug_show_arenas(true,true,true);
mi_debug_show_arenas(true);
mi_collect(true);
#endif
#endif