diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b71517ad..72783400 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,8 +6,10 @@ trigger: branches: include: - - main - - dev* + - master + - dev3 + - dev2 + - dev tags: include: - v* @@ -194,6 +196,35 @@ jobs: # Other OS versions (just debug mode) # ---------------------------------------------------------- +- job: + displayName: Windows 2019 + pool: + vmImage: + windows-2019 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + MSBuildConfiguration: Debug + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + MSBuildConfiguration: Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - task: MSBuild@1 + inputs: + solution: $(BuildType)/libmimalloc.sln + configuration: '$(MSBuildConfiguration)' + msbuildArguments: -m + - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) + workingDirectory: $(BuildType) + displayName: CTest + - job: displayName: Ubuntu 24.04 pool: diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c5252210..aeacbc1e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -8,6 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MI_INTERNAL_H #define MI_INTERNAL_H + // -------------------------------------------------------------------------- // This file contains the internal API's of mimalloc and various utility // functions and macros. @@ -17,17 +18,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "track.h" #include "bits.h" - -// -------------------------------------------------------------------------- -// Compiler defines -// -------------------------------------------------------------------------- - -#if (MI_DEBUG>0) -#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) -#else -#define mi_trace_message(...) -#endif - #define mi_decl_cache_align mi_decl_align(64) #if defined(_MSC_VER) @@ -36,59 +26,26 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_noinline __declspec(noinline) #define mi_decl_thread __declspec(thread) #define mi_decl_align(a) __declspec(align(a)) -#define mi_decl_noreturn __declspec(noreturn) #define mi_decl_weak #define mi_decl_hidden -#define mi_decl_cold #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) #define mi_decl_thread __thread #define mi_decl_align(a) __attribute__((aligned(a))) -#define mi_decl_noreturn __attribute__((noreturn)) #define mi_decl_weak __attribute__((weak)) #define mi_decl_hidden __attribute__((visibility("hidden"))) -#if (__GNUC__ >= 4) || defined(__clang__) -#define mi_decl_cold __attribute__((cold)) -#else -#define mi_decl_cold -#endif #elif __cplusplus >= 201103L // c++11 #define mi_decl_noinline #define mi_decl_thread thread_local -#define mi_decl_align(a) alignas(a) -#define mi_decl_noreturn [[noreturn]] +#define mi_decl_cache_align alignas(MI_CACHE_LINE) #define mi_decl_weak #define mi_decl_hidden -#define mi_decl_cold #else #define mi_decl_noinline #define mi_decl_thread __thread // hope for the best :-) #define mi_decl_align(a) -#define mi_decl_noreturn #define mi_decl_weak #define mi_decl_hidden -#define mi_decl_cold -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define mi_unlikely(x) (__builtin_expect(!!(x),false)) -#define mi_likely(x) (__builtin_expect(!!(x),true)) -#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) -#define mi_unlikely(x) (x) [[unlikely]] -#define mi_likely(x) (x) [[likely]] -#else -#define mi_unlikely(x) (x) -#define mi_likely(x) (x) -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#else -#define mi_decl_externc #endif #if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc @@ -110,10 +67,11 @@ terms of the MIT license. A copy of the license can be found in the file #define __wasi__ #endif - -// -------------------------------------------------------------------------- -// Internal functions -// -------------------------------------------------------------------------- +#if (MI_DEBUG>0) +#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) +#else +#define mi_trace_message(...) +#endif // "libc.c" @@ -177,7 +135,7 @@ void _mi_os_init(void); // c void* _mi_os_alloc(size_t size, mi_memid_t* memid); void* _mi_os_zalloc(size_t size, mi_memid_t* memid); void _mi_os_free(void* p, size_t size, mi_memid_t memid); -void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc ); +void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid); size_t _mi_os_page_size(void); size_t _mi_os_guard_page_size(void); @@ -188,7 +146,6 @@ size_t _mi_os_virtual_address_bits(void); bool _mi_os_reset(void* addr, size_t size); bool _mi_os_decommit(void* addr, size_t size); -void _mi_os_reuse(void* p, size_t size); mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero); mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size); mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); @@ -243,7 +200,7 @@ void _mi_page_map_register(mi_page_t* page); void _mi_page_map_unregister(mi_page_t* page); void _mi_page_map_unregister_range(void* start, size_t size); mi_page_t* _mi_safe_ptr_page(const void* p); -void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc); +void _mi_page_map_unsafe_destroy(void); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; @@ -282,7 +239,6 @@ void _mi_heap_page_reclaim(mi_heap_t* heap, mi_page_t* page); // "stats.c" void _mi_stats_init(void); void _mi_stats_done(mi_stats_t* stats); -void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out, void* arg) mi_attr_noexcept; void _mi_stats_merge_thread(mi_tld_t* tld); void _mi_stats_merge_from(mi_stats_t* to, mi_stats_t* from); mi_msecs_t _mi_clock_now(void); @@ -304,13 +260,14 @@ bool _mi_page_is_valid(mi_page_t* page); #endif -// ------------------------------------------------------ -// Assertions -// ------------------------------------------------------ + +/* ----------------------------------------------------------- + Assertions +----------------------------------------------------------- */ #if (MI_DEBUG) // use our own assertion to print without memory allocation -mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept; +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) @@ -328,7 +285,6 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const #define mi_assert_expensive(x) #endif - /* ----------------------------------------------------------- Statistics (in `stats.c`) ----------------------------------------------------------- */ @@ -387,8 +343,6 @@ typedef struct mi_option_desc_s { const char* legacy_name; // potential legacy option name } mi_option_desc_t; - - /* ----------------------------------------------------------- Inlined definitions ----------------------------------------------------------- */ diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index f015977a..011a330b 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -63,11 +63,6 @@ int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); // Returns error code or 0 on success. int _mi_prim_reset(void* addr, size_t size); -// Reuse memory. This is called for memory that is already committed but -// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false. -// Returns error code or 0 on success. On most platforms this is a no-op. -int _mi_prim_reuse(void* addr, size_t size); - // Protect memory. Returns error code or 0 on success. int _mi_prim_protect(void* addr, size_t size, bool protect); diff --git a/readme.md b/readme.md index 601a7e24..cee78898 100644 --- a/readme.md +++ b/readme.md @@ -72,14 +72,15 @@ Enjoy! ### Branches -* `main`: latest stable release (still based on `dev2`). +* `master`: latest stable release (still based on `dev2`). * `dev`: development branch for mimalloc v1. Use this branch for submitting PR's. * `dev2`: development branch for mimalloc v2. This branch is downstream of `dev` (and is essentially equal to `dev` except for `src/segment.c`). Uses larger sliced segments to manage mimalloc pages that can reduce fragmentation. * `dev3`: development branch for mimalloc v3-beta. This branch is downstream of `dev`. This version simplifies the lock-free ownership of previous versions, has no thread-local segments any more. - This improves sharing of memory between threads, and on certain large workloads may use (much) less memory. + This improves sharing of memory between threads, and on certain large workloads may use less memory + with less fragmentation. ### Releases diff --git a/src/arena.c b/src/arena.c index e46ad626..7eb5bc4a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -76,7 +76,7 @@ size_t mi_arena_min_alignment(void) { return MI_ARENA_SLICE_ALIGN; } -mi_decl_nodiscard static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) { +static bool mi_arena_commit(mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) { if (arena != NULL && arena->commit_fun != NULL) { return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); } @@ -216,13 +216,12 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( } } else { - // already fully committed. - _mi_os_reuse(p, mi_size_of_slices(slice_count)); + // already fully commited. // if the OS has overcommit, and this is the first time we access these pages, then // count the commit now (as at arena reserve we didn't count those commits as these are on-demand) if (_mi_os_has_overcommit() && touched_slices > 0) { mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices)); - } + } } // tool support if (memid->initially_zero) { @@ -678,10 +677,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(size_t slice_count, size_t block_si commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE); if (commit_size > page_noguard_size) { commit_size = page_noguard_size; } bool is_zero; - if (!mi_arena_commit( mi_memid_arena(memid), page, commit_size, &is_zero, 0)) { - _mi_arenas_free(page, alloc_size, memid); - return NULL; - } + mi_arena_commit( mi_memid_arena(memid), page, commit_size, &is_zero, 0); if (!memid.initially_zero && !is_zero) { _mi_memzero_aligned(page, commit_size); } @@ -1078,7 +1074,6 @@ bool _mi_arenas_contain(const void* p) { // destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` // for dynamic libraries that are unloaded and need to release all their allocated memory. static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { - mi_assert_internal(subproc != NULL); const size_t max_arena = mi_arenas_get_count(subproc); size_t new_max_arena = 0; for (size_t i = 0; i < max_arena; i++) { @@ -1087,7 +1082,7 @@ static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { // mi_lock_done(&arena->abandoned_visit_lock); mi_atomic_store_ptr_release(mi_arena_t, &subproc->arenas[i], NULL); if (mi_memkind_is_os(arena->memid.memkind)) { - _mi_os_free_ex(mi_arena_start(arena), mi_arena_size(arena), true, arena->memid, subproc); // pass `subproc` to avoid accessing the heap pointer (in `_mi_subproc()`) + _mi_os_free(mi_arena_start(arena), mi_arena_size(arena), arena->memid); } } } @@ -1101,7 +1096,7 @@ static void mi_arenas_unsafe_destroy(mi_subproc_t* subproc) { // destroy owned arenas; this is unsafe and should only be done using `mi_option_destroy_on_exit` // for dynamic libraries that are unloaded and need to release all their allocated memory. void _mi_arenas_unsafe_destroy_all(mi_tld_t* tld) { - mi_arenas_unsafe_destroy(tld->subproc); + mi_arenas_unsafe_destroy(_mi_subproc()); _mi_arenas_collect(true /* force purge */, true /* visit all*/, tld); // purge non-owned arenas } @@ -1309,7 +1304,7 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com void* start = _mi_os_alloc_aligned(size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid); if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { - _mi_os_free_ex(start, size, commit, memid, NULL); + _mi_os_free_ex(start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); return ENOMEM; } diff --git a/src/init.c b/src/init.c index 036fc5ad..8336fdb9 100644 --- a/src/init.c +++ b/src/init.c @@ -779,12 +779,11 @@ void mi_cdecl _mi_process_done(void) { mi_heap_collect(heap, true /* force */); _mi_heap_unsafe_destroy_all(heap); // forcefully release all memory held by all heaps (of this thread only!) _mi_arenas_unsafe_destroy_all(heap->tld); - _mi_page_map_unsafe_destroy(_mi_subproc_main()); + _mi_page_map_unsafe_destroy(); } - //_mi_page_map_unsafe_destroy(_mi_subproc_main()); if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { - _mi_stats_print(&_mi_subproc_main()->stats, NULL, NULL); // use always main subproc at process exit to avoid dereferencing the heap (as it may be destroyed by now) + mi_stats_print(NULL); } _mi_allocator_done(); _mi_verbose_message("process done: 0x%zx\n", tld_main.thread_id); diff --git a/src/options.c b/src/options.c index 7fa90f83..f9188a0f 100644 --- a/src/options.c +++ b/src/options.c @@ -405,10 +405,8 @@ void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept { // add stderr to the delayed output after the module is loaded static void mi_add_stderr_output(void) { mi_assert_internal(mi_out_default == NULL); - if (mi_out_default==NULL) { - mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr - mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output - } + mi_out_buf_flush(&mi_out_stderr, false, NULL); // flush current contents to stderr + mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output } // -------------------------------------------------------- @@ -546,7 +544,7 @@ void _mi_warning_message(const char* fmt, ...) { #if MI_DEBUG -mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept { +void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) { _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion); abort(); } diff --git a/src/os.c b/src/os.c index e65f6c90..035bc1ee 100644 --- a/src/os.c +++ b/src/os.c @@ -168,23 +168,22 @@ bool _mi_os_secure_guard_page_reset_before(void* addr, mi_memid_t memid) { Free memory -------------------------------------------------------------- */ -static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc); +static void mi_os_free_huge_os_pages(void* p, size_t size); -static void mi_os_prim_free(void* addr, size_t size, size_t commit_size, mi_subproc_t* subproc) { +static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) { mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr) int err = _mi_prim_free(addr, size); // allow size==0 (issue #1041) if (err != 0) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } - if (subproc == NULL) { subproc = _mi_subproc(); } // from `mi_arenas_unsafe_destroy` we pass subproc_main explicitly as we can no longer use the heap pointer if (commit_size > 0) { - mi_subproc_stat_decrease(subproc, committed, commit_size); + mi_os_stat_decrease(committed, commit_size); } - mi_subproc_stat_decrease(subproc, reserved, size); + mi_os_stat_decrease(reserved, size); } -void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_subproc_t* subproc /* can be NULL */) { +void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) { if (mi_memkind_is_os(memid.memkind)) { size_t csize = memid.mem.os.size; if (csize==0) { csize = _mi_os_good_alloc_size(size); } @@ -205,10 +204,10 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me // free it if (memid.memkind == MI_MEM_OS_HUGE) { mi_assert(memid.is_pinned); - mi_os_free_huge_os_pages(base, csize, subproc); + mi_os_free_huge_os_pages(base, csize); } else { - mi_os_prim_free(base, csize, (still_committed ? commit_size : 0), subproc); + mi_os_prim_free(base, csize, (still_committed ? commit_size : 0)); } } else { @@ -218,7 +217,7 @@ void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t me } void _mi_os_free(void* p, size_t size, mi_memid_t memid) { - _mi_os_free_ex(p, size, true, memid, NULL); + _mi_os_free_ex(p, size, true, memid); } @@ -293,7 +292,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); } #endif - if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0), NULL); } + if (p != NULL) { mi_os_prim_free(p, size, (commit ? size : 0)); } if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -311,7 +310,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit // explicitly commit only the aligned part if (commit) { if (!_mi_os_commit(p, size, NULL)) { - mi_os_prim_free(*base, over_size, 0, NULL); + mi_os_prim_free(*base, over_size, 0); return NULL; } } @@ -327,8 +326,8 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit size_t mid_size = _mi_align_up(size, _mi_os_page_size()); size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0), NULL); } - if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), NULL); } + if (pre_size > 0) { mi_os_prim_free(p, pre_size, (commit ? pre_size : 0)); } + if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); } // we can return the aligned pointer on `mmap` systems p = aligned_p; *base = aligned_p; // since we freed the pre part, `*base == p`. @@ -532,17 +531,6 @@ bool _mi_os_reset(void* addr, size_t size) { } -void _mi_os_reuse( void* addr, size_t size ) { - // page align conservatively within the range - size_t csize = 0; - void* const start = mi_os_page_align_area_conservative(addr, size, &csize); - if (csize == 0) return; - const int err = _mi_prim_reuse(start, csize); - if (err != 0) { - _mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); - } -} - // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, mi_commit_fun_t* commit_fun, void* commit_fun_arg) @@ -680,7 +668,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // no success, issue a warning and break if (p != NULL) { _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); - mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, NULL); + mi_os_prim_free(p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); } break; } @@ -722,11 +710,11 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse // free every huge page in a range individually (as we allocated per page) // note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. -static void mi_os_free_huge_os_pages(void* p, size_t size, mi_subproc_t* subproc) { +static void mi_os_free_huge_os_pages(void* p, size_t size) { if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { - mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, subproc); + mi_os_prim_free(base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); size -= MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE; } diff --git a/src/page-map.c b/src/page-map.c index 3ff6154c..36ef3e82 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -78,11 +78,10 @@ bool _mi_page_map_init(void) { return true; } -void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) { - mi_assert_internal(subproc != NULL); +void _mi_page_map_unsafe_destroy(void) { mi_assert_internal(_mi_page_map != NULL); if (_mi_page_map == NULL) return; - _mi_os_free_ex(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc); + _mi_os_free(mi_page_map_memid.mem.os.base, mi_page_map_memid.mem.os.size, mi_page_map_memid); _mi_page_map = NULL; mi_page_map_commit = NULL; mi_page_map_max_address = NULL; @@ -266,8 +265,7 @@ bool _mi_page_map_init(void) { } -void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) { - mi_assert_internal(subproc != NULL); +void _mi_page_map_unsafe_destroy(void) { mi_assert_internal(_mi_page_map != NULL); if (_mi_page_map == NULL) return; for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) @@ -276,12 +274,12 @@ void _mi_page_map_unsafe_destroy(mi_subproc_t* subproc) { mi_page_t** sub = _mi_page_map_at(idx); if (sub != NULL) { mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); - _mi_os_free_ex(memid.mem.os.base, memid.mem.os.size, true, memid, subproc); + _mi_os_free(memid.mem.os.base, memid.mem.os.size, memid); mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL); } } } - _mi_os_free_ex(_mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid, subproc); + _mi_os_free(_mi_page_map, mi_page_map_memid.mem.os.size, mi_page_map_memid); _mi_page_map = NULL; mi_page_map_count = 0; mi_page_map_memid = _mi_memid_none(); diff --git a/src/prim/emscripten/prim.c b/src/prim/emscripten/prim.c index 9ddec5df..6a78dab0 100644 --- a/src/prim/emscripten/prim.c +++ b/src/prim/emscripten/prim.c @@ -114,11 +114,6 @@ int _mi_prim_reset(void* addr, size_t size) { return 0; } -int _mi_prim_reuse(void* addr, size_t size) { - MI_UNUSED(addr); MI_UNUSED(size); - return 0; -} - int _mi_prim_protect(void* addr, size_t size, bool protect) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); return 0; diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index fef2c60a..db38c3a1 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -434,27 +434,13 @@ int _mi_prim_commit(void* start, size_t size, bool* is_zero) { return err; } -int _mi_prim_reuse(void* start, size_t size) { - #if defined(__APPLE__) && defined(MADV_FREE_REUSE) - return unix_madvise(start, size, MADV_FREE_REUSE); - #endif - return 0; -} - int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int err = 0; + // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) + err = unix_madvise(start, size, MADV_DONTNEED); #if !MI_DEBUG && MI_SECURE<=2 *needs_recommit = false; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) - err = unix_madvise(start, size, MADV_FREE_REUSABLE); - #else - // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) - err = unix_madvise(start, size, MADV_DONTNEED); - #endif #else - // note: don't use MADV_FREE_REUSABLE as the range may contain protected areas - err = unix_madvise(start, size, MADV_DONTNEED); *needs_recommit = true; mprotect(start, size, PROT_NONE); #endif @@ -469,21 +455,14 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { } int _mi_prim_reset(void* start, size_t size) { - int err = 0; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // on macOS we try to use MADV_FREE_REUSABLE as it seems the fastest - err = unix_madvise(start, size, MADV_FREE_REUSABLE); - if (err == 0) return 0; - // fall through - #endif - - #if defined(MADV_FREE) - // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it + // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it // will not reduce the `rss` stats in tools like `top` even though the memory is available // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by // default `MADV_DONTNEED` is used though. + #if defined(MADV_FREE) static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); + int err; while ((err = unix_madvise(start, size, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on @@ -491,7 +470,7 @@ int _mi_prim_reset(void* start, size_t size) { err = unix_madvise(start, size, MADV_DONTNEED); } #else - err = unix_madvise(start, size, MADV_DONTNEED); + int err = unix_madvise(start, size, MADV_DONTNEED); #endif return err; } diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index cda6bd83..def09985 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -149,11 +149,6 @@ int _mi_prim_reset(void* addr, size_t size) { return 0; } -int _mi_prim_reuse(void* addr, size_t size) { - MI_UNUSED(addr); MI_UNUSED(size); - return 0; -} - int _mi_prim_protect(void* addr, size_t size, bool protect) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); return 0; diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 5a757923..4f00294c 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -376,11 +376,6 @@ int _mi_prim_reset(void* addr, size_t size) { return (p != NULL ? 0 : (int)GetLastError()); } -int _mi_prim_reuse(void* addr, size_t size) { - MI_UNUSED(addr); MI_UNUSED(size); - return 0; -} - int _mi_prim_protect(void* addr, size_t size, bool protect) { DWORD oldprotect = 0; BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect); diff --git a/src/stats.c b/src/stats.c index 4eba519a..ab4c8d6e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -310,7 +310,7 @@ static void mi_cdecl mi_buffered_out(const char* msg, void* arg) { // Print statistics //------------------------------------------------------------ -void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { +static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr_noexcept { // wrap the output function to be line buffered char buf[256]; _mi_memzero_var(buf); buffered_t buffer = { out0, arg0, NULL, 0, 255 };