diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ea915815..b7fc59d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,8 +6,10 @@ trigger: branches: include: - - main - - dev* + - master + - dev3 + - dev2 + - dev tags: include: - v* @@ -182,6 +184,35 @@ jobs: # Other OS versions (just debug mode) # ---------------------------------------------------------- +- job: + displayName: Windows 2019 + pool: + vmImage: + windows-2019 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + MSBuildConfiguration: Debug + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + MSBuildConfiguration: Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - task: MSBuild@1 + inputs: + solution: $(BuildType)/libmimalloc.sln + configuration: '$(MSBuildConfiguration)' + msbuildArguments: -m + - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) + workingDirectory: $(BuildType) + displayName: CTest + - job: displayName: Ubuntu 24.04 pool: diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 3e57e252..e1052787 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -8,6 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_INTERNAL_H #define MIMALLOC_INTERNAL_H + // -------------------------------------------------------------------------- // This file contains the internal API's of mimalloc and various utility // functions and macros. @@ -16,88 +17,50 @@ terms of the MIT license. A copy of the license can be found in the file #include "types.h" #include "track.h" - -// -------------------------------------------------------------------------- -// Compiler defines -// -------------------------------------------------------------------------- - #if (MI_DEBUG>0) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) #else #define mi_trace_message(...) #endif -#define mi_decl_cache_align mi_decl_align(64) - +#define MI_CACHE_LINE 64 #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) #pragma warning(disable:26812) // unscoped enum warning #define mi_decl_noinline __declspec(noinline) #define mi_decl_thread __declspec(thread) -#define mi_decl_align(a) __declspec(align(a)) -#define mi_decl_noreturn __declspec(noreturn) +#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) #define mi_decl_weak #define mi_decl_hidden -#define mi_decl_cold #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) #define mi_decl_thread __thread -#define mi_decl_align(a) __attribute__((aligned(a))) -#define mi_decl_noreturn __attribute__((noreturn)) +#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) #define mi_decl_weak __attribute__((weak)) #define mi_decl_hidden __attribute__((visibility("hidden"))) -#if (__GNUC__ >= 4) || defined(__clang__) -#define mi_decl_cold __attribute__((cold)) -#else -#define mi_decl_cold -#endif #elif __cplusplus >= 201103L // c++11 #define mi_decl_noinline #define mi_decl_thread thread_local -#define mi_decl_align(a) alignas(a) -#define mi_decl_noreturn [[noreturn]] +#define mi_decl_cache_align alignas(MI_CACHE_LINE) #define mi_decl_weak #define mi_decl_hidden -#define mi_decl_cold #else #define mi_decl_noinline #define mi_decl_thread __thread // hope for the best :-) -#define mi_decl_align(a) -#define mi_decl_noreturn +#define mi_decl_cache_align #define mi_decl_weak #define mi_decl_hidden -#define mi_decl_cold -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define mi_unlikely(x) (__builtin_expect(!!(x),false)) -#define mi_likely(x) (__builtin_expect(!!(x),true)) -#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) -#define mi_unlikely(x) (x) [[unlikely]] -#define mi_likely(x) (x) [[likely]] -#else -#define mi_unlikely(x) (x) -#define mi_likely(x) (x) -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#else -#define mi_decl_externc #endif #if defined(__EMSCRIPTEN__) && !defined(__wasi__) #define __wasi__ #endif - -// -------------------------------------------------------------------------- -// Internal functions -// -------------------------------------------------------------------------- +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#else +#define mi_decl_externc +#endif // "libc.c" #include @@ -167,7 +130,6 @@ bool _mi_os_decommit(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_purge(void* p, size_t size); bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size); -void _mi_os_reuse(void* p, size_t size); mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero); mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size); mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); @@ -294,6 +256,26 @@ bool _mi_page_is_valid(mi_page_t* page); #endif +// ------------------------------------------------------ +// Branches +// ------------------------------------------------------ + +#if defined(__GNUC__) || defined(__clang__) +#define mi_unlikely(x) (__builtin_expect(!!(x),false)) +#define mi_likely(x) (__builtin_expect(!!(x),true)) +#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) +#define mi_unlikely(x) (x) [[unlikely]] +#define mi_likely(x) (x) [[likely]] +#else +#define mi_unlikely(x) (x) +#define mi_likely(x) (x) +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + + /* ----------------------------------------------------------- Error codes passed to `_mi_fatal_error` All are recoverable but EFAULT is a serious error and aborts by default in secure mode. @@ -318,32 +300,6 @@ bool _mi_page_is_valid(mi_page_t* page); #endif -// ------------------------------------------------------ -// Assertions -// ------------------------------------------------------ - -#if (MI_DEBUG) -// use our own assertion to print without memory allocation -mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept; -#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) -#else -#define mi_assert(x) -#endif - -#if (MI_DEBUG>1) -#define mi_assert_internal mi_assert -#else -#define mi_assert_internal(x) -#endif - -#if (MI_DEBUG>2) -#define mi_assert_expensive mi_assert -#else -#define mi_assert_expensive(x) -#endif - - - /* ----------------------------------------------------------- Inlined definitions ----------------------------------------------------------- */ diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index c71678cc..3d8f1806 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -63,11 +63,6 @@ int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit); // Returns error code or 0 on success. int _mi_prim_reset(void* addr, size_t size); -// Reuse memory. This is called for memory that is already committed but -// may have been reset (`_mi_prim_reset`) or decommitted (`_mi_prim_decommit`) where `needs_recommit` was false. -// Returns error code or 0 on success. On most platforms this is a no-op. -int _mi_prim_reuse(void* addr, size_t size); - // Protect memory. Returns error code or 0 on success. int _mi_prim_protect(void* addr, size_t size, bool protect); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 855374e5..e2b5d318 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -574,6 +574,7 @@ struct mi_tld_s { }; + // ------------------------------------------------------ // Debug // ------------------------------------------------------ @@ -588,6 +589,26 @@ struct mi_tld_s { #define MI_DEBUG_PADDING (0xDE) #endif +#if (MI_DEBUG) +// use our own assertion to print without memory allocation +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); +#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) +#else +#define mi_assert(x) +#endif + +#if (MI_DEBUG>1) +#define mi_assert_internal mi_assert +#else +#define mi_assert_internal(x) +#endif + +#if (MI_DEBUG>2) +#define mi_assert_expensive mi_assert +#else +#define mi_assert_expensive(x) +#endif + // ------------------------------------------------------ // Statistics diff --git a/readme.md b/readme.md index 601a7e24..cee78898 100644 --- a/readme.md +++ b/readme.md @@ -72,14 +72,15 @@ Enjoy! ### Branches -* `main`: latest stable release (still based on `dev2`). +* `master`: latest stable release (still based on `dev2`). * `dev`: development branch for mimalloc v1. Use this branch for submitting PR's. * `dev2`: development branch for mimalloc v2. This branch is downstream of `dev` (and is essentially equal to `dev` except for `src/segment.c`). Uses larger sliced segments to manage mimalloc pages that can reduce fragmentation. * `dev3`: development branch for mimalloc v3-beta. This branch is downstream of `dev`. This version simplifies the lock-free ownership of previous versions, has no thread-local segments any more. - This improves sharing of memory between threads, and on certain large workloads may use (much) less memory. + This improves sharing of memory between threads, and on certain large workloads may use less memory + with less fragmentation. ### Releases diff --git a/src/arena.c b/src/arena.c index 25cef886..aa01ffcb 100644 --- a/src/arena.c +++ b/src/arena.c @@ -266,12 +266,12 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar else if (commit) { // commit requested, but the range may not be committed as a whole: ensure it is committed now memid->initially_committed = true; - const size_t commit_size = mi_arena_block_size(needed_bcount); bool any_uncommitted; size_t already_committed = 0; _mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted, &already_committed); if (any_uncommitted) { mi_assert_internal(already_committed < needed_bcount); + const size_t commit_size = mi_arena_block_size(needed_bcount); const size_t stat_commit_size = commit_size - mi_arena_block_size(already_committed); bool commit_zero = false; if (!_mi_os_commit_ex(p, commit_size, &commit_zero, stat_commit_size)) { @@ -281,10 +281,6 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(mi_arena_t* arena, size_t ar if (commit_zero) { memid->initially_zero = true; } } } - else { - // all are already committed: signal that we are reusing memory in case it was purged before - _mi_os_reuse( p, commit_size ); - } } else { // no need to commit, but check if already fully committed diff --git a/src/options.c b/src/options.c index 9bb5d1b3..772dfe66 100644 --- a/src/options.c +++ b/src/options.c @@ -525,7 +525,7 @@ void _mi_warning_message(const char* fmt, ...) { #if MI_DEBUG -mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept { +void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) { _mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion); abort(); } diff --git a/src/os.c b/src/os.c index 4c99d625..580b8af0 100644 --- a/src/os.c +++ b/src/os.c @@ -512,17 +512,6 @@ bool _mi_os_reset(void* addr, size_t size) { } -void _mi_os_reuse( void* addr, size_t size ) { - // page align conservatively within the range - size_t csize = 0; - void* const start = mi_os_page_align_area_conservative(addr, size, &csize); - if (csize == 0) return; - const int err = _mi_prim_reuse(start, csize); - if (err != 0) { - _mi_warning_message("cannot reuse OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize); - } -} - // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size) diff --git a/src/prim/emscripten/prim.c b/src/prim/emscripten/prim.c index c4cfc35d..a8677cbc 100644 --- a/src/prim/emscripten/prim.c +++ b/src/prim/emscripten/prim.c @@ -114,11 +114,6 @@ int _mi_prim_reset(void* addr, size_t size) { return 0; } -int _mi_prim_reuse(void* addr, size_t size) { - MI_UNUSED(addr); MI_UNUSED(size); - return 0; -} - int _mi_prim_protect(void* addr, size_t size, bool protect) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); return 0; diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 9ac855a5..a90fa659 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -433,27 +433,13 @@ int _mi_prim_commit(void* start, size_t size, bool* is_zero) { return err; } -int _mi_prim_reuse(void* start, size_t size) { - #if defined(__APPLE__) && defined(MADV_FREE_REUSE) - return unix_madvise(start, size, MADV_FREE_REUSE); - #endif - return 0; -} - int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int err = 0; + // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) + err = unix_madvise(start, size, MADV_DONTNEED); #if !MI_DEBUG && MI_SECURE<=2 *needs_recommit = false; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // decommit on macOS: use MADV_FREE_REUSABLE as it does immediate rss accounting (issue #1097) - err = unix_madvise(start, size, MADV_FREE_REUSABLE); - #else - // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) - err = unix_madvise(start, size, MADV_DONTNEED); - #endif #else - // note: don't use MADV_FREE_REUSABLE as the range may contain protected areas - err = unix_madvise(start, size, MADV_DONTNEED); *needs_recommit = true; mprotect(start, size, PROT_NONE); #endif @@ -468,21 +454,14 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { } int _mi_prim_reset(void* start, size_t size) { - int err = 0; - #if defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - // on macOS we try to use MADV_FREE_REUSABLE as it seems the fastest - err = unix_madvise(start, size, MADV_FREE_REUSABLE); - if (err == 0) return 0; - // fall through - #endif - - #if defined(MADV_FREE) - // Otherwise, we try to use `MADV_FREE` as that is the fastest. A drawback though is that it + // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it // will not reduce the `rss` stats in tools like `top` even though the memory is available // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by // default `MADV_DONTNEED` is used though. + #if defined(MADV_FREE) static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); + int err; while ((err = unix_madvise(start, size, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on @@ -490,7 +469,7 @@ int _mi_prim_reset(void* start, size_t size) { err = unix_madvise(start, size, MADV_DONTNEED); } #else - err = unix_madvise(start, size, MADV_DONTNEED); + int err = unix_madvise(start, size, MADV_DONTNEED); #endif return err; } diff --git a/src/prim/wasi/prim.c b/src/prim/wasi/prim.c index 745a41fd..e1e7de5e 100644 --- a/src/prim/wasi/prim.c +++ b/src/prim/wasi/prim.c @@ -149,11 +149,6 @@ int _mi_prim_reset(void* addr, size_t size) { return 0; } -int _mi_prim_reuse(void* addr, size_t size) { - MI_UNUSED(addr); MI_UNUSED(size); - return 0; -} - int _mi_prim_protect(void* addr, size_t size, bool protect) { MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(protect); return 0; diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index df941af9..b82918c1 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -352,11 +352,6 @@ int _mi_prim_reset(void* addr, size_t size) { return (p != NULL ? 0 : (int)GetLastError()); } -int _mi_prim_reuse(void* addr, size_t size) { - MI_UNUSED(addr); MI_UNUSED(size); - return 0; -} - int _mi_prim_protect(void* addr, size_t size, bool protect) { DWORD oldprotect = 0; BOOL ok = VirtualProtect(addr, size, protect ? PAGE_NOACCESS : PAGE_READWRITE, &oldprotect);