From 0f57425f808ea900ff26875f04dc5b25f6dbd204 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 30 May 2021 13:09:02 +0800 Subject: [PATCH 01/19] Distinguish SI and Binary Prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SI prefixes [the decimal prefixes] refer strictly to powers of 10. They should not be used to indicate powers of 2. e.g., one kilobit represents 1000 bits instead of 1024 bits. IEC 60027‐2 symbols are formed adding a "i" to the SI symbol (e.g. G + i = Gi). --- include/mimalloc-types.h | 22 +++++++++++----------- src/arena.c | 6 +++--- src/os.c | 6 +++--- src/segment.c | 8 ++++---- src/stats.c | 10 +++++----- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 18f1623c..502e41f8 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -105,10 +105,10 @@ terms of the MIT license. A copy of the license can be found in the file // Main tuning parameters for segment and page sizes // Sizes for 64-bit, divide by two for 32-bit -#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb -#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb -#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb -#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb +#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB +#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB +#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB // Derived constants #define MI_SEGMENT_SIZE (1UL<512kb) are put into a single page in a segment of the exact size (but still 2mb aligned) + MI_PAGE_HUGE // huge blocks (>512KiB) are put into a single page in a segment of the exact size (but still 2MiB aligned) } mi_page_kind_t; -// Segments are large allocated memory blocks (2mb on 64 bit) from +// Segments are large allocated memory blocks (2MiB on 64 bit) from // the OS. Inside segments we allocated fixed size _pages_ that // contain blocks. typedef struct mi_segment_s { diff --git a/src/arena.c b/src/arena.c index 0e6615a4..fb25a89d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -330,7 +330,7 @@ int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noe _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); return ENOMEM; } - _mi_verbose_message("reserved %zu kb memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); return 0; } @@ -347,10 +347,10 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec size_t pages_reserved = 0; void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize); if (p==NULL || pages_reserved==0) { - _mi_warning_message("failed to reserve %zu gb huge pages\n", pages); + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); return ENOMEM; } - _mi_verbose_message("numa node %i: reserved %zu gb huge pages (of the %zu gb requested)\n", numa_node, pages_reserved, pages); + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) { _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); diff --git a/src/os.c b/src/os.c index 2e2ec88d..4b87054e 100644 --- a/src/os.c +++ b/src/os.c @@ -210,7 +210,7 @@ void _mi_os_init(void) { } #elif defined(__wasi__) void _mi_os_init() { - os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KB + os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } #else @@ -964,7 +964,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) else { // fall back to regular large pages mi_huge_pages_available = false; // don't try further huge pages - _mi_warning_message("unable to allocate using huge (1gb) pages, trying large (2mb) pages instead (status 0x%lx)\n", err); + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); } } // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation @@ -1007,7 +1007,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) // see: long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { - _mi_warning_message("failed to bind huge (1gb) pages to numa node %d: %s\n", numa_node, strerror(errno)); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); } } return p; diff --git a/src/segment.c b/src/segment.c index 1d59be9d..bdf97019 100644 --- a/src/segment.c +++ b/src/segment.c @@ -17,14 +17,14 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_ /* -------------------------------------------------------------------------------- Segment allocation - We allocate pages inside bigger "segments" (4mb on 64-bit). This is to avoid + We allocate pages inside bigger "segments" (4MiB on 64-bit). This is to avoid splitting VMA's on Linux and reduce fragmentation on other OS's. Each thread owns its own segments. Currently we have: - - small pages (64kb), 64 in one segment - - medium pages (512kb), 8 in one segment - - large pages (4mb), 1 in one segment + - small pages (64KiB), 64 in one segment + - medium pages (512KiB), 8 in one segment + - large pages (4MiB), 1 in one segment - huge blocks > MI_LARGE_OBJ_SIZE_MAX become large segment with 1 page In any case the memory for a segment is virtual and usually committed on demand. diff --git a/src/stats.c b/src/stats.c index c94fbde9..07fce5e2 100644 --- a/src/stats.c +++ b/src/stats.c @@ -135,7 +135,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { char buf[32]; int len = 32; - const char* suffix = (unit <= 0 ? " " : "b"); + const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); if (unit>0) n *= unit; @@ -145,13 +145,13 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* } else { int64_t divider = base; - const char* magnitude = "k"; - if (pos >= divider*base) { divider *= base; magnitude = "m"; } - if (pos >= divider*base) { divider *= base; magnitude = "g"; } + const char* magnitude = "K"; + if (pos >= divider*base) { divider *= base; magnitude = "M"; } + if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, suffix); + snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } From a35a7d4f19b23787b67af8a49d17e18e2fc36df9 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 19 Jun 2021 09:14:43 +0000 Subject: [PATCH 02/19] haiku biuld fix proposal, warning suppression. --- src/random.c | 4 +++- src/stats.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/random.c b/src/random.c index 255bede4..723d3507 100644 --- a/src/random.c +++ b/src/random.c @@ -194,8 +194,10 @@ static bool os_random_buf(void* buf, size_t buf_len) { arc4random_buf(buf, buf_len); return true; } -#elif defined(__linux__) +#elif defined(__linux__) || defined(__HAIKU__) +#if defined(__linux__) #include +#endif #include #include #include diff --git a/src/stats.c b/src/stats.c index c94fbde9..74b783e9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -520,6 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec while (get_next_area_info(tid.team, &c, &mem) == B_OK) { *peak_rss += mem.ram_size; } + *page_faults = 0; #elif defined(__APPLE__) *peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; From 1c1571742d24b9cfbdbef4a3c537018f41756c64 Mon Sep 17 00:00:00 2001 From: hank Date: Mon, 21 Jun 2021 22:36:47 +0800 Subject: [PATCH 03/19] fix typo --- include/mimalloc-internal.h | 2 +- src/page.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..825b0ed0 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -427,7 +427,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) { return _mi_segment_page_of(_mi_ptr_segment(p), p); } -// Get the block size of a page (special cased for huge objects) +// Get the block size of a page (special case for huge objects) static inline size_t mi_page_block_size(const mi_page_t* page) { const size_t bsize = page->xblock_size; mi_assert_internal(bsize > 0); diff --git a/src/page.c b/src/page.c index c08be9c0..033976dc 100644 --- a/src/page.c +++ b/src/page.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- The core of the allocator. Every segment contains - pages of a {certain block size. The main function + pages of a certain block size. The main function exported is `mi_malloc_generic`. ----------------------------------------------------------- */ From edb0b93c6fdd2edbe4da520c1d632010e91dedcb Mon Sep 17 00:00:00 2001 From: Artur Sinila Date: Tue, 29 Jun 2021 21:07:13 +0300 Subject: [PATCH 04/19] Fix 'malloc-nomem1' test for 32-bit architectures --- test/test-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-api.c b/test/test-api.c index d3344928..739b1d18 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -83,7 +83,7 @@ int main() { void* p = mi_malloc(0); mi_free(p); }); CHECK_BODY("malloc-nomem1",{ - result = (mi_malloc(SIZE_MAX/2) == NULL); + result = (mi_malloc((size_t)PTRDIFF_MAX + (size_t)1) == NULL); }); CHECK_BODY("malloc-null",{ mi_free(NULL); From 30be78d97a4a1a25c81755fb2671428a5893a0d9 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:31:48 -0400 Subject: [PATCH 05/19] Fixed typo in headers --- include/mimalloc-types.h | 2 +- src/heap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index caf161d6..66b311c1 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -319,7 +319,7 @@ typedef struct mi_random_cxt_s { } mi_random_ctx_t; -// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows #if (MI_PADDING) typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) diff --git a/src/heap.c b/src/heap.c index bda10699..bbf4cef9 100644 --- a/src/heap.c +++ b/src/heap.c @@ -333,7 +333,7 @@ void mi_heap_destroy(mi_heap_t* heap) { Safe Heap delete ----------------------------------------------------------- */ -// Tranfer the pages from one heap to the other +// Transfer the pages from one heap to the other static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; From ee0b01c84dbaf8b65f505fbbb2debb64bfc489c6 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:46:58 -0400 Subject: [PATCH 06/19] Fixed stats in mi_os_mem_alloc_aligned on windows when re-allocate In the last try the previous mi_os_mem_free decrease stat and mi_win_virtual_alloc doesn't increase it --- src/os.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/os.c b/src/os.c index 85415232..89d166ef 100644 --- a/src/os.c +++ b/src/os.c @@ -593,6 +593,11 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); + if (p != NULL) + { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } + } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? mi_os_mem_free(p, size, commit, stats); From 9ee780894a18c3892a1c64bb2950fa862a3b7588 Mon Sep 17 00:00:00 2001 From: Masashi Fujita Date: Thu, 16 Sep 2021 03:35:56 +0900 Subject: [PATCH 07/19] fix: Fix a typo --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..385d2071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ set(mi_sources src/init.c) # ----------------------------------------------------------------------------- -# Converience: set default build type depending on the build directory +# Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- if (NOT CMAKE_BUILD_TYPE) From c21b6df51e8ff558b852e088ec9d0a99a15bff94 Mon Sep 17 00:00:00 2001 From: Jeremy Lorelli Date: Tue, 5 Oct 2021 08:41:03 -0700 Subject: [PATCH 08/19] Fix missing parameter in mi_free error message --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..1154e70a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -465,7 +465,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #endif #if (MI_DEBUG>0 || MI_SECURE>=4) if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { - _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", p); + _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); } #endif return segment; From 3c058f07a98fe1bc35390099e9674509a74a8d3f Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 8 Oct 2021 23:59:35 -0700 Subject: [PATCH 09/19] Add an option to disable automatic use of `getenv` inside options.c --- src/options.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 30025db2..2dc7bfee 100644 --- a/src/options.c +++ b/src/options.c @@ -409,6 +409,14 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest[dest_size - 1] = 0; } +#ifdef MI_NO_GETENV +static bool mi_getenv(const char* name, char* result, size_t result_size) { + UNUSED(name); + UNUSED(result); + UNUSED(result_size); + return false; +} +#else static inline int mi_strnicmp(const char* s, const char* t, size_t n) { if (n==0) return 0; for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { @@ -416,7 +424,6 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) { } return (n==0 ? 0 : *s - *t); } - #if defined _WIN32 // On Windows use GetEnvironmentVariable instead of getenv to work // reliably even when this is invoked before the C runtime is initialized. @@ -485,6 +492,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { } } #endif +#endif static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 7c73e3996d5d729c1baec2e833985166a3f5db2e Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 10:38:57 +0200 Subject: [PATCH 10/19] Fix strict function prototype warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix warning ``warning: function declaration isn’t a prototype`` when building mimalloc with ``-Wstrict-prototypes`` flag. In C argumentless functions should be declared as ``func(void)``. Reproducer: ```shell $ cmake ../.. -DCMAKE_C_FLAGS="-Wstrict-prototypes" $ make VERBOSE=1 ``` Co-authored-by: Sam Gross Co-authored-by: Neil Schemenauer Signed-off-by: Christian Heimes --- include/mimalloc-internal.h | 2 +- src/alloc.c | 8 ++++---- src/options.c | 2 +- src/os.c | 2 +- src/region.c | 2 +- test/test-api.c | 10 +++++----- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..9b097b8e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -53,7 +53,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); -bool _mi_preloading(); // true while the C runtime is not ready +bool _mi_preloading(void); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); diff --git a/src/alloc.c b/src/alloc.c index 8acff783..d2ba5c68 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -747,7 +747,7 @@ mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char } #else #include // pathconf -static size_t mi_path_max() { +static size_t mi_path_max(void) { static size_t path_max = 0; if (path_max <= 0) { long m = pathconf("/",_PC_PATH_MAX); @@ -807,13 +807,13 @@ static bool mi_try_new_handler(bool nothrow) { } } #else -typedef void (*std_new_handler_t)(); +typedef void (*std_new_handler_t)(void); #if (defined(__GNUC__) || defined(__clang__)) -std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() { +std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv(void) { return NULL; } -static std_new_handler_t mi_get_new_handler() { +static std_new_handler_t mi_get_new_handler(void) { return _ZSt15get_new_handlerv(); } #else diff --git a/src/options.c b/src/options.c index 30025db2..4274af97 100644 --- a/src/options.c +++ b/src/options.c @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file static uintptr_t mi_max_error_count = 16; // stop outputting errors after this static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this -static void mi_add_stderr_output(); +static void mi_add_stderr_output(void); int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; diff --git a/src/os.c b/src/os.c index 85415232..16659b33 100644 --- a/src/os.c +++ b/src/os.c @@ -95,7 +95,7 @@ size_t _mi_os_page_size() { } // if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) -size_t _mi_os_large_page_size() { +size_t _mi_os_large_page_size(void) { return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); } diff --git a/src/region.c b/src/region.c index 79540730..2f68b140 100644 --- a/src/region.c +++ b/src/region.c @@ -40,7 +40,7 @@ Possible issues: #include "bitmap.h" // Internal raw OS interface -size_t _mi_os_large_page_size(); +size_t _mi_os_large_page_size(void); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); diff --git a/test/test-api.c b/test/test-api.c index d3344928..53e7f3d8 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -64,15 +64,15 @@ static int failed = 0; // --------------------------------------------------------------------------- // Test functions // --------------------------------------------------------------------------- -bool test_heap1(); -bool test_heap2(); -bool test_stl_allocator1(); -bool test_stl_allocator2(); +bool test_heap1(void); +bool test_heap2(void); +bool test_stl_allocator1(void); +bool test_stl_allocator2(void); // --------------------------------------------------------------------------- // Main testing // --------------------------------------------------------------------------- -int main() { +int main(void) { mi_option_disable(mi_option_verbose); // --------------------------------------------------- From afbcf20f24729e9bf2e87a72efadeb48ddcdd6f7 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 15:07:52 +0200 Subject: [PATCH 11/19] Define _DEFAULT_SOURCE for syscall and realpath Define ``_DEFAULT_SOURCE`` in ``random.c`` and ``alloc.c``. The macro is required for ``syscall()`` and ``realpath()``. Other files like ``os.c`` already define the macro. Signed-off-by: Christian Heimes --- src/alloc.c | 4 ++++ src/random.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..5e45e76f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for realpath() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" diff --git a/src/random.c b/src/random.c index 255bede4..0e2331bd 100644 --- a/src/random.c +++ b/src/random.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for syscall() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" From 60937b5bc8ac142ebc0ed2df5aca429fabd01c60 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:39:33 -0700 Subject: [PATCH 12/19] add -Wstrict-prototypes flag during compilation --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..ac4c0569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,7 +165,7 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -Wstrict-prototypes -fvisibility=hidden) if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() From f0f9aecfe49a1a14e6247770e6b677016902fab4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:52:20 -0700 Subject: [PATCH 13/19] add comment on #if ending --- src/options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/options.c b/src/options.c index 8d5f1556..c40a187b 100644 --- a/src/options.c +++ b/src/options.c @@ -491,8 +491,8 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { return false; } } -#endif -#endif +#endif // !MI_USE_ENVIRON +#endif // !MI_NO_GETENV static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 5b9409f4d6c1b010c79eaa81117ba9b7e57672e3 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:17:30 -0700 Subject: [PATCH 14/19] add space after _Atomic to prevent errors on msvc without /TP (see PR #452) --- include/mimalloc-atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index dc48f0a2..e07df84d 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -25,7 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_memory_order(name) std::memory_order_##name #elif defined(_MSC_VER) // Use MSVC C wrapper for C11 atomics -#define _Atomic(tp) tp +#define _Atomic(tp) tp #define ATOMIC_VAR_INIT(x) x #define mi_atomic(name) mi_atomic_##name #define mi_memory_order(name) mi_memory_order_##name @@ -173,7 +173,7 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt } static inline void mi_atomic_thread_fence(mi_memory_order mo) { (void)(mo); - _Atomic(uintptr_t)x = 0; + _Atomic(uintptr_t) x = 0; mi_atomic_exchange_explicit(&x, 1, mo); } static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { From adeef8d4035c91b34a97fb83acf1c6bde1f1e4c9 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:31:49 -0700 Subject: [PATCH 15/19] formatting --- src/os.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index d7d3d33d..ba6cc5e3 100644 --- a/src/os.c +++ b/src/os.c @@ -641,10 +641,9 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); - if (p != NULL) - { - _mi_stat_increase(&stats->reserved, size); - if (commit) { _mi_stat_increase(&stats->committed, size); } + if (p != NULL) { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? From 2d821003b07b4ccdfbf5259592f7831684f996ee Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:56:13 -0700 Subject: [PATCH 16/19] don't add prefix if build type is None (PR #427) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d1d9191..abab5a05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ else() endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel)$")) +if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$")) set(mi_basename "${mi_basename}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version endif() if(MI_BUILD_SHARED) From 9a724889ead0ab9ad04f66b91b3e22064ccd8d52 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 12:50:16 -0700 Subject: [PATCH 17/19] refine stat output with new SI units --- src/stats.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/stats.c b/src/stats.c index f85285e1..dda55cda 100644 --- a/src/stats.c +++ b/src/stats.c @@ -133,7 +133,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // unit == 0: count as decimal // unit < 0 : count in binary static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { - char buf[32]; + char buf[32]; buf[0] = 0; int len = 32; const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); @@ -141,17 +141,21 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t pos = (n < 0 ? -n : n); if (pos < base) { - snprintf(buf, len, "%d %s ", (int)n, suffix); + if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column + snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix)); + } } else { - int64_t divider = base; + int64_t divider = base; const char* magnitude = "K"; if (pos >= divider*base) { divider *= base; magnitude = "M"; } if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); + char unitdesc[16]; + snprintf(unitdesc, 16, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } @@ -221,7 +225,7 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* static void mi_print_header(mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); + _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); } #if MI_STAT>1 From bb00a5a32e53067e6e23215d883fbece5ec0a74f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 13:09:43 -0700 Subject: [PATCH 18/19] remove MADV_FREE_REUSABLE from mi_os_reset as it needs MADV_FREE_REUSE to reuse the memory again. --- src/os.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/os.c b/src/os.c index 08068c9d..bf1d4716 100644 --- a/src/os.c +++ b/src/os.c @@ -796,9 +796,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } - #if defined(MADV_FREE_REUSE) - while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } - #endif + //#if defined(MADV_FREE_REUSE) + // while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } + //#endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -860,17 +860,12 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - #if defined(MADV_FREE_REUSABLE) - #define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE - #else - #define KK_MADV_FREE_INITIAL MADV_FREE - #endif - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL); + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; - if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) { - // if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on + if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } From ae5aae7e105eaccef37be0cda601b005264d2d29 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 14:00:40 -0700 Subject: [PATCH 19/19] allow decommit in arena separate from initial commit --- src/arena.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/arena.c b/src/arena.c index fb25a89d..2f5a349f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -62,11 +62,11 @@ typedef struct mi_arena_s { size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) int numa_node; // associated NUMA node bool is_zero_init; // is the arena zero initialized? - bool is_committed; // is the memory fully committed? (if so, block_committed == NULL) + bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? - mi_bitmap_field_t* blocks_committed; // if `!is_committed`, are the blocks committed? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) } mi_arena_t; @@ -129,8 +129,8 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n *memid = mi_arena_id_create(arena_index, bitmap_index); *is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); *large = arena->is_large; - *is_pinned = (arena->is_large || arena->is_committed); - if (arena->is_committed) { + *is_pinned = (arena->is_large || !arena->allow_decommit); + if (arena->blocks_committed == NULL) { // always committed *commit = true; } @@ -245,12 +245,13 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s return; } // potentially decommit - if (arena->is_committed) { - mi_assert_internal(all_committed); + if (!arena->allow_decommit || arena->blocks_committed == NULL) { + mi_assert_internal(all_committed); // note: may be not true as we may "pretend" to be not committed (in segment.c) } else { mi_assert_internal(arena->blocks_committed != NULL); _mi_os_decommit(p, blocks * MI_ARENA_BLOCK_SIZE, stats); // ok if this fails + // todo: use reset instead of decommit on windows? _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); } // and make it available to others again @@ -300,12 +301,16 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->is_large = is_large; arena->is_zero_init = is_zero; - arena->is_committed = is_committed; + arena->allow_decommit = !is_large && !is_committed; // only allow decommit for initially uncommitted memory arena->search_idx = 0; arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap - arena->blocks_committed = (is_committed ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap + arena->blocks_committed = (!arena->allow_decommit ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap // the bitmaps are already zero initialized due to os_alloc - // just claim leftover blocks if needed + // initialize committed bitmap? + if (arena->blocks_committed != NULL && is_committed) { + memset(arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); + } + // and claim leftover blocks if needed (so we never allocate there) ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; mi_assert_internal(post >= 0); if (post > 0) {