This commit is contained in:
daanx 2025-03-26 17:28:27 -07:00
commit d3a0bf320f
13 changed files with 106 additions and 59 deletions

View file

@ -442,7 +442,7 @@ endif()
# Compiler and architecture specific flags
if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku")
if(MI_OPT_ARCH)
if(APPLE AND CMAKE_C_COMPILER_ID STREQUAL "AppleClang" AND CMAKE_OSX_ARCHITECTURES) # to support multi-arch binaries (#999)
if(APPLE AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND CMAKE_OSX_ARCHITECTURES) # to support multi-arch binaries (#999)
if("arm64" IN_LIST CMAKE_OSX_ARCHITECTURES)
list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.1-a")
endif()
@ -553,7 +553,9 @@ if(MI_TRACK_ASAN)
endif()
string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC)
list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $<CONFIG> ?
if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$"))
if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$")
list(APPEND mi_defines MI_BUILD_RELEASE)
else()
set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version
endif()
@ -603,7 +605,7 @@ if(MI_BUILD_SHARED)
install(TARGETS mimalloc EXPORT mimalloc ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir})
if(WIN32)
if(WIN32 AND NOT MINGW)
# On windows, the import library name for the dll would clash with the static mimalloc.lib library
# so we postfix the dll import library with `.dll.lib` (and also the .pdb debug file)
set_property(TARGET mimalloc PROPERTY ARCHIVE_OUTPUT_NAME "${mi_libname}.dll" )
@ -613,6 +615,9 @@ if(MI_BUILD_SHARED)
# install(FILES "$<TARGET_FILE_DIR:mimalloc>/${mi_libname}.dll.pdb" DESTINATION ${CMAKE_INSTALL_LIBDIR})
endif()
if(WIN32 AND MI_WIN_REDIRECT)
if(MINGW)
set_property(TARGET mimalloc PROPERTY PREFIX "")
endif()
# On windows, link and copy the mimalloc redirection dll too.
if(CMAKE_GENERATOR_PLATFORM STREQUAL "arm64ec")
set(MIMALLOC_REDIRECT_SUFFIX "-arm64ec")

View file

@ -7,9 +7,9 @@ trigger:
branches:
include:
- master
- dev
- dev2
- dev3
- dev2
- dev
tags:
include:
- v*
@ -38,6 +38,14 @@ jobs:
BuildType: secure
cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON
MSBuildConfiguration: Release
Debug x86:
BuildType: debug
cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -A Win32
MSBuildConfiguration: Debug
Release x86:
BuildType: release
cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -A Win32
MSBuildConfiguration: Release
steps:
- task: CMake@1
inputs:

View file

@ -386,7 +386,7 @@ static inline void mi_atomic_yield(void) {
static inline void mi_atomic_yield(void) {
__asm__ volatile("wfe");
}
#elif (defined(__arm__) && __ARM_ARCH__ >= 7)
#elif (defined(__arm__) && __ARM_ARCH >= 7)
static inline void mi_atomic_yield(void) {
__asm__ volatile("yield" ::: "memory");
}

View file

@ -320,7 +320,7 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
#define mi_heap_stat_counter_increase(heap,stat,amount) mi_tld_stat_counter_increase(heap->tld, stat, amount)
#define mi_heap_stat_increase(heap,stat,amount) mi_tld_stat_increase( heap->tld, stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) mi_tld_stat_decrease( heap->tld, stat, amount)
#define mi_heap_stat_adjust_decrease(heap,stat,amount) mi_tld_stat_adjust_decrease( heap->tld, stat, amount)
/* -----------------------------------------------------------
Options (exposed for the debugger)

View file

@ -69,10 +69,10 @@ terms of the MIT license. A copy of the license can be found in the file
// #define MI_DEBUG 2 // + internal assertion checks
// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON)
#if !defined(MI_DEBUG)
#if !defined(NDEBUG) || defined(_DEBUG)
#define MI_DEBUG 2
#else
#if defined(MI_BUILD_RELEASE) || defined(NDEBUG)
#define MI_DEBUG 0
#else
#define MI_DEBUG 2
#endif
#endif

View file

@ -193,9 +193,6 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t
const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0;
if mi_likely(is_aligned)
{
#if MI_STAT>1
mi_heap_stat_increase(heap, malloc_requested, size);
#endif
void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen
mi_assert_internal(p != NULL);
mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);

View file

@ -92,6 +92,7 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_
#if (MI_STAT>1)
const size_t bin = _mi_bin(bsize);
mi_heap_stat_increase(heap, malloc_bins[bin], 1);
mi_heap_stat_increase(heap, malloc_requested, size - MI_PADDING_SIZE);
#endif
}
#endif
@ -150,12 +151,6 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap,
void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero);
mi_track_malloc(p,size,zero);
#if MI_STAT>1
if (p != NULL) {
if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }
mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p));
}
#endif
#if MI_DEBUG>3
if (p != NULL && zero) {
mi_assert_expensive(mi_mem_is_zero(p, size));
@ -192,12 +187,6 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z
void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic
mi_track_malloc(p,size,zero);
#if MI_STAT>1
if (p != NULL) {
if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }
mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p));
}
#endif
#if MI_DEBUG>3
if (p != NULL && zero) {
mi_assert_expensive(mi_mem_is_zero(p, size));
@ -672,7 +661,8 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo
if (p != NULL) {
if (!mi_heap_is_initialized(heap)) { heap = mi_prim_get_default_heap(); }
#if MI_STAT>1
mi_heap_stat_increase(heap, malloc_requested, mi_usable_size(p));
mi_heap_stat_adjust_decrease(heap, malloc_requested, req_size);
mi_heap_stat_increase(heap, malloc_requested, size);
#endif
mi_heap_stat_counter_increase(heap, malloc_guarded_count, 1);
}

View file

@ -532,16 +532,14 @@ static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) {
// only maintain stats for smaller objects if requested
#if (MI_STAT>0)
void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
#if (MI_STAT < 2)
static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) {
MI_UNUSED(block);
#endif
mi_heap_t* const heap = mi_heap_get_default();
const size_t bsize = mi_page_usable_block_size(page);
#if (MI_STAT>1)
const size_t usize = mi_page_usable_size_of(page, block);
mi_heap_stat_decrease(heap, malloc_requested, usize);
#endif
// #if (MI_STAT>1)
// const size_t usize = mi_page_usable_size_of(page, block);
// mi_heap_stat_decrease(heap, malloc_requested, usize);
// #endif
if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
mi_heap_stat_decrease(heap, malloc_normal, bsize);
#if (MI_STAT > 1)

View file

@ -342,7 +342,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
if (bsize > MI_LARGE_MAX_OBJ_SIZE) {
mi_heap_stat_decrease(heap, malloc_huge, bsize);
}
#if (MI_STAT)
#if (MI_STAT>0)
_mi_page_free_collect(page, false); // update used count
const size_t inuse = page->used;
if (bsize <= MI_LARGE_MAX_OBJ_SIZE) {
@ -351,7 +351,7 @@ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_
mi_heap_stat_decrease(heap, malloc_bins[_mi_bin(bsize)], inuse);
#endif
}
mi_heap_stat_decrease(heap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...
// mi_heap_stat_decrease(heap, malloc_requested, bsize * inuse); // todo: off for aligned blocks...
#endif
/// pretend it is all free now

View file

@ -31,11 +31,12 @@ terms of the MIT license. A copy of the license can be found in the file
#if defined(__linux__)
#include <features.h>
#include <linux/prctl.h> // PR_SET_VMA
//#if defined(MI_NO_THP)
#include <sys/prctl.h> // THP disable
#include <sys/prctl.h> // THP disable
//#endif
#if defined(__GLIBC__)
#include <linux/mman.h> // linux mmap flags
#include <linux/mman.h> // linux mmap flags
#else
#include <sys/mman.h>
#endif
@ -207,7 +208,17 @@ static int unix_madvise(void* addr, size_t size, int advice) {
return (res==0 ? 0 : errno);
}
static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
static void* unix_mmap_prim(void* addr, size_t size, int protect_flags, int flags, int fd) {
void* p = mmap(addr, size, protect_flags, flags, fd, 0 /* offset */);
#if (defined(__linux__) && defined(PR_SET_VMA))
if (p!=MAP_FAILED && p!=NULL) {
prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, size, "mimalloc");
}
#endif
return p;
}
static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
MI_UNUSED(try_alignment);
void* p = NULL;
#if defined(MAP_ALIGNED) // BSD
@ -215,7 +226,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
size_t idx;
size_t n = mi_bsr(try_alignment, &idx);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0);
p = unix_mmap_prim(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd);
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
int err = errno;
_mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
@ -226,7 +237,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
}
#elif defined(MAP_ALIGN) // Solaris
if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) {
p = mmap((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); // addr parameter is the required alignment
p = unix_mmap_prim((void*)try_alignment, size, protect_flags, flags | MAP_ALIGN, fd); // addr parameter is the required alignment
if (p!=MAP_FAILED) return p;
// fall back to regular mmap
}
@ -236,7 +247,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
if (addr == NULL) {
void* hint = _mi_os_get_aligned_hint(try_alignment, size);
if (hint != NULL) {
p = mmap(hint, size, protect_flags, flags, fd, 0);
p = unix_mmap_prim(hint, size, protect_flags, flags, fd);
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
#if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly?
int err = 0;
@ -251,7 +262,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
}
#endif
// regular mmap
p = mmap(addr, size, protect_flags, flags, fd, 0);
p = unix_mmap_prim(addr, size, protect_flags, flags, fd);
if (p!=MAP_FAILED) return p;
// failed to allocate
return NULL;
@ -322,7 +333,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
if (large_only || lflags != flags) {
// try large OS page allocation
*is_large = true;
p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd);
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) == MAP_HUGE_1GB) {
mi_huge_pages_available = false; // don't try huge 1GiB pages again
@ -330,7 +341,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (errno: %i)\n", errno);
}
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
p = unix_mmap_prim(addr, size, try_alignment, protect_flags, lflags, lfd);
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, lflags, lfd);
}
#endif
if (large_only) return p;
@ -343,7 +354,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec
// regular allocation
if (p == NULL) {
*is_large = false;
p = unix_mmap_prim(addr, size, try_alignment, protect_flags, flags, fd);
p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd);
if (p != NULL) {
#if defined(MADV_HUGEPAGE)
// Many Linux systems don't allow MAP_HUGETLB but they support instead

View file

@ -138,13 +138,17 @@ void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) {
uintptr_t _mi_random_next(mi_random_ctx_t* ctx) {
mi_assert_internal(mi_random_is_initialized(ctx));
#if MI_INTPTR_SIZE <= 4
return chacha_next32(ctx);
#elif MI_INTPTR_SIZE == 8
return (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
#else
# error "define mi_random_next for this platform"
#endif
uintptr_t r;
do {
#if MI_INTPTR_SIZE <= 4
r = chacha_next32(ctx);
#elif MI_INTPTR_SIZE == 8
r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx));
#else
# error "define mi_random_next for this platform"
#endif
} while (r==0);
return r;
}
@ -158,7 +162,7 @@ uintptr_t _mi_os_random_weak(uintptr_t extra_seed) {
x ^= _mi_prim_clock_now();
// and do a few randomization steps
uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1;
for (uintptr_t i = 0; i < max; i++) {
for (uintptr_t i = 0; i < max || x==0; i++, x++) {
x = _mi_random_shuffle(x);
}
mi_assert_internal(x != 0);
@ -174,7 +178,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) {
if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); }
#endif
uintptr_t x = _mi_os_random_weak(0);
for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words.
for (size_t i = 0; i < 8; i++, x++) { // key is eight 32-bit words.
x = _mi_random_shuffle(x);
((uint32_t*)key)[i] = (uint32_t)x;
}

View file

@ -227,6 +227,13 @@ static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int
_mi_fprintf(out, arg, "\n");
}
static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
_mi_fprintf(out, arg, "%10s:", msg);
_mi_fprintf(out, arg, "%12s", " "); // no peak
mi_print_amount(stat->total, unit, out, arg);
_mi_fprintf(out, arg, "\n");
}
static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) {
_mi_fprintf(out, arg, "%10s:", msg);
mi_print_amount(stat->total, -1, out, arg);
@ -243,7 +250,7 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char*
static void mi_print_header(mi_output_fun* out, void* arg ) {
_mi_fprintf(out, arg, "%10s: %11s %11s %11s %11s %11s\n", "heap stats", "peak ", "total ", "current ", "unit ", "total# ");
_mi_fprintf(out, arg, "%10s: %11s %11s %11s %11s %11s\n", "heap stats", "peak ", "total ", "current ", "block ", "total# ");
}
#if MI_STAT>1
@ -312,10 +319,10 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
// and print using that
mi_print_header(out,arg);
#if MI_STAT>1
mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, "normal",out,arg);
mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, "bin",out,arg);
#endif
#if MI_STAT
mi_stat_print(&stats->malloc_normal, "normal", (stats->malloc_normal_count.total == 0 ? 1 : -1), out, arg);
mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? 1 : -1), out, arg);
mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? 1 : -1), out, arg);
mi_stat_count_t total = { 0,0,0 };
mi_stat_count_add_mt(&total, &stats->malloc_normal);
@ -323,7 +330,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0)
mi_stat_print_ex(&total, "total", 1, out, arg, "");
#endif
#if MI_STAT>1
mi_stat_print_ex(&stats->malloc_requested, "malloc req", 1, out, arg, "");
mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg);
_mi_fprintf(out, arg, "\n");
#endif
mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, "");
@ -544,6 +551,21 @@ static void mi_heap_buf_print_count_bin(mi_heap_buf_t* hbuf, const char* prefix,
mi_heap_buf_print(hbuf, buf);
}
static void mi_heap_buf_print_count_cbin(mi_heap_buf_t* hbuf, const char* prefix, mi_stat_count_t* stat, mi_chunkbin_t bin, bool add_comma) {
const char* cbin = " ";
switch(bin) {
case MI_CBIN_SMALL: cbin = "S"; break;
case MI_CBIN_MEDIUM: cbin = "M"; break;
case MI_CBIN_LARGE: cbin = "L"; break;
case MI_CBIN_OTHER: cbin = "X"; break;
default: cbin = " "; break;
}
char buf[128];
_mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld, \"bin\": \"%s\" }%s\n", prefix, stat->total, stat->peak, stat->current, cbin, (add_comma ? "," : ""));
buf[127] = 0;
mi_heap_buf_print(hbuf, buf);
}
static void mi_heap_buf_print_count(mi_heap_buf_t* hbuf, const char* prefix, mi_stat_count_t* stat, bool add_comma) {
char buf[128];
_mi_snprintf(buf, 128, "%s{ \"total\": %lld, \"peak\": %lld, \"current\": %lld }%s\n", prefix, stat->total, stat->peak, stat->current, (add_comma ? "," : ""));
@ -630,6 +652,11 @@ char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept {
for (size_t i = 0; i <= MI_BIN_HUGE; i++) {
mi_heap_buf_print_count_bin(&hbuf, " ", &stats->page_bins[i], i, i!=MI_BIN_HUGE);
}
mi_heap_buf_print(&hbuf, " ],\n");
mi_heap_buf_print(&hbuf, " \"chunk_bins\": [\n");
for (size_t i = 0; i < MI_CBIN_COUNT; i++) {
mi_heap_buf_print_count_cbin(&hbuf, " ", &stats->chunk_bins[i], (mi_chunkbin_t)i, i!=MI_CBIN_COUNT-1);
}
mi_heap_buf_print(&hbuf, " ]\n");
mi_heap_buf_print(&hbuf, "}\n");
return hbuf.buf;

View file

@ -203,8 +203,15 @@ int main(void) {
CHECK_BODY("malloc-aligned9") { // test large alignments
bool ok = true;
void* p[8];
const int max_align_shift =
#if SIZE_MAX > UINT32_MAX
28
#else
20
#endif
;
size_t sizes[8] = { 8, 512, 1024 * 1024, MI_PAGE_MAX_OVERALLOC_ALIGN, MI_PAGE_MAX_OVERALLOC_ALIGN + 1, 2 * MI_PAGE_MAX_OVERALLOC_ALIGN, 8 * MI_PAGE_MAX_OVERALLOC_ALIGN, 0 };
for (int i = 0; i < 28 && ok; i++) {
for (int i = 0; i < max_align_shift && ok; i++) {
int align = (1 << i);
for (int j = 0; j < 8 && ok; j++) {
p[j] = mi_zalloc_aligned(sizes[j], align);