diff --git a/CMakeLists.txt b/CMakeLists.txt
index 295becb2..1387e0db 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -27,7 +27,7 @@ option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF)
option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF)
option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF)
option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF)
-option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version (deprecated)" OFF)
+option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF)
# deprecated options
option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF)
@@ -45,6 +45,7 @@ set(mi_sources
src/bitmap.c
src/heap.c
src/init.c
+ src/libc.c
src/options.c
src/os.c
src/page.c
diff --git a/ide/vs2017/mimalloc-override.vcxproj b/ide/vs2017/mimalloc-override.vcxproj
index 3d5c1f75..6d20eb57 100644
--- a/ide/vs2017/mimalloc-override.vcxproj
+++ b/ide/vs2017/mimalloc-override.vcxproj
@@ -238,6 +238,7 @@
+
diff --git a/ide/vs2017/mimalloc-override.vcxproj.filters b/ide/vs2017/mimalloc-override.vcxproj.filters
index 70f84d59..1adafcfa 100644
--- a/ide/vs2017/mimalloc-override.vcxproj.filters
+++ b/ide/vs2017/mimalloc-override.vcxproj.filters
@@ -91,5 +91,8 @@
Source Files
+
+ Source Files
+
\ No newline at end of file
diff --git a/ide/vs2017/mimalloc.vcxproj b/ide/vs2017/mimalloc.vcxproj
index 46eb05d8..ece9a14d 100644
--- a/ide/vs2017/mimalloc.vcxproj
+++ b/ide/vs2017/mimalloc.vcxproj
@@ -227,6 +227,7 @@
+
diff --git a/ide/vs2017/mimalloc.vcxproj.filters b/ide/vs2017/mimalloc.vcxproj.filters
index 0c2bd522..8359e0e4 100644
--- a/ide/vs2017/mimalloc.vcxproj.filters
+++ b/ide/vs2017/mimalloc.vcxproj.filters
@@ -62,6 +62,9 @@
Source Files
+
+ Source Files
+
diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj
index 1c5c61b7..a84a5178 100644
--- a/ide/vs2019/mimalloc-override.vcxproj
+++ b/ide/vs2019/mimalloc-override.vcxproj
@@ -238,6 +238,7 @@
+
diff --git a/ide/vs2019/mimalloc-override.vcxproj.filters b/ide/vs2019/mimalloc-override.vcxproj.filters
index 370c8ab3..046e5603 100644
--- a/ide/vs2019/mimalloc-override.vcxproj.filters
+++ b/ide/vs2019/mimalloc-override.vcxproj.filters
@@ -52,6 +52,9 @@
Source Files
+
+ Source Files
+
diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj
index 0e2eb312..0076b1db 100644
--- a/ide/vs2019/mimalloc.vcxproj
+++ b/ide/vs2019/mimalloc.vcxproj
@@ -219,6 +219,7 @@
+
true
diff --git a/ide/vs2019/mimalloc.vcxproj.filters b/ide/vs2019/mimalloc.vcxproj.filters
index 21f9c517..98f29289 100644
--- a/ide/vs2019/mimalloc.vcxproj.filters
+++ b/ide/vs2019/mimalloc.vcxproj.filters
@@ -55,6 +55,9 @@
Source Files
+
+ Source Files
+
diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj
index e2c7f71d..df2a0816 100644
--- a/ide/vs2022/mimalloc-override.vcxproj
+++ b/ide/vs2022/mimalloc-override.vcxproj
@@ -240,6 +240,7 @@
+
true
diff --git a/ide/vs2022/mimalloc-override.vcxproj.filters b/ide/vs2022/mimalloc-override.vcxproj.filters
index 0f105c1a..dd69c827 100644
--- a/ide/vs2022/mimalloc-override.vcxproj.filters
+++ b/ide/vs2022/mimalloc-override.vcxproj.filters
@@ -55,6 +55,9 @@
Sources
+
+ Sources
+
diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj
index a02a8393..11da11c3 100644
--- a/ide/vs2022/mimalloc.vcxproj
+++ b/ide/vs2022/mimalloc.vcxproj
@@ -219,6 +219,7 @@
+
true
diff --git a/ide/vs2022/mimalloc.vcxproj.filters b/ide/vs2022/mimalloc.vcxproj.filters
index b3cdb3b3..bb5c8ce9 100644
--- a/ide/vs2022/mimalloc.vcxproj.filters
+++ b/ide/vs2022/mimalloc.vcxproj.filters
@@ -55,6 +55,9 @@
Sources
+
+ Sources
+
diff --git a/include/mimalloc.h b/include/mimalloc.h
index e7947e81..f06f88e1 100644
--- a/include/mimalloc.h
+++ b/include/mimalloc.h
@@ -256,6 +256,17 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_zalloc_remappable(mi_heap_t* heap
// mi_decl_nodiscard mi_decl_export void* mi_remap(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
+// ------------------------------------------------------
+// Expandable memory reserves a virtual address range of `max_expand_size` (capped at 1TiB)
+// and commits this on-demand through `mi_realloc` and `mi_expand`. Both of those will
+// expand inplace for blocks allocated as expandable up to `max_expand_size`.
+// ------------------------------------------------------
+
+mi_decl_nodiscard void* mi_heap_malloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept;
+mi_decl_nodiscard void* mi_heap_zalloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept;
+mi_decl_nodiscard void* mi_malloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept;
+mi_decl_nodiscard void* mi_zalloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept;
+
// ------------------------------------------------------
// Analysis
diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h
index d4c9dfed..60f334e8 100644
--- a/include/mimalloc/internal.h
+++ b/include/mimalloc/internal.h
@@ -202,14 +202,17 @@ bool _mi_free_delayed_block(mi_block_t* block);
void _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration
void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size);
-// option.c, c primitives
+// "libc.c"
+#include
+void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
+void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...);
char _mi_toupper(char c);
int _mi_strnicmp(const char* s, const char* t, size_t n);
void _mi_strlcpy(char* dest, const char* src, size_t dest_size);
void _mi_strlcat(char* dest, const char* src, size_t dest_size);
size_t _mi_strlen(const char* s);
size_t _mi_strnlen(const char* s, size_t max_len);
-
+bool _mi_getenv(const char* name, char* result, size_t result_size);
#if MI_DEBUG>1
bool _mi_page_is_valid(mi_page_t* page);
diff --git a/src/alloc.c b/src/alloc.c
index c471de92..e27a7594 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -690,53 +690,78 @@ mi_decl_nodiscard mi_decl_restrict void* mi_mallocn(size_t count, size_t size) m
return mi_heap_mallocn(mi_prim_get_default_heap(),count,size);
}
+static void* mi_heap_try_expand_zero(mi_heap_t* heap, mi_segment_t* segment, void* p, size_t size, size_t newsize, bool zero);
+
// Expand (or shrink) in place (or fail)
-void* mi_expand(void* p, size_t newsize) mi_attr_noexcept {
- #if MI_PADDING
- // we do not shrink/expand with padding enabled
- MI_UNUSED(p); MI_UNUSED(newsize);
- return NULL;
- #else
+void* mi_expand(void* p, size_t newsize) mi_attr_noexcept
+{
if (p == NULL) return NULL;
+
const size_t size = _mi_usable_size(p,"mi_expand");
+ if (size == newsize) return p;
+
+ // try OS expand
+ mi_segment_t* const segment = mi_checked_ptr_segment(p, "mi_expand");
+ if mi_unlikely(segment->memid.memkind == MI_MEM_OS_EXPAND) {
+ void* newp = mi_heap_try_expand_zero(mi_prim_get_default_heap(), segment, p, size, newsize, false);
+ mi_assert_internal(newp == NULL || newp == p);
+ if (newp != NULL) return newp;
+ }
+
+ // otherwise we cannot grow inplace
if (newsize > size) return NULL;
- return p; // it fits
- #endif
+
+ // shrink padding
+ mi_page_t* page = _mi_segment_page_of(segment, p);
+ mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+ mi_padding_init(page, block, newsize);
+ mi_track_resize(p, size, newsize);
+
+ return p; // it still fits
}
static void* mi_heap_try_remap_zero(mi_heap_t* heap, mi_segment_t* segment, void* p, size_t size, size_t newsize, bool zero);
+static void mi_padding_init_ptr(void* p, size_t size) {
+ mi_segment_t* segment = mi_checked_ptr_segment(p, "_mi_padding_init_ptr");
+ mi_page_t* page = _mi_segment_page_of(segment, p);
+ mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+ mi_padding_init(page, block, size);
+}
+
void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept {
// if p == NULL then behave as malloc.
if mi_unlikely(p==NULL) return _mi_heap_malloc_zero(heap,newsize,zero);
- // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
+ // else if newsize == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)).
// (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.)
const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0)
- if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
+ if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0 && newsize < 1024) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0)
mi_assert_internal(p!=NULL);
- // todo: do not track as the usable size is still the same in the free; adjust potential padding?
- // mi_track_resize(p,size,newsize)
- // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); }
+ mi_padding_init_ptr(p, newsize);
+ mi_track_resize(p,size,newsize)
return p; // reallocation still fits and not more than 50% waste
}
- // use OS remap for large reallocations
+
+ // try OS expand
mi_segment_t* const segment = mi_checked_ptr_segment(p, "mi_realloc");
+ if mi_unlikely(segment->memid.memkind == MI_MEM_OS_EXPAND) {
+ void* newp = mi_heap_try_expand_zero(heap, segment, p, size, newsize, zero);
+ mi_assert_internal(newp == NULL || newp == p);
+ if (newp != NULL) return newp;
+ }
+
+ // try OS remap for large reallocations
const size_t remap_threshold = mi_option_get_size(mi_option_remap_threshold);
const bool use_remap = (segment->memid.memkind == MI_MEM_OS_REMAP) || (remap_threshold > 0 && newsize >= remap_threshold);
if mi_unlikely(use_remap) {
void* newp = mi_heap_try_remap_zero(heap, segment, p, size, newsize, zero);
if (newp != NULL) return newp;
}
+
// otherwise copy into a new area
- void* newp;
- if mi_unlikely(use_remap) {
- newp = mi_heap_malloc_remappable(heap, newsize);
- }
- else {
- newp = mi_heap_malloc(heap, newsize);
- }
+ void* newp = (use_remap ? mi_heap_malloc_remappable(heap, newsize) : mi_heap_malloc(heap, newsize));
if mi_likely(newp != NULL) {
if (zero && newsize > size) {
// also set last word in the previous allocation to zero to ensure any padding is zero-initialized
@@ -806,6 +831,76 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_
return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size);
}
+// ------------------------------------------------------
+// expand
+// ------------------------------------------------------
+
+static void* mi_heap_malloc_zero_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size, bool zero) mi_attr_noexcept {
+ if (max_expand_size <= size) {
+ return _mi_heap_malloc_zero(heap, size, zero);
+ }
+ size_t increments = _mi_divide_up(max_expand_size, MI_EXPAND_INCREMENT);
+ if (increments > MI_ALIGN_EXPAND_MAX) increments = MI_ALIGN_EXPAND_MAX;
+ else if (increments < MI_ALIGN_EXPAND_MIN) increments = MI_ALIGN_EXPAND_MIN;
+ return _mi_heap_malloc_zero_ex(heap, size, zero, increments);
+}
+
+mi_decl_nodiscard void* mi_heap_malloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept {
+ return mi_heap_malloc_zero_expandable(heap, size, max_expand_size, false);
+}
+
+mi_decl_nodiscard void* mi_heap_zalloc_expandable(mi_heap_t* heap, size_t size, size_t max_expand_size) mi_attr_noexcept {
+ return mi_heap_malloc_zero_expandable(heap, size, max_expand_size, true);
+}
+
+mi_decl_nodiscard void* mi_malloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept {
+ return mi_heap_malloc_expandable(mi_prim_get_default_heap(), size, max_expand_size);
+}
+
+mi_decl_nodiscard void* mi_zalloc_expandable(size_t size, size_t max_expand_size) mi_attr_noexcept {
+ return mi_heap_zalloc_expandable(mi_prim_get_default_heap(), size, max_expand_size);
+}
+
+
+static void* mi_heap_try_expand_zero(mi_heap_t* heap, mi_segment_t* segment, void* p, size_t size, size_t newsize, bool zero)
+{
+ if (newsize == 0) return NULL;
+ if (p == NULL) {
+ return mi_heap_malloc_zero_expandable(heap, newsize, zero, (newsize < PTRDIFF_MAX/2 ? 2*newsize : newsize));
+ }
+
+ // expandable memory?
+ if (segment->memid.memkind != MI_MEM_OS_EXPAND) return NULL;
+
+ // we can only expand from an owning thread (as the segment is modified temporarily)
+ const mi_threadid_t tid = _mi_prim_thread_id();
+ mi_assert(heap->thread_id == tid);
+ if (segment->thread_id != tid) return NULL;
+
+ // check size
+ const size_t padsize = newsize + MI_PADDING_SIZE;
+ mi_assert_internal(segment != NULL);
+ mi_page_t* page = _mi_segment_page_of(segment, p);
+ mi_block_t* block = _mi_page_ptr_unalign(segment, page, p);
+
+ // try to use OS expand
+ mi_assert_internal((void*)block == p);
+ block = _mi_segment_huge_page_expand(segment, page, block, padsize, &heap->tld->segments);
+ mi_assert_internal(block == NULL || (void*)block == p);
+ if (block == NULL) return NULL;
+
+ _mi_trace_message("expanded inplace (address: %p to %zu bytes)\n", p, newsize);
+ mi_padding_init(page, block, newsize);
+ mi_track_resize(p, size, newsize);
+ if (zero) {
+ // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
+ const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
+ _mi_memzero((uint8_t*)p + start, newsize - start);
+ }
+ return block;
+}
+
+
// ------------------------------------------------------
// remap
// ------------------------------------------------------
@@ -871,6 +966,7 @@ static void* mi_heap_try_remap_zero(mi_heap_t* heap, mi_segment_t* segment, void
if (bsize >= padsize && 9*(bsize/10) <= padsize) { // if smaller and not more than 10% waste, keep it
_mi_verbose_message("remapping in the same block (address: %p from %zu bytes to %zu bytes)\n", p, mi_usable_size(p), newsize);
mi_padding_init(page, block, newsize);
+ mi_track_resize(p, size, newsize);
return p;
}
@@ -879,7 +975,7 @@ static void* mi_heap_try_remap_zero(mi_heap_t* heap, mi_segment_t* segment, void
block = _mi_segment_huge_page_remap(segment, page, block, padsize, &heap->tld->segments);
if (block != NULL) {
// succes! re-establish the pointers to the potentially relocated memory
- _mi_verbose_message("used remap (address: %p to %zu bytes)\n", p, newsize);
+ _mi_trace_message("used remap (address: %p to %zu bytes)\n", p, newsize);
segment = mi_checked_ptr_segment(block, "mi_remap");
page = _mi_segment_page_of(segment, block);
mi_padding_init(page, block, newsize);
diff --git a/src/arena.c b/src/arena.c
index fe8c5440..345621e3 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -812,7 +812,7 @@ int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exc
const bool is_large = memid.is_pinned; // todo: use separate is_large field?
if (!mi_manage_os_memory_ex2(start, size, is_large, -1 /* numa node */, exclusive, memid, arena_id)) {
_mi_os_free_ex(start, size, commit, memid, &_mi_stats_main);
- _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size, 1024));
+ _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024));
return ENOMEM;
}
_mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size, 1024), is_large ? " (in large os pages)" : "");
diff --git a/src/init.c b/src/init.c
index c9aa4b0f..fe885a5b 100644
--- a/src/init.c
+++ b/src/init.c
@@ -175,9 +175,9 @@ mi_heap_t* _mi_heap_main_get(void) {
// note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size).
typedef struct mi_thread_data_s {
- mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
+ mi_heap_t heap; // must come first due to cast in `_mi_heap_done`
mi_tld_t tld;
- mi_memid_t memid;
+ mi_memid_t memid; // must come last due to zero'ing
} mi_thread_data_t;
@@ -223,7 +223,7 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) {
}
if (td != NULL && !is_zero) {
- _mi_memzero_aligned(td, sizeof(*td));
+ _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid));
}
return td;
}
diff --git a/src/libc.c b/src/libc.c
new file mode 100644
index 00000000..f1412722
--- /dev/null
+++ b/src/libc.c
@@ -0,0 +1,273 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// --------------------------------------------------------
+// This module defines various std libc functions to reduce
+// the dependency on libc, and also prevent errors caused
+// by some libc implementations when called before `main`
+// executes (due to malloc redirection)
+// --------------------------------------------------------
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "mimalloc/prim.h" // mi_prim_getenv
+
+char _mi_toupper(char c) {
+ if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
+ else return c;
+}
+
+int _mi_strnicmp(const char* s, const char* t, size_t n) {
+ if (n == 0) return 0;
+ for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
+ if (_mi_toupper(*s) != _mi_toupper(*t)) break;
+ }
+ return (n == 0 ? 0 : *s - *t);
+}
+
+void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
+ if (dest==NULL || src==NULL || dest_size == 0) return;
+ // copy until end of src, or when dest is (almost) full
+ while (*src != 0 && dest_size > 1) {
+ *dest++ = *src++;
+ dest_size--;
+ }
+ // always zero terminate
+ *dest = 0;
+}
+
+void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
+ if (dest==NULL || src==NULL || dest_size == 0) return;
+ // find end of string in the dest buffer
+ while (*dest != 0 && dest_size > 1) {
+ dest++;
+ dest_size--;
+ }
+ // and catenate
+ _mi_strlcpy(dest, src, dest_size);
+}
+
+size_t _mi_strlen(const char* s) {
+ if (s==NULL) return 0;
+ size_t len = 0;
+ while(s[len] != 0) { len++; }
+ return len;
+}
+
+size_t _mi_strnlen(const char* s, size_t max_len) {
+ if (s==NULL) return 0;
+ size_t len = 0;
+ while(s[len] != 0 && len < max_len) { len++; }
+ return len;
+}
+
+#ifdef MI_NO_GETENV
+bool _mi_getenv(const char* name, char* result, size_t result_size) {
+ MI_UNUSED(name);
+ MI_UNUSED(result);
+ MI_UNUSED(result_size);
+ return false;
+}
+#else
+bool _mi_getenv(const char* name, char* result, size_t result_size) {
+ if (name==NULL || result == NULL || result_size < 64) return false;
+ return _mi_prim_getenv(name,result,result_size);
+}
+#endif
+
+// --------------------------------------------------------
+// Define our own limited `_mi_vsnprintf` and `_mi_snprintf`
+// This is mostly to avoid calling these when libc is not yet
+// initialized (and to reduce dependencies)
+//
+// format: d i, p x u, s
+// prec: z l ll L
+// width: 10
+// align-left: -
+// fill: 0
+// plus: +
+// --------------------------------------------------------
+
+static void mi_outc(char c, char** out, char* end) {
+ char* p = *out;
+ if (p >= end) return;
+ *p = c;
+ *out = p + 1;
+}
+
+static void mi_outs(const char* s, char** out, char* end) {
+ if (s == NULL) return;
+ char* p = *out;
+ while (*s != 0 && p < end) {
+ *p++ = *s++;
+ }
+ *out = p;
+}
+
+static void mi_out_fill(char fill, size_t len, char** out, char* end) {
+ char* p = *out;
+ for (size_t i = 0; i < len && p < end; i++) {
+ *p++ = fill;
+ }
+ *out = p;
+}
+
+static void mi_out_alignright(char fill, char* start, size_t len, size_t extra, char* end) {
+ if (len == 0 || extra == 0) return;
+ if (start + len + extra >= end) return;
+ // move `len` characters to the right (in reverse since it can overlap)
+ for (size_t i = 1; i <= len; i++) {
+ start[len + extra - i] = start[len - i];
+ }
+ // and fill the start
+ for (size_t i = 0; i < extra; i++) {
+ start[i] = fill;
+ }
+}
+
+
+static void mi_out_num(uintptr_t x, size_t base, char prefix, char** out, char* end)
+{
+ if (x == 0 || base == 0 || base > 16) {
+ if (prefix != 0) { mi_outc(prefix, out, end); }
+ mi_outc('0',out,end);
+ }
+ else {
+ // output digits in reverse
+ char* start = *out;
+ while (x > 0) {
+ char digit = (char)(x % base);
+ mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end);
+ x = x / base;
+ }
+ if (prefix != 0) {
+ mi_outc(prefix, out, end);
+ }
+ size_t len = *out - start;
+ // and reverse in-place
+ for (size_t i = 0; i < (len / 2); i++) {
+ char c = start[len - i - 1];
+ start[len - i - 1] = start[i];
+ start[i] = c;
+ }
+ }
+}
+
+
+#define MI_NEXTC() c = *in; if (c==0) break; in++;
+
+void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
+ if (buf == NULL || bufsize == 0 || fmt == NULL) return;
+ buf[bufsize - 1] = 0;
+ char* const end = buf + (bufsize - 1);
+ const char* in = fmt;
+ char* out = buf;
+ while (true) {
+ if (out >= end) break;
+ char c;
+ MI_NEXTC();
+ if (c != '%') {
+ if ((c >= ' ' && c <= '~') || c=='\n' || c=='\r' || c=='\t') { // output visible ascii or standard control only
+ mi_outc(c, &out, end);
+ }
+ }
+ else {
+ MI_NEXTC();
+ char fill = ' ';
+ size_t width = 0;
+ char numtype = 'd';
+ char numplus = 0;
+ bool alignright = true;
+ if (c == '+' || c == ' ') { numplus = c; MI_NEXTC(); }
+ if (c == '-') { alignright = false; MI_NEXTC(); }
+ if (c == '0') { fill = '0'; MI_NEXTC(); }
+ if (c >= '1' && c <= '9') {
+ width = (c - '0'); MI_NEXTC();
+ while (c >= '0' && c <= '9') {
+ width = (10 * width) + (c - '0'); MI_NEXTC();
+ }
+ if (c == 0) break; // extra check due to while
+ }
+ if (c == 'z' || c == 't' || c == 'L') { numtype = c; MI_NEXTC(); }
+ else if (c == 'l') {
+ numtype = c; MI_NEXTC();
+ if (c == 'l') { numtype = 'L'; MI_NEXTC(); }
+ }
+
+ char* start = out;
+ if (c == 's') {
+ // string
+ const char* s = va_arg(args, const char*);
+ mi_outs(s, &out, end);
+ }
+ else if (c == 'p' || c == 'x' || c == 'u') {
+ // unsigned
+ uintptr_t x = 0;
+ if (c == 'x' || c == 'u') {
+ if (numtype == 'z') x = va_arg(args, size_t);
+ else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
+ else if (numtype == 'L') x = va_arg(args, unsigned long long);
+ else x = va_arg(args, unsigned long);
+ }
+ else if (c == 'p') {
+ x = va_arg(args, uintptr_t);
+ mi_outs("0x", &out, end);
+ start = out;
+ width = (width >= 2 ? width - 2 : 0);
+ }
+ if (width == 0 && (c == 'x' || c == 'p')) {
+ if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); }
+ if (width == 0) { width = 2; }
+ fill = '0';
+ }
+ mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end);
+ }
+ else if (c == 'i' || c == 'd') {
+ // signed
+ intptr_t x = 0;
+ if (numtype == 'z') x = va_arg(args, intptr_t );
+ else if (numtype == 't') x = va_arg(args, ptrdiff_t);
+ else if (numtype == 'L') x = va_arg(args, long long);
+ else x = va_arg(args, long);
+ char pre = 0;
+ if (x < 0) {
+ pre = '-';
+ if (x > INTPTR_MIN) { x = -x; }
+ }
+ else if (numplus != 0) {
+ pre = numplus;
+ }
+ mi_out_num((uintptr_t)x, 10, pre, &out, end);
+ }
+ else if (c >= ' ' && c <= '~') {
+ // unknown format
+ mi_outc('%', &out, end);
+ mi_outc(c, &out, end);
+ }
+
+ // fill & align
+ mi_assert_internal(out <= end);
+ mi_assert_internal(out >= start);
+ const size_t len = out - start;
+ if (len < width) {
+ mi_out_fill(fill, width - len, &out, end);
+ if (alignright && out <= end) {
+ mi_out_alignright(fill, start, len, width - len, end);
+ }
+ }
+ }
+ }
+ mi_assert_internal(out <= end);
+ *out = 0;
+}
+
+void _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) {
+ va_list args;
+ va_start(args, fmt);
+ _mi_vsnprintf(buf, buflen, fmt, args);
+ va_end(args);
+}
diff --git a/src/options.c b/src/options.c
index 1eadf994..44fb3c64 100644
--- a/src/options.c
+++ b/src/options.c
@@ -1,5 +1,5 @@
/* ----------------------------------------------------------------------------
-Copyright (c) 2018-2021, Microsoft Research, Daan Leijen
+Copyright (c) 2018-2023, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
@@ -9,9 +9,9 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/atomic.h"
#include "mimalloc/prim.h" // mi_prim_out_stderr
-#include // FILE
+#include // stdin/stdout
#include // abort
-#include
+
static long mi_max_error_count = 16; // stop outputting errors after this (use < 0 for no limit)
@@ -94,12 +94,13 @@ static mi_option_desc_t options[_mi_option_last] =
{ 1024,UNINIT, MI_OPTION(remap_threshold) }, // size in KiB after which realloc starts using OS remap (0 to disable auto remap)
};
-static bool mi_option_is_size_in_kib(mi_option_t option) {
- return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || option == mi_option_remap_threshold);
-}
static void mi_option_init(mi_option_desc_t* desc);
+static bool mi_option_has_size_in_kib(mi_option_t option) {
+ return (option == mi_option_reserve_os_memory || option == mi_option_arena_reserve || option == mi_option_remap_threshold);
+}
+
void _mi_options_init(void) {
// called on process load; should not be called before the CRT is initialized!
// (e.g. do not call this from process_init as that may run before CRT initialization)
@@ -110,7 +111,7 @@ void _mi_options_init(void) {
// if (option != mi_option_verbose)
{
mi_option_desc_t* desc = &options[option];
- _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value);
+ _mi_verbose_message("option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : ""));
}
}
mi_max_error_count = mi_option_get(mi_option_max_errors);
@@ -134,7 +135,7 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma
}
mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
- mi_assert_internal(mi_option_is_size_in_kib(option));
+ mi_assert_internal(mi_option_has_size_in_kib(option));
long x = mi_option_get(option);
return (x < 0 ? 0 : (size_t)x * MI_KiB);
}
@@ -317,12 +318,12 @@ void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* me
}
// Define our own limited `fprintf` that avoids memory allocation.
-// We do this using `snprintf` with a limited buffer.
+// We do this using `_mi_vsnprintf` with a limited buffer.
static void mi_vfprintf( mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args ) {
char buf[512];
if (fmt==NULL) return;
if (!mi_recurse_enter()) return;
- vsnprintf(buf,sizeof(buf)-1,fmt,args);
+ _mi_vsnprintf(buf, sizeof(buf)-1, fmt, args);
mi_recurse_exit();
_mi_fputs(out,arg,prefix,buf);
}
@@ -337,7 +338,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) {
static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) {
if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) {
char tprefix[64];
- snprintf(tprefix, sizeof(tprefix), "%sthread 0x%llx: ", prefix, (unsigned long long)_mi_thread_id());
+ _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id());
mi_vfprintf(out, arg, tprefix, fmt, args);
}
else {
@@ -440,68 +441,6 @@ void _mi_error_message(int err, const char* fmt, ...) {
// --------------------------------------------------------
// Initialize options by checking the environment
// --------------------------------------------------------
-char _mi_toupper(char c) {
- if (c >= 'a' && c <= 'z') return (c - 'a' + 'A');
- else return c;
-}
-
-int _mi_strnicmp(const char* s, const char* t, size_t n) {
- if (n == 0) return 0;
- for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) {
- if (_mi_toupper(*s) != _mi_toupper(*t)) break;
- }
- return (n == 0 ? 0 : *s - *t);
-}
-
-void _mi_strlcpy(char* dest, const char* src, size_t dest_size) {
- if (dest==NULL || src==NULL || dest_size == 0) return;
- // copy until end of src, or when dest is (almost) full
- while (*src != 0 && dest_size > 1) {
- *dest++ = *src++;
- dest_size--;
- }
- // always zero terminate
- *dest = 0;
-}
-
-void _mi_strlcat(char* dest, const char* src, size_t dest_size) {
- if (dest==NULL || src==NULL || dest_size == 0) return;
- // find end of string in the dest buffer
- while (*dest != 0 && dest_size > 1) {
- dest++;
- dest_size--;
- }
- // and catenate
- _mi_strlcpy(dest, src, dest_size);
-}
-
-size_t _mi_strlen(const char* s) {
- if (s==NULL) return 0;
- size_t len = 0;
- while(s[len] != 0) { len++; }
- return len;
-}
-
-size_t _mi_strnlen(const char* s, size_t max_len) {
- if (s==NULL) return 0;
- size_t len = 0;
- while(s[len] != 0 && len < max_len) { len++; }
- return len;
-}
-
-#ifdef MI_NO_GETENV
-static bool mi_getenv(const char* name, char* result, size_t result_size) {
- MI_UNUSED(name);
- MI_UNUSED(result);
- MI_UNUSED(result_size);
- return false;
-}
-#else
-static bool mi_getenv(const char* name, char* result, size_t result_size) {
- if (name==NULL || result == NULL || result_size < 64) return false;
- return _mi_prim_getenv(name,result,result_size);
-}
-#endif
// TODO: implement ourselves to reduce dependencies on the C runtime
#include // strtol
@@ -514,11 +453,11 @@ static void mi_option_init(mi_option_desc_t* desc) {
char buf[64+1];
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
_mi_strlcat(buf, desc->name, sizeof(buf));
- bool found = mi_getenv(buf, s, sizeof(s));
+ bool found = _mi_getenv(buf, s, sizeof(s));
if (!found && desc->legacy_name != NULL) {
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
_mi_strlcat(buf, desc->legacy_name, sizeof(buf));
- found = mi_getenv(buf, s, sizeof(s));
+ found = _mi_getenv(buf, s, sizeof(s));
if (found) {
_mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
}
@@ -541,7 +480,7 @@ static void mi_option_init(mi_option_desc_t* desc) {
else {
char* end = buf;
long value = strtol(buf, &end, 10);
- if (mi_option_is_size_in_kib(desc->option)) {
+ if (mi_option_has_size_in_kib(desc->option)) {
// this option is interpreted in KiB to prevent overflow of `long`
if (*end == 'K') { end++; }
else if (*end == 'M') { value *= MI_KiB; end++; }
diff --git a/src/os.c b/src/os.c
index 9db16fa1..8af15e20 100644
--- a/src/os.c
+++ b/src/os.c
@@ -421,11 +421,15 @@ bool _mi_os_expand(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_
if (p == NULL) return false;
if (memid->memkind != MI_MEM_OS_EXPAND) return false;
if (newsize > size) {
- mi_assert(memid->mem.os.size <= newsize);
- return _mi_os_commit((uint8_t*)p + size, newsize - size, NULL, stats);
+ if (memid->mem.os.size < newsize) {
+ return false;
+ }
+ else {
+ return _mi_os_commit((uint8_t*)p + size, newsize - size, NULL, stats);
+ }
}
else if (newsize < size) {
- mi_assert(memid->mem.os.size <= size);
+ mi_assert(memid->mem.os.size >= size);
return _mi_os_decommit((uint8_t*)p + newsize, size - newsize, stats);
}
else {
diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c
index 43ac1fe9..9ef5b555 100644
--- a/src/prim/unix/prim.c
+++ b/src/prim/unix/prim.c
@@ -483,8 +483,6 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo
#if defined(__linux__)
-#include // snprintf
-
size_t _mi_prim_numa_node(void) {
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu)
unsigned long node = 0;
@@ -502,7 +500,7 @@ size_t _mi_prim_numa_node_count(void) {
unsigned node = 0;
for(node = 0; node < 256; node++) {
// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
- snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
+ _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
if (mi_prim_access(buf,R_OK) != 0) break;
}
return (node+1);
diff --git a/src/static.c b/src/static.c
index 34f0db42..3c6d41c8 100644
--- a/src/static.c
+++ b/src/static.c
@@ -31,6 +31,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include "bitmap.c"
#include "heap.c"
#include "init.c"
+#include "libc.c"
#include "options.c"
#include "os.c"
#include "page.c" // includes page-queue.c
diff --git a/src/stats.c b/src/stats.c
index a8eac648..fa947e5d 100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -9,7 +9,6 @@ terms of the MIT license. A copy of the license can be found in the file
#include "mimalloc/atomic.h"
#include "mimalloc/prim.h"
-#include // snprintf
#include // memset
#if defined(_MSC_VER) && (_MSC_VER < 1920)
@@ -146,7 +145,7 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*
const int64_t pos = (n < 0 ? -n : n);
if (pos < base) {
if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column
- snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix));
+ _mi_snprintf(buf, len, "%lld %-3s", (long long)n, (n==0 ? "" : suffix));
}
}
else {
@@ -158,8 +157,8 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void*
const long whole = (long)(tens/10);
const long frac1 = (long)(tens%10);
char unitdesc[8];
- snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
- snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
+ _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix);
+ _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc);
}
_mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf);
}
@@ -255,7 +254,7 @@ static void mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, const c
if (bins[i].allocated > 0) {
found = true;
int64_t unit = _mi_bin_size((uint8_t)i);
- snprintf(buf, 64, "%s %3lu", fmt, (long)i);
+ _mi_snprintf(buf, 64, "%s %3lu", fmt, (long)i);
mi_stat_print(&bins[i], buf, unit, out, arg);
}
}
diff --git a/test/main-override-static.c b/test/main-override-static.c
index fb368309..a33c23ae 100644
--- a/test/main-override-static.c
+++ b/test/main-override-static.c
@@ -21,14 +21,16 @@ static void test_heap_walk(void);
static void test_remap(bool start_remappable);
static void test_remap2(void);
static void test_remap3(void);
+static void test_expandable(bool use_realloc);
int main() {
mi_version();
mi_stats_reset();
//test_remap2();
- test_remap3();
- //test_remap(true);
+ //test_remap3();
+ //test_remap(false);
+ test_expandable(true);
// detect double frees and heap corruption
// double_free1();
@@ -247,15 +249,21 @@ static void test_remap3(void) { // by Jason Gibson
mi_free(x);
}
-static void test_remap(bool start_remappable) {
+static void test_remap_expand(bool start_remappable, bool start_expandable, bool use_expand) {
const size_t iterN = 100;
const size_t size0 = 64 * 1024 * 1024;
const size_t inc = 1024 * 1024;
+ const size_t expand_size = size0 + 2 * inc; // (iterN * inc);
size_t size = size0;
- uint8_t* p = (uint8_t*)(start_remappable ? mi_malloc_remappable(size) : mi_malloc(size));
+ uint8_t* p = (uint8_t*)(start_remappable ? mi_malloc_remappable(size) : (start_expandable ? mi_malloc_expandable(size, expand_size) : mi_malloc(size)));
memset(p, 1, size);
for (int i = 2; i < iterN; i++) {
- p = mi_realloc(p, size + inc);
+ void* newp = (use_expand ? mi_expand(p, size + inc) : mi_realloc(p, size + inc));
+ if (use_expand && newp != p) {
+ printf("error: could not expand in place: i=%i, size %zu\n", i, size + inc);
+ abort();
+ }
+ p = newp;
memset(p + size, i, inc);
size += inc;
printf("%3d: increased to size %zu\n", i, size);
@@ -271,6 +279,14 @@ static void test_remap(bool start_remappable) {
mi_free(p);
}
+static void test_remap(bool start_remappable) {
+ test_remap_expand(start_remappable, false, false);
+}
+
+static void test_expandable(bool use_realloc) {
+ test_remap_expand(false, true, !use_realloc);
+}
+
// ----------------------------
// bin size experiments