diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9550f77f..a9e098c7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -49,7 +49,6 @@ set(mi_sources
src/os.c
src/page.c
src/random.c
- src/region.c
src/segment.c
src/stats.c
src/prim/prim.c)
diff --git a/ide/vs2022/mimalloc-override.vcxproj b/ide/vs2022/mimalloc-override.vcxproj
index 50a3d6b9..81a3fc70 100644
--- a/ide/vs2022/mimalloc-override.vcxproj
+++ b/ide/vs2022/mimalloc-override.vcxproj
@@ -247,7 +247,6 @@
true
true
-
diff --git a/ide/vs2022/mimalloc.vcxproj b/ide/vs2022/mimalloc.vcxproj
index 9a7bf18c..d7e147b8 100644
--- a/ide/vs2022/mimalloc.vcxproj
+++ b/ide/vs2022/mimalloc.vcxproj
@@ -226,7 +226,6 @@
true
true
-
true
diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h
index 5904198f..b9fe5453 100644
--- a/include/mimalloc/internal.h
+++ b/include/mimalloc/internal.h
@@ -119,7 +119,9 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_o
bool _mi_arena_memid_is_suitable(size_t arena_memid, mi_arena_id_t request_arena_id);
bool _mi_arena_is_os_allocated(size_t arena_memid);
void _mi_arena_collect(bool free_arenas, bool force_decommit, mi_stats_t* stats);
+bool _mi_arena_contains(const void* p);
+/*
// memory.c
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld);
void _mi_mem_free(void* p, size_t size, size_t alignment, size_t align_offset, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld);
@@ -132,6 +134,7 @@ bool _mi_mem_protect(void* addr, size_t size);
bool _mi_mem_unprotect(void* addr, size_t size);
void _mi_mem_collect(mi_os_tld_t* tld);
+*/
// "segment.c"
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
diff --git a/src/arena.c b/src/arena.c
index 64cb1624..724fbaf4 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -128,6 +128,10 @@ static size_t mi_block_count_of_size(size_t size) {
return _mi_divide_up(size, MI_ARENA_BLOCK_SIZE);
}
+static size_t mi_arena_block_size(size_t bcount) {
+ return (bcount * MI_ARENA_BLOCK_SIZE);
+}
+
/* -----------------------------------------------------------
Thread safe allocation in an arena
----------------------------------------------------------- */
@@ -158,7 +162,7 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren
if (!mi_arena_alloc(arena, needed_bcount, &bitmap_index)) return NULL;
// claimed it!
- void* p = arena->start + (mi_bitmap_index_bit(bitmap_index)*MI_ARENA_BLOCK_SIZE);
+ void* p = arena->start + mi_arena_block_size(mi_bitmap_index_bit(bitmap_index));
*memid = mi_arena_memid_create(arena->id, arena->exclusive, bitmap_index);
*large = arena->is_large;
*is_pinned = (arena->is_large || !arena->allow_decommit);
@@ -183,7 +187,7 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren
_mi_bitmap_claim_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index, &any_uncommitted);
if (any_uncommitted) {
bool commit_zero;
- _mi_os_commit(p, needed_bcount * MI_ARENA_BLOCK_SIZE, &commit_zero, tld->stats);
+ _mi_os_commit(p, mi_arena_block_size(needed_bcount), &commit_zero, tld->stats);
if (commit_zero) { *is_zero = true; }
}
}
@@ -192,7 +196,7 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren
*commit = _mi_bitmap_is_claimed_across(arena->blocks_committed, arena->field_count, needed_bcount, bitmap_index);
}
- // mi_track_mem_undefined(p,needed_bcount*MI_ARENA_BLOCK_SIZE);
+ // mi_track_mem_undefined(p,mi_arena_block_size(needed_bcount));
return p;
}
@@ -207,7 +211,7 @@ static void* mi_arena_alloc_in(mi_arena_id_t arena_id, int numa_node, size_t siz
const size_t bcount = mi_block_count_of_size(size);
const size_t arena_index = mi_arena_id_index(arena_id);
mi_assert_internal(arena_index < max_arena);
- mi_assert_internal(size <= bcount * MI_ARENA_BLOCK_SIZE);
+ mi_assert_internal(size <= mi_arena_block_size(bcount));
if (arena_index >= max_arena) return NULL;
mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[arena_index]);
@@ -228,7 +232,7 @@ static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size
const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count);
const size_t bcount = mi_block_count_of_size(size);
if mi_likely(max_arena == 0) return NULL;
- mi_assert_internal(size <= bcount * MI_ARENA_BLOCK_SIZE);
+ mi_assert_internal(size <= mi_arena_block_size(bcount));
size_t arena_index = mi_arena_id_index(req_arena_id);
if (arena_index < MI_MAX_ARENAS) {
@@ -335,7 +339,7 @@ void* mi_arena_area(mi_arena_id_t arena_id, size_t* size) {
if (arena_index >= MI_MAX_ARENAS) return NULL;
mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[arena_index]);
if (arena == NULL) return NULL;
- if (size != NULL) *size = arena->block_count * MI_ARENA_BLOCK_SIZE;
+ if (size != NULL) { *size = mi_arena_block_size(arena->block_count); }
return arena->start;
}
@@ -348,8 +352,8 @@ static void mi_arena_purge(mi_arena_t* arena, size_t bitmap_idx, size_t blocks,
mi_assert_internal(arena->blocks_committed != NULL);
mi_assert_internal(arena->blocks_purge != NULL);
mi_assert_internal(arena->allow_decommit);
- const size_t size = blocks * MI_ARENA_BLOCK_SIZE;
- void* const p = arena->start + (mi_bitmap_index_bit(bitmap_idx) * MI_ARENA_BLOCK_SIZE);
+ const size_t size = mi_arena_block_size(blocks);
+ void* const p = arena->start + mi_arena_block_size(mi_bitmap_index_bit(bitmap_idx));
const bool decommitted = _mi_os_purge(p, size, stats);
// clear the purged blocks
_mi_bitmap_unclaim_across(arena->blocks_purge, arena->field_count, blocks, bitmap_idx);
@@ -557,6 +561,22 @@ void _mi_arena_collect(bool free_arenas, bool force_decommit, mi_stats_t* stats)
mi_arenas_try_purge(force_decommit, true, stats);
}
+
+bool _mi_arena_contains(const void* p) {
+ const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count);
+ for (size_t i = 0; i < max_arena; i++) {
+ mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[i]);
+ if (arena->start <= (const uint8_t*)p && arena->start + mi_arena_block_size(arena->block_count) > (const uint8_t*)p) {
+ return true;
+ }
+ }
+ return false;
+}
+
+mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
+ return _mi_arena_contains(p); // todo: extend to track os allocated memory as well
+}
+
/* -----------------------------------------------------------
Add an arena.
----------------------------------------------------------- */
diff --git a/src/heap.c b/src/heap.c
index 31a8b660..99316bb8 100644
--- a/src/heap.c
+++ b/src/heap.c
@@ -158,7 +158,6 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
// collect regions on program-exit (or shared library unload)
if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
- _mi_mem_collect(&heap->tld->os);
_mi_arena_collect(false,true,&heap->tld->stats);
}
}
diff --git a/src/init.c b/src/init.c
index 61245cd1..8c79561f 100644
--- a/src/init.c
+++ b/src/init.c
@@ -590,7 +590,6 @@ static void mi_cdecl mi_process_done(void) {
// or C-runtime termination code.
if (mi_option_is_enabled(mi_option_destroy_on_exit)) {
_mi_heap_destroy_all(); // forcefully release all memory held by all heaps (of this thread only!)
- _mi_mem_collect(&_mi_heap_main_get()->tld->os); // release all regions
_mi_arena_collect(true,true,&_mi_heap_main_get()->tld->stats);
}
diff --git a/src/options.c b/src/options.c
index 79e3560e..5d4af7ba 100644
--- a/src/options.c
+++ b/src/options.c
@@ -41,10 +41,11 @@ typedef struct mi_option_desc_s {
mi_init_t init; // is it initialized yet? (from the environment)
mi_option_t option; // for debugging: the option index should match the option
const char* name; // option name without `mimalloc_` prefix
+ const char* legacy_name; // potential legacy option name
} mi_option_desc_t;
-#define MI_OPTION(opt) mi_option_##opt, #opt
-#define MI_OPTION_DESC(opt) {0, UNINIT, MI_OPTION(opt) }
+#define MI_OPTION(opt) mi_option_##opt, #opt, NULL
+#define MI_OPTION_LEGACY(opt,legacy) mi_option_##opt, #opt, #legacy
static mi_option_desc_t options[_mi_option_last] =
{
@@ -58,14 +59,9 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(verbose) },
// the following options are experimental and not all combinations make sense.
- { 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`)
- #if defined(_WIN32) || (MI_INTPTR_SIZE <= 4) // and other OS's without overcommit?
- { 0, UNINIT, MI_OPTION(eager_region_commit) },
- { 1, UNINIT, MI_OPTION(reset_decommits) }, // reset decommits memory
- #else
- { 1, UNINIT, MI_OPTION(eager_region_commit) },
- { 0, UNINIT, MI_OPTION(reset_decommits) }, // reset uses MADV_FREE/MADV_DONTNEED
- #endif
+ { 1, UNINIT, MI_OPTION_LEGACY(segment_eager_commit,eager_commit) }, // commit per segment directly (4MiB) (but see also `eager_commit_delay`)
+ { 2, UNINIT, MI_OPTION_LEGACY(arena_eager_commit,eager_region_commit) },
+ { 1, UNINIT, MI_OPTION_LEGACY(purge_decommits,reset_decommits) }, // purge decommits memory (instead of reset)
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
{ -1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
@@ -87,7 +83,12 @@ static mi_option_desc_t options[_mi_option_last] =
{ 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
{ 8, UNINIT, MI_OPTION(max_segment_reclaim)},// max. number of segment reclaims from the abandoned segments per try.
{ 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
- { 0, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (disable for now in v1.x due to regions)
+ #if (MI_INTPTR_SIZE>4)
+ { 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time
+ #else
+ { 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) },
+ #endif
+
{ 500, UNINIT, MI_OPTION(arena_purge_delay) }, // reset/decommit delay in milli-seconds for arena allocation
{ 1, UNINIT, MI_OPTION(allow_purge) } // allow decommit/reset to free (physical) memory back to the OS
};
@@ -504,18 +505,27 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) {
static void mi_option_init(mi_option_desc_t* desc) {
// Read option value from the environment
+ char s[64 + 1];
char buf[64+1];
_mi_strlcpy(buf, "mimalloc_", sizeof(buf));
_mi_strlcat(buf, desc->name, sizeof(buf));
- char s[64+1];
- if (mi_getenv(buf, s, sizeof(s))) {
- size_t len = _mi_strnlen(s,64);
- if (len >= sizeof(buf)) len = sizeof(buf) - 1;
+ bool found = mi_getenv(buf, s, sizeof(s));
+ if (!found && desc->legacy_name != NULL) {
+ _mi_strlcpy(buf, "mimalloc_", sizeof(buf));
+ _mi_strlcat(buf, desc->legacy_name, sizeof(buf));
+ found = mi_getenv(buf, s, sizeof(s));
+ if (found) {
+ _mi_warning_message("environment option \"mimalloc_%s\" is deprecated -- use \"mimalloc_%s\" instead.\n", desc->legacy_name, desc->name);
+ }
+ }
+
+ if (found) {
+ size_t len = _mi_strnlen(s, sizeof(buf) - 1);
for (size_t i = 0; i < len; i++) {
buf[i] = _mi_toupper(s[i]);
}
buf[len] = 0;
- if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) {
+ if (buf[0] == 0 || strstr("1;TRUE;YES;ON", buf) != NULL) {
desc->value = 1;
desc->init = INITIALIZED;
}
@@ -546,11 +556,11 @@ static void mi_option_init(mi_option_desc_t* desc) {
// if the 'mimalloc_verbose' env var has a bogus value we'd never know
// (since the value defaults to 'off') so in that case briefly enable verbose
desc->value = 1;
- _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name );
+ _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
desc->value = 0;
}
else {
- _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name );
+ _mi_warning_message("environment option mimalloc_%s has an invalid value.\n", desc->name);
}
}
}
diff --git a/src/region.c b/src/region.c
deleted file mode 100644
index 36226eff..00000000
--- a/src/region.c
+++ /dev/null
@@ -1,502 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019-2020, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-
-/* ----------------------------------------------------------------------------
-This implements a layer between the raw OS memory (VirtualAlloc/mmap/sbrk/..)
-and the segment and huge object allocation by mimalloc. There may be multiple
-implementations of this (one could be the identity going directly to the OS,
-another could be a simple cache etc), but the current one uses large "regions".
-In contrast to the rest of mimalloc, the "regions" are shared between threads and
-need to be accessed using atomic operations.
-We need this memory layer between the raw OS calls because of:
-1. on `sbrk` like systems (like WebAssembly) we need our own memory maps in order
- to reuse memory effectively.
-2. It turns out that for large objects, between 1MiB and 32MiB (?), the cost of
- an OS allocation/free is still (much) too expensive relative to the accesses
- in that object :-( (`malloc-large` tests this). This means we need a cheaper
- way to reuse memory.
-3. This layer allows for NUMA aware allocation.
-
-Possible issues:
-- (2) can potentially be addressed too with a small cache per thread which is much
- simpler. Generally though that requires shrinking of huge pages, and may overuse
- memory per thread. (and is not compatible with `sbrk`).
-- Since the current regions are per-process, we need atomic operations to
- claim blocks which may be contended
-- In the worst case, we need to search the whole region map (16KiB for 256GiB)
- linearly. At what point will direct OS calls be faster? Is there a way to
- do this better without adding too much complexity?
------------------------------------------------------------------------------*/
-#include "mimalloc.h"
-#include "mimalloc/internal.h"
-#include "mimalloc/atomic.h"
-
-#include // memset
-
-#include "bitmap.h"
-
-// os.c
-bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats);
-
-// Constants
-#if (MI_INTPTR_SIZE==8)
-#define MI_HEAP_REGION_MAX_SIZE (256 * MI_GiB) // 64KiB for the region map
-#elif (MI_INTPTR_SIZE==4)
-#define MI_HEAP_REGION_MAX_SIZE (3 * MI_GiB) // ~ KiB for the region map
-#else
-#error "define the maximum heap space allowed for regions on this platform"
-#endif
-
-#define MI_REGION_MAX_BLOCKS MI_BITMAP_FIELD_BITS
-#define MI_REGION_SIZE (MI_SEGMENT_SIZE * MI_BITMAP_FIELD_BITS) // 256MiB (64MiB on 32 bits)
-#define MI_REGION_MAX (MI_HEAP_REGION_MAX_SIZE / MI_REGION_SIZE) // 1024 (48 on 32 bits)
-#define MI_REGION_MAX_OBJ_BLOCKS (MI_REGION_MAX_BLOCKS/4) // 64MiB
-#define MI_REGION_MAX_OBJ_SIZE (MI_REGION_MAX_OBJ_BLOCKS*MI_SEGMENT_SIZE)
-
-// Region info
-typedef union mi_region_info_u {
- size_t value;
- struct {
- bool valid; // initialized?
- bool is_large:1; // allocated in fixed large/huge OS pages
- bool is_pinned:1; // pinned memory cannot be decommitted
- short numa_node; // the associated NUMA node (where -1 means no associated node)
- } x;
-} mi_region_info_t;
-
-
-// A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with
-// a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block.
-typedef struct mem_region_s {
- _Atomic(size_t) info; // mi_region_info_t.value
- _Atomic(void*) start; // start of the memory area
- mi_bitmap_field_t in_use; // bit per in-use block
- mi_bitmap_field_t dirty; // track if non-zero per block
- mi_bitmap_field_t commit; // track if committed per block
- mi_bitmap_field_t reset; // track if reset per block
- _Atomic(size_t) arena_memid; // if allocated from a (huge page) arena
- _Atomic(size_t) padding; // round to 8 fields (needs to be atomic for msvc, see issue #508)
-} mem_region_t;
-
-// The region map
-static mem_region_t regions[MI_REGION_MAX];
-
-// Allocated regions
-static _Atomic(size_t) regions_count; // = 0;
-
-
-/* ----------------------------------------------------------------------------
-Utility functions
------------------------------------------------------------------------------*/
-
-// Blocks (of 4MiB) needed for the given size.
-static size_t mi_region_block_count(size_t size) {
- return _mi_divide_up(size, MI_SEGMENT_SIZE);
-}
-
-/*
-// Return a rounded commit/reset size such that we don't fragment large OS pages into small ones.
-static size_t mi_good_commit_size(size_t size) {
- if (size > (SIZE_MAX - _mi_os_large_page_size())) return size;
- return _mi_align_up(size, _mi_os_large_page_size());
-}
-*/
-
-// Return if a pointer points into a region reserved by us.
-mi_decl_nodiscard bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
- if (p==NULL) return false;
- size_t count = mi_atomic_load_relaxed(®ions_count);
- for (size_t i = 0; i < count; i++) {
- uint8_t* start = (uint8_t*)mi_atomic_load_ptr_relaxed(uint8_t, ®ions[i].start);
- if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true;
- }
- return false;
-}
-
-
-static void* mi_region_blocks_start(const mem_region_t* region, mi_bitmap_index_t bit_idx) {
- uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t, &((mem_region_t*)region)->start);
- mi_assert_internal(start != NULL);
- return (start + (bit_idx * MI_SEGMENT_SIZE));
-}
-
-static size_t mi_memid_create(mem_region_t* region, mi_bitmap_index_t bit_idx) {
- mi_assert_internal(bit_idx < MI_BITMAP_FIELD_BITS);
- size_t idx = region - regions;
- mi_assert_internal(®ions[idx] == region);
- return (idx*MI_BITMAP_FIELD_BITS + bit_idx)<<1;
-}
-
-static size_t mi_memid_create_from_arena(size_t arena_memid) {
- return (arena_memid << 1) | 1;
-}
-
-
-static bool mi_memid_is_arena(size_t id, mem_region_t** region, mi_bitmap_index_t* bit_idx, size_t* arena_memid) {
- if ((id&1)==1) {
- if (arena_memid != NULL) *arena_memid = (id>>1);
- return true;
- }
- else {
- size_t idx = (id >> 1) / MI_BITMAP_FIELD_BITS;
- *bit_idx = (mi_bitmap_index_t)(id>>1) % MI_BITMAP_FIELD_BITS;
- *region = ®ions[idx];
- return false;
- }
-}
-
-
-/* ----------------------------------------------------------------------------
- Allocate a region is allocated from the OS (or an arena)
------------------------------------------------------------------------------*/
-
-static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, mem_region_t** region, mi_bitmap_index_t* bit_idx, mi_os_tld_t* tld)
-{
- // not out of regions yet?
- if (mi_atomic_load_relaxed(®ions_count) >= MI_REGION_MAX - 1) return false;
-
- // try to allocate a fresh region from the OS
- bool region_commit = (commit && mi_option_is_enabled(mi_option_eager_region_commit));
- bool region_large = (commit && allow_large);
- bool is_zero = false;
- bool is_pinned = false;
- size_t arena_memid = 0;
- void* const start = _mi_arena_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, 0, ®ion_commit, ®ion_large, &is_pinned, &is_zero, _mi_arena_id_none(), & arena_memid, tld);
- if (start == NULL) return false;
- mi_assert_internal(!(region_large && !allow_large));
- mi_assert_internal(!region_large || region_commit);
-
- // claim a fresh slot
- const size_t idx = mi_atomic_increment_acq_rel(®ions_count);
- if (idx >= MI_REGION_MAX) {
- mi_atomic_decrement_acq_rel(®ions_count);
- _mi_arena_free(start, MI_REGION_SIZE, MI_SEGMENT_ALIGN, 0, arena_memid, region_commit, tld->stats);
- _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, MI_GiB));
- return false;
- }
-
- // allocated, initialize and claim the initial blocks
- mem_region_t* r = ®ions[idx];
- r->arena_memid = arena_memid;
- mi_atomic_store_release(&r->in_use, (size_t)0);
- mi_atomic_store_release(&r->dirty, (is_zero ? 0 : MI_BITMAP_FIELD_FULL));
- mi_atomic_store_release(&r->commit, (region_commit ? MI_BITMAP_FIELD_FULL : 0));
- mi_atomic_store_release(&r->reset, (size_t)0);
- *bit_idx = 0;
- _mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL);
- mi_atomic_store_ptr_release(void,&r->start, start);
-
- // and share it
- mi_region_info_t info;
- info.value = 0; // initialize the full union to zero
- info.x.valid = true;
- info.x.is_large = region_large;
- info.x.is_pinned = is_pinned;
- info.x.numa_node = (short)_mi_os_numa_node(tld);
- mi_atomic_store_release(&r->info, info.value); // now make it available to others
- *region = r;
- return true;
-}
-
-/* ----------------------------------------------------------------------------
- Try to claim blocks in suitable regions
------------------------------------------------------------------------------*/
-
-static bool mi_region_is_suitable(const mem_region_t* region, int numa_node, bool allow_large ) {
- // initialized at all?
- mi_region_info_t info;
- info.value = mi_atomic_load_relaxed(&((mem_region_t*)region)->info);
- if (info.value==0) return false;
-
- // numa correct
- if (numa_node >= 0) { // use negative numa node to always succeed
- int rnode = info.x.numa_node;
- if (rnode >= 0 && rnode != numa_node) return false;
- }
-
- // check allow-large
- if (!allow_large && info.x.is_large) return false;
-
- return true;
-}
-
-
-static bool mi_region_try_claim(int numa_node, size_t blocks, bool allow_large, mem_region_t** region, mi_bitmap_index_t* bit_idx, mi_os_tld_t* tld)
-{
- // try all regions for a free slot
- const size_t count = mi_atomic_load_relaxed(®ions_count); // monotonic, so ok to be relaxed
- size_t idx = tld->region_idx; // Or start at 0 to reuse low addresses? Starting at 0 seems to increase latency though
- for (size_t visited = 0; visited < count; visited++, idx++) {
- if (idx >= count) idx = 0; // wrap around
- mem_region_t* r = ®ions[idx];
- // if this region suits our demand (numa node matches, large OS page matches)
- if (mi_region_is_suitable(r, numa_node, allow_large)) {
- // then try to atomically claim a segment(s) in this region
- if (_mi_bitmap_try_find_claim_field(&r->in_use, 0, blocks, bit_idx)) {
- tld->region_idx = idx; // remember the last found position
- *region = r;
- return true;
- }
- }
- }
- return false;
-}
-
-
-static void* mi_region_try_alloc(size_t blocks, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
-{
- mi_assert_internal(blocks <= MI_BITMAP_FIELD_BITS);
- mem_region_t* region;
- mi_bitmap_index_t bit_idx;
- const int numa_node = (_mi_os_numa_node_count() <= 1 ? -1 : _mi_os_numa_node(tld));
- // try to claim in existing regions
- if (!mi_region_try_claim(numa_node, blocks, *large, ®ion, &bit_idx, tld)) {
- // otherwise try to allocate a fresh region and claim in there
- if (!mi_region_try_alloc_os(blocks, *commit, *large, ®ion, &bit_idx, tld)) {
- // out of regions or memory
- return NULL;
- }
- }
-
- // ------------------------------------------------
- // found a region and claimed `blocks` at `bit_idx`, initialize them now
- mi_assert_internal(region != NULL);
- mi_assert_internal(_mi_bitmap_is_claimed(®ion->in_use, 1, blocks, bit_idx));
-
- mi_region_info_t info;
- info.value = mi_atomic_load_acquire(®ion->info);
- uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ion->start);
- mi_assert_internal(!(info.x.is_large && !*large));
- mi_assert_internal(start != NULL);
-
- *is_zero = _mi_bitmap_claim(®ion->dirty, 1, blocks, bit_idx, NULL);
- *large = info.x.is_large;
- *is_pinned = info.x.is_pinned;
- *memid = mi_memid_create(region, bit_idx);
- void* p = start + (mi_bitmap_index_bit_in_field(bit_idx) * MI_SEGMENT_SIZE);
-
- // commit
- if (*commit) {
- // ensure commit
- bool any_uncommitted;
- _mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, &any_uncommitted);
- if (any_uncommitted) {
- mi_assert_internal(!info.x.is_large && !info.x.is_pinned);
- bool commit_zero = false;
- if (!_mi_mem_commit(p, blocks * MI_SEGMENT_SIZE, &commit_zero, tld)) {
- // failed to commit! unclaim and return
- _mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx);
- return NULL;
- }
- if (commit_zero) *is_zero = true;
- }
- }
- else {
- // no need to commit, but check if already fully committed
- *commit = _mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx);
- }
- mi_assert_internal(!*commit || _mi_bitmap_is_claimed(®ion->commit, 1, blocks, bit_idx));
-
- // unreset reset blocks
- if (_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx)) {
- // some blocks are still reset
- mi_assert_internal(!info.x.is_large && !info.x.is_pinned);
- mi_assert_internal(!mi_option_is_enabled(mi_option_eager_commit) || *commit || mi_option_get(mi_option_eager_commit_delay) > 0);
- _mi_bitmap_unclaim(®ion->reset, 1, blocks, bit_idx);
- if (*commit || !mi_option_is_enabled(mi_option_reset_decommits)) { // only if needed
- bool reset_zero = false;
- _mi_mem_unreset(p, blocks * MI_SEGMENT_SIZE, &reset_zero, tld);
- if (reset_zero) *is_zero = true;
- }
- }
- mi_assert_internal(!_mi_bitmap_is_any_claimed(®ion->reset, 1, blocks, bit_idx));
-
- #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED // && !MI_TSAN
- if (*commit) { ((uint8_t*)p)[0] = 0; }
- #endif
-
- // and return the allocation
- mi_assert_internal(p != NULL);
- return p;
-}
-
-
-/* ----------------------------------------------------------------------------
- Allocation
------------------------------------------------------------------------------*/
-
-// Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`.
-// (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`)
-void* _mi_mem_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld)
-{
- mi_assert_internal(memid != NULL && tld != NULL);
- mi_assert_internal(size > 0);
- *memid = 0;
- *is_zero = false;
- *is_pinned = false;
- bool default_large = false;
- if (large==NULL) large = &default_large; // ensure `large != NULL`
- if (size == 0) return NULL;
- size = _mi_align_up(size, _mi_os_page_size());
-
- // allocate from regions if possible
- void* p = NULL;
- size_t arena_memid;
- const size_t blocks = mi_region_block_count(size);
- if (blocks <= MI_REGION_MAX_OBJ_BLOCKS && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) {
- p = mi_region_try_alloc(blocks, commit, large, is_pinned, is_zero, memid, tld);
- if (p == NULL) {
- _mi_warning_message("unable to allocate from region: size %zu\n", size);
- }
- }
- if (p == NULL) {
- // and otherwise fall back to the OS
- p = _mi_arena_alloc_aligned(size, alignment, align_offset, commit, large, is_pinned, is_zero, _mi_arena_id_none(), & arena_memid, tld);
- *memid = mi_memid_create_from_arena(arena_memid);
- }
-
- if (p != NULL) {
- mi_assert_internal(((uintptr_t)p + align_offset) % alignment == 0);
- #if (MI_DEBUG>=2) && !MI_TRACK_ENABLED // && !MI_TSAN
- if (*commit) { ((uint8_t*)p)[0] = 0; } // ensure the memory is committed
- #endif
- }
- return p;
-}
-
-
-
-/* ----------------------------------------------------------------------------
-Free
------------------------------------------------------------------------------*/
-
-// Free previously allocated memory with a given id.
-void _mi_mem_free(void* p, size_t size, size_t alignment, size_t align_offset, size_t id, bool full_commit, bool any_reset, mi_os_tld_t* tld) {
- mi_assert_internal(size > 0 && tld != NULL);
- if (p==NULL) return;
- if (size==0) return;
- size = _mi_align_up(size, _mi_os_page_size());
-
- size_t arena_memid = 0;
- mi_bitmap_index_t bit_idx;
- mem_region_t* region;
- if (mi_memid_is_arena(id,®ion,&bit_idx,&arena_memid)) {
- // was a direct arena allocation, pass through
- _mi_arena_free(p, size, alignment, align_offset, arena_memid, full_commit, tld->stats);
- }
- else {
- // allocated in a region
- mi_assert_internal(align_offset == 0);
- mi_assert_internal(size <= MI_REGION_MAX_OBJ_SIZE); if (size > MI_REGION_MAX_OBJ_SIZE) return;
- const size_t blocks = mi_region_block_count(size);
- mi_assert_internal(blocks + bit_idx <= MI_BITMAP_FIELD_BITS);
- mi_region_info_t info;
- info.value = mi_atomic_load_acquire(®ion->info);
- mi_assert_internal(info.value != 0);
- void* blocks_start = mi_region_blocks_start(region, bit_idx);
- mi_assert_internal(blocks_start == p); // not a pointer in our area?
- mi_assert_internal(bit_idx + blocks <= MI_BITMAP_FIELD_BITS);
- if (blocks_start != p || bit_idx + blocks > MI_BITMAP_FIELD_BITS) return; // or `abort`?
-
- // committed?
- if (full_commit && (size % MI_SEGMENT_SIZE) == 0) {
- _mi_bitmap_claim(®ion->commit, 1, blocks, bit_idx, NULL);
- }
-
- if (any_reset) {
- // set the is_reset bits if any pages were reset
- _mi_bitmap_claim(®ion->reset, 1, blocks, bit_idx, NULL);
- }
-
- // reset the blocks to reduce the working set.
- if (!info.x.is_large && !info.x.is_pinned && mi_option_is_enabled(mi_option_segment_reset)
- && (mi_option_is_enabled(mi_option_eager_commit) ||
- mi_option_is_enabled(mi_option_reset_decommits))) // cannot reset halfway committed segments, use only `option_page_reset` instead
- {
- bool any_unreset;
- _mi_bitmap_claim(®ion->reset, 1, blocks, bit_idx, &any_unreset);
- if (any_unreset) {
- _mi_abandoned_await_readers(); // ensure no more pending write (in case reset = decommit)
- _mi_mem_reset(p, blocks * MI_SEGMENT_SIZE, tld);
- }
- }
-
- // and unclaim
- bool all_unclaimed = _mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx);
- mi_assert_internal(all_unclaimed); MI_UNUSED(all_unclaimed);
- }
-}
-
-
-/* ----------------------------------------------------------------------------
- collection
------------------------------------------------------------------------------*/
-void _mi_mem_collect(mi_os_tld_t* tld) {
- // free every region that has no segments in use.
- size_t rcount = mi_atomic_load_relaxed(®ions_count);
- for (size_t i = 0; i < rcount; i++) {
- mem_region_t* region = ®ions[i];
- if (mi_atomic_load_relaxed(®ion->info) != 0) {
- // if no segments used, try to claim the whole region
- size_t m = mi_atomic_load_relaxed(®ion->in_use);
- while (m == 0 && !mi_atomic_cas_weak_release(®ion->in_use, &m, MI_BITMAP_FIELD_FULL)) { /* nothing */ };
- if (m == 0) {
- // on success, free the whole region
- uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ions[i].start);
- size_t arena_memid = mi_atomic_load_relaxed(®ions[i].arena_memid);
- size_t commit = mi_atomic_load_relaxed(®ions[i].commit);
- memset((void*)®ions[i], 0, sizeof(mem_region_t)); // cast to void* to avoid atomic warning
- // and release the whole region
- mi_atomic_store_release(®ion->info, (size_t)0);
- if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) {
- _mi_abandoned_await_readers(); // ensure no pending reads
- _mi_arena_free(start, MI_REGION_SIZE, MI_SEGMENT_ALIGN, 0, arena_memid, (~commit == 0), tld->stats);
- }
- }
- }
- }
-}
-
-
-/* ----------------------------------------------------------------------------
- Other
------------------------------------------------------------------------------*/
-
-bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld) {
- if (mi_option_is_enabled(mi_option_reset_decommits)) {
- return _mi_os_decommit(p, size, tld->stats);
- }
- else {
- return _mi_os_reset(p, size, tld->stats);
- }
-}
-
-bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
- if (mi_option_is_enabled(mi_option_reset_decommits)) {
- return _mi_os_commit(p, size, is_zero, tld->stats);
- }
- else {
- // return _mi_os_unreset(p, size, is_zero, tld->stats);
- return true;
- }
-}
-
-bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
- return _mi_os_commit(p, size, is_zero, tld->stats);
-}
-
-bool _mi_mem_decommit(void* p, size_t size, mi_os_tld_t* tld) {
- return _mi_os_decommit(p, size, tld->stats);
-}
-
-bool _mi_mem_protect(void* p, size_t size) {
- return _mi_os_protect(p, size);
-}
-
-bool _mi_mem_unprotect(void* p, size_t size) {
- return _mi_os_unprotect(p, size);
-}
diff --git a/src/segment.c b/src/segment.c
index 56b7a06c..af325fe7 100644
--- a/src/segment.c
+++ b/src/segment.c
@@ -177,10 +177,10 @@ static bool mi_page_not_in_queue(const mi_page_t* page, mi_segments_tld_t* tld)
static void mi_segment_protect_range(void* p, size_t size, bool protect) {
if (protect) {
- _mi_mem_protect(p, size);
+ _mi_os_protect(p, size);
}
else {
- _mi_mem_unprotect(p, size);
+ _mi_os_unprotect(p, size);
}
}
@@ -202,7 +202,7 @@ static void mi_segment_protect(mi_segment_t* segment, bool protect, mi_os_tld_t*
if (protect && !segment->mem_is_committed) {
if (protect) {
// ensure secure page is committed
- if (_mi_mem_commit(start, os_psize, NULL, tld)) { // if this fails that is ok (as it is an unaccessible page)
+ if (_mi_os_commit(start, os_psize, NULL, tld->stats)) { // if this fails that is ok (as it is an unaccessible page)
mi_segment_protect_range(start, os_psize, protect);
}
}
@@ -238,26 +238,29 @@ static void mi_page_reset(mi_segment_t* segment, mi_page_t* page, size_t size, m
page->is_reset = true;
mi_assert_internal(size <= psize);
size_t reset_size = ((size == 0 || size > psize) ? psize : size);
- if (reset_size > 0) _mi_mem_reset(start, reset_size, tld->os);
+ if (reset_size > 0) { _mi_os_reset(start, reset_size, tld->stats); }
}
static bool mi_page_unreset(mi_segment_t* segment, mi_page_t* page, size_t size, mi_segments_tld_t* tld)
{
+ MI_UNUSED(size); MI_UNUSED(tld);
mi_assert_internal(page->is_reset);
mi_assert_internal(page->is_committed);
mi_assert_internal(!segment->mem_is_pinned);
if (segment->mem_is_pinned || !page->is_committed || !page->is_reset) return true;
page->is_reset = false;
+ /*
size_t psize;
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
size_t unreset_size = (size == 0 || size > psize ? psize : size);
- bool is_zero = false;
- bool ok = true;
- if (unreset_size > 0) {
- ok = _mi_mem_unreset(start, unreset_size, &is_zero, tld->os);
- }
- if (is_zero) page->is_zero_init = true;
- return ok;
+ */
+ // bool is_zero = false;
+ // bool ok = true;
+ // if (unreset_size > 0) {
+ // ok = _mi_mem_unreset(start, unreset_size, &is_zero, tld->os);
+ // }
+ // if (is_zero) page->is_zero_init = true;
+ return true;
}
@@ -477,7 +480,8 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
if (any_reset && mi_option_is_enabled(mi_option_reset_decommits)) {
fully_committed = false;
}
- _mi_mem_free(segment, segment_size, segment->mem_alignment, segment->mem_align_offset, segment->memid, fully_committed, any_reset, tld->os);
+
+ _mi_arena_free(segment, segment_size, segment->mem_alignment, segment->mem_align_offset, segment->memid, fully_committed, tld->stats);
}
// called by threads that are terminating to free cached segments
@@ -510,17 +514,18 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme
*segment_size = *segment_size + (align_offset - pre_size);
}
- mi_segment_t* segment = (mi_segment_t*)_mi_mem_alloc_aligned(*segment_size, alignment, align_offset, commit, &mem_large, &is_pinned, is_zero, &memid, tld_os);
+ // mi_segment_t* segment = (mi_segment_t*)_mi_mem_alloc_aligned(*segment_size, alignment, align_offset, commit, &mem_large, &is_pinned, is_zero, &memid, tld_os);
+ mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(*segment_size, alignment, align_offset, commit, &mem_large, &is_pinned, is_zero, _mi_arena_id_none(), &memid, tld_os);
if (segment == NULL) return NULL; // failed to allocate
if (!(*commit)) {
// ensure the initial info is committed
mi_assert_internal(!mem_large && !is_pinned);
bool commit_zero = false;
- bool ok = _mi_mem_commit(segment, pre_size, &commit_zero, tld_os);
- if (commit_zero) *is_zero = true;
+ bool ok = _mi_os_commit(segment, pre_size, &commit_zero, tld_os->stats);
+ if (commit_zero) { *is_zero = true; }
if (!ok) {
// commit failed; we cannot touch the memory: free the segment directly and return `NULL`
- _mi_mem_free(segment, *segment_size, alignment, align_offset, memid, false, false, tld_os);
+ _mi_arena_free(segment, *segment_size, alignment, align_offset, memid, false, tld_os->stats);
return NULL;
}
}
@@ -651,7 +656,7 @@ static bool mi_segment_page_claim(mi_segment_t* segment, mi_page_t* page, mi_seg
uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
bool is_zero = false;
const size_t gsize = (MI_SECURE >= 2 ? _mi_os_page_size() : 0);
- bool ok = _mi_mem_commit(start, psize + gsize, &is_zero, tld->os);
+ bool ok = _mi_os_commit(start, psize + gsize, &is_zero, tld->stats);
if (!ok) return false; // failed to commit!
if (gsize > 0) { mi_segment_protect_range(start + psize, gsize, true); }
if (is_zero) { page->is_zero_init = true; }
diff --git a/src/static.c b/src/static.c
index 090f0c25..483b1b41 100644
--- a/src/static.c
+++ b/src/static.c
@@ -31,7 +31,6 @@ terms of the MIT license. A copy of the license can be found in the file
#include "os.c"
#include "page.c" // includes page-queue.c
#include "random.c"
-#include "region.c"
#include "segment.c"
#include "stats.c"
#include "prim/prim.c"