kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip: remappable memory

Browse source
This commit is contained in:
daanx 2023-04-26 12:23:38 -07:00
parent a0bd338d96
commit be2f35641a
13 changed files with 406 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
include/mimalloc.h
View file

@ -242,6 +242,17 @@ mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap,
mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4);
// ------------------------------------------------------
// Remappable memory (uses `mremap` if possible)
// ------------------------------------------------------
mi_decl_nodiscard mi_decl_export void* mi_malloc_remappable(size_t size) mi_attr_noexcept mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export void* mi_zalloc_remappable(size_t size) mi_attr_noexcept mi_attr_alloc_size(1);
mi_decl_nodiscard mi_decl_export void* mi_remap(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void* mi_heap_malloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_alloc_size(2);
mi_decl_nodiscard mi_decl_export void* mi_heap_zalloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_alloc_size(2);
// ------------------------------------------------------
// Analysis
// ------------------------------------------------------

13
include/mimalloc/internal.h
View file

@ -56,6 +56,8 @@ terms of the MIT license. A copy of the license can be found in the file
#include <pthread.h>
#endif
#define MI_PAGE_ALIGN_REMAPPABLE (1)
// "options.c"
void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
@ -114,6 +116,10 @@ size_t _mi_os_large_page_size(void);
void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid);
void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats);
void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats);
// arena.c
mi_arena_id_t _mi_arena_id_none(void);
void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats);
@ -144,6 +150,8 @@ void _mi_segment_thread_collect(mi_segments_tld_t* tld);
void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
void _mi_abandoned_await_readers(void);
mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld);
// "page.c"
void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc;
@ -671,8 +679,11 @@ static inline mi_memid_t _mi_memid_none(void) {
return _mi_memid_create(MI_MEM_NONE);
}
static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool is_large) {
static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, size_t alignment, bool committed, bool is_zero, bool is_large) {
mi_memid_t memid = _mi_memid_create(MI_MEM_OS);
memid.mem.os.base = base;
memid.mem.os.size = size;
memid.mem.os.alignment = alignment;
memid.initially_committed = committed;
memid.initially_zero = is_zero;
memid.is_pinned = is_large;

18
include/mimalloc/prim.h
View file

@ -69,6 +69,24 @@ int _mi_prim_protect(void* addr, size_t size, bool protect);
// numa_node is either negative (don't care), or a numa node number.
int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bool* is_zero, void** addr);
// Allocate remappable memory that can be used with `_mi_prim_remap`.
// Return `EINVAL` if this is not supported.
// The returned memory is always committed.
// If `is_pinned` is `true` the memory cannot be decommitted or reset.
// The `remap_info` argument can be used to store OS specific information that is passed to `_mi_prim_realloc_remappable` and `_mi_prim_free_remappable`.
int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info );
// Remap remappable memory. Return `EINVAL` if this is not supported.
// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`.
// `newsize > 0`, `size > 0`, `alignment > 0`, `allow_large != NULL`, `newaddr != NULL`.
int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info );
// Free remappable memory. Return `EINVAL` if this is not supported.
// pre: `addr != NULL` and previously allocated using `_mi_prim_realloc_remappable` or `_mi_prim_alloc_remappable`.
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info );
// Return the current NUMA node
size_t _mi_prim_numa_node(void);

4
include/mimalloc/types.h
View file

@ -341,7 +341,9 @@ static inline bool mi_memkind_is_os(mi_memkind_t memkind) {
typedef struct mi_memid_os_info {
void* base; // actual base address of the block (used for offset aligned allocations)
size_t alignment; // alignment at allocation
size_t size; // allocated size (the full extent from base)
size_t alignment; // requested alignment at allocation (may be offset aligned, so base may not be aligned at this value)
void* prim_info; // potential OS dependent information (used for remappable memory on Windows)
} mi_memid_os_info_t;
typedef struct mi_memid_arena_info {

49
src/alloc.c
View file

@ -780,7 +780,56 @@ mi_decl_nodiscard void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_
return mi_heap_recalloc(mi_prim_get_default_heap(), p, count, size);
}
// ------------------------------------------------------
// remap
// ------------------------------------------------------
static void* mi_heap_malloc_zero_remappable(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept {
return _mi_heap_malloc_zero_ex(heap, size, zero, MI_PAGE_ALIGN_REMAPPABLE);
}
mi_decl_nodiscard void* mi_heap_malloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept {
return mi_heap_malloc_zero_remappable(heap, size, false);
}
mi_decl_nodiscard void* mi_heap_zalloc_remappable(mi_heap_t* heap, size_t size) mi_attr_noexcept {
return mi_heap_malloc_zero_remappable(heap, size, true);
}
mi_decl_nodiscard void* mi_malloc_remappable(size_t size) mi_attr_noexcept {
return mi_heap_malloc_remappable(mi_prim_get_default_heap(), size);
}
mi_decl_nodiscard void* mi_zalloc_remappable(size_t size) mi_attr_noexcept {
return mi_heap_zalloc_remappable(mi_prim_get_default_heap(), size);
}
mi_decl_nodiscard void* mi_remap(void* p, size_t newsize) mi_attr_noexcept {
if (p == NULL) return mi_malloc(newsize);
mi_segment_t* segment = mi_checked_ptr_segment(p, "mi_remap");
mi_assert_internal(segment != NULL);
mi_page_t* const page = _mi_segment_page_of(segment, p);
const size_t bsize = mi_page_usable_block_size(page);
if (bsize >= newsize) {
// TODO: adjust padding
return p;
}
mi_heap_t* heap = mi_prim_get_default_heap();
if (segment->thread_id == heap->thread_id &&
segment->memid.memkind == MI_MEM_OS_REMAP)
{
mi_heap_t* heap = mi_prim_get_default_heap();
mi_block_t* block = _mi_segment_huge_page_remap(segment, page, (mi_block_t*)p, newsize, &heap->tld->segments);
if (block != NULL) {
// TODO: adjust padding?
return block;
}
}
_mi_warning_message("unable to remap block, fall back to reallocation (address: %p from %zu bytes to %zu bytes)\n", p, 0, newsize);
return mi_realloc(p, newsize);
}
// ------------------------------------------------------
// strdup, strndup, and realpath

211
src/os.c
View file

@ -50,6 +50,10 @@ bool _mi_os_use_large_page(size_t size, size_t alignment) {
return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0);
}
static size_t mi_os_alloc_size(size_t size) {
return _mi_align_up(size, mi_os_mem_config.alloc_granularity);
}
// round to a good OS allocation size (bounded by max 12.5% waste)
size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size;
@ -58,12 +62,16 @@ size_t _mi_os_good_alloc_size(size_t size) {
else if (size < 8*MI_MiB) align_size = 256*MI_KiB;
else if (size < 32*MI_MiB) align_size = 1*MI_MiB;
else align_size = 4*MI_MiB;
if (align_size < mi_os_mem_config.alloc_granularity) align_size = mi_os_mem_config.alloc_granularity;
if mi_unlikely(size >= (SIZE_MAX - align_size)) return size; // possible overflow?
return _mi_align_up(size, align_size);
}
void _mi_os_init(void) {
_mi_prim_mem_init(&mi_os_mem_config);
if (mi_os_mem_config.alloc_granularity < mi_os_mem_config.page_size) {
mi_os_mem_config.alloc_granularity = mi_os_mem_config.page_size;
}
}
@ -117,23 +125,23 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
{
if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL;
size = _mi_align_up(size, MI_SEGMENT_SIZE);
if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
#if (MI_SECURE>0)
if (size > 1 * MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096.
#if (MI_SECURE>0)
size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas.
#endif
#endif
uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size);
if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize
uintptr_t init = MI_HINT_BASE;
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
#endif
init = init + ((MI_SEGMENT_SIZE * ((r >> 17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
#endif
uintptr_t expected = hint + size;
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all
}
if (hint%try_alignment != 0) return NULL;
if (hint % try_alignment != 0) return NULL;
return (void*)hint;
}
#else
@ -163,22 +171,39 @@ static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_st
_mi_stat_decrease(&stats->reserved, size);
}
static void mi_os_prim_free_remappable(void* addr, size_t size, bool still_committed, void* remap_info, mi_stats_t* tld_stats) {
MI_UNUSED(tld_stats);
mi_assert_internal((size % _mi_os_page_size()) == 0);
if (addr == NULL || size == 0) return; // || _mi_os_is_huge_reserved(addr)
int err = _mi_prim_free_remappable(addr, size, remap_info);
if (err != 0) {
_mi_warning_message("unable to free remappable OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr);
}
mi_stats_t* stats = &_mi_stats_main;
if (still_committed) { _mi_stat_decrease(&stats->committed, size); }
_mi_stat_decrease(&stats->reserved, size);
}
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) {
if (mi_memkind_is_os(memid.memkind)) {
size_t csize = _mi_os_good_alloc_size(size);
size_t csize = mi_os_alloc_size(size);
void* base = addr;
// different base? (due to alignment)
if (memid.mem.os.base != NULL) {
mi_assert(memid.mem.os.base <= addr);
mi_assert((uint8_t*)memid.mem.os.base + memid.mem.os.alignment >= (uint8_t*)addr);
mi_assert(memid.mem.os.size >= csize);
base = memid.mem.os.base;
csize += ((uint8_t*)addr - (uint8_t*)memid.mem.os.base);
csize = memid.mem.os.size;
}
// free it
if (memid.memkind == MI_MEM_OS_HUGE) {
mi_assert(memid.is_pinned);
mi_os_free_huge_os_pages(base, csize, tld_stats);
}
else if (memid.memkind == MI_MEM_OS_REMAP) {
mi_os_prim_free_remappable(base, csize, still_committed, memid.mem.os.prim_info, tld_stats);
}
else {
mi_os_prim_free(base, csize, still_committed, tld_stats);
}
@ -200,7 +225,7 @@ void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats)
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(size > 0 && size == mi_os_alloc_size(size));
mi_assert_internal(is_zero != NULL);
mi_assert_internal(is_large != NULL);
if (size == 0) return NULL;
@ -228,18 +253,44 @@ static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bo
return p;
}
static void* mi_os_align_within(void* base, size_t over_size, size_t alignment, size_t size, bool committed, bool is_pinned, mi_stats_t* stats)
{
//mi_assert_internal((size + alignment - 1) <= over_size);
void* p = mi_align_up_ptr(base, alignment);
mi_assert_internal((uintptr_t)p + size <= (uintptr_t)base + over_size);
if (!is_pinned) {
size_t pre_size = (uint8_t*)p - (uint8_t*)base;
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size);
if (mi_os_mem_config.must_free_whole) {
// decommit the pre- and post part (if needed)
if (committed) {
if (pre_size > 0) { _mi_os_decommit(base, pre_size, stats); }
if (post_size > 0) { _mi_os_decommit((uint8_t*)p + mid_size, post_size, stats); }
}
}
else {
// free the pre- and post part
if (pre_size > 0) { mi_os_prim_free(base, pre_size, committed, stats); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)p + mid_size, post_size, committed, stats); }
}
}
mi_assert_internal(_mi_is_aligned(p, alignment));
return p;
}
// Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned.
static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) {
static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, size_t* fullsize, mi_stats_t* stats) {
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(size > 0 && size == mi_os_alloc_size(size));
mi_assert_internal(is_large != NULL);
mi_assert_internal(is_zero != NULL);
mi_assert_internal(base != NULL);
if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = _mi_align_up(size, _mi_os_page_size());
size = mi_os_alloc_size(size);
// try first with a hint (this will be aligned directly on Win 10+ or BSD)
void* p = mi_os_prim_alloc(size, alignment, commit, allow_large, is_large, is_zero, stats);
@ -248,47 +299,21 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
// aligned already?
if (((uintptr_t)p % alignment) == 0) {
*base = p;
*fullsize = size;
return p;
}
else {
// if not aligned, free it, overallocate, and unmap around it
// if not aligned, free the original allocation, overallocate, and unmap around it
_mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit);
mi_os_prim_free(p, size, commit, stats);
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
const size_t over_size = size + alignment;
if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block
// over-allocate uncommitted (virtual) memory
p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats);
if (p == NULL) return NULL;
// set p to the aligned part in the full region
// note: this is dangerous on Windows as VirtualFree needs the actual base pointer
// this is handled though by having the `base` field in the memid's
*base = p; // remember the base
p = mi_align_up_ptr(p, alignment);
// explicitly commit only the aligned part
if (commit) {
_mi_os_commit(p, size, NULL, stats);
}
}
else { // mmap can free inside an allocation
// overallocate...
p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats);
if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area. (noop on sbrk)
void* aligned_p = mi_align_up_ptr(p, alignment);
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
size_t post_size = over_size - pre_size - mid_size;
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); }
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); }
// we can return the aligned pointer on `mmap` (and sbrk) systems
p = aligned_p;
*base = aligned_p; // since we freed the pre part, `*base == p`.
}
const size_t oversize = size + alignment - 1;
p = mi_os_prim_alloc(oversize, 1 /* alignment */, commit, false /* allow_large */, is_large, is_zero, stats);
if (p == NULL) return NULL;
*base = p;
*fullsize = oversize;
return mi_os_align_within(base, oversize, alignment, size, commit, *is_large, stats);
}
mi_assert_internal(p == NULL || (p != NULL && *base != NULL && ((uintptr_t)p % alignment) == 0));
@ -310,7 +335,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* tld_stats) {
bool os_is_zero = false;
void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero, stats);
if (p != NULL) {
*memid = _mi_memid_create_os(true, os_is_zero, os_is_large);
*memid = _mi_memid_create_os(p, size, 0, true, os_is_zero, os_is_large);
}
return p;
}
@ -327,11 +352,10 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo
bool os_is_large = false;
bool os_is_zero = false;
void* os_base = NULL;
void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &_mi_stats_main /*tld->stats*/ );
size_t os_size = 0;
void* p = mi_os_prim_alloc_aligned(size, alignment, commit, allow_large, &os_is_large, &os_is_zero, &os_base, &os_size, &_mi_stats_main /*tld->stats*/ );
if (p != NULL) {
*memid = _mi_memid_create_os(commit, os_is_zero, os_is_large);
memid->mem.os.base = os_base;
memid->mem.os.alignment = alignment;
*memid = _mi_memid_create_os(os_base, os_size, alignment, commit, os_is_zero, os_is_large);
}
return p;
}
@ -357,20 +381,86 @@ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offse
else {
// overallocate to align at an offset
const size_t extra = _mi_align_up(offset, alignment) - offset;
const size_t oversize = size + extra;
const size_t oversize = mi_os_alloc_size(size + extra);
void* const start = _mi_os_alloc_aligned(oversize, alignment, commit, allow_large, memid, tld_stats);
if (start == NULL) return NULL;
void* const p = (uint8_t*)start + extra;
mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment));
// decommit the overallocation at the start
if (commit && extra > _mi_os_page_size()) {
if (memid->initially_committed && !memid->is_pinned && (extra > _mi_os_page_size())) {
_mi_os_decommit(start, extra, tld_stats);
}
return p;
}
}
/* -----------------------------------------------------------
Remappable memory
----------------------------------------------------------- */
void* _mi_os_alloc_remappable(size_t size, size_t future_reserve, size_t alignment, mi_memid_t* memid, mi_stats_t* stats) {
mi_assert_internal(size > 0);
mi_assert_internal(memid != NULL);
*memid = _mi_memid_none();
if (alignment == 0) { alignment = 1; }
const size_t oversize = mi_os_alloc_size(size + alignment - 1);
if (future_reserve < oversize) { future_reserve = oversize; }
bool os_is_pinned = true;
bool os_is_zero = false;
void* base = NULL;
void* remap_info = NULL;
int err = _mi_prim_alloc_remappable(oversize, future_reserve, &os_is_pinned, &os_is_zero, &base, &remap_info);
if (err != 0 || base == NULL) {
// fall back to regular allocation
return _mi_os_alloc_aligned(size, alignment, true /* commit */, true /* allow_large */, memid, stats);
}
*memid = _mi_memid_create_os(base, oversize, alignment, true, os_is_zero, os_is_pinned);
memid->memkind = MI_MEM_OS_REMAP;
memid->mem.os.prim_info = remap_info;
return mi_os_align_within(base,oversize,alignment,size,true,memid->is_pinned,stats);
}
void* _mi_os_realloc(void* p, size_t size, size_t newsize, mi_memid_t* memid, mi_stats_t* stats) {
mi_assert_internal(size > 0);
mi_assert_internal(newsize > 0);
mi_assert_internal(p != NULL && memid != NULL);
mi_assert_internal(mi_memkind_is_os(memid->memkind));
if (p == NULL) return NULL;
if (!mi_memkind_is_os(memid->memkind)) return NULL;
newsize = mi_os_alloc_size(newsize);
const size_t oversize = newsize + memid->mem.os.alignment - 1;
if (memid->memkind == MI_MEM_OS_REMAP) {
bool extend_is_zero = false;
void* newp = NULL;
int err = _mi_prim_realloc_remappable(memid->mem.os.base, memid->mem.os.size, oversize, &extend_is_zero, &newp, &memid->mem.os.prim_info);
if (err == 0 && newp != NULL) {
memid->initially_committed = true;
memid->mem.os.base = newp;
memid->mem.os.size = oversize;
return mi_os_align_within(newp, oversize, memid->mem.os.alignment, newsize, memid->initially_committed, memid->is_pinned, stats);
}
else {
_mi_warning_message("failed to remap OS re-allocation (error %d (0x%x) at %p of size %zu to size %zu)\n", err, err, p, size, newsize);
}
}
// fall back to regular realloc
mi_memid_t newmemid = _mi_memid_none();
void* newp = _mi_os_alloc_aligned(newsize, memid->mem.os.alignment, memid->initially_committed, true /* allow large? */, &newmemid, stats);
if (newp == NULL) return NULL;
size_t csize = (size > newsize ? newsize : size);
_mi_memcpy_aligned(newp, p, csize);
_mi_os_free(p, size, *memid, stats);
*memid = newmemid;
return newp;
}
/* -----------------------------------------------------------
OS memory API: reset, commit, decommit, protect, unprotect.
----------------------------------------------------------- */
@ -640,16 +730,17 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
}
}
}
mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size);
const size_t alloc_size = page * MI_HUGE_OS_PAGE_SIZE;
mi_assert_internal(alloc_size <= size);
if (pages_reserved != NULL) { *pages_reserved = page; }
if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; }
if (psize != NULL) { *psize = alloc_size; }
if (page != 0) {
mi_assert(start != NULL);
*memid = _mi_memid_create_os(true /* is committed */, all_zero, true /* is_large */);
*memid = _mi_memid_create_os(start, alloc_size, _mi_os_page_size(), true /* is committed */, all_zero, true /* is_large */);
memid->memkind = MI_MEM_OS_HUGE;
mi_assert(memid->is_pinned);
#ifdef MI_TRACK_ASAN
if (all_zero) { mi_track_mem_defined(start,size); }
if (all_zero) { mi_track_mem_defined(start,alloc_size); }
#endif
}
return (page == 0 ? NULL : start);

13
src/page.c
View file

@ -814,11 +814,10 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex
General allocation
----------------------------------------------------------- */
// A huge page is allocated directly without being in a queue.
// Because huge pages contain just one block, and the segment contains
// just that page, we always treat them as abandoned and any thread
// that frees the block can free the whole page and segment directly.
// Huge pages are also use if the requested alignment is very large (> MI_ALIGNMENT_MAX).
// A huge page always occupies a single segment.
// It is used for large allocations, (very) large alignments (> MI_ALIGNMENT_MAX), or remappable blocks.
// When a huge page is freed from another thread, it is immediately reset to reduce memory pressure.
// We use a page_alignment of 1 for remappable memory.
static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size, size_t page_alignment) {
size_t block_size = _mi_os_good_alloc_size(size);
mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE || page_alignment > 0);
@ -918,7 +917,9 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al
// note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case.
void* p = _mi_page_malloc(heap, page, size, false);
mi_assert_internal(p != NULL);
_mi_memzero_aligned(p, mi_page_usable_block_size(page));
if (!page->free_is_zero) {
_mi_memzero_aligned(p, mi_page_usable_block_size(page));
}
return p;
}
else {

42
src/prim/unix/prim.c
View file

@ -11,6 +11,10 @@ terms of the MIT license. A copy of the license can be found in the file
#define _DEFAULT_SOURCE // ensure mmap flags and syscall are defined
#endif
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // ensure mremap is defined
#endif
#if defined(__sun)
// illumos provides new mman.h api when any of these are defined
// otherwise the old api based on caddr_t which predates the void pointers one.
@ -861,3 +865,41 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
}
#endif
//----------------------------------------------------------------
// Remappable memory
//----------------------------------------------------------------
int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
#if !defined(MREMAP_MAYMOVE)
MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
return EINVAL;
#else
MI_UNUSED(future_reserve);
*remap_info = NULL;
return _mi_prim_alloc(size,1,true /* commit? */, true /* allow_large */, is_pinned /* is_large? */, is_zero, addr);
#endif
}
int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
#if !defined(MREMAP_MAYMOVE)
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
return EINVAL;
#else
mi_assert_internal(*remap_info == NULL); MI_UNUSED(remap_info);
void* p = mremap(addr,size,newsize,MREMAP_MAYMOVE);
if (p == MAP_FAILED) { return errno; }
*extend_is_zero = true;
*newaddr = p;
return 0;
#endif
}
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
MI_UNUSED(remap_info);
mi_assert_internal(remap_info == NULL);
return _mi_prim_free(addr,size);
}

20
src/prim/wasi/prim.c
View file

@ -273,3 +273,23 @@ void _mi_prim_thread_done_auto_done(void) {
void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
MI_UNUSED(heap);
}
//----------------------------------------------------------------
// Remappable memory
//----------------------------------------------------------------
int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
return EINVAL;
}
int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
return EINVAL;
}
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info);
return EINVAL;
}

21
src/prim/windows/prim.c
View file

@ -620,3 +620,24 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
}
#endif
//----------------------------------------------------------------
// Remappable memory
//----------------------------------------------------------------
int _mi_prim_alloc_remappable(size_t size, size_t future_reserve, bool* is_pinned, bool* is_zero, void** addr, void** remap_info ) {
MI_UNUSED(size); MI_UNUSED(future_reserve); MI_UNUSED(is_pinned); MI_UNUSED(is_zero); MI_UNUSED(addr); MI_UNUSED(remap_info);
return EINVAL;
}
int _mi_prim_realloc_remappable(void* addr, size_t size, size_t newsize, bool* extend_is_zero, void** newaddr, void** remap_info ) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(newsize); MI_UNUSED(extend_is_zero); MI_UNUSED(newaddr); MI_UNUSED(remap_info);
return EINVAL;
}
int _mi_prim_free_remappable(void* addr, size_t size, void* remap_info ) {
MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(remap_info);
return EINVAL;
}

57
src/segment.c
View file

@ -517,13 +517,19 @@ static mi_segment_t* mi_segment_os_alloc(bool eager_delayed, size_t page_alignme
bool allow_large = (!eager_delayed && (MI_SECURE == 0)); // only allow large OS pages once we are no longer lazy
size_t align_offset = 0;
size_t alignment = MI_SEGMENT_SIZE;
if (page_alignment > 0) {
if (page_alignment > MI_PAGE_ALIGN_REMAPPABLE) {
alignment = page_alignment;
align_offset = _mi_align_up(pre_size, MI_SEGMENT_SIZE);
segment_size = segment_size + (align_offset - pre_size); // adjust the segment size
}
mi_segment_t* segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os);
mi_segment_t* segment = NULL;
if (page_alignment == MI_PAGE_ALIGN_REMAPPABLE) {
segment = (mi_segment_t*)_mi_os_alloc_remappable(segment_size, 0, alignment, &memid, tld_os->stats);
}
else {
segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, alignment, align_offset, commit, allow_large, req_arena_id, &memid, tld_os);
}
if (segment == NULL) {
return NULL; // failed to allocate
}
@ -1224,7 +1230,7 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment,
page->xblock_size = (psize > MI_HUGE_BLOCK_SIZE ? MI_HUGE_BLOCK_SIZE : (uint32_t)psize);
// reset the part of the page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE)
if (page_alignment > 0 && segment->allow_decommit && page->is_committed) {
if (page_alignment > MI_PAGE_ALIGN_REMAPPABLE && segment->allow_decommit && page->is_committed) {
uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment);
mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment));
mi_assert_internal(psize - (aligned_p - start) >= size);
@ -1283,17 +1289,52 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc
}
#endif
mi_block_t* _mi_segment_huge_page_remap(mi_segment_t* segment, mi_page_t* page, mi_block_t* block, size_t newsize, mi_segments_tld_t* tld) {
mi_assert_internal(segment == _mi_page_segment(page));
mi_assert_internal(page->used == 1);
mi_assert_internal(page->free == NULL);
mi_assert_internal(page->local_free == NULL);
mi_assert_internal(mi_page_thread_free(page) == NULL);
mi_assert_internal(segment->next == NULL && segment->prev == NULL);
mi_assert_internal(page->next == NULL && page->prev == NULL);
const size_t bsize = mi_page_block_size(page);
const size_t newssize = _mi_align_up(_mi_align_up(newsize, _mi_os_page_size()) + (mi_segment_size(segment) - bsize), MI_SEGMENT_SIZE);
mi_memid_t memid = segment->memid;
const ptrdiff_t block_ofs = (uint8_t*)block - (uint8_t*)segment;
mi_segment_protect(segment, false, tld->os);
if (segment->allow_decommit && !page->is_committed) { // TODO: always commit fully regardless of the page?
_mi_os_commit(segment, mi_segment_size(segment), NULL, tld->stats);
page->is_committed = true;
}
mi_segment_t* newsegment = (mi_segment_t*)_mi_os_realloc(segment, mi_segment_size(segment), newssize, &memid, tld->stats);
if (newsegment == NULL) {
mi_segment_protect(segment, true, tld->os);
return NULL;
}
else {
newsegment->memid = memid;
newsegment->segment_size = newssize;
newsegment->cookie = _mi_ptr_cookie(newsegment);
mi_segment_protect(newsegment, true, tld->os);
}
mi_block_t* newblock = (mi_block_t*)((uint8_t*)newsegment + block_ofs);
mi_assert_internal(_mi_ptr_segment(newblock) == newsegment);
mi_page_t* newpage = _mi_ptr_page(newblock);
mi_assert_internal(mi_page_block_size(newpage) >= newsize); MI_UNUSED(newpage);
return newblock;
}
/* -----------------------------------------------------------
Page allocation
----------------------------------------------------------- */
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) {
mi_page_t* page;
if mi_unlikely(page_alignment > MI_ALIGNMENT_MAX) {
mi_assert_internal(_mi_is_power_of_two(page_alignment));
mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE);
//mi_assert_internal((MI_SEGMENT_SIZE % page_alignment) == 0);
if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; }
if mi_unlikely(page_alignment > 0) {
mi_assert_internal(page_alignment == MI_PAGE_ALIGN_REMAPPABLE || (_mi_is_power_of_two(page_alignment) && page_alignment >= MI_SEGMENT_SIZE));
if (page_alignment != MI_PAGE_ALIGN_REMAPPABLE && page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; }
page = mi_segment_huge_page_alloc(block_size, page_alignment, heap->arena_id, tld, os_tld);
}
else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) {

6
src/static.c
View file

@ -5,8 +5,12 @@ terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution.
-----------------------------------------------------------------------------*/
#ifndef _DEFAULT_SOURCE
#define _DEFAULT_SOURCE
#define _DEFAULT_SOURCE // ensure mmap is defined
#endif
#ifndef _GNU_SOURCE
#define _GNU_SOURCE // ensure mremap is defined
#endif
#if defined(__sun)
// same remarks as os.c for the static's context.
#undef _XOPEN_SOURCE

19
test/main-override-static.c
View file

@ -18,11 +18,14 @@ static void test_reserved(void);
static void negative_stat(void);
static void alloc_huge(void);
static void test_heap_walk(void);
static void test_remap(void);
int main() {
mi_version();
mi_stats_reset();
test_remap();
// detect double frees and heap corruption
// double_free1();
// double_free2();
@ -216,6 +219,20 @@ static void test_heap_walk(void) {
mi_heap_visit_blocks(heap, true, &test_visit, NULL);
}
static void test_remap(void) {
size_t size = 64 * 1024 * 1024;
size_t inc = 1024 * 1024;
uint8_t* p = (uint8_t*)mi_malloc_remappable(size);
memset(p, 1, size);
for (int i = 2; i < 100; i++) {
p = mi_remap(p, size + inc);
memset(p + size, i, inc);
size += inc;
}
mi_free(p);
}
// ----------------------------
// bin size experiments
// ------------------------------