kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-05 15:09:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove delayed reset option (for now)

Browse source
This commit is contained in:
Daan Leijen 2019-11-20 14:13:02 -08:00
parent 56b9fac4bf
commit 30e2c54adb
4 changed files with 142 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/mimalloc.h
View file

@ -273,7 +273,7 @@ typedef enum mi_option_e {
mi_option_page_reset, mi_option_page_reset,
mi_option_segment_reset, mi_option_segment_reset,
mi_option_eager_commit_delay, mi_option_eager_commit_delay,
mi_option_reset_delay, mi_option_reset_decommits,
mi_option_use_numa_nodes, mi_option_use_numa_nodes,
mi_option_os_tag, mi_option_os_tag,
mi_option_max_errors, mi_option_max_errors,

139
src/memory.c
View file

@ -53,9 +53,6 @@ void _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats);
void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
// local
static bool mi_delay_remove(mi_delay_slots_t* delay_slots, void* p, size_t size);
// Constants // Constants
#if (MI_INTPTR_SIZE==8) #if (MI_INTPTR_SIZE==8)
@ -354,8 +351,6 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_os_tld_t* tld) {
if (p==NULL) return; if (p==NULL) return;
if (size==0) return; if (size==0) return;
mi_delay_remove(tld->reset_delay, p, size);
size_t arena_memid = 0; size_t arena_memid = 0;
mi_bitmap_index_t bit_idx; mi_bitmap_index_t bit_idx;
mem_region_t* region; mem_region_t* region;
@ -424,7 +419,6 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
bool is_eager_committed; bool is_eager_committed;
void* start = mi_region_info_read(mi_atomic_read(&regions[i].info), NULL, &is_eager_committed); void* start = mi_region_info_read(mi_atomic_read(&regions[i].info), NULL, &is_eager_committed);
if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) {
mi_delay_remove(tld->reset_delay, start, MI_REGION_SIZE);
_mi_arena_free(start, MI_REGION_SIZE, region->arena_memid, tld->stats); _mi_arena_free(start, MI_REGION_SIZE, region->arena_memid, tld->stats);
} }
// and release // and release
@ -434,142 +428,23 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
} }
} }
/* ----------------------------------------------------------------------------
Delay slots
-----------------------------------------------------------------------------*/
typedef void (mi_delay_resolve_fun)(void* addr, size_t size, void* arg);
static void mi_delay_insert(mi_delay_slots_t* ds,
mi_msecs_t delay, uint8_t* addr, size_t size,
mi_delay_resolve_fun* resolve, void* arg)
{
if (ds == NULL || delay==0 || addr==NULL || size==0) {
resolve(addr, size, arg);
return;
}
mi_msecs_t now = _mi_clock_now();
mi_delay_slot_t* oldest = &ds->slots[0];
// walk through all slots, resolving expired ones.
// remember the oldest slot to insert the new entry in.
size_t newcount = 0;
for (size_t i = 0; i < ds->count; i++) {
mi_delay_slot_t* slot = &ds->slots[i];
if (slot->expire == 0) {
// empty slot
oldest = slot;
}
// TODO: should we handle overlapping areas too?
else if (slot->addr <= addr && slot->addr + slot->size >= addr + size) {
// earlier slot encompasses new area, increase expiration
slot->expire = now + delay;
delay = 0;
}
else if (addr <= slot->addr && addr + size >= slot->addr + slot->size) {
// new one encompasses old slot, overwrite
slot->expire = now + delay;
slot->addr = addr;
slot->size = size;
delay = 0;
}
else if (slot->expire < now) {
// expired slot, resolve now
slot->expire = 0;
resolve(slot->addr, slot->size, arg);
}
else if (oldest->expire > slot->expire) {
oldest = slot;
newcount = i+1;
}
else {
newcount = i+1;
}
}
ds->count = newcount;
if (delay>0) {
// not yet registered, use the oldest slot (or a new one if there is space)
if (ds->count < ds->capacity) {
oldest = &ds->slots[ds->count];
ds->count++;
}
else if (oldest->expire > 0) {
resolve(oldest->addr, oldest->size, arg); // evict if not empty
}
mi_assert_internal((oldest - ds->slots) < (ptrdiff_t)ds->count);
oldest->expire = now + delay;
oldest->addr = addr;
oldest->size = size;
}
}
static bool mi_delay_remove(mi_delay_slots_t* ds, void* p, size_t size)
{
if (ds == NULL || p==NULL || size==0) return false;
uint8_t* addr = (uint8_t*)p;
bool done = false;
size_t newcount = 0;
// walk through all valid slots
for (size_t i = 0; i < ds->count; i++) {
mi_delay_slot_t* slot = &ds->slots[i];
if (slot->addr <= addr && slot->addr + slot->size >= addr + size) {
// earlier slot encompasses the area; remove it
slot->expire = 0;
done = true;
}
else if (addr <= slot->addr && addr + size >= slot->addr + slot->size) {
// new one encompasses old slot, remove it
slot->expire = 0;
}
else if ((addr <= slot->addr && addr + size > slot->addr) ||
(addr < slot->addr + slot->size && addr + size >= slot->addr + slot->size)) {
// partial overlap
// can happen with a large object spanning onto some partial end block
// mi_assert_internal(false);
slot->expire = 0;
}
else {
newcount = i + 1;
}
}
ds->count = newcount;
return done;
}
static void mi_resolve_reset(void* p, size_t size, void* vtld) {
mi_os_tld_t* tld = (mi_os_tld_t*)vtld;
_mi_os_reset(p, size, tld->stats);
}
bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld) {
mi_delay_insert(tld->reset_delay, mi_option_get(mi_option_reset_delay),
(uint8_t*)p, size, &mi_resolve_reset, tld);
return true;
}
bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
if (!mi_delay_remove(tld->reset_delay, (uint8_t*)p, size)) {
return _mi_os_unreset(p, size, is_zero, tld->stats);
}
return true;
}
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
Other Other
-----------------------------------------------------------------------------*/ -----------------------------------------------------------------------------*/
bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld) {
return _mi_os_reset(p, size, tld->stats);
}
bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
return _mi_os_unreset(p, size, is_zero, tld->stats);
}
bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) { bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
mi_delay_remove(tld->reset_delay,p, size);
return _mi_os_commit(p, size, is_zero, tld->stats); return _mi_os_commit(p, size, is_zero, tld->stats);
} }
bool _mi_mem_decommit(void* p, size_t size, mi_os_tld_t* tld) { bool _mi_mem_decommit(void* p, size_t size, mi_os_tld_t* tld) {
mi_delay_remove(tld->reset_delay, p, size);
return _mi_os_decommit(p, size, tld->stats); return _mi_os_decommit(p, size, tld->stats);
} }

4
src/options.c
View file

@ -65,10 +65,10 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
{ 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread
{ 0, UNINIT, MI_OPTION(page_reset) }, // reset pages on free { 1, UNINIT, MI_OPTION(page_reset) }, // reset pages on free
{ 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit) { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit)
{ 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed
{ 500, UNINIT, MI_OPTION(reset_delay) }, // reset delay in milli-seconds { 1, UNINIT, MI_OPTION(reset_decommits) }, // reset uses decommit/commit
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
{ 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output { 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output

213
src/os.c
View file

@ -77,11 +77,11 @@ static bool use_large_os_page(size_t size, size_t alignment) {
// round to a good OS allocation size (bounded by max 12.5% waste) // round to a good OS allocation size (bounded by max 12.5% waste)
size_t _mi_os_good_alloc_size(size_t size) { size_t _mi_os_good_alloc_size(size_t size) {
size_t align_size; size_t align_size;
if (size < 512*KiB) align_size = _mi_os_page_size(); if (size < 512 * KiB) align_size = _mi_os_page_size();
else if (size < 2*MiB) align_size = 64*KiB; else if (size < 2 * MiB) align_size = 64 * KiB;
else if (size < 8*MiB) align_size = 256*KiB; else if (size < 8 * MiB) align_size = 256 * KiB;
else if (size < 32*MiB) align_size = 1*MiB; else if (size < 32 * MiB) align_size = 1 * MiB;
else align_size = 4*MiB; else align_size = 4 * MiB;
if (size >= (SIZE_MAX - align_size)) return size; // possible overflow? if (size >= (SIZE_MAX - align_size)) return size; // possible overflow?
return _mi_align_up(size, align_size); return _mi_align_up(size, align_size);
} }
@ -92,8 +92,8 @@ size_t _mi_os_good_alloc_size(size_t size) {
// NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB) // NtAllocateVirtualAllocEx is used for huge OS page allocation (1GiB)
// We hide MEM_EXTENDED_PARAMETER to compile with older SDK's. // We hide MEM_EXTENDED_PARAMETER to compile with older SDK's.
#include <winternl.h> #include <winternl.h>
typedef PVOID (__stdcall *PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG); typedef PVOID(__stdcall* PVirtualAlloc2)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ void*, ULONG);
typedef NTSTATUS (__stdcall *PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ PVOID, ULONG); typedef NTSTATUS(__stdcall* PNtAllocateVirtualMemoryEx)(HANDLE, PVOID*, SIZE_T*, ULONG, ULONG, /* MEM_EXTENDED_PARAMETER* */ PVOID, ULONG);
static PVirtualAlloc2 pVirtualAlloc2 = NULL; static PVirtualAlloc2 pVirtualAlloc2 = NULL;
static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL; static PNtAllocateVirtualMemoryEx pNtAllocateVirtualMemoryEx = NULL;
@ -129,7 +129,7 @@ static bool mi_win_enable_large_os_pages()
if (err == 0) err = GetLastError(); if (err == 0) err = GetLastError();
_mi_warning_message("cannot enable large OS page support, error %lu\n", err); _mi_warning_message("cannot enable large OS page support, error %lu\n", err);
} }
return (ok!=0); return (ok != 0);
} }
void _mi_os_init(void) { void _mi_os_init(void) {
@ -144,7 +144,7 @@ void _mi_os_init(void) {
if (hDll != NULL) { if (hDll != NULL) {
// use VirtualAlloc2FromApp if possible as it is available to Windows store apps // use VirtualAlloc2FromApp if possible as it is available to Windows store apps
pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp");
if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); if (pVirtualAlloc2 == NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2");
FreeLibrary(hDll); FreeLibrary(hDll);
} }
hDll = LoadLibrary(TEXT("ntdll.dll")); hDll = LoadLibrary(TEXT("ntdll.dll"));
@ -170,7 +170,7 @@ void _mi_os_init() {
os_alloc_granularity = os_page_size; os_alloc_granularity = os_page_size;
} }
if (mi_option_is_enabled(mi_option_large_os_pages)) { if (mi_option_is_enabled(mi_option_large_os_pages)) {
large_os_page_size = 2*MiB; large_os_page_size = 2 * MiB;
} }
} }
#endif #endif
@ -210,7 +210,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
#if (MI_INTPTR_SIZE >= 8) #if (MI_INTPTR_SIZE >= 8)
// on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations // on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations
void* hint; void* hint;
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) {
return VirtualAlloc(hint, size, flags, PAGE_READWRITE); return VirtualAlloc(hint, size, flags, PAGE_READWRITE);
} }
#endif #endif
@ -233,7 +233,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0; static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
void* p = NULL; void* p = NULL;
if ((large_only || use_large_os_page(size, try_alignment)) if ((large_only || use_large_os_page(size, try_alignment))
&& allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { && allow_large && (flags & MEM_COMMIT) != 0 && (flags & MEM_RESERVE) != 0) {
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
if (!large_only && try_ok > 0) { if (!large_only && try_ok > 0) {
// if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
@ -247,12 +247,12 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
if (large_only) return p; if (large_only) return p;
// fall back to non-large page allocation on error (`p == NULL`). // fall back to non-large page allocation on error (`p == NULL`).
if (p == NULL) { if (p == NULL) {
mi_atomic_write(&large_page_try_ok,10); // on error, don't try again for the next N allocations mi_atomic_write(&large_page_try_ok, 10); // on error, don't try again for the next N allocations
} }
} }
} }
if (p == NULL) { if (p == NULL) {
*is_large = ((flags&MEM_LARGE_PAGES) != 0); *is_large = ((flags & MEM_LARGE_PAGES) != 0);
p = mi_win_virtual_allocx(addr, size, try_alignment, flags); p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
} }
if (p == NULL) { if (p == NULL) {
@ -264,8 +264,8 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
#elif defined(__wasi__) #elif defined(__wasi__)
static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size(); uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size();
uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); uintptr_t aligned_base = _mi_align_up(base, (uintptr_t)try_alignment);
size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); size_t alloc_size = _mi_align_up(aligned_base - base + size, _mi_os_page_size());
mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0);
if (alloc_size < size) return NULL; if (alloc_size < size) return NULL;
if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) { if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) {
@ -278,47 +278,50 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) {
#define MI_OS_USE_MMAP #define MI_OS_USE_MMAP
static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) {
void* p = NULL; void* p = NULL;
#if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED)
// on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations // on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations
void* hint; void* hint;
if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) {
p = mmap(hint,size,protect_flags,flags,fd,0); p = mmap(hint, size, protect_flags, flags, fd, 0);
if (p==MAP_FAILED) p = NULL; // fall back to regular mmap if (p == MAP_FAILED) p = NULL; // fall back to regular mmap
} }
#else #else
UNUSED(try_alignment); UNUSED(try_alignment);
#endif #endif
if (p==NULL) { if (p == NULL) {
p = mmap(addr,size,protect_flags,flags,fd,0); p = mmap(addr, size, protect_flags, flags, fd, 0);
if (p==MAP_FAILED) p = NULL; if (p == MAP_FAILED) p = NULL;
} }
return p; return p;
} }
static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
void* p = NULL; void* p = NULL;
#if !defined(MAP_ANONYMOUS) #if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON #define MAP_ANONYMOUS MAP_ANON
#endif #endif
int flags = MAP_PRIVATE | MAP_ANONYMOUS; #if !defined(MAP_NORESERVE)
#define MAP_NORESERVE 0
#endif
int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
int fd = -1; int fd = -1;
#if defined(MAP_ALIGNED) // BSD #if defined(MAP_ALIGNED) // BSD
if (try_alignment > 0) { if (try_alignment > 0) {
size_t n = _mi_bsr(try_alignment); size_t n = _mi_bsr(try_alignment);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
flags |= MAP_ALIGNED(n); flags |= MAP_ALIGNED(n);
} }
} }
#endif #endif
#if defined(PROT_MAX) #if defined(PROT_MAX)
protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD
#endif #endif
#if defined(VM_MAKE_TAG) #if defined(VM_MAKE_TAG)
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
int os_tag = (int)mi_option_get(mi_option_os_tag); int os_tag = (int)mi_option_get(mi_option_os_tag);
if (os_tag < 100 || os_tag > 255) os_tag = 100; if (os_tag < 100 || os_tag > 255) os_tag = 100;
fd = VM_MAKE_TAG(os_tag); fd = VM_MAKE_TAG(os_tag);
#endif #endif
if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) {
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0; static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
@ -332,39 +335,39 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
else { else {
int lflags = flags; int lflags = flags;
int lfd = fd; int lfd = fd;
#ifdef MAP_ALIGNED_SUPER #ifdef MAP_ALIGNED_SUPER
lflags |= MAP_ALIGNED_SUPER; lflags |= MAP_ALIGNED_SUPER;
#endif #endif
#ifdef MAP_HUGETLB #ifdef MAP_HUGETLB
lflags |= MAP_HUGETLB; lflags |= MAP_HUGETLB;
#endif #endif
#ifdef MAP_HUGE_1GB #ifdef MAP_HUGE_1GB
static bool mi_huge_pages_available = true; static bool mi_huge_pages_available = true;
if ((size % GiB) == 0 && mi_huge_pages_available) { if ((size % GiB) == 0 && mi_huge_pages_available) {
lflags |= MAP_HUGE_1GB; lflags |= MAP_HUGE_1GB;
} }
else else
#endif #endif
{ {
#ifdef MAP_HUGE_2MB #ifdef MAP_HUGE_2MB
lflags |= MAP_HUGE_2MB; lflags |= MAP_HUGE_2MB;
#endif #endif
} }
#ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB #ifdef VM_FLAGS_SUPERPAGE_SIZE_2MB
lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB; lfd |= VM_FLAGS_SUPERPAGE_SIZE_2MB;
#endif #endif
if (large_only || lflags != flags) { if (large_only || lflags != flags) {
// try large OS page allocation // try large OS page allocation
*is_large = true; *is_large = true;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd); p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB #ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) {
mi_huge_pages_available = false; // don't try huge 1GiB pages again mi_huge_pages_available = false; // don't try huge 1GiB pages again
_mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno); _mi_warning_message("unable to allocate huge (1GiB) page, trying large (2MiB) pages instead (error %i)\n", errno);
lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB); lflags = ((lflags & ~MAP_HUGE_1GB) | MAP_HUGE_2MB);
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd); p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
} }
#endif #endif
if (large_only) return p; if (large_only) return p;
if (p == NULL) { if (p == NULL) {
mi_atomic_write(&large_page_try_ok, 10); // on error, don't try again for the next N allocations mi_atomic_write(&large_page_try_ok, 10); // on error, don't try again for the next N allocations
@ -375,7 +378,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
if (p == NULL) { if (p == NULL) {
*is_large = false; *is_large = false;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd);
#if defined(MADV_HUGEPAGE) #if defined(MADV_HUGEPAGE)
// Many Linux systems don't allow MAP_HUGETLB but they support instead // Many Linux systems don't allow MAP_HUGETLB but they support instead
// transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE // transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE
// though since properly aligned allocations will already use large pages if available // though since properly aligned allocations will already use large pages if available
@ -387,7 +390,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
*is_large = true; // possibly *is_large = true; // possibly
}; };
} }
#endif #endif
} }
return p; return p;
} }
@ -401,18 +404,18 @@ static volatile _Atomic(intptr_t) aligned_base;
// Return a 4MiB aligned address that is probably available // Return a 4MiB aligned address that is probably available
static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) {
if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL;
if ((size%MI_SEGMENT_SIZE) != 0) return NULL; if ((size % MI_SEGMENT_SIZE) != 0) return NULL;
intptr_t hint = mi_atomic_add(&aligned_base, size); intptr_t hint = mi_atomic_add(&aligned_base, size);
if (hint == 0 || hint > ((intptr_t)30<<40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages) if (hint == 0 || hint > ((intptr_t)30 << 40)) { // try to wrap around after 30TiB (area after 32TiB is used for huge OS pages)
intptr_t init = ((intptr_t)4 << 40); // start at 4TiB area intptr_t init = ((intptr_t)4 << 40); // start at 4TiB area
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_os_get_aligned_hint ^ hint); uintptr_t r = _mi_random_init((uintptr_t)&mi_os_get_aligned_hint ^ hint);
init = init + (MI_SEGMENT_SIZE * ((r>>17) & 0xFFFF)); // (randomly 0-64k)*4MiB == 0 to 256GiB init = init + (MI_SEGMENT_SIZE * ((r >> 17) & 0xFFFF)); // (randomly 0-64k)*4MiB == 0 to 256GiB
#endif #endif
mi_atomic_cas_strong(mi_atomic_cast(uintptr_t, &aligned_base), init, hint + size); mi_atomic_cas_strong(mi_atomic_cast(uintptr_t, &aligned_base), init, hint + size);
hint = mi_atomic_add(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all hint = mi_atomic_add(&aligned_base, size); // this may still give 0 or > 30TiB but that is ok, it is a hint after all
} }
if (hint%try_alignment != 0) return NULL; if (hint % try_alignment != 0) return NULL;
return (void*)hint; return (void*)hint;
} }
#else #else
@ -441,17 +444,17 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo
} }
*/ */
#if defined(_WIN32) #if defined(_WIN32)
int flags = MEM_RESERVE; int flags = MEM_RESERVE;
if (commit) flags |= MEM_COMMIT; if (commit) flags |= MEM_COMMIT;
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
#elif defined(__wasi__) #elif defined(__wasi__)
*is_large = false; *is_large = false;
p = mi_wasm_heap_grow(size, try_alignment); p = mi_wasm_heap_grow(size, try_alignment);
#else #else
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
#endif #endif
mi_stat_counter_increase(stats->mmap_calls, 1); mi_stat_counter_increase(stats->mmap_calls, 1);
if (p != NULL) { if (p != NULL) {
_mi_stat_increase(&stats->reserved, size); _mi_stat_increase(&stats->reserved, size);
@ -561,7 +564,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* lar
allow_large = *large; allow_large = *large;
*large = false; *large = false;
} }
return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large!=NULL?large:&allow_large), tld->stats); return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large != NULL ? large : &allow_large), tld->stats);
} }
@ -613,7 +616,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
_mi_stat_decrease(&stats->committed, csize); _mi_stat_decrease(&stats->committed, csize);
} }
#if defined(_WIN32) #if defined(_WIN32)
if (commit) { if (commit) {
// if the memory was already committed, the call succeeds but it is not zero'd // if the memory was already committed, the call succeeds but it is not zero'd
// *is_zero = true; // *is_zero = true;
@ -624,28 +627,42 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
BOOL ok = VirtualFree(start, csize, MEM_DECOMMIT); BOOL ok = VirtualFree(start, csize, MEM_DECOMMIT);
err = (ok ? 0 : GetLastError()); err = (ok ? 0 : GetLastError());
} }
#elif defined(__wasi__) #elif defined(__wasi__)
// WebAssembly guests can't control memory protection // WebAssembly guests can't control memory protection
#else #elif defined(MAP_FIXED)
if (!commit) {
// use mmap with MAP_FIXED to discard the existing memory (and reduce commit charge)
void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
if (p != start) { err = errno; }
}
else {
// for commit, just change the protection
err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
if (err != 0) { err = errno; }
}
#else
err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
if (err != 0) { err = errno; } if (err != 0) { err = errno; }
#endif #endif
if (err != 0) { if (err != 0) {
_mi_warning_message("commit/decommit error: start: 0x%p, csize: 0x%x, err: %i\n", start, csize, err); _mi_warning_message("%s error: start: 0x%p, csize: 0x%x, err: %i\n", commit ? "commit" : "decommit", start, csize, err);
} }
mi_assert_internal(err == 0); mi_assert_internal(err == 0);
return (err == 0); return (err == 0);
} }
bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) { bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, false /* conservative? */, is_zero, stats); return mi_os_commitx(addr, size, true, false /* liberal */, is_zero, stats);
} }
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
bool is_zero; bool is_zero;
return mi_os_commitx(addr, size, false, true /* conservative? */, &is_zero, stats); return mi_os_commitx(addr, size, false, true /* conservative */, &is_zero, stats);
} }
bool _mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
return mi_os_commitx(addr, size, true, true /* conservative */, is_zero, stats);
}
// Signal to the OS that the address range is no longer in use // Signal to the OS that the address range is no longer in use
// but may be used later again. This will release physical memory // but may be used later again. This will release physical memory
@ -660,21 +677,21 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
else _mi_stat_decrease(&stats->reset, csize); else _mi_stat_decrease(&stats->reset, csize);
if (!reset) return true; // nothing to do on unreset! if (!reset) return true; // nothing to do on unreset!
#if (MI_DEBUG>1) #if (MI_DEBUG>1)
if (MI_SECURE==0) { if (MI_SECURE == 0) {
memset(start, 0, csize); // pretend it is eagerly reset memset(start, 0, csize); // pretend it is eagerly reset
} }
#endif #endif
#if defined(_WIN32) #if defined(_WIN32)
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == start); mi_assert_internal(p == start);
#if 1 #if 1
if (p == start && start != NULL) { if (p == start && start != NULL) {
VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set VirtualUnlock(start, csize); // VirtualUnlock after MEM_RESET removes the memory from the working set
} }
#endif #endif
if (p != start) return false; if (p != start) return false;
#else #else
#if defined(MADV_FREE) #if defined(MADV_FREE)
@ -704,12 +721,22 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
// pages and reduce swapping while keeping the memory committed. // pages and reduce swapping while keeping the memory committed.
// We page align to a conservative area inside the range to reset. // We page align to a conservative area inside the range to reset.
bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
if (mi_option_is_enabled(mi_option_reset_decommits)) {
return _mi_os_decommit(addr, size, stats);
}
else {
return mi_os_resetx(addr, size, true, stats); return mi_os_resetx(addr, size, true, stats);
}
} }
bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) { bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
if (mi_option_is_enabled(mi_option_reset_decommits)) {
return _mi_os_commit_unreset(addr, size, is_zero, stats); // re-commit it (conservatively!)
}
else {
*is_zero = false; *is_zero = false;
return mi_os_resetx(addr, size, false, stats); return mi_os_resetx(addr, size, false, stats);
}
} }
@ -753,7 +780,7 @@ bool _mi_os_unprotect(void* addr, size_t size) {
bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
// page align conservatively within the range // page align conservatively within the range
mi_assert_internal(oldsize > newsize && p != NULL); mi_assert_internal(oldsize > newsize&& p != NULL);
if (oldsize < newsize || p == NULL) return false; if (oldsize < newsize || p == NULL) return false;
if (oldsize == newsize) return true; if (oldsize == newsize) return true;
@ -781,20 +808,20 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`)
#if defined(WIN32) && (MI_INTPTR_SIZE >= 8) #if defined(WIN32) && (MI_INTPTR_SIZE >= 8)
static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
{ {
mi_assert_internal(size%GiB == 0); mi_assert_internal(size % GiB == 0);
mi_assert_internal(addr != NULL); mi_assert_internal(addr != NULL);
const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE;
mi_win_enable_large_os_pages(); mi_win_enable_large_os_pages();
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
MEM_EXTENDED_PARAMETER params[3] = { {0,0},{0,0},{0,0} }; MEM_EXTENDED_PARAMETER params[3] = { {0,0},{0,0},{0,0} };
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
static bool mi_huge_pages_available = true; static bool mi_huge_pages_available = true;
if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) { if (pNtAllocateVirtualMemoryEx != NULL && mi_huge_pages_available) {
#ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE #ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
#define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE (0x10) #define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE (0x10)
#endif #endif
params[0].Type = 5; // == MemExtendedParameterAttributeFlags; params[0].Type = 5; // == MemExtendedParameterAttributeFlags;
params[0].ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; params[0].ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
ULONG param_count = 1; ULONG param_count = 1;
@ -821,7 +848,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
params[0].ULong = (unsigned)numa_node; params[0].ULong = (unsigned)numa_node;
return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
} }
#endif #endif
// otherwise use regular virtual alloc on older windows // otherwise use regular virtual alloc on older windows
return VirtualAlloc(addr, size, flags, PAGE_READWRITE); return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
} }
@ -842,16 +869,16 @@ static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, cons
} }
#endif #endif
static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) {
mi_assert_internal(size%GiB == 0); mi_assert_internal(size % GiB == 0);
bool is_large = true; bool is_large = true;
void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large);
if (p == NULL) return NULL; if (p == NULL) return NULL;
if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes if (numa_node >= 0 && numa_node < 8 * MI_INTPTR_SIZE) { // at most 64 nodes
uintptr_t numa_mask = (1UL << numa_node); uintptr_t numa_mask = (1UL << numa_node);
// TODO: does `mbind` work correctly for huge OS pages? should we // TODO: does `mbind` work correctly for huge OS pages? should we
// use `set_mempolicy` before calling mmap instead? // use `set_mempolicy` before calling mmap instead?
// see: <https://lkml.org/lkml/2017/2/9/875> // see: <https://lkml.org/lkml/2017/2/9/875>
long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8 * MI_INTPTR_SIZE, 0);
if (err != 0) { if (err != 0) {
_mi_warning_message("failed to bind huge (1GiB) pages to NUMA node %d: %s\n", numa_node, strerror(errno)); _mi_warning_message("failed to bind huge (1GiB) pages to NUMA node %d: %s\n", numa_node, strerror(errno));
} }
@ -883,7 +910,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
start = ((uintptr_t)32 << 40); // 32TiB virtual start address start = ((uintptr_t)32 << 40); // 32TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_random_init((uintptr_t)&mi_os_claim_huge_pages); uintptr_t r = _mi_random_init((uintptr_t)&mi_os_claim_huge_pages);
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r >> 17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB
#endif #endif
} }
end = start + size; end = start + size;
@ -936,8 +963,8 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
if (max_msecs > 0) { if (max_msecs > 0) {
mi_msecs_t elapsed = _mi_clock_end(start_t); mi_msecs_t elapsed = _mi_clock_end(start_t);
if (page >= 1) { if (page >= 1) {
mi_msecs_t estimate = ((elapsed / (page+1)) * pages); mi_msecs_t estimate = ((elapsed / (page + 1)) * pages);
if (estimate > 2*max_msecs) { // seems like we are going to timeout, break if (estimate > 2 * max_msecs) { // seems like we are going to timeout, break
elapsed = max_msecs + 1; elapsed = max_msecs + 1;
} }
} }
@ -947,7 +974,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
} }
} }
} }
mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size); mi_assert_internal(page * MI_HUGE_OS_PAGE_SIZE <= size);
if (pages_reserved != NULL) *pages_reserved = page; if (pages_reserved != NULL) *pages_reserved = page;
if (psize != NULL) *psize = page * MI_HUGE_OS_PAGE_SIZE; if (psize != NULL) *psize = page * MI_HUGE_OS_PAGE_SIZE;
return (page == 0 ? NULL : start); return (page == 0 ? NULL : start);
@ -956,7 +983,7 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse
// free every huge page in a range individually (as we allocated per page) // free every huge page in a range individually (as we allocated per page)
// note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. // note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems.
void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) { void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
if (p==NULL || size==0) return; if (p == NULL || size == 0) return;
uint8_t* base = (uint8_t*)p; uint8_t* base = (uint8_t*)p;
while (size >= MI_HUGE_OS_PAGE_SIZE) { while (size >= MI_HUGE_OS_PAGE_SIZE) {
_mi_os_free(base, MI_HUGE_OS_PAGE_SIZE, stats); _mi_os_free(base, MI_HUGE_OS_PAGE_SIZE, stats);
@ -972,7 +999,7 @@ static size_t mi_os_numa_nodex() {
PROCESSOR_NUMBER pnum; PROCESSOR_NUMBER pnum;
USHORT numa_node = 0; USHORT numa_node = 0;
GetCurrentProcessorNumberEx(&pnum); GetCurrentProcessorNumberEx(&pnum);
GetNumaProcessorNodeEx(&pnum,&numa_node); GetNumaProcessorNodeEx(&pnum, &numa_node);
return numa_node; return numa_node;
} }
@ -999,12 +1026,12 @@ static size_t mi_os_numa_nodex(void) {
static size_t mi_os_numa_node_countx(void) { static size_t mi_os_numa_node_countx(void) {
char buf[128]; char buf[128];
unsigned node = 0; unsigned node = 0;
for(node = 0; node < 256; node++) { for (node = 0; node < 256; node++) {
// enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation)
snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1);
if (access(buf,R_OK) != 0) break; if (access(buf, R_OK) != 0) break;
} }
return (node+1); return (node + 1);
} }
#else #else
static size_t mi_os_numa_nodex(void) { static size_t mi_os_numa_nodex(void) {
@ -1031,7 +1058,7 @@ size_t _mi_os_numa_node_count_get(void) {
int _mi_os_numa_node_get(mi_os_tld_t* tld) { int _mi_os_numa_node_get(mi_os_tld_t* tld) {
UNUSED(tld); UNUSED(tld);
size_t numa_count = _mi_os_numa_node_count(); size_t numa_count = _mi_os_numa_node_count();
if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 if (numa_count <= 1) return 0; // optimize on single numa node systems: always node 0
// never more than the node count and >= 0 // never more than the node count and >= 0
size_t numa_node = mi_os_numa_nodex(); size_t numa_node = mi_os_numa_nodex();
if (numa_node >= numa_count) { numa_node = numa_node % numa_count; } if (numa_node >= numa_count) { numa_node = numa_node % numa_count; }