kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge from dev

Browse source
This commit is contained in:
Daan 2024-05-18 16:57:35 -07:00
commit d4a7c0ffcc
4 changed files with 43 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
include/mimalloc/atomic.h
View file

@ -132,7 +132,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
#elif defined(_MSC_VER) #elif defined(_MSC_VER)
// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. // Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
#ifndef WIN32_LEAN_AND_MEAN #ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN
#endif #endif
@ -201,7 +201,7 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_
#else #else
uintptr_t x = *p; uintptr_t x = *p;
if (mo > mi_memory_order_relaxed) { if (mo > mi_memory_order_relaxed) {
while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
} }
return x; return x;
#endif #endif

4
include/mimalloc/internal.h
View file

@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file
// functions and macros. // functions and macros.
// -------------------------------------------------------------------------- // --------------------------------------------------------------------------
#include "mimalloc/types.h" #include "types.h"
#include "mimalloc/track.h" #include "track.h"
#if (MI_DEBUG>0) #if (MI_DEBUG>0)
#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)

2
include/mimalloc/types.h
View file

@ -24,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <stddef.h> // ptrdiff_t #include <stddef.h> // ptrdiff_t
#include <stdint.h> // uintptr_t, uint16_t, etc #include <stdint.h> // uintptr_t, uint16_t, etc
#include "mimalloc/atomic.h" // _Atomic #include "atomic.h" // _Atomic
#ifdef _MSC_VER #ifdef _MSC_VER
#pragma warning(disable:4214) // bitfield is not int #pragma warning(disable:4214) // bitfield is not int

77
src/prim/unix/prim.c
View file

@ -28,7 +28,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <sys/mman.h> // mmap #include <sys/mman.h> // mmap
#include <unistd.h> // sysconf #include <unistd.h> // sysconf
#include <fcntl.h> // open, close, read, access #include <fcntl.h> // open, close, read, access
#if defined(__linux__) #if defined(__linux__)
#include <features.h> #include <features.h>
#if defined(MI_NO_THP) #if defined(MI_NO_THP)
@ -57,7 +57,7 @@ terms of the MIT license. A copy of the license can be found in the file
#include <sys/sysctl.h> #include <sys/sysctl.h>
#endif #endif
#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(__OpenBSD__) && !defined(__sun) #if defined(__linux__) || defined(__FreeBSD__)
#define MI_HAS_SYSCALL_H #define MI_HAS_SYSCALL_H
#include <sys/syscall.h> #include <sys/syscall.h>
#endif #endif
@ -65,39 +65,38 @@ terms of the MIT license. A copy of the license can be found in the file
//------------------------------------------------------------------------------------ //------------------------------------------------------------------------------------
// Use syscalls for some primitives to allow for libraries that override open/read/close etc. // Use syscalls for some primitives to allow for libraries that override open/read/close etc.
// and do allocation themselves; using syscalls prevents recursion when mimalloc is // and do allocation themselves; using syscalls prevents recursion when mimalloc is
// still initializing (issue #713) // still initializing (issue #713)
// Declare inline to avoid unused function warnings.
//------------------------------------------------------------------------------------ //------------------------------------------------------------------------------------
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) #if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)
static int mi_prim_open(const char* fpath, int open_flags) { static inline int mi_prim_open(const char* fpath, int open_flags) {
return syscall(SYS_open,fpath,open_flags,0); return syscall(SYS_open,fpath,open_flags,0);
} }
static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
return syscall(SYS_read,fd,buf,bufsize); return syscall(SYS_read,fd,buf,bufsize);
} }
static int mi_prim_close(int fd) { static inline int mi_prim_close(int fd) {
return syscall(SYS_close,fd); return syscall(SYS_close,fd);
} }
static int mi_prim_access(const char *fpath, int mode) { static inline int mi_prim_access(const char *fpath, int mode) {
return syscall(SYS_access,fpath,mode); return syscall(SYS_access,fpath,mode);
} }
#elif !defined(__sun) && \ #else
(!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS and Solaris
static int mi_prim_open(const char* fpath, int open_flags) { static inline int mi_prim_open(const char* fpath, int open_flags) {
return open(fpath,open_flags); return open(fpath,open_flags);
} }
static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) { static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
return read(fd,buf,bufsize); return read(fd,buf,bufsize);
} }
static int mi_prim_close(int fd) { static inline int mi_prim_close(int fd) {
return close(fd); return close(fd);
} }
static int mi_prim_access(const char *fpath, int mode) { static inline int mi_prim_access(const char *fpath, int mode) {
return access(fpath,mode); return access(fpath,mode);
} }
@ -130,12 +129,12 @@ static bool unix_detect_overcommit(void) {
os_overcommit = (val != 0); os_overcommit = (val != 0);
} }
#else #else
// default: overcommit is true // default: overcommit is true
#endif #endif
return os_overcommit; return os_overcommit;
} }
void _mi_prim_mem_init( mi_os_mem_config_t* config ) void _mi_prim_mem_init( mi_os_mem_config_t* config )
{ {
long psize = sysconf(_SC_PAGESIZE); long psize = sysconf(_SC_PAGESIZE);
if (psize > 0) { if (psize > 0) {
@ -197,12 +196,12 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
size_t n = mi_bsr(try_alignment); size_t n = mi_bsr(try_alignment);
if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB
p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0);
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
int err = errno; int err = errno;
_mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr); _mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
} }
if (p!=MAP_FAILED) return p; if (p!=MAP_FAILED) return p;
// fall back to regular mmap // fall back to regular mmap
} }
} }
#elif defined(MAP_ALIGN) // Solaris #elif defined(MAP_ALIGN) // Solaris
@ -218,16 +217,16 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
void* hint = _mi_os_get_aligned_hint(try_alignment, size); void* hint = _mi_os_get_aligned_hint(try_alignment, size);
if (hint != NULL) { if (hint != NULL) {
p = mmap(hint, size, protect_flags, flags, fd, 0); p = mmap(hint, size, protect_flags, flags, fd, 0);
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) { if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
#if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly? #if MI_TRACK_ENABLED // asan sometimes does not instrument errno correctly?
int err = 0; int err = 0;
#else #else
int err = errno; int err = errno;
#endif #endif
_mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint); _mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
} }
if (p!=MAP_FAILED) return p; if (p!=MAP_FAILED) return p;
// fall back to regular mmap // fall back to regular mmap
} }
} }
#endif #endif
@ -356,9 +355,9 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
mi_assert_internal(commit || !allow_large); mi_assert_internal(commit || !allow_large);
mi_assert_internal(try_alignment > 0); mi_assert_internal(try_alignment > 0);
*is_zero = true; *is_zero = true;
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
*addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); *addr = unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
return (*addr != NULL ? 0 : errno); return (*addr != NULL ? 0 : errno);
} }
@ -386,19 +385,19 @@ int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
// was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but // was either from mmap PROT_NONE, or from decommit MADV_DONTNEED, but
// we sometimes call commit on a range with still partially committed // we sometimes call commit on a range with still partially committed
// memory and `mprotect` does not zero the range. // memory and `mprotect` does not zero the range.
*is_zero = false; *is_zero = false;
int err = mprotect(start, size, (PROT_READ | PROT_WRITE)); int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
if (err != 0) { if (err != 0) {
err = errno; err = errno;
unix_mprotect_hint(err); unix_mprotect_hint(err);
} }
return err; return err;
} }
int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
int err = 0; int err = 0;
// decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE)
err = unix_madvise(start, size, MADV_DONTNEED); err = unix_madvise(start, size, MADV_DONTNEED);
#if !MI_DEBUG && !MI_SECURE #if !MI_DEBUG && !MI_SECURE
*needs_recommit = false; *needs_recommit = false;
#else #else
@ -410,15 +409,15 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) {
*needs_recommit = true; *needs_recommit = true;
const int fd = unix_mmap_fd(); const int fd = unix_mmap_fd();
void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0); void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), fd, 0);
if (p != start) { err = errno; } if (p != start) { err = errno; }
*/ */
return err; return err;
} }
int _mi_prim_reset(void* start, size_t size) { int _mi_prim_reset(void* start, size_t size) {
// We try to use `MADV_FREE` as that is the fastest. A drawback though is that it // We try to use `MADV_FREE` as that is the fastest. A drawback though is that it
// will not reduce the `rss` stats in tools like `top` even though the memory is available // will not reduce the `rss` stats in tools like `top` even though the memory is available
// to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by // to other processes. With the default `MIMALLOC_PURGE_DECOMMITS=1` we ensure that by
// default `MADV_DONTNEED` is used though. // default `MADV_DONTNEED` is used though.
#if defined(MADV_FREE) #if defined(MADV_FREE)
static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE); static _Atomic(size_t) advice = MI_ATOMIC_VAR_INIT(MADV_FREE);
@ -438,7 +437,7 @@ int _mi_prim_reset(void* start, size_t size) {
int _mi_prim_protect(void* start, size_t size, bool protect) { int _mi_prim_protect(void* start, size_t size, bool protect) {
int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE)); int err = mprotect(start, size, protect ? PROT_NONE : (PROT_READ | PROT_WRITE));
if (err != 0) { err = errno; } if (err != 0) { err = errno; }
unix_mprotect_hint(err); unix_mprotect_hint(err);
return err; return err;
} }
@ -479,7 +478,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo
if (err != 0) { if (err != 0) {
err = errno; err = errno;
_mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err); _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %d (0x%x))\n", numa_node, err, err);
} }
} }
return (*addr != NULL ? 0 : errno); return (*addr != NULL ? 0 : errno);
} }
@ -594,9 +593,9 @@ mi_msecs_t _mi_prim_clock_now(void) {
// low resolution timer // low resolution timer
mi_msecs_t _mi_prim_clock_now(void) { mi_msecs_t _mi_prim_clock_now(void) {
#if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0) #if !defined(CLOCKS_PER_SEC) || (CLOCKS_PER_SEC == 1000) || (CLOCKS_PER_SEC == 0)
return (mi_msecs_t)clock(); return (mi_msecs_t)clock();
#elif (CLOCKS_PER_SEC < 1000) #elif (CLOCKS_PER_SEC < 1000)
return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC); return (mi_msecs_t)clock() * (1000 / (mi_msecs_t)CLOCKS_PER_SEC);
#else #else
return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000); return (mi_msecs_t)clock() / ((mi_msecs_t)CLOCKS_PER_SEC / 1000);
#endif #endif
@ -636,7 +635,7 @@ void _mi_prim_process_info(mi_process_info_t* pinfo)
pinfo->stime = timeval_secs(&rusage.ru_stime); pinfo->stime = timeval_secs(&rusage.ru_stime);
#if !defined(__HAIKU__) #if !defined(__HAIKU__)
pinfo->page_faults = rusage.ru_majflt; pinfo->page_faults = rusage.ru_majflt;
#endif #endif
#if defined(__HAIKU__) #if defined(__HAIKU__)
// Haiku does not have (yet?) a way to // Haiku does not have (yet?) a way to
// get these stats per process // get these stats per process
@ -763,7 +762,7 @@ bool _mi_prim_getenv(const char* name, char* result, size_t result_size) {
bool _mi_prim_random_buf(void* buf, size_t buf_len) { bool _mi_prim_random_buf(void* buf, size_t buf_len) {
// We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf
// may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html> // may fail silently on macOS. See PR #390, and <https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html>
return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess);
} }
#elif defined(__ANDROID__) || defined(__DragonFly__) || \ #elif defined(__ANDROID__) || defined(__DragonFly__) || \
@ -862,7 +861,7 @@ void _mi_prim_thread_associate_default_heap(mi_heap_t* heap) {
} }
} }
#else #else
void _mi_prim_thread_init_auto_done(void) { void _mi_prim_thread_init_auto_done(void) {
// nothing // nothing