From 53f88dce76df21061f5b1a83603b50d9e401ce84 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 10 Jun 2025 11:00:20 -0700 Subject: [PATCH 1/2] add physical memory detection on macOS/freeBSD --- src/prim/unix/prim.c | 78 ++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 780d254f..bdd6298c 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -49,6 +49,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !defined(MAC_OS_X_VERSION_10_7) #define MAC_OS_X_VERSION_10_7 1070 #endif + #include #elif defined(__FreeBSD__) || defined(__DragonFly__) #include #if __FreeBSD_version >= 1200000 @@ -119,43 +120,64 @@ static inline int mi_prim_access(const char *fpath, int mode) { static bool unix_detect_overcommit(void) { bool os_overcommit = true; -#if defined(__linux__) - int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd >= 0) { - char buf[32]; - ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); - mi_prim_close(fd); - // - // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) - if (nread >= 1) { - os_overcommit = (buf[0] == '0' || buf[0] == '1'); + #if defined(__linux__) + int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd >= 0) { + char buf[32]; + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } } - } -#elif defined(__FreeBSD__) - int val = 0; - size_t olen = sizeof(val); - if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { - os_overcommit = (val != 0); - } -#else - // default: overcommit is true -#endif + #elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } + #else + // default: overcommit is true + #endif return os_overcommit; } +// try to detect the physical memory dynamically (if possible) +static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) { + #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS + MI_UNUSED(page_size); + int64_t physical_memory = 0; + size_t length = sizeof(int64_t); + #if defined(HW_PHYSMEM64) + int mib[2] = { CTL_HW, HW_PHYSMEM64 }; + #else + int mib[2] = { CTL_HW, HW_MEMSIZE }; + #endif + const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0); + if (err == 0 && physical_memory > 0) { + const int64_t phys_in_kib = physical_memory / MI_KiB; + if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) { + *physical_memory_in_kib = (size_t)phys_in_kib; + } + } + #elif defined(_SC_PHYS_PAGES) // linux + const long pphys = sysconf(_SC_PHYS_PAGES); + const size_t psize_in_kib = page_size / MI_KiB; + if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys < SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { + *physical_memory_in_kib = (size_t)pphys * psize_in_kib; + } + #endif +} + void _mi_prim_mem_init( mi_os_mem_config_t* config ) { long psize = sysconf(_SC_PAGESIZE); - if (psize > 0) { + if (psize > 0 && (unsigned long)psize < SIZE_MAX) { config->page_size = (size_t)psize; config->alloc_granularity = (size_t)psize; - #if defined(_SC_PHYS_PAGES) - long pphys = sysconf(_SC_PHYS_PAGES); - const size_t psize_in_kib = (size_t)psize / MI_KiB; - if (psize_in_kib > 0 && pphys > 0 && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { - config->physical_memory_in_kib = (size_t)pphys * psize_in_kib; - } - #endif + unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib); } config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE; config->has_overcommit = unix_detect_overcommit(); From c9541ac6e88082f5976fc1aa08178a72dd6a8708 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 10 Jun 2025 11:28:03 -0700 Subject: [PATCH 2/2] use sysinfo call on linux to determine physical memory (as _SC_PHYSPAGES may cause allocation) (issue #1100) --- src/prim/unix/prim.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index bdd6298c..cfcaa8a3 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -32,6 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__linux__) #include #include // THP disable, PR_SET_VMA + #include // sysinfo #if defined(__GLIBC__) && !defined(PR_SET_VMA) #include #endif @@ -156,18 +157,25 @@ static void unix_detect_physical_memory( size_t page_size, size_t* physical_memo int mib[2] = { CTL_HW, HW_MEMSIZE }; #endif const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0); - if (err == 0 && physical_memory > 0) { + if (err==0 && physical_memory > 0) { const int64_t phys_in_kib = physical_memory / MI_KiB; if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) { *physical_memory_in_kib = (size_t)phys_in_kib; - } + } } - #elif defined(_SC_PHYS_PAGES) // linux + #elif defined(__linux__) + MI_UNUSED(page_size); + struct sysinfo info; _mi_memzero_var(info); + const int err = sysinfo(&info); + if (err==0 && info.totalram > 0 && info.totalram <= SIZE_MAX) { + *physical_memory_in_kib = (size_t)info.totalram / MI_KiB; + } + #elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100) const long pphys = sysconf(_SC_PHYS_PAGES); const size_t psize_in_kib = page_size / MI_KiB; - if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys < SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { + if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { *physical_memory_in_kib = (size_t)pphys * psize_in_kib; - } + } #endif } @@ -472,7 +480,7 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { #else // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) err = unix_madvise(start, size, MADV_DONTNEED); - #endif + #endif #if !MI_DEBUG && MI_SECURE<=2 *needs_recommit = false; #else @@ -493,8 +501,8 @@ int _mi_prim_reset(void* start, size_t size) { int err = 0; // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower) - #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - err = unix_madvise(start, size, MADV_FREE_REUSABLE); + #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + err = unix_madvise(start, size, MADV_FREE_REUSABLE); if (err==0) return 0; // fall through #endif