merge from dev-exp

CMakeLists.txt

@@ -1,6 +1,8 @@
 cmake_minimum_required(VERSION 3.0)
 project(libmimalloc C CXX)
 include("cmake/mimalloc-config-version.cmake")
+include("CheckIncludeFile")
+
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_CXX_STANDARD 17)
 
@@ -87,6 +89,16 @@ if(MI_USE_CXX MATCHES "ON")
   set_source_files_properties(src/static.c test/test-api.c PROPERTIES LANGUAGE CXX )
 endif()
 
+CHECK_INCLUDE_FILE("numaif.h" MI_HAVE_NUMA_H)
+if(MI_HAVE_NUMA_H)
+  list(APPEND mi_defines MI_HAS_NUMA)
+  list(APPEND mi_libraries numa)
+else()
+  if (NOT(WIN32))
+    message(WARNING "Compiling without using NUMA optimized allocation (on Linux, install libnuma-dev?)")
+  endif()
+endif()
+
 # Compiler flags
 if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU")
   list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas)

include/mimalloc-internal.h

@@ -57,7 +57,7 @@ void       _mi_os_init(void);                                      // called fro
 void*      _mi_os_alloc(size_t size, mi_stats_t* stats);           // to allocate thread local data
 void       _mi_os_free(void* p, size_t size, mi_stats_t* stats);   // to free thread local data
 size_t     _mi_os_good_alloc_size(size_t size);
-int        _mi_os_numa_node(void);
+int        _mi_os_numa_node(mi_os_tld_t* tld);
 
 bool      _mi_os_protect(void* addr, size_t size);
 bool      _mi_os_unprotect(void* addr, size_t size);

include/mimalloc-types.h

@@ -432,6 +432,7 @@ typedef struct mi_segments_tld_s {
 // OS thread local data
 typedef struct mi_os_tld_s {
   size_t              region_idx;   // start point for next allocation
+  int                 numa_node;    // numa node associated with this thread
   mi_stats_t*         stats;        // points to tld stats
 } mi_os_tld_t;
 

src/arena.c

@@ -268,7 +268,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment,
   {
     size_t asize = _mi_align_up(size, MI_ARENA_BLOCK_SIZE);
     size_t bcount = asize / MI_ARENA_BLOCK_SIZE;
-    int numa_node = _mi_os_numa_node(); // current numa node
+    int numa_node = _mi_os_numa_node(tld); // current numa node
 
     mi_assert_internal(size <= bcount*MI_ARENA_BLOCK_SIZE);
     // try numa affine allocation

src/init.c

@@ -109,7 +109,7 @@ static const mi_tld_t tld_empty = {
   false,
   NULL,
   { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_empty_stats }, // segments
-  { 0, tld_empty_stats },             // os
+  { 0, -1, tld_empty_stats },         // os
   { MI_STATS_NULL }                   // stats
 };
 
@@ -123,8 +123,8 @@ static mi_tld_t tld_main = {
   0, false,
   &_mi_heap_main,
   { MI_SEGMENT_SPAN_QUEUES_EMPTY, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats }, // segments
-  { 0, tld_main_stats },              // os
+  { 0, -1, tld_main_stats },   // os
-  { MI_STATS_NULL }                   // stats
+  { MI_STATS_NULL }            // stats
 };
 
 mi_heap_t _mi_heap_main = {
@@ -242,6 +242,7 @@ static bool _mi_heap_init(void) {
     heap->tld = tld;    
     tld->heap_backing = heap;
     tld->segments.stats = &tld->stats;
+    tld->os.numa_node = -1;
     tld->os.stats = &tld->stats;
     _mi_heap_default = heap;
   }

src/os.c

@@ -144,7 +144,7 @@ void _mi_os_init(void) {
     FreeLibrary(hDll);
   }
   hDll = LoadLibrary(TEXT("ntdll.dll"));
-  if (hDll != NULL) {    
+  if (hDll != NULL) {
     pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx");
     FreeLibrary(hDll);
   }
@@ -206,7 +206,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size);
 #define MEM_COMMIT_RESERVE  (MEM_COMMIT|MEM_RESERVE)
 
 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) {
-#if (MI_INTPTR_SIZE >= 8) 
+#if (MI_INTPTR_SIZE >= 8)
   // on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations
   void* hint;
   if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) {
@@ -784,14 +784,14 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) {
 
 
 /* ----------------------------------------------------------------------------
-Support for allocating huge OS pages (1Gib) that are reserved up-front 
+Support for allocating huge OS pages (1Gib) that are reserved up-front
 and possibly associated with a specific NUMA node. (use `numa_node>=0`)
 -----------------------------------------------------------------------------*/
-#define MI_HUGE_OS_PAGE_SIZE  (GiB)  
+#define MI_HUGE_OS_PAGE_SIZE  (GiB)
 
 #if defined(WIN32) && (MI_INTPTR_SIZE >= 8)
-static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node) 
+static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node)
-{  
+{
   mi_assert_internal(size%GiB == 0);
 
   #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
@@ -801,8 +801,8 @@ static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node)
   reqs.HighestEndingAddress = NULL;
   reqs.LowestStartingAddress = NULL;
   reqs.Alignment = MI_SEGMENT_SIZE;
-  
+
-  // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages  
+  // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
   if (pNtAllocateVirtualMemoryEx != NULL) {
     #ifndef MEM_EXTENDED_PARAMETER_NONPAGED_HUGE
     #define MEM_EXTENDED_PARAMETER_NONPAGED_HUGE  (0x10)
@@ -811,7 +811,7 @@ static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node)
     params[0].Pointer = &reqs;
     params[1].Type = 5; // == MemExtendedParameterAttributeFlags;
     params[1].ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
-    size_t param_count = 2;
+    ULONG param_count = 2;
     if (numa_node >= 0) {
       param_count++;
       params[2].Type = MemExtendedParameterNumaNode;
@@ -824,15 +824,15 @@ static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node)
       return base;
     }
     else {
-      // fall back to regular huge pages    
+      // fall back to regular huge pages
       _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (error 0x%lx)\n", err);
     }
-  }  
+  }
   // on modern Windows try use VirtualAlloc2 for aligned large OS page allocation
   if (pVirtualAlloc2 != NULL) {
     params[0].Type = MemExtendedParameterAddressRequirements;
     params[0].Pointer = &reqs;
-    size_t param_count = 1;
+    ULONG param_count = 1;
     if (numa_node >= 0) {
       param_count++;
       params[1].Type = MemExtendedParameterNumaNode;
@@ -841,7 +841,7 @@ static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node)
     return (*pVirtualAlloc2)(GetCurrentProcess(), NULL, size, flags, PAGE_READWRITE, params, param_count);
   }
   #endif
-  return NULL; // give up on older Windows.. 
+  return NULL; // give up on older Windows..
 }
 #elif defined(MI_OS_USE_MMAP) && (MI_INTPTR_SIZE >= 8)
 #ifdef MI_HAS_NUMA
@@ -852,7 +852,7 @@ static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node) {
   bool is_large = true;
   void* p = mi_unix_mmap(NULL, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large);
   if (p == NULL) return NULL;
-  #ifdef MI_HAS_NUMA  
+  #ifdef MI_HAS_NUMA
   if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) {
     uintptr_t numa_mask = (1UL << numa_node);
     long err = mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0);
@@ -865,7 +865,7 @@ static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node) {
   #endif
   return p;
 }
-#else 
+#else
 static void* mi_os_alloc_huge_os_pagesx(size_t size, int numa_node) {
   return NULL;
 }
@@ -883,12 +883,12 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, size_t* psize) {
 }
 
 #ifdef WIN32
-static int mi_os_numa_nodex(void) {
+static int mi_os_numa_nodex() {
   PROCESSOR_NUMBER pnum;
   USHORT numa_node = 0;
   GetCurrentProcessorNumberEx(&pnum);
   GetNumaProcessorNodeEx(&pnum,&numa_node);
-  return (int)numa_node; 
+  return (int)numa_node;
 }
 
 static int mi_os_numa_node_countx(void) {
@@ -897,12 +897,42 @@ static int mi_os_numa_node_countx(void) {
   return (int)(numa_max + 1);
 }
 #elif MI_HAS_NUMA
-#include <numa.h>
+#include <dirent.h>
+#include <stdlib.h>
+#include <numaif.h>
 static int mi_os_numa_nodex(void) {
-  return numa_preferred();
+  #define MI_MAX_MASK (4)          // support at most 256 nodes
+  unsigned long mask[MI_MAX_MASK];
+  memset(mask,0,MI_MAX_MASK*sizeof(long));
+  int mode = 0;
+  long err = get_mempolicy(&mode, mask, MI_MAX_MASK*sizeof(long)*8, NULL, 0 /* thread policy */);
+  if (err != 0) return 0;
+  // find the lowest bit that is set
+  for(int i = 0; i < MI_MAX_MASK; i++) {
+    for(int j = 0; j < (int)(sizeof(long)*8); j++) {
+      if ((mask[i] & (1UL << j)) != 0) {
+        return (i*sizeof(long)*8 + j);
+      }
+    }
+  }
+	return 0;
 }
+
 static int mi_os_numa_node_countx(void) {
-  return (numa_max_node() + 1);
+  DIR* d = opendir("/sys/devices/system/node");
+  if (d==NULL) return 1;
+  
+  struct dirent* de;
+  int max_node_num = 0;
+  while ((de = readdir(d)) != NULL) {
+  	int node_num;
+  	if (strncmp(de->d_name, "node", 4) == 0) {
+		  node_num = (int)strtol(de->d_name+4, NULL, 0);
+			if (max_node_num < node_num) max_node_num = node_num;
+    }
+  }
+  closedir(d);
+  return (max_node_num + 1);
 }
 #else
 static int mi_os_numa_nodex(void) {
@@ -914,19 +944,29 @@ static int mi_os_numa_node_countx(void) {
 #endif
 
 int _mi_os_numa_node_count(void) {
-  long ncount = mi_os_numa_node_countx();
+  static int numa_node_count = 0;
-  // never more than max numa node and at least 1
+  if (mi_unlikely(numa_node_count <= 0)) {
-  long nmax  = 1 + mi_option_get(mi_option_max_numa_node);
+    int ncount = mi_os_numa_node_countx();
-  if (ncount > nmax) ncount = nmax;
+    // never more than max numa node and at least 1
-  if (ncount <= 0) ncount = 1;
+    int nmax = 1 + (int)mi_option_get(mi_option_max_numa_node);
-  return ncount;
+    if (ncount > nmax) ncount = nmax;
+    if (ncount <= 0)   ncount = 1;
+    numa_node_count = ncount;
+  }
+  mi_assert_internal(numa_node_count >= 1);
+  return numa_node_count;
 }
 
-int _mi_os_numa_node(void) {
+int _mi_os_numa_node(mi_os_tld_t* tld) {
-  int nnode = mi_os_numa_nodex();
+  if (mi_unlikely(tld->numa_node < 0)) {
-  // never more than the node count
+    int nnode = mi_os_numa_nodex();
-  int ncount = _mi_os_numa_node_count();
+    // never more than the node count
-  if (nnode >= ncount) { nnode = nnode % ncount; }  
+    int ncount = _mi_os_numa_node_count();
-  return nnode;
+    if (nnode >= ncount) { nnode = nnode % ncount; }
+    if (nnode < 0) nnode = 0;
+    tld->numa_node = nnode;
+  }
+  mi_assert_internal(tld->numa_node >= 0 && tld->numa_node < _mi_os_numa_node_count());
+  return tld->numa_node;
 }