kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'dev' into dev-slice

Browse source
This commit is contained in:
daan 2022-01-01 16:24:41 -08:00
commit 3eac4a912c
2 changed files with 69 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

122
include/mimalloc-internal.h
View file

@ -786,8 +786,10 @@ static inline size_t _mi_os_numa_node_count(void) {
// ------------------------------------------------------------------- // -------------------------------------------------------------------
// Getting the thread id should be performant as it is called in the // Getting the thread id should be performant as it is called in the
// fast path of `_mi_free` and we specialize for various platforms. // fast path of `_mi_free` and we specialize for various platforms.
// We only require _mi_threadid() to return a unique id for each thread.
// ------------------------------------------------------------------- // -------------------------------------------------------------------
#if defined(_WIN32) #if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN
#include <windows.h> #include <windows.h>
static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
@ -795,81 +797,93 @@ static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
return (uintptr_t)NtCurrentTeb(); return (uintptr_t)NtCurrentTeb();
} }
#elif defined(__GNUC__) && \ // We use assembly for a fast thread id on the main platforms. The TLS layout depends on
(defined(__x86_64__) || defined(__i386__) || defined(__aarch64__)) // both the OS and libc implementation so we use specific tests for each main platform.
// If you test on another platform and it works please send a PR :-)
// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. // see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register.
#elif defined(__GNUC__) && ( \
(defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
|| (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \
|| (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \
|| (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \
)
static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept {
void* res; void* res;
const size_t ofs = (slot*sizeof(void*)); const size_t ofs = (slot*sizeof(void*));
#if defined(__i386__) #if defined(__i386__)
__asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS
#elif defined(__APPLE__) && defined(__x86_64__) #elif defined(__APPLE__) && defined(__x86_64__)
__asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
__asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI
#elif defined(__x86_64__) #elif defined(__x86_64__)
__asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS
#elif defined(__arm__) // arm32: defined but currently not used (see issue #495) #elif defined(__arm__)
void** tcb; MI_UNUSED(ofs); void** tcb; MI_UNUSED(ofs);
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
res = tcb[slot]; res = tcb[slot];
#elif defined(__aarch64__) #elif defined(__aarch64__)
void** tcb; MI_UNUSED(ofs); void** tcb; MI_UNUSED(ofs);
#if defined(__APPLE__) // M1, issue #343 #if defined(__APPLE__) // M1, issue #343
__asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits #else
#else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif
res = tcb[slot];
#endif #endif
res = tcb[slot];
#endif
return res; return res;
} }
// setting a tls slot is only used on macOSX for now // setting a tls slot is only used on macOS for now
static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept {
const size_t ofs = (slot*sizeof(void*)); const size_t ofs = (slot*sizeof(void*));
#if defined(__i386__) #if defined(__i386__)
__asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS
#elif defined(__APPLE__) && defined(__x86_64__) #elif defined(__APPLE__) && defined(__x86_64__)
__asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS
#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4)
__asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI
#elif defined(__x86_64__) #elif defined(__x86_64__)
__asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS
#elif defined(__arm__) #elif defined(__arm__)
void** tcb; MI_UNUSED(ofs); void** tcb; MI_UNUSED(ofs);
__asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb));
tcb[slot] = value; tcb[slot] = value;
#elif defined(__aarch64__) #elif defined(__aarch64__)
void** tcb; MI_UNUSED(ofs); void** tcb; MI_UNUSED(ofs);
#if defined(__APPLE__) // M1, issue #343 #if defined(__APPLE__) // M1, issue #343
__asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb));
tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits #else
#else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb));
__asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif
tcb[slot] = value;
#endif #endif
tcb[slot] = value;
#endif
} }
static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
#if defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__)) #if defined(__BIONIC__)
// issue #384, #495: on arm Android, slot 1 is the thread ID (pointer to pthread internal struct) // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
return (uintptr_t)mi_tls_slot(1); // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
#else return (uintptr_t)mi_tls_slot(1);
// in all our other targets, slot 0 is the pointer to the thread control block #else
return (uintptr_t)mi_tls_slot(0); // in all our other targets, slot 0 is the thread id
#endif // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h
// apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36
return (uintptr_t)mi_tls_slot(0);
#endif
} }
#else #else
// otherwise use portable C
// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept {
return (uintptr_t)&_mi_heap_default; return (uintptr_t)&_mi_heap_default;
} }
#endif #endif
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) // Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero)
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------

2
include/mimalloc.h
View file

@ -254,7 +254,7 @@ typedef struct mi_heap_area_s {
void* blocks; // start of the area containing heap blocks void* blocks; // start of the area containing heap blocks
size_t reserved; // bytes reserved for this area (virtual) size_t reserved; // bytes reserved for this area (virtual)
size_t committed; // current available bytes for this area size_t committed; // current available bytes for this area
size_t used; // bytes in use by allocated blocks size_t used; // number of allocated blocks
size_t block_size; // size in bytes of each block size_t block_size; // size in bytes of each block
} mi_heap_area_t; } mi_heap_area_t;