combine flags and xthread_id

include/mimalloc/internal.h

@@ -622,11 +622,16 @@ static inline mi_thread_free_t mi_tf_create(mi_block_t* block, bool owned) {
 }
 
 
-// Thread id of thread that owns this page
-static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) {
+// Thread id of thread that owns this page (with flags in the bottom 2 bits)
+static inline mi_threadid_t mi_page_xthread_id(const mi_page_t* page) {
   return mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id);
 }
 
+// Plain thread id of the thread that owns this page
+static inline mi_threadid_t mi_page_thread_id(const mi_page_t* page) {
+  return (mi_page_xthread_id(page) & ~MI_PAGE_FLAG_MASK);
+}
+
 // Thread free access
 static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
   return mi_tf_block(mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_free));
@@ -695,19 +700,21 @@ static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) {
 
 static inline bool mi_page_is_abandoned(const mi_page_t* page) {
   // note: the xheap field of an abandoned heap is set to the subproc (for fast reclaim-on-free)
-  return (mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id) <= 1);
+  return (mi_page_xthread_id(page) <= MI_PAGE_IS_ABANDONED_MAPPED);
 }
 
 static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) {
-  return (mi_atomic_load_relaxed(&((mi_page_t*)page)->xthread_id) == 1);
+  return (mi_page_xthread_id(page) == MI_PAGE_IS_ABANDONED_MAPPED);
 }
 
 static inline void mi_page_set_abandoned_mapped(mi_page_t* page) {
-  mi_atomic_or_relaxed(&page->xthread_id, (uintptr_t)1);
+  mi_assert_internal(mi_page_is_abandoned(page));
+  mi_atomic_or_relaxed(&page->xthread_id, MI_PAGE_IS_ABANDONED_MAPPED);
 }
 
 static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {
-  mi_atomic_and_relaxed(&page->xthread_id, ~(uintptr_t)1);
+  mi_assert_internal(mi_page_is_abandoned_mapped(page));
+  mi_atomic_and_relaxed(&page->xthread_id, ~MI_PAGE_IS_ABANDONED_MAPPED);
 }
 
 
@@ -766,15 +773,15 @@ static inline bool _mi_page_unown(mi_page_t* page) {
 // Page flags
 //-----------------------------------------------------------
 static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) {
-  return mi_atomic_load_relaxed(&((mi_page_t*)page)->xflags);
+  return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK);
 }
 
 static inline void mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
   if (set) {
-    mi_atomic_or_relaxed(&page->xflags, newflag);
+    mi_atomic_or_relaxed(&page->xthread_id, newflag);
   }
   else {
-    mi_atomic_and_relaxed(&page->xflags, ~newflag);
+    mi_atomic_and_relaxed(&page->xthread_id, ~newflag);
   }
 }
 

include/mimalloc/prim.h

@@ -270,35 +270,42 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce
 
 
 // defined in `init.c`; do not use these directly
-extern mi_decl_thread mi_heap_t* _mi_heap_default;  // default heap to allocate from
-extern bool _mi_process_is_initialized;             // has mi_process_init been called?
+extern mi_decl_hidden mi_decl_thread mi_heap_t* _mi_heap_default;  // default heap to allocate from
+extern mi_decl_hidden bool _mi_process_is_initialized;             // has mi_process_init been called?
 
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept;
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept;
+
+static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+  const mi_threadid_t tid = __mi_prim_thread_id();
+  mi_assert_internal(tid > 1);
+  mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0);  // bottom 2 bits are clear?
+  return tid;
+}
 
 // Get a unique id for the current thread.
 #if defined(MI_PRIM_THREAD_ID)
 
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
   return MI_PRIM_THREAD_ID();  // used for example by CPython for a free threaded build (see python/cpython#115488)
 }
 
 #elif defined(_WIN32)
 
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
   // Windows: works on Intel and ARM in both 32- and 64-bit
   return (uintptr_t)NtCurrentTeb();
 }
 
 #elif MI_USE_BUILTIN_THREAD_POINTER
 
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
   // Works on most Unix based platforms with recent compilers
   return (uintptr_t)__builtin_thread_pointer();
 }
 
 #elif MI_HAS_TLS_SLOT
 
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
   #if defined(__BIONIC__)
     // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id
     // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86
@@ -314,7 +321,7 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
 #else
 
 // otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms).
-static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
   return (uintptr_t)&_mi_heap_default;
 }
 

include/mimalloc/types.h

@@ -241,14 +241,16 @@ typedef struct mi_block_s {
 } mi_block_t;
 
 
-// The `in_full` and `has_aligned` page flags are put in the same field
-// to efficiently test if both are false (`full_aligned == 0`) in the `mi_free` routine.
+// The `in_full` and `has_aligned` page flags are put in the bottom bits of the thread_id (for fast test in `mi_free`)
 // `has_aligned` is true if the page has pointers at an offset in a block (so we unalign before free-ing)
 // `in_full_queue` is true if the page is full and resides in the full queue (so we move it to a regular queue on free-ing)
-#define MI_PAGE_IN_FULL_QUEUE  MI_ZU(0x01)
-#define MI_PAGE_HAS_ALIGNED    MI_ZU(0x02)
+#define MI_PAGE_IN_FULL_QUEUE         MI_ZU(0x01)
+#define MI_PAGE_HAS_ALIGNED           MI_ZU(0x02)
+#define MI_PAGE_IS_ABANDONED_MAPPED   MI_ZU(0x04)
+#define MI_PAGE_FLAG_MASK             MI_ZU(0x07)
 typedef size_t mi_page_flags_t;
 
+
 // Thread free list.
 // Points to a list of blocks that are freed by other threads.
 // The low-bit is set if the page is owned by the current thread. (`mi_page_is_owned`).
@@ -296,7 +298,6 @@ typedef struct mi_page_s {
 
   mi_block_t*               local_free;        // list of deferred free blocks by this thread (migrates to `free`)
   _Atomic(mi_thread_free_t) xthread_free;      // list of deferred free blocks freed by other threads
-  _Atomic(mi_page_flags_t)  xflags;            // `in_full_queue` and `has_aligned` flags
 
   size_t                    block_size;        // size available in each block (always `>0`)
   uint8_t*                  page_start;        // start of the blocks