mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-07-06 19:38:41 +03:00
wip: initial work on mimalloc3 without segments
This commit is contained in:
daanx
parent
9b7537755a
commit
71cfa45e76
15 changed files with 3001 additions and 289 deletions
313
include/mimalloc/bits.h
Normal file
313
include/mimalloc/bits.h
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Copyright (c) 2019-2024 Microsoft Research, Daan Leijen
|
||||
This is free software; you can redistribute it and/or modify it under the
|
||||
terms of the MIT license. A copy of the license can be found in the file
|
||||
"LICENSE" at the root of this distribution.
|
||||
-----------------------------------------------------------------------------*/
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
Bit operation, and platform dependent definition (MI_INTPTR_SIZE etc)
|
||||
---------------------------------------------------------------------------- */
|
||||
|
||||
#pragma once
|
||||
#ifndef MI_BITS_H
|
||||
#define MI_BITS_H
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Size of a pointer.
|
||||
// We assume that `sizeof(void*)==sizeof(intptr_t)`
|
||||
// and it holds for all platforms we know of.
|
||||
//
|
||||
// However, the C standard only requires that:
|
||||
// p == (void*)((intptr_t)p))
|
||||
// but we also need:
|
||||
// i == (intptr_t)((void*)i)
|
||||
// or otherwise one might define an intptr_t type that is larger than a pointer...
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if INTPTR_MAX > INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)
|
||||
#elif INTPTR_MAX == INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (3)
|
||||
#elif INTPTR_MAX == INT32_MAX
|
||||
# define MI_INTPTR_SHIFT (2)
|
||||
#else
|
||||
#error platform pointers must be 32, 64, or 128 bits
|
||||
#endif
|
||||
|
||||
#if SIZE_MAX == UINT64_MAX
|
||||
# define MI_SIZE_SHIFT (3)
|
||||
typedef int64_t mi_ssize_t;
|
||||
#elif SIZE_MAX == UINT32_MAX
|
||||
# define MI_SIZE_SHIFT (2)
|
||||
typedef int32_t mi_ssize_t;
|
||||
#else
|
||||
#error platform objects must be 32 or 64 bits
|
||||
#endif
|
||||
|
||||
#if (SIZE_MAX/2) > LONG_MAX
|
||||
# define MI_ZU(x) x##ULL
|
||||
# define MI_ZI(x) x##LL
|
||||
#else
|
||||
# define MI_ZU(x) x##UL
|
||||
# define MI_ZI(x) x##L
|
||||
#endif
|
||||
|
||||
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
|
||||
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
|
||||
|
||||
#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT)
|
||||
#define MI_SIZE_BITS (MI_SIZE_SIZE*8)
|
||||
|
||||
#define MI_KiB (MI_ZU(1024))
|
||||
#define MI_MiB (MI_KiB*MI_KiB)
|
||||
#define MI_GiB (MI_MiB*MI_KiB)
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Architecture
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64)
|
||||
#define MI_ARCH_X64 1
|
||||
#elif defined(__i386__) || defined(__i386) || defined(_M_IX86) || defined(_X86_) || defined(__X86__)
|
||||
#define MI_ARCH_X86 1
|
||||
#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_HYBRID_X86_ARM64) || defined(_M_ARM64EC)
|
||||
#define MI_ARCH_ARM64 1
|
||||
#elif defined(__arm__) || defined(_ARM) || defined(_M_ARM) || defined(_M_ARMT) || defined(__arm)
|
||||
#define MI_ARCH_ARM32 1
|
||||
#elif defined(__riscv) || defined(_M_RISCV)
|
||||
#define MI_ARCH_RISCV 1
|
||||
#if (LONG_MAX == INT32_MAX)
|
||||
#define MI_ARCH_RISCV32 1
|
||||
#else
|
||||
#define MI_ARCH_RISCV64 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if MI_ARCH_X64 && defined(__AVX2__)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
#include <intrin.h>
|
||||
#endif
|
||||
|
||||
#if defined(__AVX2__) && !defined(__BMI2__) // msvc
|
||||
#define __BMI2__ 1
|
||||
#endif
|
||||
#if (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
|
||||
#define __BMI1__ 1
|
||||
#endif
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Builtin's
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
#define mi_builtin(name) __builtin_##name
|
||||
#define mi_has_builtin(name) __has_builtin(__builtin_##name)
|
||||
|
||||
#if (LONG_MAX == INT32_MAX)
|
||||
#define mi_builtin32(name) mi_builtin(name##l)
|
||||
#define mi_has_builtin32(name) mi_has_builtin(name##l)
|
||||
#else
|
||||
#define mi_builtin32(name) mi_builtin(name)
|
||||
#define mi_has_builtin32(name) mi_has_builtin(name)
|
||||
#endif
|
||||
#if (LONG_MAX == INT64_MAX)
|
||||
#define mi_builtin64(name) mi_builtin(name##l)
|
||||
#define mi_has_builtin64(name) mi_has_builtin(name##l)
|
||||
#else
|
||||
#define mi_builtin64(name) mi_builtin(name##ll)
|
||||
#define mi_has_builtin64(name) mi_has_builtin(name##ll)
|
||||
#endif
|
||||
|
||||
#if (MI_SIZE_BITS == 32)
|
||||
#define mi_builtin_size(name) mi_builtin32(name)
|
||||
#define mi_has_builtin_size(name) mi_has_builtin32(name)
|
||||
#elif (MI_SIZE_BITS == 64)
|
||||
#define mi_builtin_size(name) mi_builtin64(name)
|
||||
#define mi_has_builtin_size(name) mi_has_builtin64(name)
|
||||
#endif
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
Count trailing/leading zero's
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
size_t _mi_clz_generic(size_t x);
|
||||
size_t _mi_ctz_generic(size_t x);
|
||||
|
||||
static inline size_t mi_ctz(size_t x) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__)
|
||||
uint64_t r;
|
||||
__asm volatile ("tzcnt\t%1, %0" : "=&r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(__GNUC__) && MI_ARCH_ARM64
|
||||
uint64_t r;
|
||||
__asm volatile ("rbit\t%0, %1\n\tclz\t%0, %0" : "=&r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(__GNUC__) && MI_ARCH_RISCV
|
||||
size_t r;
|
||||
__asm volatile ("ctz\t%0, %1" : "=&r"(r) : "r"(x) : );
|
||||
return r;
|
||||
#elif MI_ARCH_X64 && defined(__BMI1__)
|
||||
return (size_t)_tzcnt_u64(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long idx;
|
||||
#if MI_SIZE_BITS==32
|
||||
return (_BitScanForward(&idx, x) ? (size_t)idx : 32);
|
||||
#else
|
||||
return (_BitScanForward64(&idx, x) ? (size_t)idx : 64);
|
||||
#endif
|
||||
#elif mi_has_builtin_size(ctz)
|
||||
return (x!=0 ? (size_t)mi_builtin_size(ctz)(x) : MI_SIZE_BITS);
|
||||
#else
|
||||
#define MI_HAS_FAST_BITSCAN 0
|
||||
return _mi_ctz_generic(x);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t mi_clz(size_t x) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__)
|
||||
uint64_t r;
|
||||
__asm volatile ("lzcnt\t%1, %0" : "=&r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(__GNUC__) && MI_ARCH_ARM64
|
||||
uint64_t r;
|
||||
__asm volatile ("clz\t%0, %1" : "=&r"(r) : "r"(x) : "cc");
|
||||
return r;
|
||||
#elif defined(__GNUC__) && MI_ARCH_RISCV
|
||||
size_t r;
|
||||
__asm volatile ("clz\t%0, %1" : "=&r"(r) : "r"(x) : );
|
||||
return r;
|
||||
#elif MI_ARCH_X64 && defined(__BMI1__)
|
||||
return (size_t)_lzcnt_u64(x);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long idx;
|
||||
#if MI_SIZE_BITS==32
|
||||
return (_BitScanReverse(&idx, x) ? 31 - (size_t)idx : 32);
|
||||
#else
|
||||
return (_BitScanReverse64(&idx, x) ? 63 - (size_t)idx : 64);
|
||||
#endif
|
||||
#elif mi_has_builtin_size(clz)
|
||||
return (x!=0 ? (size_t)mi_builtin_size(clz)(x) : MI_SIZE_BITS);
|
||||
#else
|
||||
#define MI_HAS_FAST_BITSCAN 0
|
||||
return _mi_clz_generic(x);
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifndef MI_HAS_FAST_BITSCAN
|
||||
#define MI_HAS_FAST_BITSCAN 1
|
||||
#endif
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
find trailing/leading zero (bit scan forward/reverse)
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
// Bit scan forward: find the least significant bit that is set (i.e. count trailing zero's)
|
||||
// return false if `x==0` (with `*idx` undefined) and true otherwise,
|
||||
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
|
||||
static inline bool mi_bsf(size_t x, size_t* idx) {
|
||||
#if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__)
|
||||
// on x64 the carry flag is set on zero which gives better codegen
|
||||
bool is_zero;
|
||||
__asm ( "tzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc" );
|
||||
return !is_zero;
|
||||
#else
|
||||
*idx = mi_ctz(x);
|
||||
return (x!=0);
|
||||
#endif
|
||||
}
|
||||
|
||||
// Bit scan reverse: find the most significant bit that is set
|
||||
// return false if `x==0` (with `*idx` undefined) and true otherwise,
|
||||
// with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
|
||||
static inline bool mi_bsr(size_t x, size_t* idx) {
|
||||
#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long i;
|
||||
#if MI_SIZE_BITS==32
|
||||
return (_BitScanReverse(&i, x) ? (*idx = i, true) : false);
|
||||
#else
|
||||
return (_BitScanReverse64(&i, x) ? (*idx = i, true) : false);
|
||||
#endif
|
||||
#else
|
||||
const size_t r = mi_clz(x);
|
||||
*idx = (~r & (MI_SIZE_BITS - 1));
|
||||
return (x!=0);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
find least/most significant bit position
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
// Find most significant bit index, or MI_SIZE_BITS if 0
|
||||
static inline size_t mi_find_msb(size_t x) {
|
||||
#if defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
unsigned long i;
|
||||
#if MI_SIZE_BITS==32
|
||||
return (_BitScanReverse(&i, x) ? i : 32);
|
||||
#else
|
||||
return (_BitScanReverse64(&i, x) ? i : 64);
|
||||
#endif
|
||||
#else
|
||||
return (x==0 ? MI_SIZE_BITS : MI_SIZE_BITS - 1 - mi_clz(x));
|
||||
#endif
|
||||
}
|
||||
|
||||
// Find least significant bit index, or MI_SIZE_BITS if 0 (this equals `mi_ctz`, count trailing zero's)
|
||||
static inline size_t mi_find_lsb(size_t x) {
|
||||
return mi_ctz(x);
|
||||
}
|
||||
|
||||
|
||||
/* --------------------------------------------------------------------------------
|
||||
rotate
|
||||
-------------------------------------------------------------------------------- */
|
||||
|
||||
static inline size_t mi_rotr(size_t x, size_t r) {
|
||||
#if (mi_has_builtin(rotateright64) && MI_SIZE_BITS==64)
|
||||
return mi_builtin(rotateright64)(x,r);
|
||||
#elif (mi_has_builtin(rotateright32) && MI_SIZE_BITS==32)
|
||||
return mi_builtin(rotateright32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
#if MI_BFIELD_SIZE==4
|
||||
return _lrotr(x,(int)r);
|
||||
#else
|
||||
return _rotr64(x,(int)r);
|
||||
#endif
|
||||
#else
|
||||
// The term `(-rshift)&(MI_BFIELD_BITS-1)` is written instead of `MI_BFIELD_BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
|
||||
return (x >> rshift) | (x << ((-rshift) & (MI_SIZE_BITS-1)));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t mi_rotl(size_t x, size_t r) {
|
||||
#if (mi_has_builtin(rotateleft64) && MI_SIZE_BITS==64)
|
||||
return mi_builtin(rotateleft64)(x,r);
|
||||
#elif (mi_has_builtin(rotateleft32) && MI_SIZE_BITS==32)
|
||||
return mi_builtin(rotateleft32)(x,r);
|
||||
#elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
|
||||
#if MI_SIZE_BITS==32
|
||||
return _lrotl(x,(int)r);
|
||||
#else
|
||||
return _rotl64(x,(int)r);
|
||||
#endif
|
||||
#else
|
||||
// The term `(-rshift)&(MI_BFIELD_BITS-1)` is written instead of `MI_BFIELD_BITS - rshift` to
|
||||
// avoid UB when `rshift==0`. See <https://blog.regehr.org/archives/1063>
|
||||
const unsigned int rshift = (unsigned int)(r) & (MI_SIZE_BITS-1);
|
||||
return (x << rshift) | (x >> ((-rshift) & (MI_SIZE_BITS-1)))
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif // MI_BITS_H
|
||||
|
|
@ -16,6 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
|
||||
#include "types.h"
|
||||
#include "track.h"
|
||||
#include "bits.h"
|
||||
|
||||
#if (MI_DEBUG>0)
|
||||
#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)
|
||||
|
|
@ -23,26 +24,28 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_trace_message(...)
|
||||
#endif
|
||||
|
||||
#define MI_CACHE_LINE 64
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
|
||||
#pragma warning(disable:26812) // unscoped enum warning
|
||||
#define mi_decl_noinline __declspec(noinline)
|
||||
#define mi_decl_thread __declspec(thread)
|
||||
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
|
||||
#define mi_decl_align(a) __declspec(align(a))
|
||||
#define mi_decl_weak
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
|
||||
#define mi_decl_noinline __attribute__((noinline))
|
||||
#define mi_decl_thread __thread
|
||||
#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
|
||||
#define mi_decl_align(a) __attribute__((aligned(a)))
|
||||
#define mi_decl_weak __attribute__((weak))
|
||||
#else
|
||||
#define mi_decl_noinline
|
||||
#define mi_decl_thread __thread // hope for the best :-)
|
||||
#define mi_decl_cache_align
|
||||
#define mi_decl_align(a)
|
||||
#define mi_decl_weak
|
||||
#endif
|
||||
|
||||
#define mi_decl_cache_align mi_decl_align(64)
|
||||
|
||||
|
||||
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
|
||||
#define __wasi__
|
||||
#endif
|
||||
|
|
@ -89,6 +92,7 @@ void _mi_thread_done(mi_heap_t* heap);
|
|||
void _mi_thread_data_collect(void);
|
||||
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
|
||||
mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
|
||||
size_t _mi_thread_seq_id(void) mi_attr_noexcept;
|
||||
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
|
||||
mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
|
||||
void _mi_heap_guarded_init(mi_heap_t* heap);
|
||||
|
|
@ -96,6 +100,7 @@ void _mi_heap_guarded_init(mi_heap_t* heap);
|
|||
// os.c
|
||||
void _mi_os_init(void); // called from process init
|
||||
void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats);
|
||||
void* _mi_os_zalloc(size_t size, mi_memid_t* memid, mi_stats_t* stats);
|
||||
void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats);
|
||||
void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats);
|
||||
|
||||
|
|
@ -675,15 +680,6 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) {
|
|||
return (idxp == idxq);
|
||||
}
|
||||
|
||||
static inline uintptr_t mi_rotl(uintptr_t x, uintptr_t shift) {
|
||||
shift %= MI_INTPTR_BITS;
|
||||
return (shift==0 ? x : ((x << shift) | (x >> (MI_INTPTR_BITS - shift))));
|
||||
}
|
||||
static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) {
|
||||
shift %= MI_INTPTR_BITS;
|
||||
return (shift==0 ? x : ((x >> shift) | (x << (MI_INTPTR_BITS - shift))));
|
||||
}
|
||||
|
||||
static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
|
||||
void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);
|
||||
return (p==null ? NULL : p);
|
||||
|
|
@ -821,112 +817,6 @@ static inline size_t _mi_os_numa_node_count(void) {
|
|||
}
|
||||
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#if defined(__GNUC__)
|
||||
|
||||
#include <limits.h> // LONG_MAX
|
||||
#define MI_HAVE_FAST_BITSCAN
|
||||
static inline size_t mi_clz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
#if (INTPTR_MAX == LONG_MAX)
|
||||
return __builtin_clzl(x);
|
||||
#else
|
||||
return __builtin_clzll(x);
|
||||
#endif
|
||||
}
|
||||
static inline size_t mi_ctz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
#if (INTPTR_MAX == LONG_MAX)
|
||||
return __builtin_ctzl(x);
|
||||
#else
|
||||
return __builtin_ctzll(x);
|
||||
#endif
|
||||
}
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
#include <limits.h> // LONG_MAX
|
||||
#include <intrin.h> // BitScanReverse64
|
||||
#define MI_HAVE_FAST_BITSCAN
|
||||
static inline size_t mi_clz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
unsigned long idx;
|
||||
#if (INTPTR_MAX == LONG_MAX)
|
||||
_BitScanReverse(&idx, x);
|
||||
#else
|
||||
_BitScanReverse64(&idx, x);
|
||||
#endif
|
||||
return ((MI_INTPTR_BITS - 1) - idx);
|
||||
}
|
||||
static inline size_t mi_ctz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
unsigned long idx;
|
||||
#if (INTPTR_MAX == LONG_MAX)
|
||||
_BitScanForward(&idx, x);
|
||||
#else
|
||||
_BitScanForward64(&idx, x);
|
||||
#endif
|
||||
return idx;
|
||||
}
|
||||
|
||||
#else
|
||||
static inline size_t mi_ctz32(uint32_t x) {
|
||||
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
|
||||
static const unsigned char debruijn[32] = {
|
||||
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
|
||||
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
|
||||
};
|
||||
if (x==0) return 32;
|
||||
return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27];
|
||||
}
|
||||
static inline size_t mi_clz32(uint32_t x) {
|
||||
// de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf>
|
||||
static const uint8_t debruijn[32] = {
|
||||
31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1,
|
||||
23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0
|
||||
};
|
||||
if (x==0) return 32;
|
||||
x |= x >> 1;
|
||||
x |= x >> 2;
|
||||
x |= x >> 4;
|
||||
x |= x >> 8;
|
||||
x |= x >> 16;
|
||||
return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27];
|
||||
}
|
||||
|
||||
static inline size_t mi_clz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
#if (MI_INTPTR_BITS <= 32)
|
||||
return mi_clz32((uint32_t)x);
|
||||
#else
|
||||
size_t count = mi_clz32((uint32_t)(x >> 32));
|
||||
if (count < 32) return count;
|
||||
return (32 + mi_clz32((uint32_t)x));
|
||||
#endif
|
||||
}
|
||||
static inline size_t mi_ctz(uintptr_t x) {
|
||||
if (x==0) return MI_INTPTR_BITS;
|
||||
#if (MI_INTPTR_BITS <= 32)
|
||||
return mi_ctz32((uint32_t)x);
|
||||
#else
|
||||
size_t count = mi_ctz32((uint32_t)x);
|
||||
if (count < 32) return count;
|
||||
return (32 + mi_ctz32((uint32_t)(x>>32)));
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero)
|
||||
static inline size_t mi_bsr(uintptr_t x) {
|
||||
return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x));
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------------
|
||||
// Provide our own `_mi_memcpy` for potential performance optimizations.
|
||||
//
|
||||
|
|
@ -947,20 +837,20 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
|
|||
memcpy(dst, src, n);
|
||||
}
|
||||
}
|
||||
static inline void _mi_memzero(void* dst, size_t n) {
|
||||
static inline void _mi_memset(void* dst, int val, size_t n) {
|
||||
if ((_mi_cpu_has_fsrm && n <= 128) || (_mi_cpu_has_erms && n > 128)) {
|
||||
__stosb((unsigned char*)dst, 0, n);
|
||||
__stosb((unsigned char*)dst, (uint8_t)val, n);
|
||||
}
|
||||
else {
|
||||
memset(dst, 0, n);
|
||||
memset(dst, val, n);
|
||||
}
|
||||
}
|
||||
#else
|
||||
static inline void _mi_memcpy(void* dst, const void* src, size_t n) {
|
||||
memcpy(dst, src, n);
|
||||
}
|
||||
static inline void _mi_memzero(void* dst, size_t n) {
|
||||
memset(dst, 0, n);
|
||||
static inline void _mi_memset(void* dst, int val, size_t n) {
|
||||
memset(dst, val, n);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -978,10 +868,10 @@ static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
|
|||
_mi_memcpy(adst, asrc, n);
|
||||
}
|
||||
|
||||
static inline void _mi_memzero_aligned(void* dst, size_t n) {
|
||||
static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
|
||||
mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
|
||||
void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE);
|
||||
_mi_memzero(adst, n);
|
||||
_mi_memset(adst, val, n);
|
||||
}
|
||||
#else
|
||||
// Default fallback on `_mi_memcpy`
|
||||
|
|
@ -990,11 +880,19 @@ static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) {
|
|||
_mi_memcpy(dst, src, n);
|
||||
}
|
||||
|
||||
static inline void _mi_memzero_aligned(void* dst, size_t n) {
|
||||
static inline void _mi_memset_aligned(void* dst, int val, size_t n) {
|
||||
mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0);
|
||||
_mi_memzero(dst, n);
|
||||
_mi_memset(dst, val, n);
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline void _mi_memzero(void* dst, size_t n) {
|
||||
_mi_memset(dst, 0, n);
|
||||
}
|
||||
|
||||
static inline void _mi_memzero_aligned(void* dst, size_t n) {
|
||||
_mi_memset_aligned(dst, 0, n);
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -369,7 +369,4 @@ static inline mi_heap_t* mi_prim_get_default_heap(void) {
|
|||
#endif // mi_prim_get_default_heap()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
#endif // MIMALLOC_PRIM_H
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
|
||||
#include <stddef.h> // ptrdiff_t
|
||||
#include <stdint.h> // uintptr_t, uint16_t, etc
|
||||
#include "bits.h" // bit ops, size defines
|
||||
#include "atomic.h" // _Atomic
|
||||
|
||||
#ifdef _MSC_VER
|
||||
|
|
@ -106,61 +107,6 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
// #define MI_HUGE_PAGE_ABANDON 1
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Platform specific values
|
||||
// ------------------------------------------------------
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Size of a pointer.
|
||||
// We assume that `sizeof(void*)==sizeof(intptr_t)`
|
||||
// and it holds for all platforms we know of.
|
||||
//
|
||||
// However, the C standard only requires that:
|
||||
// p == (void*)((intptr_t)p))
|
||||
// but we also need:
|
||||
// i == (intptr_t)((void*)i)
|
||||
// or otherwise one might define an intptr_t type that is larger than a pointer...
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if INTPTR_MAX > INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example)
|
||||
#elif INTPTR_MAX == INT64_MAX
|
||||
# define MI_INTPTR_SHIFT (3)
|
||||
#elif INTPTR_MAX == INT32_MAX
|
||||
# define MI_INTPTR_SHIFT (2)
|
||||
#else
|
||||
#error platform pointers must be 32, 64, or 128 bits
|
||||
#endif
|
||||
|
||||
#if SIZE_MAX == UINT64_MAX
|
||||
# define MI_SIZE_SHIFT (3)
|
||||
typedef int64_t mi_ssize_t;
|
||||
#elif SIZE_MAX == UINT32_MAX
|
||||
# define MI_SIZE_SHIFT (2)
|
||||
typedef int32_t mi_ssize_t;
|
||||
#else
|
||||
#error platform objects must be 32 or 64 bits
|
||||
#endif
|
||||
|
||||
#if (SIZE_MAX/2) > LONG_MAX
|
||||
# define MI_ZU(x) x##ULL
|
||||
# define MI_ZI(x) x##LL
|
||||
#else
|
||||
# define MI_ZU(x) x##UL
|
||||
# define MI_ZI(x) x##L
|
||||
#endif
|
||||
|
||||
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
|
||||
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
|
||||
|
||||
#define MI_SIZE_SIZE (1<<MI_SIZE_SHIFT)
|
||||
#define MI_SIZE_BITS (MI_SIZE_SIZE*8)
|
||||
|
||||
#define MI_KiB (MI_ZU(1024))
|
||||
#define MI_MiB (MI_KiB*MI_KiB)
|
||||
#define MI_GiB (MI_MiB*MI_KiB)
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Main internal data-structures
|
||||
// ------------------------------------------------------
|
||||
|
|
@ -202,6 +148,9 @@ typedef int32_t mi_ssize_t;
|
|||
|
||||
// Maximum number of size classes. (spaced exponentially in 12.5% increments)
|
||||
#define MI_BIN_HUGE (73U)
|
||||
#define MI_BIN_FULL (MI_BIN_HUGE+1)
|
||||
#define MI_BIN_COUNT (MI_BIN_FULL+1)
|
||||
|
||||
|
||||
#if (MI_LARGE_OBJ_WSIZE_MAX >= 655360)
|
||||
#error "mimalloc internal: define more bins"
|
||||
|
|
@ -461,8 +410,6 @@ typedef struct mi_page_queue_s {
|
|||
size_t block_size;
|
||||
} mi_page_queue_t;
|
||||
|
||||
#define MI_BIN_FULL (MI_BIN_HUGE+1)
|
||||
|
||||
// Random context
|
||||
typedef struct mi_random_cxt_s {
|
||||
uint32_t input[16];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue