kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-03 05:59:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge from dev

Browse source
This commit is contained in:
Daan 2025-02-11 16:30:03 -08:00
commit b7779c7770
6 changed files with 88 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
include/mimalloc/internal.h
View file

@ -215,8 +215,8 @@ void _mi_deferred_free(mi_heap_t* heap, bool force);
void _mi_page_free_collect(mi_page_t* page,bool force);
void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments
size_t _mi_bin_size(uint8_t bin); // for stats
uint8_t _mi_bin(size_t size); // for stats
size_t _mi_bin_size(size_t bin); // for stats
size_t _mi_bin(size_t size); // for stats
// "heap.c"
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag);
@ -922,21 +922,21 @@ static inline size_t _mi_os_numa_node_count(void) {
#include <limits.h> // LONG_MAX
#define MI_HAVE_FAST_BITSCAN
static inline size_t mi_clz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
#if (INTPTR_MAX == LONG_MAX)
return __builtin_clzl(x);
#else
return __builtin_clzll(x);
#endif
static inline size_t mi_clz(size_t x) {
if (x==0) return MI_SIZE_BITS;
#if (SIZE_MAX == ULONG_MAX)
return __builtin_clzl(x);
#else
return __builtin_clzll(x);
#endif
}
static inline size_t mi_ctz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
#if (INTPTR_MAX == LONG_MAX)
return __builtin_ctzl(x);
#else
return __builtin_ctzll(x);
#endif
static inline size_t mi_ctz(size_t x) {
if (x==0) return MI_SIZE_BITS;
#if (SIZE_MAX == ULONG_MAX)
return __builtin_ctzl(x);
#else
return __builtin_ctzll(x);
#endif
}
#elif defined(_MSC_VER)
@ -944,24 +944,24 @@ static inline size_t mi_ctz(uintptr_t x) {
#include <limits.h> // LONG_MAX
#include <intrin.h> // BitScanReverse64
#define MI_HAVE_FAST_BITSCAN
static inline size_t mi_clz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
static inline size_t mi_clz(size_t x) {
if (x==0) return MI_SIZE_BITS;
unsigned long idx;
#if (INTPTR_MAX == LONG_MAX)
_BitScanReverse(&idx, x);
#else
_BitScanReverse64(&idx, x);
#endif
return ((MI_INTPTR_BITS - 1) - idx);
#if (SIZE_MAX == ULONG_MAX)
_BitScanReverse(&idx, x);
#else
_BitScanReverse64(&idx, x);
#endif
return ((MI_SIZE_BITS - 1) - idx);
}
static inline size_t mi_ctz(uintptr_t x) {
if (x==0) return MI_INTPTR_BITS;
static inline size_t mi_ctz(size_t x) {
if (x==0) return MI_SIZE_BITS;
unsigned long idx;
#if (INTPTR_MAX == LONG_MAX)
_BitScanForward(&idx, x);
#else
_BitScanForward64(&idx, x);
#endif
#if (SIZE_MAX == ULONG_MAX)
_BitScanForward(&idx, x);
#else
_BitScanForward64(&idx, x);
#endif
return idx;
}
@ -1024,9 +1024,9 @@ static inline size_t mi_clz(size_t x) {
#endif
// "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero)
static inline size_t mi_bsr(uintptr_t x) {
return (x==0 ? MI_INTPTR_BITS : MI_INTPTR_BITS - 1 - mi_clz(x));
// "bit scan reverse": Return index of the highest bit (or MI_SIZE_BITS if `x` is zero)
static inline size_t mi_bsr(size_t x) {
return (x==0 ? MI_SIZE_BITS : MI_SIZE_BITS - 1 - mi_clz(x));
}

10
src/bitmap.c
View file

@ -81,7 +81,7 @@ inline bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, cons
// on to the next bit range
#ifdef MI_HAVE_FAST_BITSCAN
mi_assert_internal(mapm != 0);
const size_t shift = (count == 1 ? 1 : (MI_INTPTR_BITS - mi_clz(mapm) - bitidx));
const size_t shift = (count == 1 ? 1 : (MI_SIZE_BITS - mi_clz(mapm) - bitidx));
mi_assert_internal(shift > 0 && shift <= count);
#else
const size_t shift = 1;
@ -164,7 +164,7 @@ static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size
return ((field & mask) == mask);
}
// Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically.
// Try to set `count` bits at `bitmap_idx` from 0 to 1 atomically.
// Returns `true` if successful when all previous `count` bits were 0.
bool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) {
const size_t idx = mi_bitmap_index_field(bitmap_idx);
@ -172,9 +172,9 @@ bool _mi_bitmap_try_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count
const size_t mask = mi_bitmap_mask_(count, bitidx);
mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields);
size_t expected = mi_atomic_load_relaxed(&bitmap[idx]);
do {
do {
if ((expected & mask) != 0) return false;
}
}
while (!mi_atomic_cas_strong_acq_rel(&bitmap[idx], &expected, expected | mask));
mi_assert_internal((expected & mask) == 0);
return true;
@ -212,7 +212,7 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit
if (initial == 0) return false;
if (initial >= count) return _mi_bitmap_try_find_claim_field(bitmap, idx, count, bitmap_idx); // no need to cross fields (this case won't happen for us)
if (_mi_divide_up(count - initial, MI_BITMAP_FIELD_BITS) >= (bitmap_fields - idx)) return false; // not enough entries
// scan ahead
size_t found = initial;
size_t mask = 0; // mask bits for the final field

2
src/heap.c
View file

@ -564,7 +564,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) {
static void mi_get_fast_divisor(size_t divisor, uint64_t* magic, size_t* shift) {
mi_assert_internal(divisor > 0 && divisor <= UINT32_MAX);
*shift = MI_INTPTR_BITS - mi_clz(divisor - 1);
*shift = MI_SIZE_BITS - mi_clz(divisor - 1);
*magic = ((((uint64_t)1 << 32) * (((uint64_t)1 << *shift) - divisor)) / divisor + 1);
}

49
src/page-queue.c
View file

@ -57,27 +57,23 @@ static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) {
// Returns MI_BIN_HUGE if the size is too large.
// We use `wsize` for the size in "machine word sizes",
// i.e. byte size == `wsize*sizeof(void*)`.
static inline uint8_t mi_bin(size_t size) {
size_t wsize = _mi_wsize_from_size(size);
uint8_t bin;
if (wsize <= 1) {
bin = 1;
static inline size_t mi_bin(size_t size) {
size_t wsize = _mi_wsize_from_size(size);
#if defined(MI_ALIGN4W)
if mi_likely(wsize <= 4) {
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
}
#if defined(MI_ALIGN4W)
else if (wsize <= 4) {
bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
#elif defined(MI_ALIGN2W)
if mi_likely(wsize <= 8) {
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
}
#elif defined(MI_ALIGN2W)
else if (wsize <= 8) {
bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
#else
if mi_likely(wsize <= 8) {
return (wsize == 0 ? 1 : wsize);
}
#else
else if (wsize <= 8) {
bin = (uint8_t)wsize;
}
#endif
else if (wsize > MI_MEDIUM_OBJ_WSIZE_MAX) {
bin = MI_BIN_HUGE;
#endif
else if mi_unlikely(wsize > MI_MEDIUM_OBJ_WSIZE_MAX) {
return MI_BIN_HUGE;
}
else {
#if defined(MI_ALIGN4W)
@ -85,15 +81,14 @@ static inline uint8_t mi_bin(size_t size) {
#endif
wsize--;
// find the highest bit
uint8_t b = (uint8_t)mi_bsr(wsize); // note: wsize != 0
const size_t b = (MI_SIZE_BITS - 1 - mi_clz(wsize)); // note: wsize != 0
// and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
// - adjust with 3 because we use do not round the first 8 sizes
// which each get an exact bin
bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3;
mi_assert_internal(bin < MI_BIN_HUGE);
const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
mi_assert_internal(bin > 0 && bin < MI_BIN_HUGE);
return bin;
}
mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE);
return bin;
}
@ -102,11 +97,11 @@ static inline uint8_t mi_bin(size_t size) {
Queue of pages with free blocks
----------------------------------------------------------- */
uint8_t _mi_bin(size_t size) {
size_t _mi_bin(size_t size) {
return mi_bin(size);
}
size_t _mi_bin_size(uint8_t bin) {
size_t _mi_bin_size(size_t bin) {
return _mi_heap_empty.pages[bin].block_size;
}
@ -147,7 +142,7 @@ static inline bool mi_page_is_large_or_huge(const mi_page_t* page) {
static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) {
mi_assert_internal(heap!=NULL);
uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))));
mi_assert_internal(bin <= MI_BIN_FULL);
mi_page_queue_t* pq = &heap->pages[bin];
mi_assert_internal((mi_page_block_size(page) == pq->block_size) ||
@ -189,7 +184,7 @@ static inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_que
}
else {
// find previous size; due to minimal alignment upto 3 previous bins may need to be skipped
uint8_t bin = mi_bin(size);
size_t bin = mi_bin(size);
const mi_page_queue_t* prev = pq - 1;
while( bin == mi_bin(prev->block_size) && prev > &heap->pages[0]) {
prev--;

2
src/prim/windows/prim.c
View file

@ -123,7 +123,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
if (si.dwAllocationGranularity > 0) { config->alloc_granularity = si.dwAllocationGranularity; }
// get virtual address bits
if ((uintptr_t)si.lpMaximumApplicationAddress > 0) {
const size_t vbits = MI_INTPTR_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
const size_t vbits = MI_SIZE_BITS - mi_clz((uintptr_t)si.lpMaximumApplicationAddress);
config->virtual_address_bits = vbits;
}

42
test/main-override-static.c
View file

@ -32,7 +32,10 @@ static void test_manage_os_memory(void);
int main() {
mi_version();
mi_stats_reset();
test_manage_os_memory();
// mi_bins();
// test_manage_os_memory();
// test_large_pages();
// detect double frees and heap corruption
// double_free1();
@ -40,7 +43,7 @@ int main() {
// corrupt_free();
// block_overflow1();
// block_overflow2();
test_canary_leak();
// test_canary_leak();
// test_aslr();
// invalid_free();
// test_reserved();
@ -397,31 +400,34 @@ static inline size_t _mi_wsize_from_size(size_t size) {
return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
}
// #define MI_ALIGN2W
// Return the bin for a given field size.
// Returns MI_BIN_HUGE if the size is too large.
// We use `wsize` for the size in "machine word sizes",
// i.e. byte size == `wsize*sizeof(void*)`.
extern inline uint8_t _mi_bin8(size_t size) {
size_t wsize = _mi_wsize_from_size(size);
uint8_t bin;
if (wsize <= 1) {
static inline size_t mi_bin(size_t wsize) {
// size_t wsize = _mi_wsize_from_size(size);
// size_t bin;
/*if (wsize <= 1) {
bin = 1;
}
*/
#if defined(MI_ALIGN4W)
else if (wsize <= 4) {
bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
if (wsize <= 4) {
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
}
#elif defined(MI_ALIGN2W)
else if (wsize <= 8) {
bin = (uint8_t)((wsize+1)&~1); // round to double word sizes
if (wsize <= 8) {
return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes
}
#else
else if (wsize <= 8) {
bin = (uint8_t)wsize;
if (wsize <= 8) {
return (wsize == 0 ? 1 : wsize);
}
#endif
else if (wsize > MI_LARGE_WSIZE_MAX) {
bin = MI_BIN_HUGE;
return MI_BIN_HUGE;
}
else {
#if defined(MI_ALIGN4W)
@ -429,15 +435,17 @@ extern inline uint8_t _mi_bin8(size_t size) {
#endif
wsize--;
// find the highest bit
uint8_t b = mi_bsr32((uint32_t)wsize);
const size_t b = _mi_bsr(wsize); // note: wsize != 0
// and use the top 3 bits to determine the bin (~12.5% worst internal fragmentation).
// - adjust with 3 because we use do not round the first 8 sizes
// which each get an exact bin
bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3;
const size_t bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03)) - 3;
assert(bin > 0 && bin < MI_BIN_HUGE);
return bin;
}
return bin;
}
static inline uint8_t _mi_bin4(size_t size) {
size_t wsize = _mi_wsize_from_size(size);
uint8_t bin;
@ -493,7 +501,7 @@ static size_t _mi_binx8(size_t bsize) {
}
static inline size_t mi_bin(size_t wsize) {
static inline size_t mi_binx(size_t wsize) {
uint8_t bin;
if (wsize <= 1) {
bin = 1;