kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-04 22:49:32 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

commend and make at least 8 tries for reclaim

Browse source
This commit is contained in:
Daan Leijen 2024-03-25 15:25:04 -07:00
parent 006ae2d055
commit 0022802177
8 changed files with 96 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/mimalloc/internal.h
View file

@ -147,7 +147,7 @@ void _mi_segment_map_freed_at(const mi_segment_t* segment);
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size, size_t* pre_size); // page start for any page uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size);
#if MI_HUGE_PAGE_ABANDON #if MI_HUGE_PAGE_ABANDON
void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
@ -454,6 +454,7 @@ static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const
// Quick page start for initialized pages // Quick page start for initialized pages
static inline uint8_t* mi_page_start(const mi_page_t* page) { static inline uint8_t* mi_page_start(const mi_page_t* page) {
mi_assert_internal(page->page_start != NULL); mi_assert_internal(page->page_start != NULL);
mi_assert_expensive(_mi_segment_page_start(_mi_page_segment(page),page,NULL) == page->page_start);
return page->page_start; return page->page_start;
} }
@ -466,7 +467,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) {
// Get the block size of a page (special case for huge objects) // Get the block size of a page (special case for huge objects)
static inline size_t mi_page_block_size(const mi_page_t* page) { static inline size_t mi_page_block_size(const mi_page_t* page) {
mi_assert_internal(page->block_size > 0); mi_assert_internal(page->block_size > 0);
return page->block_size; return page->block_size;
} }
static inline bool mi_page_is_huge(const mi_page_t* page) { static inline bool mi_page_is_huge(const mi_page_t* page) {

53
include/mimalloc/types.h
View file

@ -16,6 +16,8 @@ terms of the MIT license. A copy of the license can be found in the file
// are allocated. // are allocated.
// mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from // mi_page_t : a mimalloc page (usually 64KiB or 512KiB) from
// where objects are allocated. // where objects are allocated.
// Note: we write "OS page" for OS memory pages while
// using plain "page" for mimalloc pages (`mi_page_t`).
// -------------------------------------------------------------------------- // --------------------------------------------------------------------------
@ -89,10 +91,11 @@ terms of the MIT license. A copy of the license can be found in the file
#endif #endif
// We used to abandon huge pages but to eagerly deallocate if freed from another thread, // We used to abandon huge pages in order to eagerly deallocate it if freed from another thread.
// but that makes it not possible to visit them during a heap walk or include them in a // Unfortunately, that makes it not possible to visit them during a heap walk or include them in a
// `mi_heap_destroy`. We therefore instead reset/decommit the huge blocks if freed from // `mi_heap_destroy`. We therefore instead reset/decommit the huge blocks nowadays if freed from
// another thread so most memory is available until it gets properly freed by the owning thread. // another thread so the memory becomes "virtually" available (and eventually gets properly freed by
// the owning thread).
// #define MI_HUGE_PAGE_ABANDON 1 // #define MI_HUGE_PAGE_ABANDON 1
@ -160,7 +163,7 @@ typedef int32_t mi_ssize_t;
#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB #define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB
#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB #define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB
#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB #define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB
#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB #define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB -- must be equal to `MI_LARGE_PAGE_SHIFT`
// Derived constants // Derived constants
#define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT) #define MI_SEGMENT_SIZE (MI_ZU(1)<<MI_SEGMENT_SHIFT)
@ -215,7 +218,7 @@ typedef enum mi_delayed_e {
MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list MI_USE_DELAYED_FREE = 0, // push on the owning heap thread delayed list
MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap MI_DELAYED_FREEING = 1, // temporary: another thread is accessing the owning heap
MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list MI_NO_DELAYED_FREE = 2, // optimize: push on page local thread free queue if another block is already in the heap thread delayed free list
MI_NEVER_DELAYED_FREE = 3 // sticky, only resets on page reclaim MI_NEVER_DELAYED_FREE = 3 // sticky: used for abondoned pages without a owning heap; this only resets on page reclaim
} mi_delayed_t; } mi_delayed_t;
@ -264,7 +267,7 @@ typedef uintptr_t mi_thread_free_t;
// - Access is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` // - Access is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc`
// - Using `uint16_t` does not seem to slow things down // - Using `uint16_t` does not seem to slow things down
// - The size is 10 words on 64-bit which helps the page index calculations // - The size is 10 words on 64-bit which helps the page index calculations
// (and 14 words on 32-bit, and encoded free lists add 2 words) // (and 12 words on 32-bit, and encoded free lists add 2 words)
// - `xthread_free` uses the bottom bits as a delayed-free flags to optimize // - `xthread_free` uses the bottom bits as a delayed-free flags to optimize
// concurrent frees where only the first concurrent free adds to the owning // concurrent frees where only the first concurrent free adds to the owning
// heap `thread_delayed_free` list (see `free.c:mi_free_block_mt`). // heap `thread_delayed_free` list (see `free.c:mi_free_block_mt`).
@ -283,14 +286,15 @@ typedef struct mi_page_s {
// layout like this to optimize access in `mi_malloc` and `mi_free` // layout like this to optimize access in `mi_malloc` and `mi_free`
uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear`
uint16_t reserved; // number of blocks reserved in memory uint16_t reserved; // number of blocks reserved in memory
uint16_t used; // number of blocks in use (including blocks in `thread_free`)
mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits) mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits)
uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`) uint8_t free_is_zero:1; // `true` if the blocks in the free list are zero initialized
uint8_t free_is_zero:1; // `true` if the blocks in the free list are zero initialized
uint8_t retire_expire:7; // expiration count for retired blocks uint8_t retire_expire:7; // expiration count for retired blocks
// padding
mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list)
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
uint16_t used; // number of blocks in use (including blocks in `thread_free`)
uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`)
// padding
size_t block_size; // size available in each block (always `>0`) size_t block_size; // size available in each block (always `>0`)
uint8_t* page_start; // start of the page area containing the blocks uint8_t* page_start; // start of the page area containing the blocks
@ -304,7 +308,7 @@ typedef struct mi_page_s {
struct mi_page_s* next; // next page owned by the heap with the same `block_size` struct mi_page_s* next; // next page owned by the heap with the same `block_size`
struct mi_page_s* prev; // previous page owned by the heap with the same `block_size` struct mi_page_s* prev; // previous page owned by the heap with the same `block_size`
#if MI_INTPTR_SIZE==4 // pad to 14 words on 32-bit #if MI_INTPTR_SIZE==4 // pad to 12 words on 32-bit
void* padding[1]; void* padding[1];
#endif #endif
} mi_page_t; } mi_page_t;
@ -319,17 +323,22 @@ typedef enum mi_page_kind_e {
MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment MI_PAGE_SMALL, // small blocks go into 64KiB pages inside a segment
MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages inside a segment MI_PAGE_MEDIUM, // medium blocks go into 512KiB pages inside a segment
MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment
MI_PAGE_HUGE // huge blocks (>512KiB) are put into a single page in a segment of the exact size (but still 2MiB aligned) MI_PAGE_HUGE // a huge page is a single page in a segment of variable size (but still 2MiB aligned)
// used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or an aligment `> MI_BLOCK_ALIGNMENT_MAX`.
} mi_page_kind_t; } mi_page_kind_t;
// ---------------------------------------------------------------
// a memory id tracks the provenance of arena/OS allocated memory
// ---------------------------------------------------------------
// Memory can reside in arena's, direct OS allocated, or statically allocated. The memid keeps track of this. // Memory can reside in arena's, direct OS allocated, or statically allocated. The memid keeps track of this.
typedef enum mi_memkind_e { typedef enum mi_memkind_e {
MI_MEM_NONE, // not allocated MI_MEM_NONE, // not allocated
MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example)
MI_MEM_STATIC, // allocated in a static area and should not be freed (for arena meta data for example) MI_MEM_STATIC, // allocated in a static area and should not be freed (for arena meta data for example)
MI_MEM_OS, // allocated from the OS MI_MEM_OS, // allocated from the OS
MI_MEM_OS_HUGE, // allocated as huge os pages MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory)
MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`)
MI_MEM_ARENA // allocated from an arena (the usual case) MI_MEM_ARENA // allocated from an arena (the usual case)
} mi_memkind_t; } mi_memkind_t;
@ -346,7 +355,7 @@ typedef struct mi_memid_os_info {
typedef struct mi_memid_arena_info { typedef struct mi_memid_arena_info {
size_t block_index; // index in the arena size_t block_index; // index in the arena
mi_arena_id_t id; // arena id (>= 1) mi_arena_id_t id; // arena id (>= 1)
bool is_exclusive; // the arena can only be used for specific arena allocations bool is_exclusive; // this arena can only be used for specific arena allocations
} mi_memid_arena_info_t; } mi_memid_arena_info_t;
typedef struct mi_memid_s { typedef struct mi_memid_s {
@ -354,19 +363,22 @@ typedef struct mi_memid_s {
mi_memid_os_info_t os; // only used for MI_MEM_OS mi_memid_os_info_t os; // only used for MI_MEM_OS
mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA
} mem; } mem;
bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large OS pages) bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages)
bool initially_committed;// `true` if the memory was originally allocated as committed bool initially_committed;// `true` if the memory was originally allocated as committed
bool initially_zero; // `true` if the memory was originally zero initialized bool initially_zero; // `true` if the memory was originally zero initialized
mi_memkind_t memkind; mi_memkind_t memkind;
} mi_memid_t; } mi_memid_t;
// Segments are large allocated memory blocks (2MiB on 64 bit) from // ---------------------------------------------------------------
// the OS. Inside segments we allocated fixed size _pages_ that // Segments contain mimalloc pages
// contain blocks. // ---------------------------------------------------------------
// Segments are large allocated memory blocks (2MiB on 64 bit) from the OS.
// Inside segments we allocated fixed size _pages_ that contain blocks.
typedef struct mi_segment_s { typedef struct mi_segment_s {
// constant fields // constant fields
mi_memid_t memid; // id for the os-level memory manager mi_memid_t memid; // memory id to track provenance
bool allow_decommit; bool allow_decommit;
bool allow_purge; bool allow_purge;
size_t segment_size; // for huge pages this may be different from `MI_SEGMENT_SIZE` size_t segment_size; // for huge pages this may be different from `MI_SEGMENT_SIZE`
@ -572,6 +584,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
#define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) #define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount)
#define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) #define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount)
// ------------------------------------------------------ // ------------------------------------------------------
// Thread Local data // Thread Local data
// ------------------------------------------------------ // ------------------------------------------------------

2
src/heap.c
View file

@ -508,7 +508,7 @@ static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_v
const size_t bsize = mi_page_block_size(page); const size_t bsize = mi_page_block_size(page);
const size_t ubsize = mi_page_usable_block_size(page); // without padding const size_t ubsize = mi_page_usable_block_size(page); // without padding
size_t psize; size_t psize;
uint8_t* pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize, NULL); uint8_t* pstart = _mi_segment_page_start(_mi_page_segment(page), page, &psize);
if (page->capacity == 1) { if (page->capacity == 1) {
// optimize page with one block // optimize page with one block

6
src/init.c
View file

@ -14,17 +14,17 @@ terms of the MIT license. A copy of the license can be found in the file
// Empty page used to initialize the small free pages array // Empty page used to initialize the small free pages array
const mi_page_t _mi_page_empty = { const mi_page_t _mi_page_empty = {
0, 0,
false, false, false, false, false, false, false, false,
0, // capacity 0, // capacity
0, // reserved capacity 0, // reserved capacity
0, // used
{ 0 }, // flags { 0 }, // flags
0, // block size shift
false, // is_zero false, // is_zero
0, // retire_expire 0, // retire_expire
NULL, // free NULL, // free
NULL, // local_free NULL, // local_free
0, // used
0, // block size shift
0, // block_size 0, // block_size
NULL, // page_start NULL, // page_start
#if (MI_PADDING || MI_ENCODE_FREELIST) #if (MI_PADDING || MI_ENCODE_FREELIST)

4
src/libc.c
View file

@ -210,7 +210,7 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
if (c == 'x' || c == 'u') { if (c == 'x' || c == 'u') {
if (numtype == 'z') x = va_arg(args, size_t); if (numtype == 'z') x = va_arg(args, size_t);
else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t else if (numtype == 't') x = va_arg(args, uintptr_t); // unsigned ptrdiff_t
else if (numtype == 'L') x = va_arg(args, unsigned long long); else if (numtype == 'L') x = (uintptr_t)va_arg(args, unsigned long long);
else x = va_arg(args, unsigned long); else x = va_arg(args, unsigned long);
} }
else if (c == 'p') { else if (c == 'p') {
@ -231,7 +231,7 @@ void _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) {
intptr_t x = 0; intptr_t x = 0;
if (numtype == 'z') x = va_arg(args, intptr_t ); if (numtype == 'z') x = va_arg(args, intptr_t );
else if (numtype == 't') x = va_arg(args, ptrdiff_t); else if (numtype == 't') x = va_arg(args, ptrdiff_t);
else if (numtype == 'L') x = va_arg(args, long long); else if (numtype == 'L') x = (intptr_t)va_arg(args, long long);
else x = va_arg(args, long); else x = va_arg(args, long);
char pre = 0; char pre = 0;
if (x < 0) { if (x < 0) {

9
src/page.c
View file

@ -59,7 +59,7 @@ static inline uint8_t* mi_page_area(const mi_page_t* page) {
static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) {
size_t psize; size_t psize;
uint8_t* page_area = _mi_segment_page_start(_mi_page_segment(page), page, &psize, NULL); uint8_t* page_area = _mi_segment_page_start(_mi_page_segment(page), page, &psize);
mi_block_t* start = (mi_block_t*)page_area; mi_block_t* start = (mi_block_t*)page_area;
mi_block_t* end = (mi_block_t*)(page_area + psize); mi_block_t* end = (mi_block_t*)(page_area + psize);
while(p != NULL) { while(p != NULL) {
@ -85,7 +85,8 @@ static bool mi_page_is_valid_init(mi_page_t* page) {
// const size_t bsize = mi_page_block_size(page); // const size_t bsize = mi_page_block_size(page);
mi_segment_t* segment = _mi_page_segment(page); mi_segment_t* segment = _mi_page_segment(page);
uint8_t* start = mi_page_start(page); uint8_t* start = mi_page_start(page);
mi_assert_internal(start == _mi_segment_page_start(segment,page,NULL,NULL)); mi_assert_internal(start == _mi_segment_page_start(segment,page,NULL));
mi_assert_internal(page->is_huge == (segment->page_kind == MI_PAGE_HUGE));
//mi_assert_internal(start + page->capacity*page->block_size == page->top); //mi_assert_internal(start + page->capacity*page->block_size == page->top);
mi_assert_internal(mi_page_list_is_valid(page,page->free)); mi_assert_internal(mi_page_list_is_valid(page,page->free));
@ -616,7 +617,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
size_t page_size; size_t page_size;
//uint8_t* page_start = //uint8_t* page_start =
_mi_segment_page_start(_mi_page_segment(page), page, &page_size, NULL); _mi_segment_page_start(_mi_page_segment(page), page, &page_size);
mi_stat_counter_increase(tld->stats.pages_extended, 1); mi_stat_counter_increase(tld->stats.pages_extended, 1);
// calculate the extend count // calculate the extend count
@ -660,7 +661,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
mi_page_set_heap(page, heap); mi_page_set_heap(page, heap);
page->block_size = block_size; page->block_size = block_size;
size_t page_size; size_t page_size;
page->page_start = _mi_segment_page_start(segment, page, &page_size, NULL); page->page_start = _mi_segment_page_start(segment, page, &page_size);
mi_track_mem_noaccess(page->page_start,page_size); mi_track_mem_noaccess(page->page_start,page_size);
mi_assert_internal(page_size / block_size < (1L<<16)); mi_assert_internal(page_size / block_size < (1L<<16));
page->reserved = (uint16_t)(page_size / block_size); page->reserved = (uint16_t)(page_size / block_size);

51
src/segment.c
View file

@ -1,5 +1,5 @@
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
Copyright (c) 2018-2020, Microsoft Research, Daan Leijen Copyright (c) 2018-2024, Microsoft Research, Daan Leijen
This is free software; you can redistribute it and/or modify it under the This is free software; you can redistribute it and/or modify it under the
terms of the MIT license. A copy of the license can be found in the file terms of the MIT license. A copy of the license can be found in the file
"LICENSE" at the root of this distribution. "LICENSE" at the root of this distribution.
@ -25,14 +25,15 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_
- small pages (64KiB), 64 in one segment - small pages (64KiB), 64 in one segment
- medium pages (512KiB), 8 in one segment - medium pages (512KiB), 8 in one segment
- large pages (4MiB), 1 in one segment - large pages (4MiB), 1 in one segment
- huge blocks > MI_LARGE_OBJ_SIZE_MAX become large segment with 1 page - huge segments have 1 page in one segment that can be larger than `MI_SEGMENT_SIZE`.
it is used for blocks `> MI_LARGE_OBJ_SIZE_MAX` or with alignment `> MI_BLOCK_ALIGNMENT_MAX`.
In any case the memory for a segment is virtual and usually committed on demand. The memory for a segment is usually committed on demand.
(i.e. we are careful to not touch the memory until we actually allocate a block there) (i.e. we are careful to not touch the memory until we actually allocate a block there)
If a thread ends, it "abandons" pages with used blocks If a thread ends, it "abandons" pages that still contain live blocks.
and there is an abandoned segment list whose segments can Such segments are abondoned and these can be reclaimed by still running threads,
be reclaimed by still running threads, much like work-stealing. (much like work-stealing).
-------------------------------------------------------------------------------- */ -------------------------------------------------------------------------------- */
@ -142,7 +143,7 @@ static bool mi_segment_is_valid(const mi_segment_t* segment, mi_segments_tld_t*
mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(segment->used <= segment->capacity); mi_assert_internal(segment->used <= segment->capacity);
mi_assert_internal(segment->abandoned <= segment->used); mi_assert_internal(segment->abandoned <= segment->used);
mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || segment->capacity == 1); mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || segment->capacity == 1); // one large or huge page per segment
size_t nfree = 0; size_t nfree = 0;
for (size_t i = 0; i < segment->capacity; i++) { for (size_t i = 0; i < segment->capacity; i++) {
const mi_page_t* const page = &segment->pages[i]; const mi_page_t* const page = &segment->pages[i];
@ -152,7 +153,7 @@ static bool mi_segment_is_valid(const mi_segment_t* segment, mi_segments_tld_t*
if (page->segment_in_use) { if (page->segment_in_use) {
mi_assert_expensive(!mi_pages_purge_contains(page, tld)); mi_assert_expensive(!mi_pages_purge_contains(page, tld));
} }
if (segment->page_kind == MI_PAGE_HUGE) mi_assert_internal(page->is_huge); mi_assert_internal(page->is_huge == (segment->page_kind == MI_PAGE_HUGE));
} }
mi_assert_internal(nfree + segment->used == segment->capacity); mi_assert_internal(nfree + segment->used == segment->capacity);
// mi_assert_internal(segment->thread_id == _mi_thread_id() || (segment->thread_id==0)); // or 0 // mi_assert_internal(segment->thread_id == _mi_thread_id() || (segment->thread_id==0)); // or 0
@ -420,11 +421,11 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_
} }
// Start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set) // Start of the page available memory; can be used on uninitialized pages (only `segment_idx` must be set)
static uint8_t* mi_segment_page_start_ex(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size, size_t* pre_size) uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size)
{ {
size_t psize; size_t psize;
uint8_t* p = mi_segment_raw_page_start(segment, page, &psize); uint8_t* p = mi_segment_raw_page_start(segment, page, &psize);
if (pre_size != NULL) *pre_size = 0; const size_t block_size = mi_page_block_size(page);
if (page->segment_idx == 0 && block_size > 0 && segment->page_kind <= MI_PAGE_MEDIUM) { if (page->segment_idx == 0 && block_size > 0 && segment->page_kind <= MI_PAGE_MEDIUM) {
// for small and medium objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore) // for small and medium objects, ensure the page start is aligned with the block size (PR#66 by kickunderscore)
size_t adjust = block_size - ((uintptr_t)p % block_size); size_t adjust = block_size - ((uintptr_t)p % block_size);
@ -432,7 +433,7 @@ static uint8_t* mi_segment_page_start_ex(const mi_segment_t* segment, const mi_p
if (adjust < block_size) { if (adjust < block_size) {
p += adjust; p += adjust;
psize -= adjust; psize -= adjust;
if (pre_size != NULL) *pre_size = adjust; // if (pre_size != NULL) *pre_size = adjust;
} }
mi_assert_internal((uintptr_t)p % block_size == 0); mi_assert_internal((uintptr_t)p % block_size == 0);
} }
@ -444,9 +445,6 @@ static uint8_t* mi_segment_page_start_ex(const mi_segment_t* segment, const mi_p
return p; return p;
} }
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size, size_t* pre_size) {
return mi_segment_page_start_ex(segment, page, mi_page_block_size(page), page_size, pre_size);
}
static size_t mi_segment_calculate_sizes(size_t capacity, size_t required, size_t* pre_size, size_t* info_size) static size_t mi_segment_calculate_sizes(size_t capacity, size_t required, size_t* pre_size, size_t* info_size)
{ {
@ -961,26 +959,31 @@ void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) {
} }
static long mi_segment_get_reclaim_tries(void) { static long mi_segment_get_reclaim_tries(void) {
// limit the tries to 10% (default) of the abandoned segments with at least 8 tries, and at most 1024. // limit the tries to 10% (default) of the abandoned segments with at least 8 and at most 1024 tries.
const size_t perc = (size_t)mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 100); const size_t perc = (size_t)mi_option_get_clamp(mi_option_max_segment_reclaim, 0, 100);
if (perc <= 0) return 0; if (perc <= 0) return 0;
const size_t total_count = _mi_arena_segment_abandoned_count(); const size_t total_count = _mi_arena_segment_abandoned_count();
if (total_count == 0) return 0;
const size_t relative_count = (total_count > 10000 ? (total_count / 100) * perc : (total_count * perc) / 100); // avoid overflow const size_t relative_count = (total_count > 10000 ? (total_count / 100) * perc : (total_count * perc) / 100); // avoid overflow
long max_tries = (long)(relative_count < 8 ? 8 : (relative_count > 1024 ? 1024 : relative_count)); long max_tries = (long)(relative_count <= 1 ? 1 : (relative_count > 1024 ? 1024 : relative_count));
if (max_tries < 8 && total_count > 8) { max_tries = 8; }
return max_tries; return max_tries;
} }
static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, bool* reclaimed, mi_segments_tld_t* tld) static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t block_size, mi_page_kind_t page_kind, bool* reclaimed, mi_segments_tld_t* tld)
{ {
*reclaimed = false; *reclaimed = false;
mi_segment_t* segment;
mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap,&current);
long max_tries = mi_segment_get_reclaim_tries(); long max_tries = mi_segment_get_reclaim_tries();
if (max_tries <= 0) return NULL;
mi_segment_t* segment;
mi_arena_field_cursor_t current; _mi_arena_field_cursor_init(heap, &current);
while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(&current)) != NULL)) while ((max_tries-- > 0) && ((segment = _mi_arena_segment_clear_abandoned_next(&current)) != NULL))
{ {
segment->abandoned_visits++; segment->abandoned_visits++;
// todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments // todo: should we respect numa affinity for abondoned reclaim? perhaps only for the first visit?
// and push them into the visited list and use many tries. Perhaps we can skip non-suitable ones in a better way? // todo: an arena exclusive heap will potentially visit many abandoned unsuitable segments and use many tries
// Perhaps we can skip non-suitable ones in a better way?
bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid); bool is_suitable = _mi_heap_memid_is_suitable(heap, segment->memid);
bool all_pages_free; bool all_pages_free;
bool has_page = mi_segment_check_free(segment,block_size,&all_pages_free); // try to free up pages (due to concurrent frees) bool has_page = mi_segment_check_free(segment,block_size,&all_pages_free); // try to free up pages (due to concurrent frees)
@ -1088,7 +1091,7 @@ static mi_page_t* mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, mi_p
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
#if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN #if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN
// verify it is committed // verify it is committed
mi_segment_page_start_ex(_mi_page_segment(page), page, sizeof(void*), NULL, NULL)[0] = 0; mi_segment_raw_page_start(_mi_page_segment(page), page, NULL)[0] = 0;
#endif #endif
return page; return page;
} }
@ -1111,7 +1114,7 @@ static mi_page_t* mi_segment_large_page_alloc(mi_heap_t* heap, size_t block_size
mi_page_t* page = mi_segment_find_free(segment, tld); mi_page_t* page = mi_segment_find_free(segment, tld);
mi_assert_internal(page != NULL); mi_assert_internal(page != NULL);
#if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN #if MI_DEBUG>=2 && !MI_TRACK_ENABLED // && !MI_TSAN
mi_segment_page_start_ex(segment, page, sizeof(void*), NULL, NULL)[0] = 0; mi_segment_raw_page_start(segment, page, NULL)[0] = 0;
#endif #endif
return page; return page;
} }
@ -1132,9 +1135,9 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment,
// for huge pages we initialize the block_size as we may // for huge pages we initialize the block_size as we may
// overallocate to accommodate large alignments. // overallocate to accommodate large alignments.
size_t psize; size_t psize;
uint8_t* start = mi_segment_page_start_ex(segment, page, 0, &psize, NULL); uint8_t* start = mi_segment_raw_page_start(segment, page, &psize);
page->block_size = psize; page->block_size = psize;
// reset the part of the page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE) // reset the part of the page that will not be used; this can be quite large (close to MI_SEGMENT_SIZE)
if (page_alignment > 0 && segment->allow_decommit && page->is_committed) { if (page_alignment > 0 && segment->allow_decommit && page->is_committed) {
uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment); uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment);

46
src/stats.c
View file

@ -174,13 +174,28 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar
static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) {
_mi_fprintf(out, arg,"%10s:", msg); _mi_fprintf(out, arg,"%10s:", msg);
if (unit > 0) { if (unit != 0) {
mi_print_amount(stat->peak, unit, out, arg); if (unit > 0) {
mi_print_amount(stat->allocated, unit, out, arg); mi_print_amount(stat->peak, unit, out, arg);
mi_print_amount(stat->freed, unit, out, arg); mi_print_amount(stat->allocated, unit, out, arg);
mi_print_amount(stat->current, unit, out, arg); mi_print_amount(stat->freed, unit, out, arg);
mi_print_amount(unit, 1, out, arg); mi_print_amount(stat->current, unit, out, arg);
mi_print_count(stat->allocated, unit, out, arg); mi_print_amount(unit, 1, out, arg);
mi_print_count(stat->allocated, unit, out, arg);
}
else {
mi_print_amount(stat->peak, -1, out, arg);
mi_print_amount(stat->allocated, -1, out, arg);
mi_print_amount(stat->freed, -1, out, arg);
mi_print_amount(stat->current, -1, out, arg);
if (unit == -1) {
_mi_fprintf(out, arg, "%24s", "");
}
else {
mi_print_amount(-unit, 1, out, arg);
mi_print_count((stat->allocated / -unit), 0, out, arg);
}
}
if (stat->allocated > stat->freed) { if (stat->allocated > stat->freed) {
_mi_fprintf(out, arg, " "); _mi_fprintf(out, arg, " ");
_mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok)); _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok));
@ -190,23 +205,6 @@ static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64
_mi_fprintf(out, arg, " ok\n"); _mi_fprintf(out, arg, " ok\n");
} }
} }
else if (unit<0) {
mi_print_amount(stat->peak, -1, out, arg);
mi_print_amount(stat->allocated, -1, out, arg);
mi_print_amount(stat->freed, -1, out, arg);
mi_print_amount(stat->current, -1, out, arg);
if (unit==-1) {
_mi_fprintf(out, arg, "%24s", "");
}
else {
mi_print_amount(-unit, 1, out, arg);
mi_print_count((stat->allocated / -unit), 0, out, arg);
}
if (stat->allocated > stat->freed)
_mi_fprintf(out, arg, " not all freed!\n");
else
_mi_fprintf(out, arg, " ok\n");
}
else { else {
mi_print_amount(stat->peak, 1, out, arg); mi_print_amount(stat->peak, 1, out, arg);
mi_print_amount(stat->allocated, 1, out, arg); mi_print_amount(stat->allocated, 1, out, arg);