From 9afc253726fbe28015b3c37841e41c9202d382ef Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 14:03:16 -0800 Subject: [PATCH] add comments, renaming --- include/mimalloc-internal.h | 50 +++++---------------------- include/mimalloc-types.h | 21 ++++++++---- src/segment.c | 68 +++++++++++++++++++++++++------------ 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index caf5a784..6b416b17 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -19,6 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_CACHE_LINE 64 #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) +#pragma warning(disable:26812) // unscoped enum warning #define mi_decl_noinline __declspec(noinline) #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) @@ -696,72 +697,39 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c // commit mask // ------------------------------------------------------------------- - static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { cm->mask[i] = 0; } } static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { cm->mask[i] = ~((size_t)0); } } static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { if (cm->mask[i] != 0) return false; } return true; } static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { if (cm->mask[i] != 0) return false; } return true; } -static inline bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { - if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false; - } - return true; -} - -static inline bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { - if ((commit->mask[i] & cm->mask[i]) != 0) return true; - } - return false; -} - -static inline void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { - res->mask[i] = (commit->mask[i] & cm->mask[i]); - } -} - -static inline void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { - res->mask[i] &= ~(cm->mask[i]); - } -} - -static inline void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { - res->mask[i] |= cm->mask[i]; - } -} - -void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm); -size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total); -ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx); +// defined in `segment.c`: +size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total); +ptrdiff_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx); #define mi_commit_mask_foreach(cm,idx,count) \ idx = 0; \ - while ((count = mi_commit_mask_next_run(cm,&idx)) > 0) { + while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) { #define mi_commit_mask_foreach_end() \ idx += count; \ diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index c60457c8..5bf779fa 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -287,17 +287,26 @@ typedef enum mi_segment_kind_e { MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside. } mi_segment_kind_t; -#define MI_COMMIT_SIZE (4*64*1024) -#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) -#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS -#define MI_COMMIT_MASK_N (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) +// ------------------------------------------------------ +// A segment holds a commit mask where a bit is set if +// the corresponding MI_COMMIT_SIZE area is committed. +// The MI_COMMIT_SIZE must be a multiple of the slice +// size. We define it as equal so we can decommit on a +// slice level which helps with (real) memory fragmentation +// over time. +// ------------------------------------------------------ -#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_N * MI_COMMIT_MASK_FIELD_BITS)) +#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) +#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) +#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS +#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) + +#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS)) #error "the segment size must be exactly divisible by the (commit size * size_t bits)" #endif typedef struct mi_commit_mask_s { - size_t mask[MI_COMMIT_MASK_N]; + size_t mask[MI_COMMIT_MASK_FIELD_COUNT]; } mi_commit_mask_t; typedef mi_page_t mi_slice_t; diff --git a/src/segment.c b/src/segment.c index da1664b0..d953438a 100644 --- a/src/segment.c +++ b/src/segment.c @@ -15,11 +15,44 @@ terms of the MIT license. A copy of the license can be found in the file static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_stats_t* stats); + // ------------------------------------------------------------------- -// commit mask +// commit mask // ------------------------------------------------------------------- -void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm) { +static bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false; + } + return true; +} + +static bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + if ((commit->mask[i] & cm->mask[i]) != 0) return true; + } + return false; +} + +static void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + res->mask[i] = (commit->mask[i] & cm->mask[i]); + } +} + +static void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + res->mask[i] &= ~(cm->mask[i]); + } +} + +static void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { + res->mask[i] |= cm->mask[i]; + } +} + +static void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm) { mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS); mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS); if (bitcount == MI_COMMIT_MASK_BITS) { @@ -34,7 +67,7 @@ void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_ ptrdiff_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS; ptrdiff_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS; while (bitcount > 0) { - mi_assert_internal(i < MI_COMMIT_MASK_N); + mi_assert_internal(i < MI_COMMIT_MASK_FIELD_COUNT); ptrdiff_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs; ptrdiff_t count = (bitcount > avail ? avail : bitcount); size_t mask = (count >= MI_COMMIT_MASK_FIELD_BITS ? ~((size_t)0) : (((size_t)1 << count) - 1) << ofs); @@ -46,11 +79,10 @@ void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_ } } - -size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) { +size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) { mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0); size_t count = 0; - for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) { + for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) { size_t mask = cm->mask[i]; if (~mask == 0) { count += MI_COMMIT_MASK_FIELD_BITS; @@ -66,12 +98,12 @@ size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) { } -ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) { +ptrdiff_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) { ptrdiff_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS; ptrdiff_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS; size_t mask = 0; // find first ones - while (i < MI_COMMIT_MASK_N) { + while (i < MI_COMMIT_MASK_FIELD_COUNT) { mask = cm->mask[i]; mask >>= ofs; if (mask != 0) { @@ -84,7 +116,7 @@ ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) { i++; ofs = 0; } - if (i >= MI_COMMIT_MASK_N) { + if (i >= MI_COMMIT_MASK_FIELD_COUNT) { // not found *idx = MI_COMMIT_MASK_BITS; return 0; @@ -101,7 +133,7 @@ ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) { } while ((mask&1) == 1); if ((((*idx + count) % MI_COMMIT_MASK_FIELD_BITS) == 0)) { i++; - if (i >= MI_COMMIT_MASK_N) break; + if (i >= MI_COMMIT_MASK_FIELD_COUNT) break; mask = cm->mask[i]; ofs = 0; } @@ -111,14 +143,6 @@ ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) { } } -#define mi_commit_mask_foreach(cm,idx,count) \ - idx = 0; \ - while ((count = mi_commit_mask_next_run(cm,&idx)) > 0) { - -#define mi_commit_mask_foreach_end() \ - idx += count; \ - } - /* -------------------------------------------------------------------------------- Segment allocation @@ -289,7 +313,7 @@ static size_t mi_segment_info_size(mi_segment_t* segment) { static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t xblock_size, size_t* page_size) { ptrdiff_t idx = slice - segment->slices; - size_t psize = slice->slice_count*MI_SEGMENT_SLICE_SIZE; + size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE; // make the start not OS page aligned for smaller blocks to avoid page/cache effects size_t start_offset = (xblock_size >= MI_INTPTR_SIZE && xblock_size <= 1024 ? MI_MAX_ALIGN_GUARANTEE : 0); if (page_size != NULL) *page_size = psize - start_offset; @@ -362,7 +386,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { // _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats); const size_t size = mi_segment_size(segment); if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, &segment->decommit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) { - const size_t csize = mi_commit_mask_committed_size(&segment->commit_mask, size); + const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size); if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize); _mi_abandoned_await_readers(); // wait until safe to free _mi_arena_free(segment, mi_segment_size(segment), segment->memid, segment->mem_is_pinned /* pretend not committed to not double count decommits */, tld->os); @@ -502,7 +526,7 @@ static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s bool is_zero = false; mi_commit_mask_t cmask; mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); - _mi_stat_decrease(&_mi_stats_main.committed, mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap + _mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false; mi_commit_mask_set(&segment->commit_mask, &mask); } @@ -512,7 +536,7 @@ static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s mi_commit_mask_t cmask; mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); - _mi_stat_increase(&_mi_stats_main.committed, full_size - mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap + _mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap if (segment->allow_decommit) { _mi_os_decommit(start, full_size, stats); // ok if this fails }