From fb418831dfffaf9f89dce9f0793294995d839a1e Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 4 Feb 2022 16:10:51 -0800 Subject: [PATCH 1/7] only delay eager commit after the first thread --- src/segment.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/segment.c b/src/segment.c index e9d30510..9f474ca5 100644 --- a/src/segment.c +++ b/src/segment.c @@ -578,11 +578,11 @@ static void mi_segment_perhaps_decommit(mi_segment_t* segment, uint8_t* p, size_ else if (segment->decommit_expire <= now) { // previous decommit mask already expired // mi_segment_delayed_decommit(segment, true, stats); - segment->decommit_expire = now + (mi_option_get(mi_option_decommit_delay) / 8); // wait a tiny bit longer in case there is a series of free's + segment->decommit_expire = now + mi_option_get(mi_option_decommit_extend_delay); // (mi_option_get(mi_option_decommit_delay) / 8); // wait a tiny bit longer in case there is a series of free's } else { // previous decommit mask is not yet expired, increase the expiration by a bit. - segment->decommit_expire += (mi_option_get(mi_option_decommit_delay) / 8); + segment->decommit_expire += mi_option_get(mi_option_decommit_extend_delay); } } } @@ -809,7 +809,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ // Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little) const bool eager_delay = (!_mi_os_has_overcommit() && // never delay on overcommit systems - _mi_current_thread_count() > 2 && // do not delay for the first N threads + _mi_current_thread_count() > 1 && // do not delay for the first N threads tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delay && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager || (required > 0); From 0e2df71829597295c94426539d00c08414d2725b Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 4 Feb 2022 16:11:38 -0800 Subject: [PATCH 2/7] increase minimal commit size to 8*slice-size and add decommit_extend_delay as option --- include/mimalloc-types.h | 4 ++-- include/mimalloc.h | 1 + src/options.c | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 41364286..8cd3a4c3 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -320,10 +320,10 @@ typedef enum mi_segment_kind_e { // the corresponding MI_COMMIT_SIZE area is committed. // The MI_COMMIT_SIZE must be a multiple of the slice // size. If it is equal we have the most fine grained -// decommit (but in practice 2x seems to perform better). +// decommit but setting it higher can be more efficient. // ------------------------------------------------------ -#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) +#define MI_COMMIT_SIZE (8*MI_SEGMENT_SLICE_SIZE) #define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) #define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS #define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) diff --git a/include/mimalloc.h b/include/mimalloc.h index 06597e9a..08805845 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -328,6 +328,7 @@ typedef enum mi_option_e { mi_option_max_warnings, mi_option_allow_decommit, mi_option_segment_decommit_delay, + mi_option_decommit_extend_delay, _mi_option_last } mi_option_t; diff --git a/src/options.c b/src/options.c index 388be2e6..d2e61218 100644 --- a/src/options.c +++ b/src/options.c @@ -92,7 +92,8 @@ static mi_option_desc_t options[_mi_option_last] = { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output { 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit slices when no longer used (after decommit_delay milli-seconds) - { 500, UNINIT, MI_OPTION(segment_decommit_delay) } // decommit delay in milli-seconds for freed segments + { 500, UNINIT, MI_OPTION(segment_decommit_delay) }, // decommit delay in milli-seconds for freed segments + { 2, UNINIT, MI_OPTION(decommit_extend_delay) } }; static void mi_option_init(mi_option_desc_t* desc); From f2bfaa74fe9b05231848f06b7d9ede38c5f00811 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 4 Feb 2022 16:12:22 -0800 Subject: [PATCH 3/7] enable eager delay even on overcommit systems --- src/segment.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/segment.c b/src/segment.c index 1aadf5c2..8a83ceed 100644 --- a/src/segment.c +++ b/src/segment.c @@ -580,8 +580,8 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ // Initialize parameters const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && // don't delay for large objects - !_mi_os_has_overcommit() && // never delay on overcommit systems - _mi_current_thread_count() > 2 && // do not delay for the first N threads + // !_mi_os_has_overcommit() && // never delay on overcommit systems + _mi_current_thread_count() > 1 && // do not delay for the first N threads tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From e11100a13780297d7016eba0fcf541c85f60c16b Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 5 Feb 2022 10:57:15 -0800 Subject: [PATCH 4/7] add minimal commit size for increased efficiency (decommit fine grained, commit coarse grained) --- include/mimalloc-types.h | 9 +++++++-- src/segment.c | 7 +++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 8cd3a4c3..63549792 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -320,10 +320,15 @@ typedef enum mi_segment_kind_e { // the corresponding MI_COMMIT_SIZE area is committed. // The MI_COMMIT_SIZE must be a multiple of the slice // size. If it is equal we have the most fine grained -// decommit but setting it higher can be more efficient. +// decommit (but setting it higher can be more efficient). +// The MI_MINIMAL_COMMIT_SIZE is the minimal amount that will +// be committed in one go which can be set higher than +// MI_COMMIT_SIZE for efficiency (while the decommit mask +// is still tracked in fine-grained MI_COMMIT_SIZE chunks) // ------------------------------------------------------ -#define MI_COMMIT_SIZE (8*MI_SEGMENT_SLICE_SIZE) +#define MI_MINIMAL_COMMIT_SIZE (16*MI_SEGMENT_SLICE_SIZE) // 1MiB +#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) #define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) #define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS #define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) diff --git a/src/segment.c b/src/segment.c index e8c80d29..0970046d 100644 --- a/src/segment.c +++ b/src/segment.c @@ -470,17 +470,20 @@ static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uin if (p >= (uint8_t*)segment + segsize) return; size_t diff = (p - (uint8_t*)segment); + mi_assert_internal(diff + size <= segsize); + size_t start; size_t end; if (conservative) { + // decommit conservative start = _mi_align_up(diff, MI_COMMIT_SIZE); end = _mi_align_down(diff + size, MI_COMMIT_SIZE); } else { + // commit liberal start = _mi_align_down(diff, MI_COMMIT_SIZE); - end = _mi_align_up(diff + size, MI_COMMIT_SIZE); + end = _mi_align_up(diff + size, MI_MINIMAL_COMMIT_SIZE); } - mi_assert_internal(end <= segsize); if (end > segsize) { end = segsize; } From 8ec83f6945133e299290354ca74f65e906c8b163 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 5 Feb 2022 11:21:47 -0800 Subject: [PATCH 5/7] increase min commit to 2 mib --- include/mimalloc-types.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 63549792..310fb92b 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -327,8 +327,8 @@ typedef enum mi_segment_kind_e { // is still tracked in fine-grained MI_COMMIT_SIZE chunks) // ------------------------------------------------------ -#define MI_MINIMAL_COMMIT_SIZE (16*MI_SEGMENT_SLICE_SIZE) // 1MiB -#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) +#define MI_MINIMAL_COMMIT_SIZE (2*MI_MiB) +#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE) // 64KiB #define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE) #define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS #define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS) From 47f8caad4db06314d080b798f29b91e25dd51e76 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 5 Feb 2022 17:23:28 -0800 Subject: [PATCH 6/7] improve commit chunk alignment --- src/segment.c | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/segment.c b/src/segment.c index 0970046d..b000e641 100644 --- a/src/segment.c +++ b/src/segment.c @@ -466,28 +466,35 @@ static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uin mi_assert_internal(segment->kind != MI_SEGMENT_HUGE); mi_commit_mask_create_empty(cm); if (size == 0 || size > MI_SEGMENT_SIZE || segment->kind == MI_SEGMENT_HUGE) return; + const size_t segstart = mi_segment_info_size(segment); const size_t segsize = mi_segment_size(segment); if (p >= (uint8_t*)segment + segsize) return; - size_t diff = (p - (uint8_t*)segment); - mi_assert_internal(diff + size <= segsize); + size_t pstart = (p - (uint8_t*)segment); + mi_assert_internal(pstart + size <= segsize); size_t start; size_t end; if (conservative) { // decommit conservative - start = _mi_align_up(diff, MI_COMMIT_SIZE); - end = _mi_align_down(diff + size, MI_COMMIT_SIZE); + start = _mi_align_up(pstart, MI_COMMIT_SIZE); + end = _mi_align_down(pstart + size, MI_COMMIT_SIZE); + mi_assert_internal(start >= segstart); + mi_assert_internal(end <= segsize); } else { // commit liberal - start = _mi_align_down(diff, MI_COMMIT_SIZE); - end = _mi_align_up(diff + size, MI_MINIMAL_COMMIT_SIZE); + start = _mi_align_down(pstart, MI_MINIMAL_COMMIT_SIZE); + end = _mi_align_up(pstart + size, MI_MINIMAL_COMMIT_SIZE); + } + if (start < segstart) { + start = segstart; } if (end > segsize) { end = segsize; } + mi_assert_internal(start <= pstart && (pstart + size) <= end); mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0); *start_p = (uint8_t*)segment + start; *full_size = (end > start ? end - start : 0); @@ -504,14 +511,19 @@ static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uin mi_commit_mask_create(bitidx, bitcount, cm); } -#define MI_COMMIT_SIZE_BATCH MiB static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, size_t size, mi_stats_t* stats) { mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask)); - //if (commit && size < MI_COMMIT_SIZE_BATCH && p + MI_COMMIT_SIZE_BATCH <= mi_segment_end(segment)) { - // size = MI_COMMIT_SIZE_BATCH; - // } + // try to commit in at least MI_MINIMAL_COMMIT_SIZE sizes. + /* + if (commit && size > 0) { + const size_t csize = _mi_align_up(size, MI_MINIMAL_COMMIT_SIZE); + if (p + csize <= mi_segment_end(segment)) { + size = csize; + } + } + */ // commit liberal, but decommit conservative uint8_t* start = NULL; size_t full_size = 0; @@ -569,13 +581,13 @@ static void mi_segment_perhaps_decommit(mi_segment_t* segment, uint8_t* p, size_ if (mi_commit_mask_is_empty(&mask) || full_size==0) return; // update delayed commit + mi_assert_internal(segment->decommit_expire > 0 || mi_commit_mask_is_empty(&segment->decommit_mask)); mi_commit_mask_t cmask; mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); // only decommit what is committed; span_free may try to decommit more mi_commit_mask_set(&segment->decommit_mask, &cmask); mi_msecs_t now = _mi_clock_now(); if (segment->decommit_expire == 0) { // no previous decommits, initialize now - mi_assert_internal(mi_commit_mask_is_empty(&segment->decommit_mask)); segment->decommit_expire = now + mi_option_get(mi_option_decommit_delay); } else if (segment->decommit_expire <= now) { @@ -609,7 +621,8 @@ static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_st mi_segment_commitx(segment, false, p, size, stats); } } - mi_commit_mask_foreach_end() + mi_commit_mask_foreach_end() + mi_assert_internal(mi_commit_mask_is_empty(&segment->decommit_mask)); } @@ -893,6 +906,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(mi_commit_mask_is_empty(&decommit_mask)); segment->decommit_expire = 0; mi_commit_mask_create_empty( &segment->decommit_mask ); + mi_assert_internal(mi_commit_mask_is_empty(&segment->decommit_mask)); } } From f2b6938d64d555f2053612da2e84fcb128bd9116 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 5 Feb 2022 17:36:14 -0800 Subject: [PATCH 7/7] fix start adjustment for the commit mask --- src/segment.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index b000e641..c4cf9875 100644 --- a/src/segment.c +++ b/src/segment.c @@ -487,7 +487,7 @@ static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uin start = _mi_align_down(pstart, MI_MINIMAL_COMMIT_SIZE); end = _mi_align_up(pstart + size, MI_MINIMAL_COMMIT_SIZE); } - if (start < segstart) { + if (pstart >= segstart && start < segstart) { // note: the mask is also calculated for an initial commit of the info area start = segstart; } if (end > segsize) {