kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 15:29:31 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

add comments, renaming

Browse source
This commit is contained in:
daan 2021-11-13 14:03:16 -08:00
parent 8bf16746e9
commit 9afc253726
3 changed files with 70 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
include/mimalloc-internal.h
View file

@ -19,6 +19,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_CACHE_LINE 64
#if defined(_MSC_VER)
#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
#pragma warning(disable:26812) // unscoped enum warning
#define mi_decl_noinline __declspec(noinline)
#define mi_decl_thread __declspec(thread)
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
@ -696,72 +697,39 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c
// commit mask
// -------------------------------------------------------------------
static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
cm->mask[i] = 0;
}
}
static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
cm->mask[i] = ~((size_t)0);
}
}
static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
if (cm->mask[i] != 0) return false;
}
return true;
}
static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
if (cm->mask[i] != 0) return false;
}
return true;
}
static inline bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false;
}
return true;
}
static inline bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
if ((commit->mask[i] & cm->mask[i]) != 0) return true;
}
return false;
}
static inline void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
res->mask[i] = (commit->mask[i] & cm->mask[i]);
}
}
static inline void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
res->mask[i] &= ~(cm->mask[i]);
}
}
static inline void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
res->mask[i] |= cm->mask[i];
}
}
void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm);
size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total);
ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx);
// defined in `segment.c`:
size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total);
ptrdiff_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx);
#define mi_commit_mask_foreach(cm,idx,count) \
idx = 0; \
while ((count = mi_commit_mask_next_run(cm,&idx)) > 0) {
while ((count = _mi_commit_mask_next_run(cm,&idx)) > 0) {
#define mi_commit_mask_foreach_end() \
idx += count; \

21
include/mimalloc-types.h
View file

@ -287,17 +287,26 @@ typedef enum mi_segment_kind_e {
MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside.
} mi_segment_kind_t;
#define MI_COMMIT_SIZE (4*64*1024)
#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE)
#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS
#define MI_COMMIT_MASK_N (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS)
// ------------------------------------------------------
// A segment holds a commit mask where a bit is set if
// the corresponding MI_COMMIT_SIZE area is committed.
// The MI_COMMIT_SIZE must be a multiple of the slice
// size. We define it as equal so we can decommit on a
// slice level which helps with (real) memory fragmentation
// over time.
// ------------------------------------------------------
#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_N * MI_COMMIT_MASK_FIELD_BITS))
#define MI_COMMIT_SIZE (MI_SEGMENT_SLICE_SIZE)
#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE)
#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS
#define MI_COMMIT_MASK_FIELD_COUNT (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS)
#if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_FIELD_COUNT * MI_COMMIT_MASK_FIELD_BITS))
#error "the segment size must be exactly divisible by the (commit size * size_t bits)"
#endif
typedef struct mi_commit_mask_s {
size_t mask[MI_COMMIT_MASK_N];
size_t mask[MI_COMMIT_MASK_FIELD_COUNT];
} mi_commit_mask_t;
typedef mi_page_t mi_slice_t;

68
src/segment.c
View file

@ -15,11 +15,44 @@ terms of the MIT license. A copy of the license can be found in the file
static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_stats_t* stats);
// -------------------------------------------------------------------
// commit mask
// commit mask
// -------------------------------------------------------------------
void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm) {
static bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false;
}
return true;
}
static bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
if ((commit->mask[i] & cm->mask[i]) != 0) return true;
}
return false;
}
static void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
res->mask[i] = (commit->mask[i] & cm->mask[i]);
}
}
static void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
res->mask[i] &= ~(cm->mask[i]);
}
}
static void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
res->mask[i] |= cm->mask[i];
}
}
static void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm) {
mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
if (bitcount == MI_COMMIT_MASK_BITS) {
@ -34,7 +67,7 @@ void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_
ptrdiff_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS;
ptrdiff_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS;
while (bitcount > 0) {
mi_assert_internal(i < MI_COMMIT_MASK_N);
mi_assert_internal(i < MI_COMMIT_MASK_FIELD_COUNT);
ptrdiff_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs;
ptrdiff_t count = (bitcount > avail ? avail : bitcount);
size_t mask = (count >= MI_COMMIT_MASK_FIELD_BITS ? ~((size_t)0) : (((size_t)1 << count) - 1) << ofs);
@ -46,11 +79,10 @@ void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_
}
}
size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) {
size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) {
mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
size_t count = 0;
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_FIELD_COUNT; i++) {
size_t mask = cm->mask[i];
if (~mask == 0) {
count += MI_COMMIT_MASK_FIELD_BITS;
@ -66,12 +98,12 @@ size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) {
}
ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) {
ptrdiff_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) {
ptrdiff_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS;
ptrdiff_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS;
size_t mask = 0;
// find first ones
while (i < MI_COMMIT_MASK_N) {
while (i < MI_COMMIT_MASK_FIELD_COUNT) {
mask = cm->mask[i];
mask >>= ofs;
if (mask != 0) {
@ -84,7 +116,7 @@ ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) {
i++;
ofs = 0;
}
if (i >= MI_COMMIT_MASK_N) {
if (i >= MI_COMMIT_MASK_FIELD_COUNT) {
// not found
*idx = MI_COMMIT_MASK_BITS;
return 0;
@ -101,7 +133,7 @@ ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) {
} while ((mask&1) == 1);
if ((((*idx + count) % MI_COMMIT_MASK_FIELD_BITS) == 0)) {
i++;
if (i >= MI_COMMIT_MASK_N) break;
if (i >= MI_COMMIT_MASK_FIELD_COUNT) break;
mask = cm->mask[i];
ofs = 0;
}
@ -111,14 +143,6 @@ ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx) {
}
}
#define mi_commit_mask_foreach(cm,idx,count) \
idx = 0; \
while ((count = mi_commit_mask_next_run(cm,&idx)) > 0) {
#define mi_commit_mask_foreach_end() \
idx += count; \
}
/* --------------------------------------------------------------------------------
Segment allocation
@ -289,7 +313,7 @@ static size_t mi_segment_info_size(mi_segment_t* segment) {
static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t xblock_size, size_t* page_size)
{
ptrdiff_t idx = slice - segment->slices;
size_t psize = slice->slice_count*MI_SEGMENT_SLICE_SIZE;
size_t psize = (size_t)slice->slice_count * MI_SEGMENT_SLICE_SIZE;
// make the start not OS page aligned for smaller blocks to avoid page/cache effects
size_t start_offset = (xblock_size >= MI_INTPTR_SIZE && xblock_size <= 1024 ? MI_MAX_ALIGN_GUARANTEE : 0);
if (page_size != NULL) *page_size = psize - start_offset;
@ -362,7 +386,7 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
// _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
const size_t size = mi_segment_size(segment);
if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, &segment->decommit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) {
const size_t csize = mi_commit_mask_committed_size(&segment->commit_mask, size);
const size_t csize = _mi_commit_mask_committed_size(&segment->commit_mask, size);
if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize);
_mi_abandoned_await_readers(); // wait until safe to free
_mi_arena_free(segment, mi_segment_size(segment), segment->memid, segment->mem_is_pinned /* pretend not committed to not double count decommits */, tld->os);
@ -502,7 +526,7 @@ static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s
bool is_zero = false;
mi_commit_mask_t cmask;
mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_decrease(&_mi_stats_main.committed, mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
_mi_stat_decrease(&_mi_stats_main.committed, _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false;
mi_commit_mask_set(&segment->commit_mask, &mask);
}
@ -512,7 +536,7 @@ static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, s
mi_commit_mask_t cmask;
mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_increase(&_mi_stats_main.committed, full_size - mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
_mi_stat_increase(&_mi_stats_main.committed, full_size - _mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
if (segment->allow_decommit) {
_mi_os_decommit(start, full_size, stats); // ok if this fails
}