kiper220/mimalloc
1
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2025-05-06 07:29:30 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

track more precisely if memory is fixed or committed

Browse source
This commit is contained in:
daan 2019-08-26 22:45:26 -07:00
parent eea093000a
commit db8d443ae6
6 changed files with 176 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
include/mimalloc-internal.h
View file

@ -45,8 +45,8 @@ void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocat
void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data
// memory.c // memory.c
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld); void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, size_t* id, mi_os_tld_t* tld);
void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld); void* _mi_mem_alloc(size_t size, bool commit, bool* large, size_t* id, mi_os_tld_t* tld);
void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats); void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats);
bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats); bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats);

11
include/mimalloc-types.h
View file

@ -167,7 +167,7 @@ typedef struct mi_page_s {
bool is_committed:1; // `true` if the page virtual memory is committed bool is_committed:1; // `true` if the page virtual memory is committed
// layout like this to optimize access in `mi_malloc` and `mi_free` // layout like this to optimize access in `mi_malloc` and `mi_free`
uint16_t capacity; // number of blocks committed uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear`
uint16_t reserved; // number of blocks reserved in memory uint16_t reserved; // number of blocks reserved in memory
mi_page_flags_t flags; // `in_full` and `has_aligned` flags (16 bits) mi_page_flags_t flags; // `in_full` and `has_aligned` flags (16 bits)
@ -207,7 +207,13 @@ typedef enum mi_page_kind_e {
// the OS. Inside segments we allocated fixed size _pages_ that // the OS. Inside segments we allocated fixed size _pages_ that
// contain blocks. // contain blocks.
typedef struct mi_segment_s { typedef struct mi_segment_s {
struct mi_segment_s* next; // memory fields
size_t memid; // id for the os-level memory manager
bool mem_is_fixed; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages)
bool mem_is_committed; // `true` if the whole segment is eagerly committed
// segment fields
struct mi_segment_s* next; // must be the first segment field -- see `segment.c:segment_alloc`
struct mi_segment_s* prev; struct mi_segment_s* prev;
volatile _Atomic(struct mi_segment_s*) abandoned_next; volatile _Atomic(struct mi_segment_s*) abandoned_next;
size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`)
@ -216,7 +222,6 @@ typedef struct mi_segment_s {
size_t segment_size;// for huge pages this may be different from `MI_SEGMENT_SIZE` size_t segment_size;// for huge pages this may be different from `MI_SEGMENT_SIZE`
size_t segment_info_size; // space we are using from the first page for segment meta-data and possible guard pages. size_t segment_info_size; // space we are using from the first page for segment meta-data and possible guard pages.
uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie`
size_t memid; // id for the os-level memory manager
// layout like this to optimize access in `mi_free` // layout like this to optimize access in `mi_free`
size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`).

106
src/memory.c
View file

@ -45,8 +45,8 @@ bool _mi_os_commit(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats); bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats);
bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats); bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats);
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld); void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld);
bool _mi_os_is_huge_reserved(void* p);
// Constants // Constants
#if (MI_INTPTR_SIZE==8) #if (MI_INTPTR_SIZE==8)
@ -66,11 +66,24 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld
#define MI_REGION_MAP_FULL UINTPTR_MAX #define MI_REGION_MAP_FULL UINTPTR_MAX
typedef uintptr_t mi_region_info_t;
static inline mi_region_info_t mi_region_info_create(void* start, bool is_large, bool is_committed) {
return ((uintptr_t)start | ((is_large?1:0) << 1) | (is_committed?1:0));
}
static inline void* mi_region_info_read(mi_region_info_t info, bool* is_large, bool* is_committed) {
if (is_large) *is_large = ((info&0x02) != 0);
if (is_committed) *is_committed = ((info&0x01) != 0);
return (void*)(info & ~0x03);
}
// A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with // A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with
// a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block. // a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block.
typedef struct mem_region_s { typedef struct mem_region_s {
volatile _Atomic(uintptr_t) map; // in-use bit per MI_SEGMENT_SIZE block volatile _Atomic(uintptr_t) map; // in-use bit per MI_SEGMENT_SIZE block
volatile _Atomic(void*) start; // start of virtual memory area volatile _Atomic(mi_region_info_t) info; // start of virtual memory area, and flags
} mem_region_t; } mem_region_t;
@ -108,7 +121,7 @@ bool mi_is_in_heap_region(const void* p) mi_attr_noexcept {
if (p==NULL) return false; if (p==NULL) return false;
size_t count = mi_atomic_read_relaxed(&regions_count); size_t count = mi_atomic_read_relaxed(&regions_count);
for (size_t i = 0; i < count; i++) { for (size_t i = 0; i < count; i++) {
uint8_t* start = (uint8_t*)mi_atomic_read_ptr_relaxed(&regions[i].start); uint8_t* start = (uint8_t*)mi_region_info_read( mi_atomic_read_relaxed(&regions[i].info), NULL, NULL);
if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true; if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true;
} }
return false; return false;
@ -123,7 +136,7 @@ Commit from a region
// Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
// if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call.
// (not being able to claim is not considered an error so check for `p != NULL` afterwards). // (not being able to claim is not considered an error so check for `p != NULL` afterwards).
static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, bool* large, void** p, size_t* id, mi_os_tld_t* tld)
{ {
size_t mask = mi_region_block_mask(blocks,bitidx); size_t mask = mi_region_block_mask(blocks,bitidx);
mi_assert_internal(mask != 0); mi_assert_internal(mask != 0);
@ -131,10 +144,14 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
mi_assert_internal(&regions[idx] == region); mi_assert_internal(&regions[idx] == region);
// ensure the region is reserved // ensure the region is reserved
void* start = mi_atomic_read_ptr(&region->start); mi_region_info_t info = mi_atomic_read(&region->info);
if (start == NULL) if (info == 0)
{ {
start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, mi_option_is_enabled(mi_option_eager_region_commit), tld); bool region_commit = mi_option_is_enabled(mi_option_eager_region_commit);
bool region_large = region_commit && *large;
void* start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, region_commit, &region_large, tld);
*large = region_large;
if (start == NULL) { if (start == NULL) {
// failure to allocate from the OS! unclaim the blocks and fail // failure to allocate from the OS! unclaim the blocks and fail
size_t map; size_t map;
@ -145,7 +162,8 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
} }
// set the newly allocated region // set the newly allocated region
if (mi_atomic_cas_ptr_strong(&region->start, start, NULL)) { info = mi_region_info_create(start,region_large,region_commit);
if (mi_atomic_cas_strong(&region->info, info, 0)) {
// update the region count // update the region count
mi_atomic_increment(&regions_count); mi_atomic_increment(&regions_count);
} }
@ -154,28 +172,27 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit
// we assign it to a later slot instead (up to 4 tries). // we assign it to a later slot instead (up to 4 tries).
// note: we don't need to increment the region count, this will happen on another allocation // note: we don't need to increment the region count, this will happen on another allocation
for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) { for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) {
void* s = mi_atomic_read_ptr(&regions[idx+i].start); if (mi_atomic_cas_strong(&regions[idx+i].info, info, 0)) {
if (s == NULL) { // quick test
if (mi_atomic_cas_ptr_strong(&regions[idx+i].start, start, NULL)) {
start = NULL; start = NULL;
break; break;
} }
} }
}
if (start != NULL) { if (start != NULL) {
// free it if we didn't succeed to save it to some other region // free it if we didn't succeed to save it to some other region
_mi_os_free(start, MI_REGION_SIZE, tld->stats); _mi_os_free(start, MI_REGION_SIZE, tld->stats);
} }
// and continue with the memory at our index // and continue with the memory at our index
start = mi_atomic_read_ptr(&region->start); info = mi_atomic_read(&region->info);
} }
} }
mi_assert_internal(start == mi_atomic_read_ptr(&region->start)); mi_assert_internal(info == mi_atomic_read(&region->info));
mi_assert_internal(start != NULL); mi_assert_internal(info != 0);
// Commit the blocks to memory // Commit the blocks to memory
bool region_is_committed = false;
void* start = mi_region_info_read(info,large,&region_is_committed);
void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE);
if (commit && !mi_option_is_enabled(mi_option_eager_region_commit)) { if (commit && !region_is_committed) {
_mi_os_commit(blocks_start, mi_good_commit_size(size), tld->stats); // only commit needed size (unless using large OS pages) _mi_os_commit(blocks_start, mi_good_commit_size(size), tld->stats); // only commit needed size (unless using large OS pages)
} }
@ -223,7 +240,7 @@ static inline size_t mi_bsr(uintptr_t x) {
// Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
// if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call.
// (not being able to claim is not considered an error so check for `p != NULL` afterwards). // (not being able to claim is not considered an error so check for `p != NULL` afterwards).
static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, bool* large, void** p, size_t* id, mi_os_tld_t* tld)
{ {
mi_assert_internal(p != NULL && id != NULL); mi_assert_internal(p != NULL && id != NULL);
mi_assert_internal(blocks < MI_REGION_MAP_BITS); mi_assert_internal(blocks < MI_REGION_MAP_BITS);
@ -253,7 +270,7 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
else { else {
// success, we claimed the bits // success, we claimed the bits
// now commit the block memory -- this can still fail // now commit the block memory -- this can still fail
return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, p, id, tld); return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, large, p, id, tld);
} }
} }
else { else {
@ -276,14 +293,14 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc
// Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written
// if the blocks were successfully claimed so ensure they are initialized to NULL/0 before the call. // if the blocks were successfully claimed so ensure they are initialized to NULL/0 before the call.
// (not being able to claim is not considered an error so check for `p != NULL` afterwards). // (not being able to claim is not considered an error so check for `p != NULL` afterwards).
static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, bool* large, void** p, size_t* id, mi_os_tld_t* tld)
{ {
// check if there are available blocks in the region.. // check if there are available blocks in the region..
mi_assert_internal(idx < MI_REGION_MAX); mi_assert_internal(idx < MI_REGION_MAX);
mem_region_t* region = &regions[idx]; mem_region_t* region = &regions[idx];
uintptr_t m = mi_atomic_read_relaxed(&region->map); uintptr_t m = mi_atomic_read_relaxed(&region->map);
if (m != MI_REGION_MAP_FULL) { // some bits are zero if (m != MI_REGION_MAP_FULL) { // some bits are zero
return mi_region_alloc_blocks(region, idx, blocks, size, commit, p, id, tld); return mi_region_alloc_blocks(region, idx, blocks, size, commit, large, p, id, tld);
} }
else { else {
return true; // no error, but no success either return true; // no error, but no success either
@ -296,15 +313,17 @@ static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, b
// Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`. // Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`.
// (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`) // (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`)
void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld) void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, size_t* id, mi_os_tld_t* tld)
{ {
mi_assert_internal(id != NULL && tld != NULL); mi_assert_internal(id != NULL && tld != NULL);
mi_assert_internal(size > 0); mi_assert_internal(size > 0);
*id = SIZE_MAX; *id = SIZE_MAX;
bool default_large = false;
if (large==NULL) large = &default_large; // ensure `large != NULL`
// use direct OS allocation for huge blocks or alignment (with `id = SIZE_MAX`) // use direct OS allocation for huge blocks or alignment (with `id = SIZE_MAX`)
if (size > MI_REGION_MAX_ALLOC_SIZE || alignment > MI_SEGMENT_ALIGN) { if (size > MI_REGION_MAX_ALLOC_SIZE || alignment > MI_SEGMENT_ALIGN) {
return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, true, tld); // round up size return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, commit, large, tld); // round up size
} }
// always round size to OS page size multiple (so commit/decommit go over the entire range) // always round size to OS page size multiple (so commit/decommit go over the entire range)
@ -318,27 +337,27 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t*
// find a range of free blocks // find a range of free blocks
void* p = NULL; void* p = NULL;
size_t count = mi_atomic_read(&regions_count); size_t count = mi_atomic_read(&regions_count);
size_t idx = 0; // tld->region_idx; // start index is per-thread to reduce contention size_t idx = 0; // tld->region_idx; // start at 0 to reuse low addresses? Or, use tld->region_idx to reduce contention?
for (size_t visited = 0; visited < count; visited++, idx++) { for (size_t visited = 0; visited < count; visited++, idx++) {
if (idx >= count) idx = 0; // wrap around if (idx >= count) idx = 0; // wrap around
if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, large, &p, id, tld)) return NULL; // error
if (p != NULL) break; if (p != NULL) break;
} }
if (p == NULL) { if (p == NULL) {
// no free range in existing regions -- try to extend beyond the count.. but at most 4 regions // no free range in existing regions -- try to extend beyond the count.. but at most 4 regions
for (idx = count; idx < count + 4 && idx < MI_REGION_MAX; idx++) { for (idx = count; idx < count + 4 && idx < MI_REGION_MAX; idx++) {
if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, large, &p, id, tld)) return NULL; // error
if (p != NULL) break; if (p != NULL) break;
} }
} }
if (p == NULL) { if (p == NULL) {
// we could not find a place to allocate, fall back to the os directly // we could not find a place to allocate, fall back to the os directly
p = _mi_os_alloc_aligned(size, alignment, commit, tld); p = _mi_os_alloc_aligned(size, alignment, commit, large, tld);
} }
else { else {
tld->region_idx = idx; // next start of search tld->region_idx = idx; // next start of search?
} }
mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0); mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0);
@ -347,8 +366,8 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t*
// Allocate `size` memory. Return non NULL on success, with a given memory `id`. // Allocate `size` memory. Return non NULL on success, with a given memory `id`.
void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld) { void* _mi_mem_alloc(size_t size, bool commit, bool* large, size_t* id, mi_os_tld_t* tld) {
return _mi_mem_alloc_aligned(size,0,commit,id,tld); return _mi_mem_alloc_aligned(size,0,commit,large,id,tld);
} }
/* ---------------------------------------------------------------------------- /* ----------------------------------------------------------------------------
@ -377,7 +396,10 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) {
mi_assert_internal(idx < MI_REGION_MAX); if (idx >= MI_REGION_MAX) return; // or `abort`? mi_assert_internal(idx < MI_REGION_MAX); if (idx >= MI_REGION_MAX) return; // or `abort`?
mem_region_t* region = &regions[idx]; mem_region_t* region = &regions[idx];
mi_assert_internal((mi_atomic_read_relaxed(&region->map) & mask) == mask ); // claimed? mi_assert_internal((mi_atomic_read_relaxed(&region->map) & mask) == mask ); // claimed?
void* start = mi_atomic_read_ptr(&region->start); mi_region_info_t info = mi_atomic_read(&region->info);
bool is_large;
bool is_eager_committed;
void* start = mi_region_info_read(info,&is_large,&is_eager_committed);
mi_assert_internal(start != NULL); mi_assert_internal(start != NULL);
void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE);
mi_assert_internal(blocks_start == p); // not a pointer in our area? mi_assert_internal(blocks_start == p); // not a pointer in our area?
@ -388,18 +410,13 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) {
// TODO: implement delayed decommit/reset as these calls are too expensive // TODO: implement delayed decommit/reset as these calls are too expensive
// if the memory is reused soon. // if the memory is reused soon.
// reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large // reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large
if (!mi_option_is_enabled(mi_option_large_os_pages)) { if (!is_large) {
if (mi_option_is_enabled(mi_option_eager_region_commit)) {
// _mi_os_reset(p,size,stats); // _mi_os_reset(p,size,stats);
} // _mi_os_decommit(p,size,stats); // if !is_committed
else {
//_mi_os_decommit(p, size, stats);
}
} }
// TODO: should we free empty regions? currently only done _mi_mem_collect. // TODO: should we free empty regions? currently only done _mi_mem_collect.
// this frees up virtual address space which // this frees up virtual address space which might be useful on 32-bit systems?
// might be useful on 32-bit systems?
// and unclaim // and unclaim
uintptr_t map; uintptr_t map;
@ -419,17 +436,20 @@ void _mi_mem_collect(mi_stats_t* stats) {
// free every region that has no segments in use. // free every region that has no segments in use.
for (size_t i = 0; i < regions_count; i++) { for (size_t i = 0; i < regions_count; i++) {
mem_region_t* region = &regions[i]; mem_region_t* region = &regions[i];
if (mi_atomic_read_relaxed(&region->map) == 0 && region->start != NULL) { if (mi_atomic_read_relaxed(&region->map) == 0) {
// if no segments used, try to claim the whole region // if no segments used, try to claim the whole region
uintptr_t m; uintptr_t m;
do { do {
m = mi_atomic_read_relaxed(&region->map); m = mi_atomic_read_relaxed(&region->map);
} while(m == 0 && !mi_atomic_cas_weak(&region->map, ~((uintptr_t)0), 0 )); } while(m == 0 && !mi_atomic_cas_weak(&region->map, ~((uintptr_t)0), 0 ));
if (m == 0) { if (m == 0) {
// on success, free the whole region // on success, free the whole region (unless it was huge reserved)
if (region->start != NULL) _mi_os_free((void*)region->start, MI_REGION_SIZE, stats); void* start = mi_region_info_read(mi_atomic_read(&region->info), NULL, NULL);
if (start != NULL && !_mi_os_is_huge_reserved(start)) {
_mi_os_free(start, MI_REGION_SIZE, stats);
}
// and release // and release
mi_atomic_write_ptr(&region->start,NULL); mi_atomic_write(&region->info,0);
mi_atomic_write(&region->map,0); mi_atomic_write(&region->map,0);
} }
} }

2
src/options.c
View file

@ -58,7 +58,7 @@ static mi_option_desc_t options[_mi_option_last] =
#endif #endif
// the following options are experimental and not all combinations make sense. // the following options are experimental and not all combinations make sense.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // note: if eager_region_commit is on, this should be on too. { 1, UNINIT, MI_OPTION(eager_commit) }, // note: needs to be on when eager_region_commit is enabled
#ifdef _WIN32 // and BSD? #ifdef _WIN32 // and BSD?
{ 1, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...) { 1, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...)
#else #else

92
src/os.c
View file

@ -36,8 +36,7 @@ terms of the MIT license. A copy of the license can be found in the file
large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
----------------------------------------------------------- */ ----------------------------------------------------------- */
bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats);
bool _mi_os_is_huge_reserved(void* p);
static bool mi_os_is_huge_reserved(void* p);
static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit); static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit);
static void* mi_align_up_ptr(void* p, size_t alignment) { static void* mi_align_up_ptr(void* p, size_t alignment) {
@ -173,7 +172,7 @@ void _mi_os_init() {
static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats) static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats)
{ {
if (addr == NULL || size == 0 || mi_os_is_huge_reserved(addr)) return true; if (addr == NULL || size == 0 || _mi_os_is_huge_reserved(addr)) return true;
bool err = false; bool err = false;
#if defined(_WIN32) #if defined(_WIN32)
err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); err = (VirtualFree(addr, 0, MEM_RELEASE) == 0);
@ -199,7 +198,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
#if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS)
// on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages
if ((size % ((uintptr_t)1 << 30)) == 0 /* 1GiB multiple */ if ((size % ((uintptr_t)1 << 30)) == 0 /* 1GiB multiple */
&& (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0 && (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0 && (flags & MEM_RESERVE) != 0
&& (addr != NULL || try_alignment == 0 || try_alignment % _mi_os_page_size() == 0) && (addr != NULL || try_alignment == 0 || try_alignment % _mi_os_page_size() == 0)
&& pNtAllocateVirtualMemoryEx != NULL) && pNtAllocateVirtualMemoryEx != NULL)
{ {
@ -211,7 +210,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
param.ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; param.ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE;
SIZE_T psize = size; SIZE_T psize = size;
void* base = addr; void* base = addr;
NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags | MEM_RESERVE, PAGE_READWRITE, &param, 1); NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, &param, 1);
if (err == 0) { if (err == 0) {
return base; return base;
} }
@ -247,10 +246,12 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment
return VirtualAlloc(addr, size, flags, PAGE_READWRITE); return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
} }
static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only) { static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
mi_assert_internal(!(large_only && !allow_large));
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0; static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
void* p = NULL; void* p = NULL;
if (large_only || use_large_os_page(size, try_alignment)) { if ((large_only || use_large_os_page(size, try_alignment))
&& allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) {
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
if (!large_only && try_ok > 0) { if (!large_only && try_ok > 0) {
// if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive.
@ -259,7 +260,8 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
} }
else { else {
// large OS pages must always reserve and commit. // large OS pages must always reserve and commit.
p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE | flags); *is_large = true;
p = mi_win_virtual_allocx(addr, size, try_alignment, flags | MEM_LARGE_PAGES);
if (large_only) return p; if (large_only) return p;
// fall back to non-large page allocation on error (`p == NULL`). // fall back to non-large page allocation on error (`p == NULL`).
if (p == NULL) { if (p == NULL) {
@ -268,6 +270,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment,
} }
} }
if (p == NULL) { if (p == NULL) {
*is_large = ((flags&MEM_LARGE_PAGES) != 0);
p = mi_win_virtual_allocx(addr, size, try_alignment, flags); p = mi_win_virtual_allocx(addr, size, try_alignment, flags);
} }
if (p == NULL) { if (p == NULL) {
@ -311,7 +314,7 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr
return p; return p;
} }
static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only) { static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) {
void* p = NULL; void* p = NULL;
#if !defined(MAP_ANONYMOUS) #if !defined(MAP_ANONYMOUS)
#define MAP_ANONYMOUS MAP_ANON #define MAP_ANONYMOUS MAP_ANON
@ -333,7 +336,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
// macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99)
fd = VM_MAKE_TAG(100); fd = VM_MAKE_TAG(100);
#endif #endif
if (large_only || use_large_os_page(size, try_alignment)) { if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) {
static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0; static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0;
uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); uintptr_t try_ok = mi_atomic_read(&large_page_try_ok);
if (!large_only && try_ok > 0) { if (!large_only && try_ok > 0) {
@ -368,6 +371,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
#endif #endif
if (large_only || lflags != flags) { if (large_only || lflags != flags) {
// try large OS page allocation // try large OS page allocation
*is_large = true;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd); p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd);
#ifdef MAP_HUGE_1GB #ifdef MAP_HUGE_1GB
if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) {
@ -384,6 +388,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
} }
} }
if (p == NULL) { if (p == NULL) {
*is_large = false;
p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd);
#if defined(MADV_HUGEPAGE) #if defined(MADV_HUGEPAGE)
// Many Linux systems don't allow MAP_HUGETLB but they support instead // Many Linux systems don't allow MAP_HUGETLB but they support instead
@ -392,8 +397,10 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
// in that case -- in particular for our large regions (in `memory.c`). // in that case -- in particular for our large regions (in `memory.c`).
// However, some systems only allow TPH if called with explicit `madvise`, so // However, some systems only allow TPH if called with explicit `madvise`, so
// when large OS pages are enabled for mimalloc, we call `madvice` anyways. // when large OS pages are enabled for mimalloc, we call `madvice` anyways.
if (use_large_os_page(size, try_alignment)) { if (allow_large && use_large_os_page(size, try_alignment)) {
madvise(p, size, MADV_HUGEPAGE); if (madvise(p, size, MADV_HUGEPAGE) == 0) {
*is_large = true; // possibly
};
} }
#endif #endif
} }
@ -403,27 +410,35 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
// Primitive allocation from the OS. // Primitive allocation from the OS.
// Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned.
static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) { static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) {
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
if (size == 0) return NULL; if (size == 0) return NULL;
if (!commit) allow_large = false;
void* p = mi_os_alloc_from_huge_reserved(size, try_alignment, commit); void* p = NULL;
if (p != NULL) return p; if (allow_large) {
p = mi_os_alloc_from_huge_reserved(size, try_alignment, commit);
if (p != NULL) {
*is_large = true;
return p;
}
}
#if defined(_WIN32) #if defined(_WIN32)
int flags = MEM_RESERVE; int flags = MEM_RESERVE;
if (commit) flags |= MEM_COMMIT; if (commit) flags |= MEM_COMMIT;
p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false); p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large);
#elif defined(__wasi__) #elif defined(__wasi__)
*is_large = false;
p = mi_wasm_heap_grow(size, try_alignment); p = mi_wasm_heap_grow(size, try_alignment);
#else #else
int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE);
p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false); p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large);
#endif #endif
_mi_stat_increase(&stats->mmap_calls, 1); _mi_stat_increase(&stats->mmap_calls, 1);
if (p != NULL) { if (p != NULL) {
_mi_stat_increase(&stats->reserved, size); _mi_stat_increase(&stats->reserved, size);
if (commit) _mi_stat_increase(&stats->committed, size); if (commit) { _mi_stat_increase(&stats->committed, size); }
} }
return p; return p;
} }
@ -431,14 +446,15 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_
// Primitive aligned allocation from the OS. // Primitive aligned allocation from the OS.
// This function guarantees the allocated memory is aligned. // This function guarantees the allocated memory is aligned.
static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) { static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) {
mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0));
mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0);
if (!commit) allow_large = false;
if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL;
size = _mi_align_up(size, _mi_os_page_size()); size = _mi_align_up(size, _mi_os_page_size());
// try first with a hint (this will be aligned directly on Win 10+ or BSD) // try first with a hint (this will be aligned directly on Win 10+ or BSD)
void* p = mi_os_mem_alloc(size, alignment, commit, stats); void* p = mi_os_mem_alloc(size, alignment, commit, allow_large, is_large, stats);
if (p == NULL) return NULL; if (p == NULL) return NULL;
// if not aligned, free it, overallocate, and unmap around it // if not aligned, free it, overallocate, and unmap around it
@ -457,7 +473,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
if (commit) flags |= MEM_COMMIT; if (commit) flags |= MEM_COMMIT;
for (int tries = 0; tries < 3; tries++) { for (int tries = 0; tries < 3; tries++) {
// over-allocate to determine a virtual memory range // over-allocate to determine a virtual memory range
p = mi_os_mem_alloc(over_size, alignment, commit, stats); p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats);
if (p == NULL) return NULL; // error if (p == NULL) return NULL; // error
if (((uintptr_t)p % alignment) == 0) { if (((uintptr_t)p % alignment) == 0) {
// if p happens to be aligned, just decommit the left-over area // if p happens to be aligned, just decommit the left-over area
@ -468,7 +484,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
// otherwise free and allocate at an aligned address in there // otherwise free and allocate at an aligned address in there
mi_os_mem_free(p, over_size, stats); mi_os_mem_free(p, over_size, stats);
void* aligned_p = mi_align_up_ptr(p, alignment); void* aligned_p = mi_align_up_ptr(p, alignment);
p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large);
if (p == aligned_p) break; // success! if (p == aligned_p) break; // success!
if (p != NULL) { // should not happen? if (p != NULL) { // should not happen?
mi_os_mem_free(p, size, stats); mi_os_mem_free(p, size, stats);
@ -478,7 +494,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
} }
#else #else
// overallocate... // overallocate...
p = mi_os_mem_alloc(over_size, alignment, commit, stats); p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats);
if (p == NULL) return NULL; if (p == NULL) return NULL;
// and selectively unmap parts around the over-allocated area. // and selectively unmap parts around the over-allocated area.
void* aligned_p = mi_align_up_ptr(p, alignment); void* aligned_p = mi_align_up_ptr(p, alignment);
@ -504,7 +520,8 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit,
void* _mi_os_alloc(size_t size, mi_stats_t* stats) { void* _mi_os_alloc(size_t size, mi_stats_t* stats) {
if (size == 0) return NULL; if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, 0); size = mi_os_good_alloc_size(size, 0);
return mi_os_mem_alloc(size, 0, true, stats); bool is_large = false;
return mi_os_mem_alloc(size, 0, true, false, &is_large, stats);
} }
void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
@ -513,12 +530,17 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats) {
mi_os_mem_free(p, size, stats); mi_os_mem_free(p, size, stats);
} }
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld) void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld)
{ {
if (size == 0) return NULL; if (size == 0) return NULL;
size = mi_os_good_alloc_size(size, alignment); size = mi_os_good_alloc_size(size, alignment);
alignment = _mi_align_up(alignment, _mi_os_page_size()); alignment = _mi_align_up(alignment, _mi_os_page_size());
return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats); bool allow_large = false;
if (large != NULL) {
allow_large = *large;
*large = false;
}
return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large!=NULL?large:&allow_large), tld->stats);
} }
@ -559,7 +581,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
// page align in the range, commit liberally, decommit conservative // page align in the range, commit liberally, decommit conservative
size_t csize; size_t csize;
void* start = mi_os_page_align_areax(conservative, addr, size, &csize); void* start = mi_os_page_align_areax(conservative, addr, size, &csize);
if (csize == 0 || mi_os_is_huge_reserved(addr)) return true; if (csize == 0 || _mi_os_is_huge_reserved(addr)) return true;
int err = 0; int err = 0;
if (commit) { if (commit) {
_mi_stat_increase(&stats->committed, csize); _mi_stat_increase(&stats->committed, csize);
@ -611,7 +633,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
// page align conservatively within the range // page align conservatively within the range
size_t csize; size_t csize;
void* start = mi_os_page_align_area_conservative(addr, size, &csize); void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0 || mi_os_is_huge_reserved(addr)) return true; if (csize == 0 || _mi_os_is_huge_reserved(addr)) return true;
if (reset) _mi_stat_increase(&stats->reset, csize); if (reset) _mi_stat_increase(&stats->reset, csize);
else _mi_stat_decrease(&stats->reset, csize); else _mi_stat_decrease(&stats->reset, csize);
if (!reset) return true; // nothing to do on unreset! if (!reset) return true; // nothing to do on unreset!
@ -626,6 +648,11 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
// Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory
void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE);
mi_assert_internal(p == start); mi_assert_internal(p == start);
#if 0
if (p == start) {
VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set
}
#endif
if (p != start) return false; if (p != start) return false;
#else #else
#if defined(MADV_FREE) #if defined(MADV_FREE)
@ -679,7 +706,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) {
size_t csize = 0; size_t csize = 0;
void* start = mi_os_page_align_area_conservative(addr, size, &csize); void* start = mi_os_page_align_area_conservative(addr, size, &csize);
if (csize == 0) return false; if (csize == 0) return false;
if (mi_os_is_huge_reserved(addr)) { if (_mi_os_is_huge_reserved(addr)) {
_mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n");
} }
int err = 0; int err = 0;
@ -742,7 +769,7 @@ typedef struct mi_huge_info_s {
static mi_huge_info_t os_huge_reserved = { NULL, 0, ATOMIC_VAR_INIT(0) }; static mi_huge_info_t os_huge_reserved = { NULL, 0, ATOMIC_VAR_INIT(0) };
static bool mi_os_is_huge_reserved(void* p) { bool _mi_os_is_huge_reserved(void* p) {
return (mi_atomic_read_ptr(&os_huge_reserved.start) != NULL && return (mi_atomic_read_ptr(&os_huge_reserved.start) != NULL &&
p >= mi_atomic_read_ptr(&os_huge_reserved.start) && p >= mi_atomic_read_ptr(&os_huge_reserved.start) &&
(uint8_t*)p < (uint8_t*)mi_atomic_read_ptr(&os_huge_reserved.start) + mi_atomic_read(&os_huge_reserved.reserved)); (uint8_t*)p < (uint8_t*)mi_atomic_read_ptr(&os_huge_reserved.start) + mi_atomic_read(&os_huge_reserved.reserved));
@ -806,10 +833,11 @@ int mi_reserve_huge_os_pages( size_t pages, double max_secs ) mi_attr_noexcept
for (size_t page = 0; page < pages; page++, addr += MI_HUGE_OS_PAGE_SIZE ) { for (size_t page = 0; page < pages; page++, addr += MI_HUGE_OS_PAGE_SIZE ) {
// allocate lorgu pages // allocate lorgu pages
void* p = NULL; void* p = NULL;
bool is_large = true;
#ifdef _WIN32 #ifdef _WIN32
p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true); p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true, true, &is_large);
#elif defined(MI_OS_USE_MMAP) #elif defined(MI_OS_USE_MMAP)
p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true); p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true, true, &is_large);
#else #else
// always fail // always fail
#endif #endif

51
src/segment.c
View file

@ -229,6 +229,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
segment->thread_id = 0; segment->thread_id = 0;
mi_segments_track_size(-((long)segment_size),tld); mi_segments_track_size(-((long)segment_size),tld);
if (mi_option_is_enabled(mi_option_secure)) { if (mi_option_is_enabled(mi_option_secure)) {
mi_assert_internal(!segment->mem_is_fixed);
_mi_mem_unprotect(segment, segment->segment_size); // ensure no more guard pages are set _mi_mem_unprotect(segment, segment->segment_size); // ensure no more guard pages are set
} }
_mi_mem_free(segment, segment_size, segment->memid, tld->stats); _mi_mem_free(segment, segment_size, segment->memid, tld->stats);
@ -277,7 +278,7 @@ static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld)
return false; return false;
} }
mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE);
if (mi_option_is_enabled(mi_option_cache_reset)) { if (!segment->mem_is_fixed && mi_option_is_enabled(mi_option_cache_reset)) {
_mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); _mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats);
} }
segment->next = tld->cache; segment->next = tld->cache;
@ -325,11 +326,13 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
size_t page_size = (page_kind == MI_PAGE_HUGE ? segment_size : (size_t)1 << page_shift); size_t page_size = (page_kind == MI_PAGE_HUGE ? segment_size : (size_t)1 << page_shift);
// Try to get it from our thread local cache first // Try to get it from our thread local cache first
bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM); bool eager = mi_option_is_enabled(mi_option_eager_commit);
bool commit = eager || (page_kind > MI_PAGE_MEDIUM);
bool protection_still_good = false; bool protection_still_good = false;
mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld); mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld);
if (segment != NULL) { if (segment != NULL) {
if (mi_option_is_enabled(mi_option_secure)) { if (mi_option_is_enabled(mi_option_secure)) {
mi_assert_internal(!segment->mem_is_fixed);
if (segment->page_kind != page_kind) { if (segment->page_kind != page_kind) {
_mi_mem_unprotect(segment, segment->segment_size); // reset protection if the page kind differs _mi_mem_unprotect(segment, segment->segment_size); // reset protection if the page kind differs
} }
@ -337,37 +340,38 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
protection_still_good = true; // otherwise, the guard pages are still in place protection_still_good = true; // otherwise, the guard pages are still in place
} }
} }
if (!mi_option_is_enabled(mi_option_eager_commit)) { if (!segment->mem_is_committed && page_kind > MI_PAGE_MEDIUM) {
if (page_kind > MI_PAGE_MEDIUM) { mi_assert_internal(!segment->mem_is_fixed);
_mi_mem_commit(segment, segment->segment_size, tld->stats); _mi_mem_commit(segment, segment->segment_size, tld->stats);
segment->mem_is_committed = true;
} }
else { if (!segment->mem_is_fixed &&
// ok, commit (and unreset) on demand again (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) {
}
}
else if (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)) {
_mi_mem_unreset(segment, segment->segment_size, tld->stats); _mi_mem_unreset(segment, segment->segment_size, tld->stats);
} }
} }
else { else {
// Allocate the segment from the OS // Allocate the segment from the OS
size_t memid; size_t memid;
segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &memid, os_tld); bool mem_large = (eager && !mi_option_is_enabled(mi_option_secure)); // only allow large OS pages once we are no longer lazy
segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &mem_large, &memid, os_tld);
if (segment == NULL) return NULL; // failed to allocate if (segment == NULL) return NULL; // failed to allocate
if (!commit) { if (!commit) {
// ensure the initial info is committed
_mi_mem_commit(segment, info_size, tld->stats); _mi_mem_commit(segment, info_size, tld->stats);
} }
segment->memid = memid; segment->memid = memid;
segment->mem_is_fixed = mem_large;
segment->mem_is_committed = commit;
mi_segments_track_size((long)segment_size, tld); mi_segments_track_size((long)segment_size, tld);
} }
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
// zero the segment info // zero the segment info (but not the `mem` fields)
{ size_t memid = segment->memid; ptrdiff_t ofs = offsetof(mi_segment_t,next);
memset(segment, 0, info_size); memset((uint8_t*)segment + ofs, 0, info_size - ofs);
segment->memid = memid;
}
// guard pages
if (mi_option_is_enabled(mi_option_secure) && !protection_still_good) { if (mi_option_is_enabled(mi_option_secure) && !protection_still_good) {
// in secure mode, we set up a protected page in between the segment info // in secure mode, we set up a protected page in between the segment info
// and the page data // and the page data
@ -386,6 +390,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind,
} }
} }
// initialize
segment->page_kind = page_kind; segment->page_kind = page_kind;
segment->capacity = capacity; segment->capacity = capacity;
segment->page_shift = page_shift; segment->page_shift = page_shift;
@ -454,12 +459,13 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats)
if (page->is_reset || !page->is_committed) { if (page->is_reset || !page->is_committed) {
size_t psize; size_t psize;
uint8_t* start = _mi_page_start(segment, page, &psize); uint8_t* start = _mi_page_start(segment, page, &psize);
mi_assert_internal(!(page->is_reset && !page->is_committed));
if (!page->is_committed) { if (!page->is_committed) {
mi_assert_internal(!segment->mem_is_fixed);
page->is_committed = true; page->is_committed = true;
_mi_mem_commit(start,psize,stats); _mi_mem_commit(start,psize,stats);
} }
if (page->is_reset) { if (page->is_reset) {
mi_assert_internal(!segment->mem_is_fixed);
page->is_reset = false; page->is_reset = false;
_mi_mem_unreset(start, psize, stats); _mi_mem_unreset(start, psize, stats);
} }
@ -488,22 +494,17 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta
_mi_stat_decrease(&stats->pages, 1); _mi_stat_decrease(&stats->pages, 1);
// reset the page memory to reduce memory pressure? // reset the page memory to reduce memory pressure?
if (!page->is_reset && mi_option_is_enabled(mi_option_page_reset)) { if (!segment->mem_is_fixed && !page->is_reset && mi_option_is_enabled(mi_option_page_reset)) {
size_t psize; size_t psize;
uint8_t* start = _mi_page_start(segment, page, &psize); uint8_t* start = _mi_page_start(segment, page, &psize);
page->is_reset = true; page->is_reset = true;
_mi_mem_reset(start, psize, stats); _mi_mem_reset(start, psize, stats);
} }
// zero the page data // zero the page data, but not the segment fields
uint8_t idx = page->segment_idx; // don't clear the index ptrdiff_t ofs = offsetof(mi_page_t,capacity);
bool is_reset = page->is_reset; // don't clear the reset flag memset((uint8_t*)page + ofs, 0, sizeof(*page) - ofs);
bool is_committed = page->is_committed; // don't clear the commit flag
memset(page, 0, sizeof(*page));
page->segment_idx = idx;
page->segment_in_use = false; page->segment_in_use = false;
page->is_reset = is_reset;
page->is_committed = is_committed;
segment->used--; segment->used--;
} }