diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 447df7df..9b3a3907 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -45,8 +45,8 @@ void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocat void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data // memory.c -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld); -void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld); +void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, size_t* id, mi_os_tld_t* tld); +void* _mi_mem_alloc(size_t size, bool commit, bool* large, size_t* id, mi_os_tld_t* tld); void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats); bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats); diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 0b2334b8..4bf51d1d 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -167,7 +167,7 @@ typedef struct mi_page_s { bool is_committed:1; // `true` if the page virtual memory is committed // layout like this to optimize access in `mi_malloc` and `mi_free` - uint16_t capacity; // number of blocks committed + uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` uint16_t reserved; // number of blocks reserved in memory mi_page_flags_t flags; // `in_full` and `has_aligned` flags (16 bits) @@ -207,7 +207,13 @@ typedef enum mi_page_kind_e { // the OS. Inside segments we allocated fixed size _pages_ that // contain blocks. typedef struct mi_segment_s { - struct mi_segment_s* next; + // memory fields + size_t memid; // id for the os-level memory manager + bool mem_is_fixed; // `true` if we cannot decommit/reset/protect in this memory (i.e. when allocated using large OS pages) + bool mem_is_committed; // `true` if the whole segment is eagerly committed + + // segment fields + struct mi_segment_s* next; // must be the first segment field -- see `segment.c:segment_alloc` struct mi_segment_s* prev; volatile _Atomic(struct mi_segment_s*) abandoned_next; size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) @@ -216,7 +222,6 @@ typedef struct mi_segment_s { size_t segment_size;// for huge pages this may be different from `MI_SEGMENT_SIZE` size_t segment_info_size; // space we are using from the first page for segment meta-data and possible guard pages. uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` - size_t memid; // id for the os-level memory manager // layout like this to optimize access in `mi_free` size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). diff --git a/src/memory.c b/src/memory.c index 222b87c2..a9b87b8e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -39,14 +39,14 @@ Possible issues: // Internal raw OS interface size_t _mi_os_large_page_size(); -bool _mi_os_protect(void* addr, size_t size); -bool _mi_os_unprotect(void* addr, size_t size); -bool _mi_os_commit(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); -bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats); -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld); - +bool _mi_os_protect(void* addr, size_t size); +bool _mi_os_unprotect(void* addr, size_t size); +bool _mi_os_commit(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); +bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats); +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld); +bool _mi_os_is_huge_reserved(void* p); // Constants #if (MI_INTPTR_SIZE==8) @@ -66,11 +66,24 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld #define MI_REGION_MAP_FULL UINTPTR_MAX +typedef uintptr_t mi_region_info_t; + +static inline mi_region_info_t mi_region_info_create(void* start, bool is_large, bool is_committed) { + return ((uintptr_t)start | ((is_large?1:0) << 1) | (is_committed?1:0)); +} + +static inline void* mi_region_info_read(mi_region_info_t info, bool* is_large, bool* is_committed) { + if (is_large) *is_large = ((info&0x02) != 0); + if (is_committed) *is_committed = ((info&0x01) != 0); + return (void*)(info & ~0x03); +} + + // A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with // a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block. typedef struct mem_region_s { - volatile _Atomic(uintptr_t) map; // in-use bit per MI_SEGMENT_SIZE block - volatile _Atomic(void*) start; // start of virtual memory area + volatile _Atomic(uintptr_t) map; // in-use bit per MI_SEGMENT_SIZE block + volatile _Atomic(mi_region_info_t) info; // start of virtual memory area, and flags } mem_region_t; @@ -108,7 +121,7 @@ bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { if (p==NULL) return false; size_t count = mi_atomic_read_relaxed(®ions_count); for (size_t i = 0; i < count; i++) { - uint8_t* start = (uint8_t*)mi_atomic_read_ptr_relaxed(®ions[i].start); + uint8_t* start = (uint8_t*)mi_region_info_read( mi_atomic_read_relaxed(®ions[i].info), NULL, NULL); if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true; } return false; @@ -123,7 +136,7 @@ Commit from a region // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. // (not being able to claim is not considered an error so check for `p != NULL` afterwards). -static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) +static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, bool* large, void** p, size_t* id, mi_os_tld_t* tld) { size_t mask = mi_region_block_mask(blocks,bitidx); mi_assert_internal(mask != 0); @@ -131,10 +144,14 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit mi_assert_internal(®ions[idx] == region); // ensure the region is reserved - void* start = mi_atomic_read_ptr(®ion->start); - if (start == NULL) + mi_region_info_t info = mi_atomic_read(®ion->info); + if (info == 0) { - start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, mi_option_is_enabled(mi_option_eager_region_commit), tld); + bool region_commit = mi_option_is_enabled(mi_option_eager_region_commit); + bool region_large = region_commit && *large; + void* start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, region_commit, ®ion_large, tld); + *large = region_large; + if (start == NULL) { // failure to allocate from the OS! unclaim the blocks and fail size_t map; @@ -145,7 +162,8 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit } // set the newly allocated region - if (mi_atomic_cas_ptr_strong(®ion->start, start, NULL)) { + info = mi_region_info_create(start,region_large,region_commit); + if (mi_atomic_cas_strong(®ion->info, info, 0)) { // update the region count mi_atomic_increment(®ions_count); } @@ -154,12 +172,9 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit // we assign it to a later slot instead (up to 4 tries). // note: we don't need to increment the region count, this will happen on another allocation for(size_t i = 1; i <= 4 && idx + i < MI_REGION_MAX; i++) { - void* s = mi_atomic_read_ptr(®ions[idx+i].start); - if (s == NULL) { // quick test - if (mi_atomic_cas_ptr_strong(®ions[idx+i].start, start, NULL)) { - start = NULL; - break; - } + if (mi_atomic_cas_strong(®ions[idx+i].info, info, 0)) { + start = NULL; + break; } } if (start != NULL) { @@ -167,15 +182,17 @@ static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bit _mi_os_free(start, MI_REGION_SIZE, tld->stats); } // and continue with the memory at our index - start = mi_atomic_read_ptr(®ion->start); + info = mi_atomic_read(®ion->info); } } - mi_assert_internal(start == mi_atomic_read_ptr(®ion->start)); - mi_assert_internal(start != NULL); + mi_assert_internal(info == mi_atomic_read(®ion->info)); + mi_assert_internal(info != 0); // Commit the blocks to memory + bool region_is_committed = false; + void* start = mi_region_info_read(info,large,®ion_is_committed); void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); - if (commit && !mi_option_is_enabled(mi_option_eager_region_commit)) { + if (commit && !region_is_committed) { _mi_os_commit(blocks_start, mi_good_commit_size(size), tld->stats); // only commit needed size (unless using large OS pages) } @@ -223,7 +240,7 @@ static inline size_t mi_bsr(uintptr_t x) { // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. // (not being able to claim is not considered an error so check for `p != NULL` afterwards). -static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) +static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, bool* large, void** p, size_t* id, mi_os_tld_t* tld) { mi_assert_internal(p != NULL && id != NULL); mi_assert_internal(blocks < MI_REGION_MAP_BITS); @@ -253,7 +270,7 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc else { // success, we claimed the bits // now commit the block memory -- this can still fail - return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, p, id, tld); + return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, large, p, id, tld); } } else { @@ -276,14 +293,14 @@ static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t bloc // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written // if the blocks were successfully claimed so ensure they are initialized to NULL/0 before the call. // (not being able to claim is not considered an error so check for `p != NULL` afterwards). -static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) +static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, bool* large, void** p, size_t* id, mi_os_tld_t* tld) { // check if there are available blocks in the region.. mi_assert_internal(idx < MI_REGION_MAX); mem_region_t* region = ®ions[idx]; uintptr_t m = mi_atomic_read_relaxed(®ion->map); if (m != MI_REGION_MAP_FULL) { // some bits are zero - return mi_region_alloc_blocks(region, idx, blocks, size, commit, p, id, tld); + return mi_region_alloc_blocks(region, idx, blocks, size, commit, large, p, id, tld); } else { return true; // no error, but no success either @@ -296,15 +313,17 @@ static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, b // Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`. // (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`) -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld) +void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, size_t* id, mi_os_tld_t* tld) { mi_assert_internal(id != NULL && tld != NULL); mi_assert_internal(size > 0); *id = SIZE_MAX; + bool default_large = false; + if (large==NULL) large = &default_large; // ensure `large != NULL` // use direct OS allocation for huge blocks or alignment (with `id = SIZE_MAX`) if (size > MI_REGION_MAX_ALLOC_SIZE || alignment > MI_SEGMENT_ALIGN) { - return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, true, tld); // round up size + return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, commit, large, tld); // round up size } // always round size to OS page size multiple (so commit/decommit go over the entire range) @@ -318,27 +337,27 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* // find a range of free blocks void* p = NULL; size_t count = mi_atomic_read(®ions_count); - size_t idx = 0; // tld->region_idx; // start index is per-thread to reduce contention + size_t idx = 0; // tld->region_idx; // start at 0 to reuse low addresses? Or, use tld->region_idx to reduce contention? for (size_t visited = 0; visited < count; visited++, idx++) { if (idx >= count) idx = 0; // wrap around - if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error + if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, large, &p, id, tld)) return NULL; // error if (p != NULL) break; } if (p == NULL) { // no free range in existing regions -- try to extend beyond the count.. but at most 4 regions for (idx = count; idx < count + 4 && idx < MI_REGION_MAX; idx++) { - if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error + if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, large, &p, id, tld)) return NULL; // error if (p != NULL) break; } } if (p == NULL) { // we could not find a place to allocate, fall back to the os directly - p = _mi_os_alloc_aligned(size, alignment, commit, tld); + p = _mi_os_alloc_aligned(size, alignment, commit, large, tld); } else { - tld->region_idx = idx; // next start of search + tld->region_idx = idx; // next start of search? } mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0); @@ -347,8 +366,8 @@ void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* // Allocate `size` memory. Return non NULL on success, with a given memory `id`. -void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld) { - return _mi_mem_alloc_aligned(size,0,commit,id,tld); +void* _mi_mem_alloc(size_t size, bool commit, bool* large, size_t* id, mi_os_tld_t* tld) { + return _mi_mem_alloc_aligned(size,0,commit,large,id,tld); } /* ---------------------------------------------------------------------------- @@ -377,7 +396,10 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) { mi_assert_internal(idx < MI_REGION_MAX); if (idx >= MI_REGION_MAX) return; // or `abort`? mem_region_t* region = ®ions[idx]; mi_assert_internal((mi_atomic_read_relaxed(®ion->map) & mask) == mask ); // claimed? - void* start = mi_atomic_read_ptr(®ion->start); + mi_region_info_t info = mi_atomic_read(®ion->info); + bool is_large; + bool is_eager_committed; + void* start = mi_region_info_read(info,&is_large,&is_eager_committed); mi_assert_internal(start != NULL); void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); mi_assert_internal(blocks_start == p); // not a pointer in our area? @@ -388,18 +410,13 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) { // TODO: implement delayed decommit/reset as these calls are too expensive // if the memory is reused soon. // reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large - if (!mi_option_is_enabled(mi_option_large_os_pages)) { - if (mi_option_is_enabled(mi_option_eager_region_commit)) { - //_mi_os_reset(p, size, stats); - } - else { - //_mi_os_decommit(p, size, stats); - } - } + if (!is_large) { + // _mi_os_reset(p,size,stats); + // _mi_os_decommit(p,size,stats); // if !is_committed + } // TODO: should we free empty regions? currently only done _mi_mem_collect. - // this frees up virtual address space which - // might be useful on 32-bit systems? + // this frees up virtual address space which might be useful on 32-bit systems? // and unclaim uintptr_t map; @@ -419,17 +436,20 @@ void _mi_mem_collect(mi_stats_t* stats) { // free every region that has no segments in use. for (size_t i = 0; i < regions_count; i++) { mem_region_t* region = ®ions[i]; - if (mi_atomic_read_relaxed(®ion->map) == 0 && region->start != NULL) { + if (mi_atomic_read_relaxed(®ion->map) == 0) { // if no segments used, try to claim the whole region uintptr_t m; do { m = mi_atomic_read_relaxed(®ion->map); } while(m == 0 && !mi_atomic_cas_weak(®ion->map, ~((uintptr_t)0), 0 )); if (m == 0) { - // on success, free the whole region - if (region->start != NULL) _mi_os_free((void*)region->start, MI_REGION_SIZE, stats); + // on success, free the whole region (unless it was huge reserved) + void* start = mi_region_info_read(mi_atomic_read(®ion->info), NULL, NULL); + if (start != NULL && !_mi_os_is_huge_reserved(start)) { + _mi_os_free(start, MI_REGION_SIZE, stats); + } // and release - mi_atomic_write_ptr(®ion->start,NULL); + mi_atomic_write(®ion->info,0); mi_atomic_write(®ion->map,0); } } diff --git a/src/options.c b/src/options.c index 11bb78d8..41bf33a0 100644 --- a/src/options.c +++ b/src/options.c @@ -58,7 +58,7 @@ static mi_option_desc_t options[_mi_option_last] = #endif // the following options are experimental and not all combinations make sense. - { 1, UNINIT, MI_OPTION(eager_commit) }, // note: if eager_region_commit is on, this should be on too. + { 1, UNINIT, MI_OPTION(eager_commit) }, // note: needs to be on when eager_region_commit is enabled #ifdef _WIN32 // and BSD? { 1, UNINIT, MI_OPTION(eager_region_commit) }, // don't commit too eagerly on windows (just for looks...) #else diff --git a/src/os.c b/src/os.c index 7648c010..e1306722 100644 --- a/src/os.c +++ b/src/os.c @@ -35,10 +35,9 @@ terms of the MIT license. A copy of the license can be found in the file On windows initializes support for aligned allocation and large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). ----------------------------------------------------------- */ -bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); - -static bool mi_os_is_huge_reserved(void* p); -static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit); +bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); +bool _mi_os_is_huge_reserved(void* p); +static void* mi_os_alloc_from_huge_reserved(size_t size, size_t try_alignment, bool commit); static void* mi_align_up_ptr(void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); @@ -173,7 +172,7 @@ void _mi_os_init() { static bool mi_os_mem_free(void* addr, size_t size, mi_stats_t* stats) { - if (addr == NULL || size == 0 || mi_os_is_huge_reserved(addr)) return true; + if (addr == NULL || size == 0 || _mi_os_is_huge_reserved(addr)) return true; bool err = false; #if defined(_WIN32) err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); @@ -199,7 +198,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) // on modern Windows try use NtAllocateVirtualMemoryEx for 1GiB huge pages if ((size % ((uintptr_t)1 << 30)) == 0 /* 1GiB multiple */ - && (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0 + && (flags & MEM_LARGE_PAGES) != 0 && (flags & MEM_COMMIT) != 0 && (flags & MEM_RESERVE) != 0 && (addr != NULL || try_alignment == 0 || try_alignment % _mi_os_page_size() == 0) && pNtAllocateVirtualMemoryEx != NULL) { @@ -211,7 +210,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment param.ULong64 = MEM_EXTENDED_PARAMETER_NONPAGED_HUGE; SIZE_T psize = size; void* base = addr; - NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags | MEM_RESERVE, PAGE_READWRITE, ¶m, 1); + NTSTATUS err = (*pNtAllocateVirtualMemoryEx)(GetCurrentProcess(), &base, &psize, flags, PAGE_READWRITE, ¶m, 1); if (err == 0) { return base; } @@ -247,10 +246,12 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment return VirtualAlloc(addr, size, flags, PAGE_READWRITE); } -static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only) { +static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { + mi_assert_internal(!(large_only && !allow_large)); static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0; void* p = NULL; - if (large_only || use_large_os_page(size, try_alignment)) { + if ((large_only || use_large_os_page(size, try_alignment)) + && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); if (!large_only && try_ok > 0) { // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. @@ -259,7 +260,8 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, } else { // large OS pages must always reserve and commit. - p = mi_win_virtual_allocx(addr, size, try_alignment, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE | flags); + *is_large = true; + p = mi_win_virtual_allocx(addr, size, try_alignment, flags | MEM_LARGE_PAGES); if (large_only) return p; // fall back to non-large page allocation on error (`p == NULL`). if (p == NULL) { @@ -268,6 +270,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, } } if (p == NULL) { + *is_large = ((flags&MEM_LARGE_PAGES) != 0); p = mi_win_virtual_allocx(addr, size, try_alignment, flags); } if (p == NULL) { @@ -311,7 +314,7 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr return p; } -static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only) { +static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { void* p = NULL; #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON @@ -333,7 +336,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) fd = VM_MAKE_TAG(100); #endif - if (large_only || use_large_os_page(size, try_alignment)) { + if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { static volatile _Atomic(uintptr_t) large_page_try_ok; // = 0; uintptr_t try_ok = mi_atomic_read(&large_page_try_ok); if (!large_only && try_ok > 0) { @@ -368,6 +371,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only || lflags != flags) { // try large OS page allocation + *is_large = true; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, lflags, lfd); #ifdef MAP_HUGE_1GB if (p == NULL && (lflags & MAP_HUGE_1GB) != 0) { @@ -384,6 +388,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro } } if (p == NULL) { + *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); #if defined(MADV_HUGEPAGE) // Many Linux systems don't allow MAP_HUGETLB but they support instead @@ -392,8 +397,10 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // in that case -- in particular for our large regions (in `memory.c`). // However, some systems only allow TPH if called with explicit `madvise`, so // when large OS pages are enabled for mimalloc, we call `madvice` anyways. - if (use_large_os_page(size, try_alignment)) { - madvise(p, size, MADV_HUGEPAGE); + if (allow_large && use_large_os_page(size, try_alignment)) { + if (madvise(p, size, MADV_HUGEPAGE) == 0) { + *is_large = true; // possibly + }; } #endif } @@ -403,27 +410,35 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // Primitive allocation from the OS. // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. -static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_stats_t* stats) { +static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); if (size == 0) return NULL; + if (!commit) allow_large = false; - void* p = mi_os_alloc_from_huge_reserved(size, try_alignment, commit); - if (p != NULL) return p; + void* p = NULL; + if (allow_large) { + p = mi_os_alloc_from_huge_reserved(size, try_alignment, commit); + if (p != NULL) { + *is_large = true; + return p; + } + } #if defined(_WIN32) int flags = MEM_RESERVE; if (commit) flags |= MEM_COMMIT; - p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false); + p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); #elif defined(__wasi__) + *is_large = false; p = mi_wasm_heap_grow(size, try_alignment); #else int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); - p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false); + p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); #endif _mi_stat_increase(&stats->mmap_calls, 1); if (p != NULL) { _mi_stat_increase(&stats->reserved, size); - if (commit) _mi_stat_increase(&stats->committed, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } } return p; } @@ -431,14 +446,15 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, mi_ // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. -static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_stats_t* stats) { +static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) { mi_assert_internal(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0)); mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); + if (!commit) allow_large = false; if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); // try first with a hint (this will be aligned directly on Win 10+ or BSD) - void* p = mi_os_mem_alloc(size, alignment, commit, stats); + void* p = mi_os_mem_alloc(size, alignment, commit, allow_large, is_large, stats); if (p == NULL) return NULL; // if not aligned, free it, overallocate, and unmap around it @@ -457,7 +473,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, if (commit) flags |= MEM_COMMIT; for (int tries = 0; tries < 3; tries++) { // over-allocate to determine a virtual memory range - p = mi_os_mem_alloc(over_size, alignment, commit, stats); + p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats); if (p == NULL) return NULL; // error if (((uintptr_t)p % alignment) == 0) { // if p happens to be aligned, just decommit the left-over area @@ -468,7 +484,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, // otherwise free and allocate at an aligned address in there mi_os_mem_free(p, over_size, stats); void* aligned_p = mi_align_up_ptr(p, alignment); - p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false); + p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? mi_os_mem_free(p, size, stats); @@ -478,7 +494,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, } #else // overallocate... - p = mi_os_mem_alloc(over_size, alignment, commit, stats); + p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats); if (p == NULL) return NULL; // and selectively unmap parts around the over-allocated area. void* aligned_p = mi_align_up_ptr(p, alignment); @@ -504,7 +520,8 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, void* _mi_os_alloc(size_t size, mi_stats_t* stats) { if (size == 0) return NULL; size = mi_os_good_alloc_size(size, 0); - return mi_os_mem_alloc(size, 0, true, stats); + bool is_large = false; + return mi_os_mem_alloc(size, 0, true, false, &is_large, stats); } void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { @@ -513,12 +530,17 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { mi_os_mem_free(p, size, stats); } -void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld) +void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_os_tld_t* tld) { if (size == 0) return NULL; size = mi_os_good_alloc_size(size, alignment); alignment = _mi_align_up(alignment, _mi_os_page_size()); - return mi_os_mem_alloc_aligned(size, alignment, commit, tld->stats); + bool allow_large = false; + if (large != NULL) { + allow_large = *large; + *large = false; + } + return mi_os_mem_alloc_aligned(size, alignment, commit, allow_large, (large!=NULL?large:&allow_large), tld->stats); } @@ -559,7 +581,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // page align in the range, commit liberally, decommit conservative size_t csize; void* start = mi_os_page_align_areax(conservative, addr, size, &csize); - if (csize == 0 || mi_os_is_huge_reserved(addr)) return true; + if (csize == 0 || _mi_os_is_huge_reserved(addr)) return true; int err = 0; if (commit) { _mi_stat_increase(&stats->committed, csize); @@ -611,7 +633,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) // page align conservatively within the range size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); - if (csize == 0 || mi_os_is_huge_reserved(addr)) return true; + if (csize == 0 || _mi_os_is_huge_reserved(addr)) return true; if (reset) _mi_stat_increase(&stats->reset, csize); else _mi_stat_decrease(&stats->reset, csize); if (!reset) return true; // nothing to do on unreset! @@ -626,6 +648,11 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) // Testing shows that for us (on `malloc-large`) MEM_RESET is 2x faster than DiscardVirtualMemory void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); mi_assert_internal(p == start); + #if 0 + if (p == start) { + VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set + } + #endif if (p != start) return false; #else #if defined(MADV_FREE) @@ -679,8 +706,8 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { size_t csize = 0; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return false; - if (mi_os_is_huge_reserved(addr)) { - _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); + if (_mi_os_is_huge_reserved(addr)) { + _mi_warning_message("cannot mprotect memory allocated in huge OS pages\n"); } int err = 0; #ifdef _WIN32 @@ -742,7 +769,7 @@ typedef struct mi_huge_info_s { static mi_huge_info_t os_huge_reserved = { NULL, 0, ATOMIC_VAR_INIT(0) }; -static bool mi_os_is_huge_reserved(void* p) { +bool _mi_os_is_huge_reserved(void* p) { return (mi_atomic_read_ptr(&os_huge_reserved.start) != NULL && p >= mi_atomic_read_ptr(&os_huge_reserved.start) && (uint8_t*)p < (uint8_t*)mi_atomic_read_ptr(&os_huge_reserved.start) + mi_atomic_read(&os_huge_reserved.reserved)); @@ -806,10 +833,11 @@ int mi_reserve_huge_os_pages( size_t pages, double max_secs ) mi_attr_noexcept for (size_t page = 0; page < pages; page++, addr += MI_HUGE_OS_PAGE_SIZE ) { // allocate lorgu pages void* p = NULL; + bool is_large = true; #ifdef _WIN32 - p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true); + p = mi_win_virtual_alloc(addr, MI_HUGE_OS_PAGE_SIZE, 0, MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE, true, true, &is_large); #elif defined(MI_OS_USE_MMAP) - p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true); + p = mi_unix_mmap(addr, MI_HUGE_OS_PAGE_SIZE, 0, PROT_READ | PROT_WRITE, true, true, &is_large); #else // always fail #endif diff --git a/src/segment.c b/src/segment.c index 9a744ea6..020d53e8 100644 --- a/src/segment.c +++ b/src/segment.c @@ -229,6 +229,7 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se segment->thread_id = 0; mi_segments_track_size(-((long)segment_size),tld); if (mi_option_is_enabled(mi_option_secure)) { + mi_assert_internal(!segment->mem_is_fixed); _mi_mem_unprotect(segment, segment->segment_size); // ensure no more guard pages are set } _mi_mem_free(segment, segment_size, segment->memid, tld->stats); @@ -277,7 +278,7 @@ static bool mi_segment_cache_push(mi_segment_t* segment, mi_segments_tld_t* tld) return false; } mi_assert_internal(segment->segment_size == MI_SEGMENT_SIZE); - if (mi_option_is_enabled(mi_option_cache_reset)) { + if (!segment->mem_is_fixed && mi_option_is_enabled(mi_option_cache_reset)) { _mi_mem_reset((uint8_t*)segment + segment->segment_info_size, segment->segment_size - segment->segment_info_size, tld->stats); } segment->next = tld->cache; @@ -325,11 +326,13 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, size_t page_size = (page_kind == MI_PAGE_HUGE ? segment_size : (size_t)1 << page_shift); // Try to get it from our thread local cache first - bool commit = mi_option_is_enabled(mi_option_eager_commit) || (page_kind > MI_PAGE_MEDIUM); + bool eager = mi_option_is_enabled(mi_option_eager_commit); + bool commit = eager || (page_kind > MI_PAGE_MEDIUM); bool protection_still_good = false; mi_segment_t* segment = mi_segment_cache_pop(segment_size, tld); if (segment != NULL) { if (mi_option_is_enabled(mi_option_secure)) { + mi_assert_internal(!segment->mem_is_fixed); if (segment->page_kind != page_kind) { _mi_mem_unprotect(segment, segment->segment_size); // reset protection if the page kind differs } @@ -337,37 +340,38 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, protection_still_good = true; // otherwise, the guard pages are still in place } } - if (!mi_option_is_enabled(mi_option_eager_commit)) { - if (page_kind > MI_PAGE_MEDIUM) { - _mi_mem_commit(segment, segment->segment_size, tld->stats); - } - else { - // ok, commit (and unreset) on demand again - } + if (!segment->mem_is_committed && page_kind > MI_PAGE_MEDIUM) { + mi_assert_internal(!segment->mem_is_fixed); + _mi_mem_commit(segment, segment->segment_size, tld->stats); + segment->mem_is_committed = true; } - else if (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset)) { + if (!segment->mem_is_fixed && + (mi_option_is_enabled(mi_option_cache_reset) || mi_option_is_enabled(mi_option_page_reset))) { _mi_mem_unreset(segment, segment->segment_size, tld->stats); } } else { // Allocate the segment from the OS size_t memid; - segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &memid, os_tld); + bool mem_large = (eager && !mi_option_is_enabled(mi_option_secure)); // only allow large OS pages once we are no longer lazy + segment = (mi_segment_t*)_mi_mem_alloc_aligned(segment_size, MI_SEGMENT_SIZE, commit, &mem_large, &memid, os_tld); if (segment == NULL) return NULL; // failed to allocate if (!commit) { + // ensure the initial info is committed _mi_mem_commit(segment, info_size, tld->stats); } segment->memid = memid; + segment->mem_is_fixed = mem_large; + segment->mem_is_committed = commit; mi_segments_track_size((long)segment_size, tld); } mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); - // zero the segment info - { size_t memid = segment->memid; - memset(segment, 0, info_size); - segment->memid = memid; - } + // zero the segment info (but not the `mem` fields) + ptrdiff_t ofs = offsetof(mi_segment_t,next); + memset((uint8_t*)segment + ofs, 0, info_size - ofs); + // guard pages if (mi_option_is_enabled(mi_option_secure) && !protection_still_good) { // in secure mode, we set up a protected page in between the segment info // and the page data @@ -386,6 +390,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, } } + // initialize segment->page_kind = page_kind; segment->capacity = capacity; segment->page_shift = page_shift; @@ -453,13 +458,14 @@ static mi_page_t* mi_segment_find_free(mi_segment_t* segment, mi_stats_t* stats) if (!page->segment_in_use) { if (page->is_reset || !page->is_committed) { size_t psize; - uint8_t* start = _mi_page_start(segment, page, &psize); - mi_assert_internal(!(page->is_reset && !page->is_committed)); + uint8_t* start = _mi_page_start(segment, page, &psize); if (!page->is_committed) { + mi_assert_internal(!segment->mem_is_fixed); page->is_committed = true; _mi_mem_commit(start,psize,stats); } if (page->is_reset) { + mi_assert_internal(!segment->mem_is_fixed); page->is_reset = false; _mi_mem_unreset(start, psize, stats); } @@ -488,22 +494,17 @@ static void mi_segment_page_clear(mi_segment_t* segment, mi_page_t* page, mi_sta _mi_stat_decrease(&stats->pages, 1); // reset the page memory to reduce memory pressure? - if (!page->is_reset && mi_option_is_enabled(mi_option_page_reset)) { + if (!segment->mem_is_fixed && !page->is_reset && mi_option_is_enabled(mi_option_page_reset)) { size_t psize; uint8_t* start = _mi_page_start(segment, page, &psize); page->is_reset = true; _mi_mem_reset(start, psize, stats); } - // zero the page data - uint8_t idx = page->segment_idx; // don't clear the index - bool is_reset = page->is_reset; // don't clear the reset flag - bool is_committed = page->is_committed; // don't clear the commit flag - memset(page, 0, sizeof(*page)); - page->segment_idx = idx; + // zero the page data, but not the segment fields + ptrdiff_t ofs = offsetof(mi_page_t,capacity); + memset((uint8_t*)page + ofs, 0, sizeof(*page) - ofs); page->segment_in_use = false; - page->is_reset = is_reset; - page->is_committed = is_committed; segment->used--; }