mirror of
https://github.com/microsoft/mimalloc.git
synced 2025-07-01 09:14:38 +03:00
merge from dev
This commit is contained in:
Daan
commit
8b761008ce
5 changed files with 100 additions and 68 deletions
|
|
@ -8,7 +8,6 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#ifndef MIMALLOC_INTERNAL_H
|
||||
#define MIMALLOC_INTERNAL_H
|
||||
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// This file contains the internal API's of mimalloc and various utility
|
||||
// functions and macros.
|
||||
|
|
@ -17,43 +16,72 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#include "types.h"
|
||||
#include "track.h"
|
||||
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Compiler defines
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
#if (MI_DEBUG>0)
|
||||
#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)
|
||||
#else
|
||||
#define mi_trace_message(...)
|
||||
#endif
|
||||
|
||||
#define MI_CACHE_LINE 64
|
||||
#define mi_decl_cache_align mi_decl_align(64)
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
|
||||
#pragma warning(disable:26812) // unscoped enum warning
|
||||
#define mi_decl_noinline __declspec(noinline)
|
||||
#define mi_decl_thread __declspec(thread)
|
||||
#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
|
||||
#define mi_decl_align(a) __declspec(align(a))
|
||||
#define mi_decl_noreturn __declspec(noreturn)
|
||||
#define mi_decl_weak
|
||||
#define mi_decl_hidden
|
||||
#define mi_decl_cold
|
||||
#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc
|
||||
#define mi_decl_noinline __attribute__((noinline))
|
||||
#define mi_decl_thread __thread
|
||||
#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
|
||||
#define mi_decl_align(a) __attribute__((aligned(a)))
|
||||
#define mi_decl_noreturn __attribute__((noreturn))
|
||||
#define mi_decl_weak __attribute__((weak))
|
||||
#define mi_decl_hidden __attribute__((visibility("hidden")))
|
||||
#if (__GNUC__ >= 4) || defined(__clang__)
|
||||
#define mi_decl_cold __attribute__((cold))
|
||||
#else
|
||||
#define mi_decl_cold
|
||||
#endif
|
||||
#elif __cplusplus >= 201103L // c++11
|
||||
#define mi_decl_noinline
|
||||
#define mi_decl_thread thread_local
|
||||
#define mi_decl_cache_align alignas(MI_CACHE_LINE)
|
||||
#define mi_decl_align(a) alignas(a)
|
||||
#define mi_decl_noreturn [[noreturn]]
|
||||
#define mi_decl_weak
|
||||
#define mi_decl_hidden
|
||||
#define mi_decl_cold
|
||||
#else
|
||||
#define mi_decl_noinline
|
||||
#define mi_decl_thread __thread // hope for the best :-)
|
||||
#define mi_decl_cache_align
|
||||
#define mi_decl_align(a)
|
||||
#define mi_decl_noreturn
|
||||
#define mi_decl_weak
|
||||
#define mi_decl_hidden
|
||||
#define mi_decl_cold
|
||||
#endif
|
||||
|
||||
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
|
||||
#define __wasi__
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define mi_unlikely(x) (__builtin_expect(!!(x),false))
|
||||
#define mi_likely(x) (__builtin_expect(!!(x),true))
|
||||
#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
|
||||
#define mi_unlikely(x) (x) [[unlikely]]
|
||||
#define mi_likely(x) (x) [[likely]]
|
||||
#else
|
||||
#define mi_unlikely(x) (x)
|
||||
#define mi_likely(x) (x)
|
||||
#endif
|
||||
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus)
|
||||
|
|
@ -62,6 +90,15 @@ terms of the MIT license. A copy of the license can be found in the file
|
|||
#define mi_decl_externc
|
||||
#endif
|
||||
|
||||
#if defined(__EMSCRIPTEN__) && !defined(__wasi__)
|
||||
#define __wasi__
|
||||
#endif
|
||||
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Internal functions
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
// "libc.c"
|
||||
#include <stdarg.h>
|
||||
int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args);
|
||||
|
|
@ -126,13 +163,13 @@ bool _mi_os_has_overcommit(void);
|
|||
bool _mi_os_has_virtual_reserve(void);
|
||||
|
||||
bool _mi_os_reset(void* addr, size_t size);
|
||||
bool _mi_os_commit(void* p, size_t size, bool* is_zero);
|
||||
bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);
|
||||
bool _mi_os_decommit(void* addr, size_t size);
|
||||
bool _mi_os_protect(void* addr, size_t size);
|
||||
bool _mi_os_unprotect(void* addr, size_t size);
|
||||
bool _mi_os_purge(void* p, size_t size);
|
||||
bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size);
|
||||
mi_decl_nodiscard bool _mi_os_commit(void* p, size_t size, bool* is_zero);
|
||||
mi_decl_nodiscard bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size);
|
||||
bool _mi_os_protect(void* addr, size_t size);
|
||||
|
||||
void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid);
|
||||
void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid);
|
||||
|
|
@ -258,26 +295,6 @@ bool _mi_page_is_valid(mi_page_t* page);
|
|||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Branches
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define mi_unlikely(x) (__builtin_expect(!!(x),false))
|
||||
#define mi_likely(x) (__builtin_expect(!!(x),true))
|
||||
#elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
|
||||
#define mi_unlikely(x) (x) [[unlikely]]
|
||||
#define mi_likely(x) (x) [[likely]]
|
||||
#else
|
||||
#define mi_unlikely(x) (x)
|
||||
#define mi_likely(x) (x)
|
||||
#endif
|
||||
|
||||
#ifndef __has_builtin
|
||||
#define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Error codes passed to `_mi_fatal_error`
|
||||
All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
|
||||
|
|
@ -302,6 +319,32 @@ bool _mi_page_is_valid(mi_page_t* page);
|
|||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Assertions
|
||||
// ------------------------------------------------------
|
||||
|
||||
#if (MI_DEBUG)
|
||||
// use our own assertion to print without memory allocation
|
||||
mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_attr_noexcept;
|
||||
#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
|
||||
#else
|
||||
#define mi_assert(x)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>1)
|
||||
#define mi_assert_internal mi_assert
|
||||
#else
|
||||
#define mi_assert_internal(x)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>2)
|
||||
#define mi_assert_expensive mi_assert
|
||||
#else
|
||||
#define mi_assert_expensive(x)
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Inlined definitions
|
||||
----------------------------------------------------------- */
|
||||
|
|
|
|||
|
|
@ -632,7 +632,6 @@ struct mi_tld_s {
|
|||
};
|
||||
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Debug
|
||||
// ------------------------------------------------------
|
||||
|
|
@ -647,26 +646,6 @@ struct mi_tld_s {
|
|||
#define MI_DEBUG_PADDING (0xDE)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG)
|
||||
// use our own assertion to print without memory allocation
|
||||
void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func );
|
||||
#define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__))
|
||||
#else
|
||||
#define mi_assert(x)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>1)
|
||||
#define mi_assert_internal mi_assert
|
||||
#else
|
||||
#define mi_assert_internal(x)
|
||||
#endif
|
||||
|
||||
#if (MI_DEBUG>2)
|
||||
#define mi_assert_expensive mi_assert
|
||||
#else
|
||||
#define mi_assert_expensive(x)
|
||||
#endif
|
||||
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Statistics
|
||||
|
|
|
|||
|
|
@ -525,7 +525,7 @@ void _mi_warning_message(const char* fmt, ...) {
|
|||
|
||||
|
||||
#if MI_DEBUG
|
||||
void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) {
|
||||
mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) mi_attr_noexcept {
|
||||
_mi_fprintf(NULL, NULL, "mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion);
|
||||
abort();
|
||||
}
|
||||
|
|
|
|||
7
src/os.c
7
src/os.c
|
|
@ -166,7 +166,7 @@ static void mi_os_prim_free(void* addr, size_t size, size_t commit_size) {
|
|||
void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid) {
|
||||
if (mi_memkind_is_os(memid.memkind)) {
|
||||
size_t csize = memid.mem.os.size;
|
||||
if (csize==0) { _mi_os_good_alloc_size(size); }
|
||||
if (csize==0) { csize = _mi_os_good_alloc_size(size); }
|
||||
size_t commit_size = (still_committed ? csize : 0);
|
||||
void* base = addr;
|
||||
// different base? (due to alignment)
|
||||
|
|
@ -285,7 +285,10 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
|
|||
|
||||
// explicitly commit only the aligned part
|
||||
if (commit) {
|
||||
_mi_os_commit(p, size, NULL);
|
||||
if (!_mi_os_commit(p, size, NULL)) {
|
||||
mi_os_prim_free(*base, over_size, 0);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
else { // mmap can free inside an allocation
|
||||
|
|
|
|||
23
src/page.c
23
src/page.c
|
|
@ -37,7 +37,7 @@ static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_sta
|
|||
}
|
||||
|
||||
static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t* tld);
|
||||
static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld);
|
||||
static bool mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld);
|
||||
|
||||
#if (MI_DEBUG>=3)
|
||||
static size_t mi_page_list_count(mi_page_t* page, mi_block_t* head) {
|
||||
|
|
@ -632,15 +632,14 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
|
|||
// Note: we also experimented with "bump" allocation on the first
|
||||
// allocations but this did not speed up any benchmark (due to an
|
||||
// extra test in malloc? or cache effects?)
|
||||
static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) {
|
||||
MI_UNUSED(tld);
|
||||
static bool mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) {
|
||||
mi_assert_expensive(mi_page_is_valid_init(page));
|
||||
#if (MI_SECURE<=2)
|
||||
mi_assert(page->free == NULL);
|
||||
mi_assert(page->local_free == NULL);
|
||||
if (page->free != NULL) return;
|
||||
if (page->free != NULL) return true;
|
||||
#endif
|
||||
if (page->capacity >= page->reserved) return;
|
||||
if (page->capacity >= page->reserved) return true;
|
||||
|
||||
mi_stat_counter_increase(tld->stats.pages_extended, 1);
|
||||
|
||||
|
|
@ -673,6 +672,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld)
|
|||
page->capacity += (uint16_t)extend;
|
||||
mi_stat_increase(tld->stats.page_committed, extend * bsize);
|
||||
mi_assert_expensive(mi_page_is_valid_init(page));
|
||||
return true;
|
||||
}
|
||||
|
||||
// Initialize a fresh page
|
||||
|
|
@ -727,8 +727,10 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
|
|||
mi_assert_expensive(mi_page_is_valid_init(page));
|
||||
|
||||
// initialize an initial free list
|
||||
mi_page_extend_free(heap,page,tld);
|
||||
if (mi_page_extend_free(heap,page,tld)) {
|
||||
mi_assert(mi_page_immediate_available(page));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -820,9 +822,14 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
|
|||
if (page_candidate != NULL) {
|
||||
page = page_candidate;
|
||||
}
|
||||
if (page != NULL && !mi_page_immediate_available(page)) {
|
||||
if (page != NULL) {
|
||||
if (!mi_page_immediate_available(page)) {
|
||||
mi_assert_internal(mi_page_is_expandable(page));
|
||||
mi_page_extend_free(heap, page, heap->tld);
|
||||
if (!mi_page_extend_free(heap, page, heap->tld)) {
|
||||
page = NULL; // failed to extend
|
||||
}
|
||||
}
|
||||
mi_assert_internal(page == NULL || mi_page_immediate_available(page));
|
||||
}
|
||||
|
||||
if (page == NULL) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue