diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f794004..bbb50311 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) option(MI_DEBUG_TRACE "Store allocation stack trace in each heap block to debug heap block overflows or corruption" OFF) +option(MI_SKIP_COLLECT_ON_EXIT, "Skip collecting memory on program exit" OFF) # deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) @@ -121,6 +122,11 @@ if(MI_CHECK_FULL) set(MI_DEBUG_FULL "ON") endif() +if (MI_SKIP_COLLECT_ON_EXIT) + message(STATUS "Skip collecting memory on program exit (MI_SKIP_COLLECT_ON_EXIT=ON)") + list(APPEND mi_defines MI_SKIP_COLLECT_ON_EXIT=1) +endif() + if(MI_DEBUG_FULL) message(STATUS "Set debug level to full internal invariant checking (MI_DEBUG_FULL=ON)") list(APPEND mi_defines MI_DEBUG=3) # full invariant checking @@ -252,7 +258,7 @@ endif() # ----------------------------------------------------------------------------- # dynamic/shared library and symlinks always go to /usr/local/lib equivalent -set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") # static libraries and object files, includes, and cmake config files # are either installed at top level, or use versioned directories for side-by-side installation (default) @@ -331,6 +337,7 @@ if(MI_BUILD_SHARED) add_custom_command(TARGET mimalloc POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" $ COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory") + install(FILES "$/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_libdir}) endif() install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) @@ -381,7 +388,7 @@ if (MI_BUILD_OBJECT) ) # the following seems to lead to cmake warnings/errors on some systems, disable for now :-( - # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_libdir}) + # install(TARGETS mimalloc-obj EXPORT mimalloc DESTINATION ${mi_install_objdir}) # the FILES expression can also be: $ # but that fails cmake versions less than 3.10 so we leave it as is for now diff --git a/bin/minject.exe b/bin/minject.exe index e576c71f..e5f87800 100644 Binary files a/bin/minject.exe and b/bin/minject.exe differ diff --git a/bin/minject32.exe b/bin/minject32.exe index 1eb8a75d..d0181028 100644 Binary files a/bin/minject32.exe and b/bin/minject32.exe differ diff --git a/doc/ds-logo.jpg b/doc/ds-logo.jpg index 853a7279..c9abb1a9 100644 Binary files a/doc/ds-logo.jpg and b/doc/ds-logo.jpg differ diff --git a/doc/ds-logo.png b/doc/ds-logo.png new file mode 100644 index 00000000..93b84e44 Binary files /dev/null and b/doc/ds-logo.png differ diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 4cf8c2c3..ea2a1ad5 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -802,20 +802,32 @@ typedef enum mi_option_e { mi_option_show_errors, ///< Print error messages to `stderr`. mi_option_show_stats, ///< Print statistics to `stderr` when the program is done. mi_option_verbose, ///< Print verbose messages to `stderr`. + // the following options are experimental mi_option_eager_commit, ///< Eagerly commit segments (4MiB) (enabled by default). - mi_option_eager_region_commit, ///< Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows) mi_option_large_os_pages, ///< Use large OS pages (2MiB in size) if possible mi_option_reserve_huge_os_pages, ///< The number of huge OS pages (1GiB in size) to reserve at the start of the program. mi_option_reserve_huge_os_pages_at, ///< Reserve huge OS pages at node N. - mi_option_segment_cache, ///< The number of segments per thread to keep cached. + mi_option_reserve_os_memory, ///< Reserve specified amount of OS memory at startup, e.g. "1g" or "512m". + mi_option_segment_cache, ///< The number of segments per thread to keep cached (0). mi_option_page_reset, ///< Reset page memory after \a mi_option_reset_delay milliseconds when it becomes free. + mi_option_abandoned_page_reset, //< Reset free page memory when a thread terminates. + mi_option_use_numa_nodes, ///< Pretend there are at most N NUMA nodes; Use 0 to use the actual detected NUMA nodes at runtime. + mi_option_eager_commit_delay, ///< the first N segments per thread are not eagerly committed (=1). + mi_option_os_tag, ///< OS tag to assign to mimalloc'd memory + mi_option_limit_os_alloc, ///< If set to 1, do not use OS memory for allocation (but only pre-reserved arenas) + + // v1.x specific options + mi_option_eager_region_commit, ///< Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows) mi_option_segment_reset, ///< Experimental mi_option_reset_delay, ///< Delay in milli-seconds before resetting a page (100ms by default) - mi_option_use_numa_nodes, ///< Pretend there are at most N NUMA nodes mi_option_reset_decommits, ///< Experimental - mi_option_eager_commit_delay, ///< Experimental - mi_option_os_tag, ///< OS tag to assign to mimalloc'd memory + + // v2.x specific options + mi_option_allow_decommit, ///< Enable decommitting memory (=on) + mi_option_decommit_delay, ///< Decommit page memory after N milli-seconds delay (25ms). + mi_option_segment_decommit_delay, ///< Decommit large segment memory after N milli-seconds delay (500ms). + _mi_option_last } mi_option_t; diff --git a/doc/spades-logo.png b/doc/spades-logo.png new file mode 100644 index 00000000..d8c73fef Binary files /dev/null and b/doc/spades-logo.png differ diff --git a/doc/unreal-logo.svg b/doc/unreal-logo.svg new file mode 100644 index 00000000..5d5192a2 --- /dev/null +++ b/doc/unreal-logo.svg @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index b4549cb6..fe1162ff 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -730,6 +730,7 @@ static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ ) static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 648d6aca..2acabfaf 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -333,7 +333,7 @@ typedef struct mi_segment_s { // layout like this to optimize access in `mi_free` size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment - mi_page_kind_t page_kind; // kind of pages: small, large, or huge + mi_page_kind_t page_kind; // kind of pages: small, medium, large, or huge mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages } mi_segment_t; diff --git a/include/mimalloc.h b/include/mimalloc.h index 6f950751..0b84f6c3 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -296,30 +296,30 @@ mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size // ------------------------------------------------------ -// Options, all `false` by default +// Options // ------------------------------------------------------ typedef enum mi_option_e { // stable options - mi_option_show_errors, - mi_option_show_stats, - mi_option_verbose, - // the following options are experimental - mi_option_eager_commit, - mi_option_eager_region_commit, + mi_option_show_errors, // print error messages + mi_option_show_stats, // print statistics on termination + mi_option_verbose, // print verbose messages + // the following options are experimental (see src/options.h) + mi_option_eager_commit, + mi_option_eager_region_commit, mi_option_reset_decommits, - mi_option_large_os_pages, // implies eager commit - mi_option_reserve_huge_os_pages, - mi_option_reserve_huge_os_pages_at, - mi_option_reserve_os_memory, - mi_option_segment_cache, - mi_option_page_reset, - mi_option_abandoned_page_reset, + mi_option_large_os_pages, // use large (2MiB) OS pages, implies eager commit + mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB) at startup + mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node + mi_option_reserve_os_memory, // reserve specified amount of OS memory at startup + mi_option_segment_cache, + mi_option_page_reset, + mi_option_abandoned_page_reset, mi_option_segment_reset, mi_option_eager_commit_delay, mi_option_reset_delay, - mi_option_use_numa_nodes, - mi_option_limit_os_alloc, + mi_option_use_numa_nodes, // 0 = use available numa nodes, otherwise use at most N nodes. + mi_option_limit_os_alloc, // 1 = do not use OS memory for allocation (but only reserved arenas) mi_option_os_tag, mi_option_max_errors, mi_option_max_warnings, diff --git a/readme.md b/readme.md index dfef82d5..635d983e 100644 --- a/readme.md +++ b/readme.md @@ -91,34 +91,7 @@ Note: the `v2.x` beta has a new algorithm for managing internal mimalloc pages t * 2021-01-31, `v1.7.0`: stable release 1.7: support explicit user provided memory regions, more precise statistics, improve macOS overriding, initial support for Apple M1, improved DragonFly support, faster memcpy on Windows, various small fixes. -### Older Releases - -* 2020-09-24, `v1.6.7`: stable release 1.6: using standard C atomics, passing tsan testing, improved - handling of failing to commit on Windows, add [`mi_process_info`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc.h#L156) api call. -* 2020-08-06, `v1.6.4`: stable release 1.6: improved error recovery in low-memory situations, - support for IllumOS and Haiku, NUMA support for Vista/XP, improved NUMA detection for AMD Ryzen, ubsan support. -* 2020-05-05, `v1.6.3`: stable release 1.6: improved behavior in out-of-memory situations, improved malloc zones on macOS, - build PIC static libraries by default, add option to abort on out-of-memory, line buffered statistics. -* 2020-04-20, `v1.6.2`: stable release 1.6: fix compilation on Android, MingW, Raspberry, and Conda, - stability fix for Windows 7, fix multiple mimalloc instances in one executable, fix `strnlen` overload, - fix aligned debug padding. -* 2020-02-17, `v1.6.1`: stable release 1.6: minor updates (build with clang-cl, fix alignment issue for small objects). -* 2020-02-09, `v1.6.0`: stable release 1.6: fixed potential memory leak, improved overriding - and thread local support on FreeBSD, NetBSD, DragonFly, and macOSX. New byte-precise - heap block overflow detection in debug mode (besides the double-free detection and free-list - corruption detection). Add `nodiscard` attribute to most allocation functions. - Enable `MIMALLOC_PAGE_RESET` by default. New reclamation strategy for abandoned heap pages - for better memory footprint. -* 2020-02-09, `v1.5.0`: stable release 1.5: improved free performance, small bug fixes. -* 2020-01-22, `v1.4.0`: stable release 1.4: improved performance for delayed OS page reset, -more eager concurrent free, addition of STL allocator, fixed potential memory leak. -* 2020-01-15, `v1.3.0`: stable release 1.3: bug fixes, improved randomness and [stronger -free list encoding](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396) in secure mode. -* 2019-12-22, `v1.2.2`: stable release 1.2: minor updates. -* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). Improved dynamic overriding on Windows. -* 2019-10-07, `v1.1.0`: stable release 1.1. -* 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. -* 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. +* [Older release notes](#older-release-notes) Special thanks to: @@ -130,7 +103,9 @@ Special thanks to: at large scale services, leading to many improvements in the mimalloc algorithms for large workloads. * Jason Gibson (@jasongibson) for exhaustive testing on large scale workloads and server environments, and finding complex bugs in (early versions of) `mimalloc`. -* Manuel Pöter (@mpoeter) and Sam Gross (@colesbury) for finding an ABA concurrency issue in abandoned segment reclamation. +* Manuel Pöter (@mpoeter) and Sam Gross(@colesbury) for finding an ABA concurrency issue in abandoned segment reclamation. Sam also created the [no GIL](https://github.com/colesbury/nogil) Python fork which + uses mimalloc internally. + [genMC]: https://plv.mpi-sws.org/genmc/ @@ -138,9 +113,12 @@ Special thanks to: mimalloc is used in various large scale low-latency services and programs, for example: - - - + + + + + + # Building @@ -647,6 +625,7 @@ see the differences in the _larsonN_, _mstressN_, and _xmalloc-testN_ benchmarks --> + # References - \[1] Emery D. Berger, Kathryn S. McKinley, Robert D. Blumofe, and Paul R. Wilson. @@ -684,7 +663,6 @@ see the differences in the _larsonN_, _mstressN_, and _xmalloc-testN_ benchmarks In Proceedings of the 2019 ACM SIGPLAN International Symposium on Memory Management, 122–135. ACM. 2019. --> - # Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a @@ -694,3 +672,34 @@ the rights to use your contribution. For details, visit https://cla.microsoft.co When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. + + +# Older Release Notes + +* 2020-09-24, `v1.6.7`: stable release 1.6: using standard C atomics, passing tsan testing, improved + handling of failing to commit on Windows, add [`mi_process_info`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc.h#L156) api call. +* 2020-08-06, `v1.6.4`: stable release 1.6: improved error recovery in low-memory situations, + support for IllumOS and Haiku, NUMA support for Vista/XP, improved NUMA detection for AMD Ryzen, ubsan support. +* 2020-05-05, `v1.6.3`: stable release 1.6: improved behavior in out-of-memory situations, improved malloc zones on macOS, + build PIC static libraries by default, add option to abort on out-of-memory, line buffered statistics. +* 2020-04-20, `v1.6.2`: stable release 1.6: fix compilation on Android, MingW, Raspberry, and Conda, + stability fix for Windows 7, fix multiple mimalloc instances in one executable, fix `strnlen` overload, + fix aligned debug padding. +* 2020-02-17, `v1.6.1`: stable release 1.6: minor updates (build with clang-cl, fix alignment issue for small objects). +* 2020-02-09, `v1.6.0`: stable release 1.6: fixed potential memory leak, improved overriding + and thread local support on FreeBSD, NetBSD, DragonFly, and macOSX. New byte-precise + heap block overflow detection in debug mode (besides the double-free detection and free-list + corruption detection). Add `nodiscard` attribute to most allocation functions. + Enable `MIMALLOC_PAGE_RESET` by default. New reclamation strategy for abandoned heap pages + for better memory footprint. +* 2020-02-09, `v1.5.0`: stable release 1.5: improved free performance, small bug fixes. +* 2020-01-22, `v1.4.0`: stable release 1.4: improved performance for delayed OS page reset, +more eager concurrent free, addition of STL allocator, fixed potential memory leak. +* 2020-01-15, `v1.3.0`: stable release 1.3: bug fixes, improved randomness and [stronger +free list encoding](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af43a0793910a9f2d01ac7863/include/mimalloc-internal.h#L396) in secure mode. +* 2019-12-22, `v1.2.2`: stable release 1.2: minor updates. +* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). Improved dynamic overriding on Windows. +* 2019-10-07, `v1.1.0`: stable release 1.1. +* 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. +* 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. + diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index b53032e2..a88186bc 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -32,8 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file extern "C" { #endif -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 +#if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) // only available from OSX 10.6 extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); #endif @@ -184,6 +183,10 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) { #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #endif +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wc99-extensions" +#endif + static malloc_introspection_t mi_introspect = { .enumerator = &intro_enumerator, .good_size = &intro_good_size, @@ -214,7 +217,7 @@ static malloc_zone_t mi_malloc_zone = { .batch_free = &zone_batch_free, .introspect = &mi_introspect, #if defined(MAC_OS_X_VERSION_10_6) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6) - #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7) + #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) .version = 10, #else .version = 9, @@ -223,7 +226,7 @@ static malloc_zone_t mi_malloc_zone = { .memalign = &zone_memalign, .free_definite_size = &zone_free_definite_size, .pressure_relief = &zone_pressure_relief, - #if defined(MAC_OS_X_VERSION_10_7) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7) + #if defined(MAC_OS_X_VERSION_10_14) && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14) .claimed_address = &zone_claimed_address, #endif #else diff --git a/src/alloc.c b/src/alloc.c index 0bc60168..8399ffb2 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -424,6 +424,8 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc const size_t avail = mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection #if (MI_DEBUG!=0) memset(block, MI_DEBUG_FREED, avail); + #else + MI_UNUSED(avail); #endif // huge page segments are always abandoned and can be freed immediately @@ -552,13 +554,13 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms // Free a block void mi_free(void* p) mi_attr_noexcept { - const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); + mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; - + if (mi_likely(tid == mi_atomic_load_relaxed(&segment->thread_id) && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned if (mi_unlikely(mi_check_is_double_free(page, block))) return; diff --git a/src/init.c b/src/init.c index ba44b45f..ce932cd6 100644 --- a/src/init.c +++ b/src/init.c @@ -439,7 +439,9 @@ static void mi_process_load(void) { MI_UNUSED(dummy); #endif os_preloading = false; - atexit(&mi_process_done); + #if !(defined(_WIN32) && defined(MI_SHARED_LIB)) // use Dll process detach (see below) instead of atexit (issue #521) + atexit(&mi_process_done); + #endif _mi_options_init(); mi_process_init(); //mi_stats_reset();- @@ -526,11 +528,13 @@ static void mi_process_done(void) { FlsFree(mi_fls_key); // call thread-done on all threads (except the main thread) to prevent dangling callback pointer if statically linked with a DLL; Issue #208 #endif - #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB) - // free all memory if possible on process exit. This is not needed for a stand-alone process - // but should be done if mimalloc is statically linked into another shared library which - // is repeatedly loaded/unloaded, see issue #281. - mi_collect(true /* force */ ); + #ifndef MI_SKIP_COLLECT_ON_EXIT + #if (MI_DEBUG != 0) || !defined(MI_SHARED_LIB) + // free all memory if possible on process exit. This is not needed for a stand-alone process + // but should be done if mimalloc is statically linked into another shared library which + // is repeatedly loaded/unloaded, see issue #281. + mi_collect(true /* force */ ); + #endif #endif if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { @@ -551,9 +555,14 @@ static void mi_process_done(void) { if (reason==DLL_PROCESS_ATTACH) { mi_process_load(); } - else if (reason==DLL_THREAD_DETACH) { - if (!mi_is_redirected()) mi_thread_done(); + else if (reason==DLL_PROCESS_DETACH) { + mi_process_done(); } + else if (reason==DLL_THREAD_DETACH) { + if (!mi_is_redirected()) { + mi_thread_done(); + } + } return TRUE; } @@ -572,7 +581,7 @@ static void mi_process_done(void) { __pragma(comment(linker, "/include:" "__mi_msvc_initu")) #endif #pragma data_seg(".CRT$XIU") - extern "C" _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init }; + mi_decl_externc _mi_crt_callback_t _mi_msvc_initu[] = { &_mi_process_init }; #pragma data_seg() #elif defined(__cplusplus) diff --git a/src/page-queue.c b/src/page-queue.c index 365257e7..26b10800 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -53,7 +53,7 @@ static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { // Returns MI_BIN_HUGE if the size is too large. // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. -extern inline uint8_t _mi_bin(size_t size) { +static inline uint8_t mi_bin(size_t size) { size_t wsize = _mi_wsize_from_size(size); uint8_t bin; if (wsize <= 1) { @@ -98,6 +98,10 @@ extern inline uint8_t _mi_bin(size_t size) { Queue of pages with free blocks ----------------------------------------------------------- */ +uint8_t _mi_bin(size_t size) { + return mi_bin(size); +} + size_t _mi_bin_size(uint8_t bin) { return _mi_heap_empty.pages[bin].block_size; } @@ -105,7 +109,7 @@ size_t _mi_bin_size(uint8_t bin) { // Good size for allocation size_t mi_good_size(size_t size) mi_attr_noexcept { if (size <= MI_LARGE_OBJ_SIZE_MAX) { - return _mi_bin_size(_mi_bin(size)); + return _mi_bin_size(mi_bin(size)); } else { return _mi_align_up(size,_mi_os_page_size()); @@ -134,7 +138,7 @@ static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* #endif static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { - uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size)); + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : mi_bin(page->xblock_size)); mi_heap_t* heap = mi_page_heap(page); mi_assert_internal(heap != NULL && bin <= MI_BIN_FULL); mi_page_queue_t* pq = &heap->pages[bin]; @@ -144,7 +148,7 @@ static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { } static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { - uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size)); + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : mi_bin(page->xblock_size)); mi_assert_internal(bin <= MI_BIN_FULL); mi_page_queue_t* pq = &heap->pages[bin]; mi_assert_internal(mi_page_is_in_full(page) || page->xblock_size == pq->block_size); @@ -177,9 +181,9 @@ static inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_que } else { // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped - uint8_t bin = _mi_bin(size); + uint8_t bin = mi_bin(size); const mi_page_queue_t* prev = pq - 1; - while( bin == _mi_bin(prev->block_size) && prev > &heap->pages[0]) { + while( bin == mi_bin(prev->block_size) && prev > &heap->pages[0]) { prev--; } start = 1 + _mi_wsize_from_size(prev->block_size); diff --git a/src/page.c b/src/page.c index 3368bad1..6efeba2a 100644 --- a/src/page.c +++ b/src/page.c @@ -574,13 +574,16 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) // calculate the extend count const size_t bsize = (page->xblock_size < MI_HUGE_BLOCK_SIZE ? page->xblock_size : page_size); size_t extend = page->reserved - page->capacity; - size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/(uint32_t)bsize); - if (max_extend < MI_MIN_EXTEND) max_extend = MI_MIN_EXTEND; + mi_assert_internal(extend > 0); + size_t max_extend = (bsize >= MI_MAX_EXTEND_SIZE ? MI_MIN_EXTEND : MI_MAX_EXTEND_SIZE/(uint32_t)bsize); + if (max_extend < MI_MIN_EXTEND) { max_extend = MI_MIN_EXTEND; } + mi_assert_internal(max_extend > 0); + if (extend > max_extend) { // ensure we don't touch memory beyond the page to reduce page commit. // the `lean` benchmark tests this. Going from 1 to 8 increases rss by 50%. - extend = (max_extend==0 ? 1 : max_extend); + extend = max_extend; } mi_assert_internal(extend > 0 && extend + page->capacity <= page->reserved); @@ -765,7 +768,7 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex // that frees the block can free the whole page and segment directly. static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { size_t block_size = _mi_os_good_alloc_size(size); - mi_assert_internal(_mi_bin(block_size) == MI_BIN_HUGE); + mi_assert_internal(mi_bin(block_size) == MI_BIN_HUGE); mi_page_t* page = mi_page_fresh_alloc(heap,NULL,block_size); if (page != NULL) { const size_t bsize = mi_page_block_size(page); // note: not `mi_page_usable_block_size` as `size` includes padding already diff --git a/src/random.c b/src/random.c index e47946a6..0b44c8b9 100644 --- a/src/random.c +++ b/src/random.c @@ -239,7 +239,7 @@ static bool os_random_buf(void* buf, size_t buf_len) { if (mi_atomic_load_acquire(&no_getrandom)==0) { ssize_t ret = syscall(SYS_getrandom, buf, buf_len, GRND_NONBLOCK); if (ret >= 0) return (buf_len == (size_t)ret); - if (ret != ENOSYS) return false; + if (errno != ENOSYS) return false; mi_atomic_store_release(&no_getrandom, 1UL); // don't call again, and fall back to /dev/urandom } #endif diff --git a/src/segment.c b/src/segment.c index 424495ca..1aadf5c2 100644 --- a/src/segment.c +++ b/src/segment.c @@ -658,8 +658,9 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ memset((uint8_t*)segment + ofs, 0, info_size - ofs); // initialize pages info - for (uint8_t i = 0; i < capacity; i++) { - segment->pages[i].segment_idx = i; + for (size_t i = 0; i < capacity; i++) { + mi_assert_internal(i <= 255); + segment->pages[i].segment_idx = (uint8_t)i; segment->pages[i].is_reset = false; segment->pages[i].is_committed = commit; segment->pages[i].is_zero_init = is_zero;