diff --git a/CMakeLists.txt b/CMakeLists.txt index d5452bd2..4f2e1841 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,9 @@ option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macO option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) +option(MI_PADDING "Enable padding to detect heap block overflow (only in debug mode)" ON) +option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) +option(MI_SHOW_ERRORS "Show error and warning messages by default" OFF) include("cmake/mimalloc-config-version.cmake") @@ -65,6 +68,7 @@ if(MI_OVERRIDE MATCHES "ON") # use zone's on macOS message(STATUS " Use malloc zone to override malloc (MI_OSX_ZONE=ON)") list(APPEND mi_sources src/alloc-override-osx.c) + list(APPEND mi_defines MI_OSX_ZONE=1) if(NOT MI_INTERPOSE MATCHES "ON") message(STATUS " (enabling INTERPOSE as well since zone's require this)") set(MI_INTERPOSE "ON") @@ -98,6 +102,21 @@ if(MI_DEBUG_FULL MATCHES "ON") list(APPEND mi_defines MI_DEBUG=3) # full invariant checking endif() +if(MI_PADDING MATCHES "OFF") + message(STATUS "Disable padding of heap blocks in debug mode (MI_PADDING=OFF)") + list(APPEND mi_defines MI_PADDING=0) +endif() + +if(MI_XMALLOC MATCHES "ON") + message(STATUS "Enable abort() calls on memory allocation failure (MI_XMALLOC=ON)") + list(APPEND mi_defines MI_XMALLOC=1) +endif() + +if(MI_SHOW_ERRORS MATCHES "ON") + message(STATUS "Enable printing of error and warning messages by default (MI_SHOW_ERRORS=ON)") + list(APPEND mi_defines MI_SHOW_ERRORS=1) +endif() + if(MI_USE_CXX MATCHES "ON") message(STATUS "Use the C++ compiler to compile (MI_USE_CXX=ON)") set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) @@ -139,10 +158,12 @@ endif() if(WIN32) list(APPEND mi_libraries psapi shell32 user32 bcrypt) else() - list(APPEND mi_libraries pthread) - find_library(LIBRT rt) - if(LIBRT) - list(APPEND mi_libraries ${LIBRT}) + if(NOT ${CMAKE_C_COMPILER} MATCHES "android") + list(APPEND mi_libraries pthread) + find_library(LIBRT rt) + if(LIBRT) + list(APPEND mi_libraries ${LIBRT}) + endif() endif() endif() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c7bad96..c81e31bd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,12 +18,15 @@ jobs: Debug: BuildType: debug cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + MSBuildConfiguration: Debug Release: BuildType: release cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + MSBuildConfiguration: Release Secure: BuildType: secure cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + MSBuildConfiguration: Release steps: - task: CMake@1 inputs: @@ -32,12 +35,13 @@ jobs: - task: MSBuild@1 inputs: solution: $(BuildType)/libmimalloc.sln + configuration: '$(MSBuildConfiguration)' - script: | cd $(BuildType) ctest displayName: CTest - - upload: $(Build.SourcesDirectory)/$(BuildType) - artifact: mimalloc-windows-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-windows-$(BuildType) - job: displayName: Linux @@ -95,8 +99,8 @@ jobs: displayName: Make - script: make test -C $(BuildType) displayName: CTest - - upload: $(Build.SourcesDirectory)/$(BuildType) - artifact: mimalloc-ubuntu-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-ubuntu-$(BuildType) - job: displayName: macOS @@ -123,5 +127,5 @@ jobs: displayName: Make - script: make test -C $(BuildType) displayName: CTest - - upload: $(Build.SourcesDirectory)/$(BuildType) - artifact: mimalloc-macos-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-macos-$(BuildType) diff --git a/doc/doxyfile b/doc/doxyfile index 91adbeb8..6c1e30a0 100644 --- a/doc/doxyfile +++ b/doc/doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = mi-malloc # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.4 +PROJECT_NUMBER = 1.6 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 3f24a623..67f4fe95 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -1009,28 +1009,31 @@ or via environment variables. - `MIMALLOC_SHOW_STATS=1`: show statistics when the program terminates. - `MIMALLOC_VERBOSE=1`: show verbose messages. - `MIMALLOC_SHOW_ERRORS=1`: show error and warning messages. -- `MIMALLOC_PAGE_RESET=1`: reset (or purge) OS pages when not in use. This can reduce - memory fragmentation in long running (server) programs. If performance is impacted, - `MIMALLOC_RESET_DELAY=`_msecs_ can be set higher (100ms by default) to make the page - reset occur less frequently. -- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages when available; for some workloads this can significantly +- `MIMALLOC_PAGE_RESET=0`: by default, mimalloc will reset (or purge) OS pages when not in use to signal to the OS + that the underlying physical memory can be reused. This can reduce memory fragmentation in long running (server) + programs. By setting it to `0` no such page resets will be done which can improve performance for programs that are not long + running. As an alternative, the `MIMALLOC_RESET_DELAY=` can be set higher (100ms by default) to make the page + reset occur less frequently instead of turning it off completely. +- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages (2MiB) when available; for some workloads this can significantly improve performance. Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs to explicitly allow large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead when possible). -- `MIMALLOC_EAGER_REGION_COMMIT=1`: on Windows, commit large (256MiB) regions eagerly. On Windows, these regions - show in the working set even though usually just a small part is committed to physical memory. This is why it - turned off by default on Windows as it looks not good in the task manager. However, in reality it is always better - to turn it on as it improves performance and has no other drawbacks. -- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where N is the number of 1GiB huge OS pages. This reserves the huge pages at - startup and can give quite a performance improvement on long running workloads. Usually it is better to not use +- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where N is the number of 1GiB _huge_ OS pages. This reserves the huge pages at + startup and sometimes this can give a large (latency) performance improvement on big workloads. + Usually it is better to not use `MIMALLOC_LARGE_OS_PAGES` in combination with this setting. Just like large OS pages, use with care as reserving - contiguous physical memory can take a long time when memory is fragmented. + contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at + startup only once). Note that we usually need to explicitly enable huge OS pages (as on [Windows][windows-huge] and [Linux][linux-huge])). With huge OS pages, it may be beneficial to set the setting - `MIMALLOC_EAGER_COMMIT_DELAY=N` (with usually `N` as 1) to delay the initial `N` segments + `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write +for all pages in the original process including the huge OS pages. When any memory is now written in that area, the +OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in big increments. + [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 @@ -1074,14 +1077,18 @@ resolved to the _mimalloc_ library. Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -Note: unfortunately, at this time, dynamic overriding on macOS seems broken but it is actively worked on to fix this -(see issue [`#50`](https://github.com/microsoft/mimalloc/issues/50)). +(Note: macOS support for dynamic overriding is recent, please report any issues.) + ### Windows -Overriding on Windows is robust but requires that you link your program explicitly with +Overriding on Windows is robust and has the +particular advantage to be able to redirect all malloc/free calls that go through +the (dynamic) C runtime allocator, including those from other DLL's or libraries. + +The overriding on Windows requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -Moreover, you need to ensure the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) is available +Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be available in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in `mimalloc-override.dll`). @@ -1090,14 +1097,15 @@ To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some call to the mimalloc API in the `main` function, like `mi_version()` (or use the `/INCLUDE:mi_version` switch on the linker). See the `mimalloc-override-test` project for an example on how to use this. For best performance on Windows with C++, it -is highly recommended to also override the `new`/`delete` operations (by including +is also recommended to also override the `new`/`delete` operations (by including [`mimalloc-new-delete.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) a single(!) source file in your project). The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. -(Note: in principle, it is possible to patch existing executables -that are linked with the dynamic C runtime (`ucrtbase.dll`) by just putting the `mimalloc-override.dll` into the import table (and putting `mimalloc-redirect.dll` in the same folder) +(Note: in principle, it is possible to even patch existing executables without any recompilation +if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the `mimalloc-override.dll` +into the import table (and put `mimalloc-redirect.dll` in the same folder) Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388)). diff --git a/doc/mimalloc-logo.svg b/doc/mimalloc-logo.svg index 7b6231bd..672c7e41 100644 --- a/doc/mimalloc-logo.svg +++ b/doc/mimalloc-logo.svg @@ -36,15 +36,15 @@ inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:zoom="23.706667" - inkscape:cx="34.419045" + inkscape:cx="24.864771" inkscape:cy="35.79485" inkscape:document-units="mm" inkscape:current-layer="layer1" showgrid="false" inkscape:window-width="3840" - inkscape:window-height="2160" - inkscape:window-x="0" - inkscape:window-y="0" + inkscape:window-height="2050" + inkscape:window-x="-12" + inkscape:window-y="-12" inkscape:window-maximized="1" inkscape:snap-object-midpoints="false" inkscape:snap-bbox="false" @@ -126,17 +126,15 @@ style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:2.28797817px;font-family:RoutedGothicEx;-inkscape-font-specification:'RoutedGothicEx, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;stroke-width:0.28599727" id="path6525" /> - m + id="text848"> + + diff --git a/docs/annotated.html b/docs/annotated.html index 5120b803..feba2438 100644 --- a/docs/annotated.html +++ b/docs/annotated.html @@ -37,7 +37,7 @@ Logo
mi-malloc -  1.4 +  1.6
diff --git a/docs/bench.html b/docs/bench.html index 6b289c04..f39fade6 100644 --- a/docs/bench.html +++ b/docs/bench.html @@ -37,7 +37,7 @@ Logo
mi-malloc -  1.4 +  1.6
diff --git a/docs/build.html b/docs/build.html index 755aad88..2bd06f13 100644 --- a/docs/build.html +++ b/docs/build.html @@ -37,7 +37,7 @@ Logo
mi-malloc -  1.4 +  1.6
diff --git a/docs/classes.html b/docs/classes.html index de960fb6..e74a0a24 100644 --- a/docs/classes.html +++ b/docs/classes.html @@ -37,7 +37,7 @@ Logo
mi-malloc -  1.4 +  1.6
diff --git a/docs/environment.html b/docs/environment.html index 1063654e..87d67e4a 100644 --- a/docs/environment.html +++ b/docs/environment.html @@ -37,7 +37,7 @@ Logo
mi-malloc -  1.4 +  1.6
@@ -107,11 +107,11 @@ $(document).ready(function(){initNavTree('environment.html','');});
  • MIMALLOC_SHOW_STATS=1: show statistics when the program terminates.
  • MIMALLOC_VERBOSE=1: show verbose messages.
  • MIMALLOC_SHOW_ERRORS=1: show error and warning messages.
    • -
  • MIMALLOC_PAGE_RESET=1: reset (or purge) OS pages when not in use. This can reduce memory fragmentation in long running (server) programs. If performance is impacted, MIMALLOC_RESET_DELAY=_msecs_ can be set higher (100ms by default) to make the page reset occur less frequently.
    • -
  • MIMALLOC_LARGE_OS_PAGES=1: use large OS pages when available; for some workloads this can significantly improve performance. Use MIMALLOC_VERBOSE to check if the large OS pages are enabled – usually one needs to explicitly allow large OS pages (as on Windows and Linux). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that can have fragmented memory (for that reason, we generally recommend to use MIMALLOC_RESERVE_HUGE_OS_PAGES instead when possible).
    • -
  • MIMALLOC_EAGER_REGION_COMMIT=1: on Windows, commit large (256MiB) regions eagerly. On Windows, these regions show in the working set even though usually just a small part is committed to physical memory. This is why it turned off by default on Windows as it looks not good in the task manager. However, in reality it is always better to turn it on as it improves performance and has no other drawbacks.
    • -
  • MIMALLOC_RESERVE_HUGE_OS_PAGES=N: where N is the number of 1GiB huge OS pages. This reserves the huge pages at startup and can give quite a performance improvement on long running workloads. Usually it is better to not use MIMALLOC_LARGE_OS_PAGES in combination with this setting. Just like large OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented. Note that we usually need to explicitly enable huge OS pages (as on Windows and Linux)). With huge OS pages, it may be beneficial to set the setting MIMALLOC_EAGER_COMMIT_DELAY=N (with usually N as 1) to delay the initial N segments of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset).
    • +
  • MIMALLOC_PAGE_RESET=0: by default, mimalloc will reset (or purge) OS pages when not in use to signal to the OS that the underlying physical memory can be reused. This can reduce memory fragmentation in long running (server) programs. By setting it to 0 no such page resets will be done which can improve performance for programs that are not long running. As an alternative, the MIMALLOC_RESET_DELAY=<msecs> can be set higher (100ms by default) to make the page reset occur less frequently instead of turning it off completely.
    • +
  • MIMALLOC_LARGE_OS_PAGES=1: use large OS pages (2MiB) when available; for some workloads this can significantly improve performance. Use MIMALLOC_VERBOSE to check if the large OS pages are enabled – usually one needs to explicitly allow large OS pages (as on Windows and Linux). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that can have fragmented memory (for that reason, we generally recommend to use MIMALLOC_RESERVE_HUGE_OS_PAGES instead when possible).
    • +
  • MIMALLOC_RESERVE_HUGE_OS_PAGES=N: where N is the number of 1GiB huge OS pages. This reserves the huge pages at startup and sometimes this can give a large (latency) performance improvement on big workloads. Usually it is better to not use MIMALLOC_LARGE_OS_PAGES in combination with this setting. Just like large OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at startup only once). Note that we usually need to explicitly enable huge OS pages (as on Windows and Linux)). With huge OS pages, it may be beneficial to set the setting MIMALLOC_EAGER_COMMIT_DELAY=N (N is 1 by default) to delay the initial N segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset).
    • +

    Use caution when using fork in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in big increments.

    diff --git a/docs/functions.html b/docs/functions.html index 43e116ed..62021210 100644 --- a/docs/functions.html +++ b/docs/functions.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/functions_vars.html b/docs/functions_vars.html index 060a18d2..7d41d10e 100644 --- a/docs/functions_vars.html +++ b/docs/functions_vars.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__aligned.html b/docs/group__aligned.html index 88c10eb4..a3eaacf0 100644 --- a/docs/group__aligned.html +++ b/docs/group__aligned.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__analysis.html b/docs/group__analysis.html index b8d644aa..2487c240 100644 --- a/docs/group__analysis.html +++ b/docs/group__analysis.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__cpp.html b/docs/group__cpp.html index caf758a8..88c75888 100644 --- a/docs/group__cpp.html +++ b/docs/group__cpp.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__extended.html b/docs/group__extended.html index 9e2a2efc..325d62bf 100644 --- a/docs/group__extended.html +++ b/docs/group__extended.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__heap.html b/docs/group__heap.html index 0973279a..1a38c936 100644 --- a/docs/group__heap.html +++ b/docs/group__heap.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__malloc.html b/docs/group__malloc.html index bee7b4eb..224c4b08 100644 --- a/docs/group__malloc.html +++ b/docs/group__malloc.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__options.html b/docs/group__options.html index 71c7ba24..5e45d7bd 100644 --- a/docs/group__options.html +++ b/docs/group__options.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__posix.html b/docs/group__posix.html index 1aea8dc8..fe3a88e1 100644 --- a/docs/group__posix.html +++ b/docs/group__posix.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/group__typed.html b/docs/group__typed.html index cf5ac5d1..5cbfbd6b 100644 --- a/docs/group__typed.html +++ b/docs/group__typed.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    @@ -146,7 +146,7 @@ Macros

    Detailed Description

    Typed allocation macros.

    -

    Macro Definition Documentation

    +

    For example:

    int* p = mi_malloc_tp(int)

    Macro Definition Documentation

    ◆ mi_calloc_tp

    diff --git a/docs/group__zeroinit.html b/docs/group__zeroinit.html index 28983138..3c04a5a6 100644 --- a/docs/group__zeroinit.html +++ b/docs/group__zeroinit.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/index.html b/docs/index.html index 0efc9c09..ce9c983d 100644 --- a/docs/index.html +++ b/docs/index.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    @@ -127,7 +127,9 @@ $(document).ready(function(){initNavTree('index.html','');});
  • Heap Allocation
  • Typed Macros
  • Heap Introspection
    • -
  • Runtime Options
    • +
  • Runtime Options
    • +
  • Posix
    • +
  • C++ wrappers
    • diff --git a/docs/mimalloc-doc_8h_source.html b/docs/mimalloc-doc_8h_source.html index f70ae81f..09c03b9b 100644 --- a/docs/mimalloc-doc_8h_source.html +++ b/docs/mimalloc-doc_8h_source.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    @@ -102,7 +102,7 @@ $(document).ready(function(){initNavTree('mimalloc-doc_8h_source.html','');});
    mimalloc-doc.h
    -
    1 /* ----------------------------------------------------------------------------
    2 Copyright (c) 2018, Microsoft Research, Daan Leijen
    3 This is free software; you can redistribute it and/or modify it under the
    4 terms of the MIT license. A copy of the license can be found in the file
    5 "LICENSE" at the root of this distribution.
    6 -----------------------------------------------------------------------------*/
    7 
    8 #error "documentation file only!"
    9 
    10 
    81 
    85 
    89 void mi_free(void* p);
    90 
    95 void* mi_malloc(size_t size);
    96 
    101 void* mi_zalloc(size_t size);
    102 
    112 void* mi_calloc(size_t count, size_t size);
    113 
    126 void* mi_realloc(void* p, size_t newsize);
    127 
    138 void* mi_recalloc(void* p, size_t count, size_t size);
    139 
    153 void* mi_expand(void* p, size_t newsize);
    154 
    164 void* mi_mallocn(size_t count, size_t size);
    165 
    175 void* mi_reallocn(void* p, size_t count, size_t size);
    176 
    193 void* mi_reallocf(void* p, size_t newsize);
    194 
    195 
    204 char* mi_strdup(const char* s);
    205 
    215 char* mi_strndup(const char* s, size_t n);
    216 
    229 char* mi_realpath(const char* fname, char* resolved_name);
    230 
    232 
    233 // ------------------------------------------------------
    234 // Extended functionality
    235 // ------------------------------------------------------
    236 
    240 
    243 #define MI_SMALL_SIZE_MAX (128*sizeof(void*))
    244 
    252 void* mi_malloc_small(size_t size);
    253 
    261 void* mi_zalloc_small(size_t size);
    262 
    277 size_t mi_usable_size(void* p);
    278 
    288 size_t mi_good_size(size_t size);
    289 
    297 void mi_collect(bool force);
    298 
    303 void mi_stats_print(void* out);
    304 
    310 void mi_stats_print(mi_output_fun* out, void* arg);
    311 
    313 void mi_stats_reset(void);
    314 
    316 void mi_stats_merge(void);
    317 
    321 void mi_thread_init(void);
    322 
    327 void mi_thread_done(void);
    328 
    334 void mi_thread_stats_print_out(mi_output_fun* out, void* arg);
    335 
    342 typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
    343 
    359 void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
    360 
    366 typedef void (mi_output_fun)(const char* msg, void* arg);
    367 
    374 void mi_register_output(mi_output_fun* out, void* arg);
    375 
    381 typedef void (mi_error_fun)(int err, void* arg);
    382 
    398 void mi_register_error(mi_error_fun* errfun, void* arg);
    399 
    404 bool mi_is_in_heap_region(const void* p);
    405 
    406 
    419 int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
    420 
    433 int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
    434 
    435 
    440 bool mi_is_redirected();
    441 
    442 
    444 
    445 // ------------------------------------------------------
    446 // Aligned allocation
    447 // ------------------------------------------------------
    448 
    454 
    467 void* mi_malloc_aligned(size_t size, size_t alignment);
    468 void* mi_zalloc_aligned(size_t size, size_t alignment);
    469 void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
    470 void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
    471 
    482 void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
    483 void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
    484 void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
    485 void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    486 
    488 
    494 
    499 struct mi_heap_s;
    500 
    505 typedef struct mi_heap_s mi_heap_t;
    506 
    508 mi_heap_t* mi_heap_new();
    509 
    517 void mi_heap_delete(mi_heap_t* heap);
    518 
    526 void mi_heap_destroy(mi_heap_t* heap);
    527 
    531 mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
    532 
    535 mi_heap_t* mi_heap_get_default();
    536 
    542 mi_heap_t* mi_heap_get_backing();
    543 
    545 void mi_heap_collect(mi_heap_t* heap, bool force);
    546 
    549 void* mi_heap_malloc(mi_heap_t* heap, size_t size);
    550 
    554 void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
    555 
    558 void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
    559 
    562 void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
    563 
    566 void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
    567 
    570 char* mi_heap_strdup(mi_heap_t* heap, const char* s);
    571 
    574 char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
    575 
    578 char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
    579 
    580 void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
    581 void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
    582 void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
    583 
    584 void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    585 void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    586 void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    587 void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    588 void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
    589 void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
    590 void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    591 void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    592 
    594 
    595 
    604 
    605 void* mi_rezalloc(void* p, size_t newsize);
    606 void* mi_recalloc(void* p, size_t newcount, size_t size) ;
    607 
    608 void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
    609 void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    610 void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
    611 void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    612 
    613 void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
    614 void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
    615 
    616 void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    617 void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    618 void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
    619 void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    620 
    622 
    628 
    640 #define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
    641 
    643 #define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
    644 
    646 #define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
    647 
    649 #define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
    650 
    652 #define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
    653 
    655 #define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
    656 
    658 #define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
    659 
    661 #define mi_heap_calloc_tp(hp,tp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
    662 
    664 #define mi_heap_mallocn_tp(hp,tp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
    665 
    667 #define mi_heap_reallocn_tp(hp,p,tp,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
    668 
    670 #define mi_heap_recalloc_tp(hp,p,tp,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
    671 
    673 
    679 
    686 bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
    687 
    696 bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
    697 
    705 bool mi_check_owned(const void* p);
    706 
    709 typedef struct mi_heap_area_s {
    710  void* blocks;
    711  size_t reserved;
    712  size_t committed;
    713  size_t used;
    714  size_t block_size;
    715 } mi_heap_area_t;
    716 
    724 typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
    725 
    737 bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
    738 
    740 
    746 
    748 typedef enum mi_option_e {
    749  // stable options
    750  mi_option_show_stats,
    751  mi_option_show_errors,
    752  mi_option_verbose,
    753  // the following options are experimental
    754  mi_option_eager_commit,
    755  mi_option_eager_region_commit,
    756  mi_option_large_os_pages,
    757  mi_option_reserve_huge_os_pages,
    758  mi_option_segment_cache,
    759  mi_option_page_reset,
    760  mi_option_segment_reset,
    761  mi_option_reset_delay,
    762  mi_option_use_numa_nodes,
    763  mi_option_reset_decommits,
    764  mi_option_eager_commit_delay,
    765  mi_option_os_tag,
    766  _mi_option_last
    767 } mi_option_t;
    768 
    769 
    770 bool mi_option_enabled(mi_option_t option);
    771 void mi_option_enable(mi_option_t option, bool enable);
    772 void mi_option_enable_default(mi_option_t option, bool enable);
    773 
    774 long mi_option_get(mi_option_t option);
    775 void mi_option_set(mi_option_t option, long value);
    776 void mi_option_set_default(mi_option_t option, long value);
    777 
    778 
    780 
    787 
    788 void* mi_recalloc(void* p, size_t count, size_t size);
    789 size_t mi_malloc_size(const void* p);
    790 size_t mi_malloc_usable_size(const void *p);
    791 
    793 void mi_cfree(void* p);
    794 
    795 int mi_posix_memalign(void** p, size_t alignment, size_t size);
    796 int mi__posix_memalign(void** p, size_t alignment, size_t size);
    797 void* mi_memalign(size_t alignment, size_t size);
    798 void* mi_valloc(size_t size);
    799 
    800 void* mi_pvalloc(size_t size);
    801 void* mi_aligned_alloc(size_t alignment, size_t size);
    802 void* mi_reallocarray(void* p, size_t count, size_t size);
    803 
    804 void mi_free_size(void* p, size_t size);
    805 void mi_free_size_aligned(void* p, size_t size, size_t alignment);
    806 void mi_free_aligned(void* p, size_t alignment);
    807 
    809 
    822 
    824 void* mi_new(std::size_t n) noexcept(false);
    825 
    827 void* mi_new_n(size_t count, size_t size) noexcept(false);
    828 
    830 void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
    831 
    833 void* mi_new_nothrow(size_t n);
    834 
    836 void* mi_new_aligned_nothrow(size_t n, size_t alignment);
    837 
    839 void* mi_new_realloc(void* p, size_t newsize);
    840 
    842 void* mi_new_reallocn(void* p, size_t newcount, size_t size);
    843 
    851 template<class T> struct mi_stl_allocator { }
    852 
    854 
    mi_option_enable_default
    void mi_option_enable_default(mi_option_t option, bool enable)
    +
    1 /* ----------------------------------------------------------------------------
    2 Copyright (c) 2018, Microsoft Research, Daan Leijen
    3 This is free software; you can redistribute it and/or modify it under the
    4 terms of the MIT license. A copy of the license can be found in the file
    5 "LICENSE" at the root of this distribution.
    6 -----------------------------------------------------------------------------*/
    7 
    8 #error "documentation file only!"
    9 
    10 
    83 
    87 
    91 void mi_free(void* p);
    92 
    97 void* mi_malloc(size_t size);
    98 
    103 void* mi_zalloc(size_t size);
    104 
    114 void* mi_calloc(size_t count, size_t size);
    115 
    128 void* mi_realloc(void* p, size_t newsize);
    129 
    140 void* mi_recalloc(void* p, size_t count, size_t size);
    141 
    155 void* mi_expand(void* p, size_t newsize);
    156 
    166 void* mi_mallocn(size_t count, size_t size);
    167 
    177 void* mi_reallocn(void* p, size_t count, size_t size);
    178 
    195 void* mi_reallocf(void* p, size_t newsize);
    196 
    197 
    206 char* mi_strdup(const char* s);
    207 
    217 char* mi_strndup(const char* s, size_t n);
    218 
    231 char* mi_realpath(const char* fname, char* resolved_name);
    232 
    234 
    235 // ------------------------------------------------------
    236 // Extended functionality
    237 // ------------------------------------------------------
    238 
    242 
    245 #define MI_SMALL_SIZE_MAX (128*sizeof(void*))
    246 
    254 void* mi_malloc_small(size_t size);
    255 
    263 void* mi_zalloc_small(size_t size);
    264 
    279 size_t mi_usable_size(void* p);
    280 
    290 size_t mi_good_size(size_t size);
    291 
    299 void mi_collect(bool force);
    300 
    305 void mi_stats_print(void* out);
    306 
    312 void mi_stats_print(mi_output_fun* out, void* arg);
    313 
    315 void mi_stats_reset(void);
    316 
    318 void mi_stats_merge(void);
    319 
    323 void mi_thread_init(void);
    324 
    329 void mi_thread_done(void);
    330 
    336 void mi_thread_stats_print_out(mi_output_fun* out, void* arg);
    337 
    344 typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
    345 
    361 void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
    362 
    368 typedef void (mi_output_fun)(const char* msg, void* arg);
    369 
    376 void mi_register_output(mi_output_fun* out, void* arg);
    377 
    383 typedef void (mi_error_fun)(int err, void* arg);
    384 
    400 void mi_register_error(mi_error_fun* errfun, void* arg);
    401 
    406 bool mi_is_in_heap_region(const void* p);
    407 
    408 
    421 int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
    422 
    435 int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
    436 
    437 
    442 bool mi_is_redirected();
    443 
    444 
    446 
    447 // ------------------------------------------------------
    448 // Aligned allocation
    449 // ------------------------------------------------------
    450 
    456 
    469 void* mi_malloc_aligned(size_t size, size_t alignment);
    470 void* mi_zalloc_aligned(size_t size, size_t alignment);
    471 void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
    472 void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
    473 
    484 void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
    485 void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
    486 void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
    487 void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    488 
    490 
    496 
    501 struct mi_heap_s;
    502 
    507 typedef struct mi_heap_s mi_heap_t;
    508 
    510 mi_heap_t* mi_heap_new();
    511 
    519 void mi_heap_delete(mi_heap_t* heap);
    520 
    528 void mi_heap_destroy(mi_heap_t* heap);
    529 
    533 mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
    534 
    537 mi_heap_t* mi_heap_get_default();
    538 
    544 mi_heap_t* mi_heap_get_backing();
    545 
    547 void mi_heap_collect(mi_heap_t* heap, bool force);
    548 
    551 void* mi_heap_malloc(mi_heap_t* heap, size_t size);
    552 
    556 void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
    557 
    560 void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
    561 
    564 void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
    565 
    568 void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
    569 
    572 char* mi_heap_strdup(mi_heap_t* heap, const char* s);
    573 
    576 char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
    577 
    580 char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
    581 
    582 void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
    583 void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
    584 void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
    585 
    586 void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    587 void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    588 void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    589 void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    590 void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
    591 void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
    592 void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    593 void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    594 
    596 
    597 
    606 
    607 void* mi_rezalloc(void* p, size_t newsize);
    608 void* mi_recalloc(void* p, size_t newcount, size_t size) ;
    609 
    610 void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
    611 void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    612 void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
    613 void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    614 
    615 void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
    616 void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
    617 
    618 void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    619 void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    620 void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
    621 void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    622 
    624 
    633 
    645 #define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
    646 
    648 #define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
    649 
    651 #define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
    652 
    654 #define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
    655 
    657 #define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
    658 
    660 #define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
    661 
    663 #define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
    664 
    666 #define mi_heap_calloc_tp(hp,tp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
    667 
    669 #define mi_heap_mallocn_tp(hp,tp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
    670 
    672 #define mi_heap_reallocn_tp(hp,p,tp,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
    673 
    675 #define mi_heap_recalloc_tp(hp,p,tp,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
    676 
    678 
    684 
    691 bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
    692 
    701 bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
    702 
    710 bool mi_check_owned(const void* p);
    711 
    714 typedef struct mi_heap_area_s {
    715  void* blocks;
    716  size_t reserved;
    717  size_t committed;
    718  size_t used;
    719  size_t block_size;
    720 } mi_heap_area_t;
    721 
    729 typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
    730 
    742 bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg);
    743 
    745 
    751 
    753 typedef enum mi_option_e {
    754  // stable options
    755  mi_option_show_stats,
    756  mi_option_show_errors,
    757  mi_option_verbose,
    758  // the following options are experimental
    759  mi_option_eager_commit,
    760  mi_option_eager_region_commit,
    761  mi_option_large_os_pages,
    762  mi_option_reserve_huge_os_pages,
    763  mi_option_segment_cache,
    764  mi_option_page_reset,
    765  mi_option_segment_reset,
    766  mi_option_reset_delay,
    767  mi_option_use_numa_nodes,
    768  mi_option_reset_decommits,
    769  mi_option_eager_commit_delay,
    770  mi_option_os_tag,
    771  _mi_option_last
    772 } mi_option_t;
    773 
    774 
    775 bool mi_option_enabled(mi_option_t option);
    776 void mi_option_enable(mi_option_t option, bool enable);
    777 void mi_option_enable_default(mi_option_t option, bool enable);
    778 
    779 long mi_option_get(mi_option_t option);
    780 void mi_option_set(mi_option_t option, long value);
    781 void mi_option_set_default(mi_option_t option, long value);
    782 
    783 
    785 
    792 
    793 void* mi_recalloc(void* p, size_t count, size_t size);
    794 size_t mi_malloc_size(const void* p);
    795 size_t mi_malloc_usable_size(const void *p);
    796 
    798 void mi_cfree(void* p);
    799 
    800 int mi_posix_memalign(void** p, size_t alignment, size_t size);
    801 int mi__posix_memalign(void** p, size_t alignment, size_t size);
    802 void* mi_memalign(size_t alignment, size_t size);
    803 void* mi_valloc(size_t size);
    804 
    805 void* mi_pvalloc(size_t size);
    806 void* mi_aligned_alloc(size_t alignment, size_t size);
    807 void* mi_reallocarray(void* p, size_t count, size_t size);
    808 
    809 void mi_free_size(void* p, size_t size);
    810 void mi_free_size_aligned(void* p, size_t size, size_t alignment);
    811 void mi_free_aligned(void* p, size_t alignment);
    812 
    814 
    827 
    829 void* mi_new(std::size_t n) noexcept(false);
    830 
    832 void* mi_new_n(size_t count, size_t size) noexcept(false);
    833 
    835 void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
    836 
    838 void* mi_new_nothrow(size_t n);
    839 
    841 void* mi_new_aligned_nothrow(size_t n, size_t alignment);
    842 
    844 void* mi_new_realloc(void* p, size_t newsize);
    845 
    847 void* mi_new_reallocn(void* p, size_t newcount, size_t size);
    848 
    856 template<class T> struct mi_stl_allocator { }
    857 
    859 
    mi_option_enable_default
    void mi_option_enable_default(mi_option_t option, bool enable)
    mi_usable_size
    size_t mi_usable_size(void *p)
    Return the available bytes in a memory block.
    mi_new_nothrow
    void * mi_new_nothrow(size_t n)
    like mi_malloc, but when out of memory, use std::get_new_handler but return NULL on failure.
    mi_reallocn
    void * mi_reallocn(void *p, size_t count, size_t size)
    Re-allocate memory to count elements of size bytes.
    @@ -118,17 +118,17 @@ $(document).ready(function(){initNavTree('mimalloc-doc_8h_source.html','');});
    mi_stats_merge
    void mi_stats_merge(void)
    Merge thread local statistics with the main statistics and reset.
    mi_new_n
    void * mi_new_n(size_t count, size_t size) noexcept(false)
    like mi_mallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exceptio...
    mi_option_set_default
    void mi_option_set_default(mi_option_t option, long value)
    -
    mi_error_fun
    void() mi_error_fun(int err, void *arg)
    Type of error callback functions.
    Definition: mimalloc-doc.h:381
    +
    mi_error_fun
    void() mi_error_fun(int err, void *arg)
    Type of error callback functions.
    Definition: mimalloc-doc.h:383
    mi_rezalloc
    void * mi_rezalloc(void *p, size_t newsize)
    -
    mi_option_eager_commit
    Eagerly commit segments (4MiB) (enabled by default).
    Definition: mimalloc-doc.h:754
    +
    mi_option_eager_commit
    Eagerly commit segments (4MiB) (enabled by default).
    Definition: mimalloc-doc.h:759
    mi_heap_zalloc
    void * mi_heap_zalloc(mi_heap_t *heap, size_t size)
    Allocate zero-initialized in a specific heap.
    mi_option_set
    void mi_option_set(mi_option_t option, long value)
    -
    mi_option_eager_region_commit
    Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows)
    Definition: mimalloc-doc.h:755
    +
    mi_option_eager_region_commit
    Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows)
    Definition: mimalloc-doc.h:760
    mi_cfree
    void mi_cfree(void *p)
    Just as free but also checks if the pointer p belongs to our heap.
    mi_recalloc_aligned
    void * mi_recalloc_aligned(void *p, size_t newcount, size_t size, size_t alignment)
    -
    _mi_option_last
    Definition: mimalloc-doc.h:766
    +
    _mi_option_last
    Definition: mimalloc-doc.h:771
    mi_realloc_aligned_at
    void * mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset)
    -
    mi_heap_area_t::blocks
    void * blocks
    start of the area containing heap blocks
    Definition: mimalloc-doc.h:710
    +
    mi_heap_area_t::blocks
    void * blocks
    start of the area containing heap blocks
    Definition: mimalloc-doc.h:715
    mi_realloc_aligned
    void * mi_realloc_aligned(void *p, size_t newsize, size_t alignment)
    mi__posix_memalign
    int mi__posix_memalign(void **p, size_t alignment, size_t size)
    mi_free
    void mi_free(void *p)
    Free previously allocated memory.
    @@ -144,37 +144,37 @@ $(document).ready(function(){initNavTree('mimalloc-doc_8h_source.html','');});
    mi_heap_rezalloc_aligned_at
    void * mi_heap_rezalloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset)
    mi_zalloc
    void * mi_zalloc(size_t size)
    Allocate zero-initialized size bytes.
    mi_heap_rezalloc
    void * mi_heap_rezalloc(mi_heap_t *heap, void *p, size_t newsize)
    -
    mi_option_segment_cache
    The number of segments per thread to keep cached.
    Definition: mimalloc-doc.h:758
    +
    mi_option_segment_cache
    The number of segments per thread to keep cached.
    Definition: mimalloc-doc.h:763
    mi_heap_calloc
    void * mi_heap_calloc(mi_heap_t *heap, size_t count, size_t size)
    Allocate count zero-initialized elements in a specific heap.
    mi_heap_calloc_aligned
    void * mi_heap_calloc_aligned(mi_heap_t *heap, size_t count, size_t size, size_t alignment)
    mi_is_redirected
    bool mi_is_redirected()
    Is the C runtime malloc API redirected?
    -
    mi_heap_area_t::block_size
    size_t block_size
    size in bytes of one block
    Definition: mimalloc-doc.h:714
    +
    mi_heap_area_t::block_size
    size_t block_size
    size in bytes of one block
    Definition: mimalloc-doc.h:719
    mi_reallocarray
    void * mi_reallocarray(void *p, size_t count, size_t size)
    mi_reserve_huge_os_pages_interleave
    int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs)
    Reserve pages of huge OS pages (1GiB) evenly divided over numa_nodes nodes, but stops after at most t...
    -
    mi_deferred_free_fun
    void() mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
    Type of deferred free functions.
    Definition: mimalloc-doc.h:342
    +
    mi_deferred_free_fun
    void() mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
    Type of deferred free functions.
    Definition: mimalloc-doc.h:344
    mi_is_in_heap_region
    bool mi_is_in_heap_region(const void *p)
    Is a pointer part of our heap?
    mi_option_enable
    void mi_option_enable(mi_option_t option, bool enable)
    mi_new_aligned
    void * mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false)
    like mi_malloc_aligned(), but when out of memory, use std::get_new_handler and raise std::bad_alloc e...
    mi_realloc
    void * mi_realloc(void *p, size_t newsize)
    Re-allocate memory to newsize bytes.
    -
    mi_option_reserve_huge_os_pages
    The number of huge OS pages (1GiB in size) to reserve at the start of the program.
    Definition: mimalloc-doc.h:757
    +
    mi_option_reserve_huge_os_pages
    The number of huge OS pages (1GiB in size) to reserve at the start of the program.
    Definition: mimalloc-doc.h:762
    mi_heap_reallocf
    void * mi_heap_reallocf(mi_heap_t *heap, void *p, size_t newsize)
    mi_free_size_aligned
    void mi_free_size_aligned(void *p, size_t size, size_t alignment)
    mi_rezalloc_aligned_at
    void * mi_rezalloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset)
    -
    mi_option_page_reset
    Reset page memory after mi_option_reset_delay milliseconds when it becomes free.
    Definition: mimalloc-doc.h:759
    +
    mi_option_page_reset
    Reset page memory after mi_option_reset_delay milliseconds when it becomes free.
    Definition: mimalloc-doc.h:764
    mi_thread_done
    void mi_thread_done(void)
    Uninitialize mimalloc on a thread.
    mi_heap_visit_blocks
    bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_all_blocks, mi_block_visit_fun *visitor, void *arg)
    Visit all areas and blocks in a heap.
    -
    mi_option_use_numa_nodes
    Pretend there are at most N NUMA nodes.
    Definition: mimalloc-doc.h:762
    +
    mi_option_use_numa_nodes
    Pretend there are at most N NUMA nodes.
    Definition: mimalloc-doc.h:767
    mi_malloc
    void * mi_malloc(size_t size)
    Allocate size bytes.
    mi_option_enabled
    bool mi_option_enabled(mi_option_t option)
    mi_register_error
    void mi_register_error(mi_error_fun *errfun, void *arg)
    Register an error callback function.
    -
    mi_option_reset_decommits
    Experimental.
    Definition: mimalloc-doc.h:763
    +
    mi_option_reset_decommits
    Experimental.
    Definition: mimalloc-doc.h:768
    mi_heap_strndup
    char * mi_heap_strndup(mi_heap_t *heap, const char *s, size_t n)
    Duplicate a string of at most length n in a specific heap.
    -
    mi_block_visit_fun
    bool() mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
    Visitor function passed to mi_heap_visit_blocks()
    Definition: mimalloc-doc.h:724
    +
    mi_block_visit_fun
    bool() mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
    Visitor function passed to mi_heap_visit_blocks()
    Definition: mimalloc-doc.h:729
    mi_heap_recalloc
    void * mi_heap_recalloc(mi_heap_t *heap, void *p, size_t newcount, size_t size)
    mi_heap_malloc_aligned_at
    void * mi_heap_malloc_aligned_at(mi_heap_t *heap, size_t size, size_t alignment, size_t offset)
    mi_realpath
    char * mi_realpath(const char *fname, char *resolved_name)
    Resolve a file path name.
    -
    mi_option_show_errors
    Print error messages to stderr.
    Definition: mimalloc-doc.h:751
    -
    mi_option_segment_reset
    Experimental.
    Definition: mimalloc-doc.h:760
    +
    mi_option_show_errors
    Print error messages to stderr.
    Definition: mimalloc-doc.h:756
    +
    mi_option_segment_reset
    Experimental.
    Definition: mimalloc-doc.h:765
    mi_heap_rezalloc_aligned
    void * mi_heap_rezalloc_aligned(mi_heap_t *heap, void *p, size_t newsize, size_t alignment)
    mi_new_aligned_nothrow
    void * mi_new_aligned_nothrow(size_t n, size_t alignment)
    like mi_malloc_aligned, but when out of memory, use std::get_new_handler but return NULL on failure.
    mi_memalign
    void * mi_memalign(size_t alignment, size_t size)
    @@ -182,11 +182,11 @@ $(document).ready(function(){initNavTree('mimalloc-doc_8h_source.html','');});
    mi_heap_contains_block
    bool mi_heap_contains_block(mi_heap_t *heap, const void *p)
    Does a heap contain a pointer to a previously allocated block?
    mi_heap_collect
    void mi_heap_collect(mi_heap_t *heap, bool force)
    Release outstanding resources in a specific heap.
    mi_heap_recalloc_aligned_at
    void * mi_heap_recalloc_aligned_at(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment, size_t offset)
    -
    mi_option_verbose
    Print verbose messages to stderr.
    Definition: mimalloc-doc.h:752
    +
    mi_option_verbose
    Print verbose messages to stderr.
    Definition: mimalloc-doc.h:757
    mi_zalloc_aligned_at
    void * mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset)
    mi_malloc_aligned_at
    void * mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset)
    Allocate size bytes aligned by alignment at a specified offset.
    mi_heap_delete
    void mi_heap_delete(mi_heap_t *heap)
    Delete a previously allocated heap.
    -
    mi_option_os_tag
    OS tag to assign to mimalloc'd memory.
    Definition: mimalloc-doc.h:765
    +
    mi_option_os_tag
    OS tag to assign to mimalloc'd memory.
    Definition: mimalloc-doc.h:770
    mi_heap_get_default
    mi_heap_t * mi_heap_get_default()
    Get the default heap that is used for mi_malloc() et al.
    mi_reserve_huge_os_pages_at
    int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs)
    Reserve pages of huge OS pages (1GiB) at a specific numa_node, but stops after at most timeout_msecs ...
    mi_aligned_alloc
    void * mi_aligned_alloc(size_t alignment, size_t size)
    @@ -194,30 +194,30 @@ $(document).ready(function(){initNavTree('mimalloc-doc_8h_source.html','');});
    mi_thread_init
    void mi_thread_init(void)
    Initialize mimalloc on a thread.
    mi_good_size
    size_t mi_good_size(size_t size)
    Return the used allocation size.
    mi_stats_print
    void mi_stats_print(void *out)
    Print the main statistics.
    -
    mi_option_eager_commit_delay
    Experimental.
    Definition: mimalloc-doc.h:764
    +
    mi_option_eager_commit_delay
    Experimental.
    Definition: mimalloc-doc.h:769
    mi_heap_recalloc_aligned
    void * mi_heap_recalloc_aligned(mi_heap_t *heap, void *p, size_t newcount, size_t size, size_t alignment)
    mi_heap_mallocn
    void * mi_heap_mallocn(mi_heap_t *heap, size_t count, size_t size)
    Allocate count elements in a specific heap.
    -
    mi_heap_area_t
    An area of heap space contains blocks of a single size.
    Definition: mimalloc-doc.h:709
    +
    mi_heap_area_t
    An area of heap space contains blocks of a single size.
    Definition: mimalloc-doc.h:714
    mi_thread_stats_print_out
    void mi_thread_stats_print_out(mi_output_fun *out, void *arg)
    Print out heap statistics for this thread.
    -
    mi_option_show_stats
    Print statistics to stderr when the program is done.
    Definition: mimalloc-doc.h:750
    +
    mi_option_show_stats
    Print statistics to stderr when the program is done.
    Definition: mimalloc-doc.h:755
    mi_zalloc_aligned
    void * mi_zalloc_aligned(size_t size, size_t alignment)
    -
    mi_heap_area_t::reserved
    size_t reserved
    bytes reserved for this area
    Definition: mimalloc-doc.h:711
    -
    mi_heap_t
    struct mi_heap_s mi_heap_t
    Type of first-class heaps.
    Definition: mimalloc-doc.h:505
    -
    mi_heap_area_t::used
    size_t used
    bytes in use by allocated blocks
    Definition: mimalloc-doc.h:713
    +
    mi_heap_area_t::reserved
    size_t reserved
    bytes reserved for this area
    Definition: mimalloc-doc.h:716
    +
    mi_heap_t
    struct mi_heap_s mi_heap_t
    Type of first-class heaps.
    Definition: mimalloc-doc.h:507
    +
    mi_heap_area_t::used
    size_t used
    bytes in use by allocated blocks
    Definition: mimalloc-doc.h:718
    mi_register_deferred_free
    void mi_register_deferred_free(mi_deferred_free_fun *deferred_free, void *arg)
    Register a deferred free function.
    mi_free_size
    void mi_free_size(void *p, size_t size)
    mi_collect
    void mi_collect(bool force)
    Eagerly free memory.
    mi_new_reallocn
    void * mi_new_reallocn(void *p, size_t newcount, size_t size)
    like mi_reallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc excepti...
    mi_heap_destroy
    void mi_heap_destroy(mi_heap_t *heap)
    Destroy a heap, freeing all its still allocated blocks.
    mi_calloc_aligned_at
    void * mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset)
    -
    mi_option_large_os_pages
    Use large OS pages (2MiB in size) if possible.
    Definition: mimalloc-doc.h:756
    +
    mi_option_large_os_pages
    Use large OS pages (2MiB in size) if possible.
    Definition: mimalloc-doc.h:761
    mi_heap_reallocn
    void * mi_heap_reallocn(mi_heap_t *heap, void *p, size_t count, size_t size)
    mi_register_output
    void mi_register_output(mi_output_fun *out, void *arg)
    Register an output function.
    -
    mi_stl_allocator
    std::allocator implementation for mimalloc for use in STL containers.
    Definition: mimalloc-doc.h:851
    +
    mi_stl_allocator
    std::allocator implementation for mimalloc for use in STL containers.
    Definition: mimalloc-doc.h:856
    mi_heap_malloc_small
    void * mi_heap_malloc_small(mi_heap_t *heap, size_t size)
    Allocate a small object in a specific heap.
    mi_heap_realloc
    void * mi_heap_realloc(mi_heap_t *heap, void *p, size_t newsize)
    mi_malloc_usable_size
    size_t mi_malloc_usable_size(const void *p)
    -
    mi_output_fun
    void() mi_output_fun(const char *msg, void *arg)
    Type of output functions.
    Definition: mimalloc-doc.h:366
    +
    mi_output_fun
    void() mi_output_fun(const char *msg, void *arg)
    Type of output functions.
    Definition: mimalloc-doc.h:368
    mi_strdup
    char * mi_strdup(const char *s)
    Allocate and duplicate a string.
    mi_heap_realloc_aligned_at
    void * mi_heap_realloc_aligned_at(mi_heap_t *heap, void *p, size_t newsize, size_t alignment, size_t offset)
    mi_reallocf
    void * mi_reallocf(void *p, size_t newsize)
    Re-allocate memory to newsize bytes,.
    @@ -230,11 +230,11 @@ $(document).ready(function(){initNavTree('mimalloc-doc_8h_source.html','');});
    mi_heap_get_backing
    mi_heap_t * mi_heap_get_backing()
    Get the backing heap.
    mi_free_aligned
    void mi_free_aligned(void *p, size_t alignment)
    mi_new
    void * mi_new(std::size_t n) noexcept(false)
    like mi_malloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception...
    -
    mi_option_reset_delay
    Delay in milli-seconds before resetting a page (100ms by default)
    Definition: mimalloc-doc.h:761
    +
    mi_option_reset_delay
    Delay in milli-seconds before resetting a page (100ms by default)
    Definition: mimalloc-doc.h:766
    mi_heap_new
    mi_heap_t * mi_heap_new()
    Create a new heap that can be used for allocation.
    mi_heap_malloc
    void * mi_heap_malloc(mi_heap_t *heap, size_t size)
    Allocate in a specific heap.
    -
    mi_heap_area_t::committed
    size_t committed
    current committed bytes of this area
    Definition: mimalloc-doc.h:712
    -
    mi_option_t
    mi_option_t
    Runtime options.
    Definition: mimalloc-doc.h:748
    +
    mi_heap_area_t::committed
    size_t committed
    current committed bytes of this area
    Definition: mimalloc-doc.h:717
    +
    mi_option_t
    mi_option_t
    Runtime options.
    Definition: mimalloc-doc.h:753
    mi_heap_check_owned
    bool mi_heap_check_owned(mi_heap_t *heap, const void *p)
    Check safely if any pointer is part of a heap.
    mi_heap_set_default
    mi_heap_t * mi_heap_set_default(mi_heap_t *heap)
    Set the default heap to use for mi_malloc() et al.
    diff --git a/docs/mimalloc-logo.svg b/docs/mimalloc-logo.svg index 7b6231bd..672c7e41 100644 --- a/docs/mimalloc-logo.svg +++ b/docs/mimalloc-logo.svg @@ -36,15 +36,15 @@ inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:zoom="23.706667" - inkscape:cx="34.419045" + inkscape:cx="24.864771" inkscape:cy="35.79485" inkscape:document-units="mm" inkscape:current-layer="layer1" showgrid="false" inkscape:window-width="3840" - inkscape:window-height="2160" - inkscape:window-x="0" - inkscape:window-y="0" + inkscape:window-height="2050" + inkscape:window-x="-12" + inkscape:window-y="-12" inkscape:window-maximized="1" inkscape:snap-object-midpoints="false" inkscape:snap-bbox="false" @@ -126,17 +126,15 @@ style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:2.28797817px;font-family:RoutedGothicEx;-inkscape-font-specification:'RoutedGothicEx, Normal';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;stroke-width:0.28599727" id="path6525" /> - m + id="text848"> + + diff --git a/docs/modules.html b/docs/modules.html index 91bf17e8..9858d6ad 100644 --- a/docs/modules.html +++ b/docs/modules.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/overrides.html b/docs/overrides.html index 3b5d9bd3..2a6c51e9 100644 --- a/docs/overrides.html +++ b/docs/overrides.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    @@ -116,12 +116,13 @@ $(document).ready(function(){initNavTree('overrides.html','');});
  • env DYLD_FORCE_FLAT_NAMESPACE=1 DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram

    • Note that certain security restrictions may apply when doing this from the shell.

    -

    Note: unfortunately, at this time, dynamic overriding on macOS seems broken but it is actively worked on to fix this (see issue #50).

    +

    (Note: macOS support for dynamic overriding is recent, please report any issues.)

    Windows

    -

    Overriding on Windows is robust but requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the /MD or /MDd switch). Moreover, you need to ensure the mimalloc-redirect.dll (or mimalloc-redirect32.dll) is available in the same folder as the main mimalloc-override.dll at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in mimalloc-override.dll).

    -

    To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some call to the mimalloc API in the main function, like mi_version() (or use the /INCLUDE:mi_version switch on the linker). See the mimalloc-override-test project for an example on how to use this. For best performance on Windows with C++, it is highly recommended to also override the new/delete operations (by including mimalloc-new-delete.h a single(!) source file in your project).

    +

    Overriding on Windows is robust and has the particular advantage to be able to redirect all malloc/free calls that go through the (dynamic) C runtime allocator, including those from other DLL's or libraries.

    +

    The overriding on Windows requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the /MD or /MDd switch). Also, the mimalloc-redirect.dll (or mimalloc-redirect32.dll) must be available in the same folder as the main mimalloc-override.dll at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in mimalloc-override.dll).

    +

    To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some call to the mimalloc API in the main function, like mi_version() (or use the /INCLUDE:mi_version switch on the linker). See the mimalloc-override-test project for an example on how to use this. For best performance on Windows with C++, it is also recommended to also override the new/delete operations (by including mimalloc-new-delete.h a single(!) source file in your project).

    The environment variable MIMALLOC_DISABLE_REDIRECT=1 can be used to disable dynamic overriding at run-time. Use MIMALLOC_VERBOSE=1 to check if mimalloc was successfully redirected.

    -

    (Note: in principle, it is possible to patch existing executables that are linked with the dynamic C runtime (ucrtbase.dll) by just putting the mimalloc-override.dll into the import table (and putting mimalloc-redirect.dll in the same folder) Such patching can be done for example with CFF Explorer).

    +

    (Note: in principle, it is possible to even patch existing executables without any recompilation if they are linked with the dynamic C runtime (ucrtbase.dll) – just put the mimalloc-override.dll into the import table (and put mimalloc-redirect.dll in the same folder) Such patching can be done for example with CFF Explorer).

    Static override

    On Unix systems, you can also statically link with mimalloc to override the standard malloc interface. The recommended way is to link the final program with the mimalloc single object file (mimalloc-override.o). We use an object file instead of a library file as linkers give preference to that over archives to resolve symbols. To ensure that the standard malloc interface resolves to the mimalloc library, link it as the first object file. For example:

    gcc -o myprogram mimalloc-override.o myfile1.c ...

    List of Overrides:

    diff --git a/docs/pages.html b/docs/pages.html index ad5549bf..8fcd6f08 100644 --- a/docs/pages.html +++ b/docs/pages.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/docs/using.html b/docs/using.html index c5dc12e7..047e35ec 100644 --- a/docs/using.html +++ b/docs/using.html @@ -37,7 +37,7 @@ Logo
    mi-malloc -  1.4 +  1.6
    diff --git a/ide/vs2019/mimalloc-override-test.vcxproj b/ide/vs2019/mimalloc-override-test.vcxproj index a2497a19..7a9202f1 100644 --- a/ide/vs2019/mimalloc-override-test.vcxproj +++ b/ide/vs2019/mimalloc-override-test.vcxproj @@ -176,14 +176,14 @@ + + + {abb5eae7-b3e6-432e-b636-333449892ea7} - - - diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj index 3dcc4aae..9488301e 100644 --- a/ide/vs2019/mimalloc-override.vcxproj +++ b/ide/vs2019/mimalloc-override.vcxproj @@ -253,4 +253,4 @@ - + \ No newline at end of file diff --git a/include/mimalloc-internal-tld.h b/include/mimalloc-internal-tld.h deleted file mode 100644 index ce67b0c7..00000000 --- a/include/mimalloc-internal-tld.h +++ /dev/null @@ -1,722 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2019, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef MIMALLOC_INTERNAL_TLD_H -#define MIMALLOC_INTERNAL_TLD_H - -#include "mimalloc-types.h" -#include "mimalloc-internal.h" - -#define MI_TLD_DECL 1 // thread local declaration -#define MI_TLD_PTHREAD 2 // ptrhead_get/setspecific -#define MI_TLD_DECL_GUARD 3 // thread local + recursion guard at initial load -#define MI_TLD_PTHREAD_GUARD 4 // ptrhead_get/setspecific + recursion guard at initial load -#define MI_TLD_SLOT 5 // steal slot from OS thread local predefined slots -#define MI_TLD_PTHREAD_SLOT 6 // steal slot from pthread structure (usually `retval`) - - -#if !defined(MI_TLD) -#if defined(_MSC_VER) || defined(__linux__) || defined(__FreeBSD__) || defined(__NetBSD__) - // on windows and linux/freeBSD/netBSD (with initial-exec) a __thread always works without recursion into malloc - #define MI_TLD MI_TLD_DECL -#elif !defined(MI_MIMALLOC_OVERRIDE) - // if not overriding, __thread declarations should be fine (use MI_TLD_PTHREAD if your OS does not have __thread) - #define MI_TLD MI_TLD_DECL -#elif // defined(MI_MALLOC_OVERRIDE) - // if overriding, some BSD variants allocate when accessing a thread local the first time - #if defined(__APPLE__) - #define MI_TLD MI_TLD_SLOT - #define MI_TLD_SLOT_NUM 89 // seems unused? (__PTK_FRAMEWORK_OLDGC_KEY9) see - // possibly unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) - // #define MI_TLD MI_TLD_PTHREAD_SLOT - // #define MI_TLD_PTHREAD_SLOT_OFS (2*sizeof(void*) + sizeof(long) + 2*sizeof(void*) /*TAILQ*/) // offset `tl_exit_value` - #elif defined(__OpenBSD__) - #define MI_TLD MI_TLD_PTHREAD_SLOT - #define MI_TLD_PTHREAD_SLOT_OFS (6*sizeof(int) + 1*sizeof(void*)) // offset `retval` - #elif defined(__DragonFly__) - #define MI_TLD MI_TLD_PTHREAD_SLOT - #define MI_TLD_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) - #endif - #endif -#endif - -#if (MI_DEBUG>0) -#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) -#else -#define mi_trace_message(...) -#endif - -#define MI_CACHE_LINE 64 -#if defined(_MSC_VER) -#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) -#define mi_decl_noinline __declspec(noinline) -#define mi_decl_thread __declspec(thread) -#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) -#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc -#define mi_decl_noinline __attribute__((noinline)) -#define mi_decl_thread __thread -#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) -#else -#define mi_decl_noinline -#define mi_decl_thread __thread // hope for the best :-) -#define mi_decl_cache_align -#endif - - -// "options.c" -void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); -void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); -void _mi_warning_message(const char* fmt, ...); -void _mi_verbose_message(const char* fmt, ...); -void _mi_trace_message(const char* fmt, ...); -void _mi_options_init(void); -void _mi_error_message(int err, const char* fmt, ...); - -// random.c -void _mi_random_init(mi_random_ctx_t* ctx); -void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); -uintptr_t _mi_random_next(mi_random_ctx_t* ctx); -uintptr_t _mi_heap_random_next(mi_heap_t* heap); -uintptr_t _os_random_weak(uintptr_t extra_seed); -static inline uintptr_t _mi_random_shuffle(uintptr_t x); - -// init.c -extern mi_stats_t _mi_stats_main; -extern const mi_page_t _mi_page_empty; -bool _mi_is_main_thread(void); -bool _mi_preloading(); // true while the C runtime is not ready - -// os.c -size_t _mi_os_page_size(void); -void _mi_os_init(void); // called from process init -void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data -void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data -size_t _mi_os_good_alloc_size(size_t size); - -// memory.c -void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* id, mi_os_tld_t* tld); -void _mi_mem_free(void* p, size_t size, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld); - -bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld); -bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld); -bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld); -bool _mi_mem_protect(void* addr, size_t size); -bool _mi_mem_unprotect(void* addr, size_t size); - -void _mi_mem_collect(mi_os_tld_t* tld); - -// "segment.c" -mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_wsize, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); -void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); -void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); -uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size, size_t* pre_size); // page start for any page -void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); - -void _mi_segment_thread_collect(mi_segments_tld_t* tld); -void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); -void _mi_abandoned_await_readers(void); - - - -// "page.c" -void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc; - -void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks -void _mi_page_unfull(mi_page_t* page); -void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page -void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... -void _mi_heap_delayed_free(mi_heap_t* heap); -void _mi_heap_collect_retired(mi_heap_t* heap, bool force); - -void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); -size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); -void _mi_deferred_free(mi_heap_t* heap, bool force); - -void _mi_page_free_collect(mi_page_t* page,bool force); -void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments - -size_t _mi_bin_size(uint8_t bin); // for stats -uint8_t _mi_bin(size_t size); // for stats -uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD in "os.c" - -// "heap.c" -void _mi_heap_destroy_pages(mi_heap_t* heap); -void _mi_heap_collect_abandon(mi_heap_t* heap); -void _mi_heap_set_default_direct(mi_heap_t* heap); - -// "stats.c" -void _mi_stats_done(mi_stats_t* stats); - -mi_msecs_t _mi_clock_now(void); -mi_msecs_t _mi_clock_end(mi_msecs_t start); -mi_msecs_t _mi_clock_start(void); - -// "alloc.c" -void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic` -void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero); -void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero); -mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); -bool _mi_free_delayed_block(mi_block_t* block); -void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size); - -#if MI_DEBUG>1 -bool _mi_page_is_valid(mi_page_t* page); -#endif - - -// ------------------------------------------------------ -// Branches -// ------------------------------------------------------ - -#if defined(__GNUC__) || defined(__clang__) -#define mi_unlikely(x) __builtin_expect((x),0) -#define mi_likely(x) __builtin_expect((x),1) -#else -#define mi_unlikely(x) (x) -#define mi_likely(x) (x) -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - - -/* ----------------------------------------------------------- - Error codes passed to `_mi_fatal_error` - All are recoverable but EFAULT is a serious error and aborts by default in secure mode. - For portability define undefined error codes using common Unix codes: - ------------------------------------------------------------ */ -#include -#ifndef EAGAIN // double free -#define EAGAIN (11) -#endif -#ifndef ENOMEM // out of memory -#define ENOMEM (12) -#endif -#ifndef EFAULT // corrupted free-list or meta-data -#define EFAULT (14) -#endif -#ifndef EINVAL // trying to free an invalid pointer -#define EINVAL (22) -#endif -#ifndef EOVERFLOW // count*size overflow -#define EOVERFLOW (75) -#endif - - -/* ----------------------------------------------------------- - Inlined definitions ------------------------------------------------------------ */ -#define UNUSED(x) (void)(x) -#if (MI_DEBUG>0) -#define UNUSED_RELEASE(x) -#else -#define UNUSED_RELEASE(x) UNUSED(x) -#endif - -#define MI_INIT4(x) x(),x(),x(),x() -#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) -#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) -#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) -#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) -#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) -#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) - - -// Is `x` a power of two? (0 is considered a power of two) -static inline bool _mi_is_power_of_two(uintptr_t x) { - return ((x & (x - 1)) == 0); -} - -// Align upwards -static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { - mi_assert_internal(alignment != 0); - uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) { // power of two? - return ((sz + mask) & ~mask); - } - else { - return (((sz + mask)/alignment)*alignment); - } -} - -// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. -static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { - mi_assert_internal(divider != 0); - return (divider == 0 ? size : ((size + divider - 1) / divider)); -} - -// Is memory zero initialized? -static inline bool mi_mem_is_zero(void* p, size_t size) { - for (size_t i = 0; i < size; i++) { - if (((uint8_t*)p)[i] != 0) return false; - } - return true; -} - -// Align a byte size to a size in _machine words_, -// i.e. byte size == `wsize*sizeof(void*)`. -static inline size_t _mi_wsize_from_size(size_t size) { - mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); - return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); -} - - -// Overflow detecting multiply -static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { -#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 -#include // UINT_MAX, ULONG_MAX -#if (SIZE_MAX == UINT_MAX) - return __builtin_umul_overflow(count, size, total); -#elif (SIZE_MAX == ULONG_MAX) - return __builtin_umull_overflow(count, size, total); -#else - return __builtin_umulll_overflow(count, size, total); -#endif -#else /* __builtin_umul_overflow is unavailable */ - #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) - *total = count * size; - return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) - && size > 0 && (SIZE_MAX / size) < count); -#endif -} - -// Safe multiply `count*size` into `total`; return `true` on overflow. -static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { - if (count==1) { // quick check for the case where count is one (common for C++ allocators) - *total = size; - return false; - } - else if (mi_unlikely(mi_mul_overflow(count, size, total))) { - _mi_error_message(EOVERFLOW, "allocation request too large (%zu * %zu bytes)\n", count, size); - *total = SIZE_MAX; - return true; - } - else return false; -} - - -/* ----------------------------------------------------------- - The thread local default heap ------------------------------------------------------------ */ - -extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap -extern mi_heap_t _mi_heap_main; // statically allocated main backing heap -extern bool _mi_process_is_initialized; - -#if defined(MI_TLS_OSX_FAST) -#define MI_TLS_OSX_OFFSET (MI_TLS_OSX_SLOT*sizeof(void*)) -static inline void* mi_tls_osx_fast_get(void) { - void* ret; - __asm__("mov %%gs:%1, %0" : "=r" (ret) : "m" (*(void**)(MI_TLS_OSX_OFFSET))); - return ret; -} -static inline void mi_tls_osx_fast_set(void* value) { - __asm__("movq %1,%%gs:%0" : "=m" (*(void**)(MI_TLS_OSX_OFFSET)) : "rn" (value)); -} -#elif defined(MI_TLS_PTHREADS) -extern pthread_key_t _mi_heap_default_key; -#else -extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from -#endif - - -static inline mi_heap_t* mi_get_default_heap(void) { -#if defined(MI_TLS_OSX_FAST) - // Use a fixed slot in the TSD on MacOSX to avoid recursion (since the loader calls malloc). - // We use slot 94 (__PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4) - // which seems unused except for the more recent Webkit - // Use with care. - mi_heap_t* heap = (mi_heap_t*)mi_tls_osx_fast_get(); - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); -#elif defined(MI_TLS_PTHREADS) - // Use pthreads for TLS; this is used on macOSX with interpose as the loader calls `malloc` - // to allocate TLS storage leading to recursive calls if __thread declared variables are accessed. - // Using pthreads allows us to initialize without recursive calls. (performance seems still quite good). - mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? (mi_heap_t*)&_mi_heap_empty : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); -#else - #if defined(MI_TLS_RECURSE_GUARD) - // On some BSD platforms, like openBSD, the dynamic loader calls `malloc` - // to initialize thread local data (before our module is loaded). - // To avoid recursion, we need to avoid accessing the thread local `_mi_default_heap` - // until our module is loaded and use the statically allocated main heap until that time. - // TODO: patch ourselves dynamically to avoid this check every time? - // if (mi_unlikely(!_mi_process_is_initialized)) return &_mi_heap_main; - #endif - return _mi_heap_default; -#endif -} - -static inline bool mi_heap_is_default(const mi_heap_t* heap) { - return (heap == mi_get_default_heap()); -} - -static inline bool mi_heap_is_backing(const mi_heap_t* heap) { - return (heap->tld->heap_backing == heap); -} - -static inline bool mi_heap_is_initialized(mi_heap_t* heap) { - mi_assert_internal(heap != NULL); - return (heap != &_mi_heap_empty); -} - -static inline uintptr_t _mi_ptr_cookie(const void* p) { - mi_assert_internal(_mi_heap_main.cookie != 0); - return ((uintptr_t)p ^ _mi_heap_main.cookie); -} - -/* ----------------------------------------------------------- - Pages ------------------------------------------------------------ */ - -static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { - mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - const size_t idx = _mi_wsize_from_size(size); - mi_assert_internal(idx < MI_PAGES_DIRECT); - return heap->pages_free_direct[idx]; -} - -// Get the page belonging to a certain size class -static inline mi_page_t* _mi_get_free_small_page(size_t size) { - return _mi_heap_get_free_small_page(mi_get_default_heap(), size); -} - -// Segment that contains the pointer -static inline mi_segment_t* _mi_ptr_segment(const void* p) { - // mi_assert_internal(p != NULL); - return (mi_segment_t*)((uintptr_t)p & ~MI_SEGMENT_MASK); -} - -// Segment belonging to a page -static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { - mi_segment_t* segment = _mi_ptr_segment(page); - mi_assert_internal(segment == NULL || page == &segment->pages[page->segment_idx]); - return segment; -} - -// used internally -static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { - // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages - ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; - mi_assert_internal(diff >= 0 && (size_t)diff < MI_SEGMENT_SIZE); - uintptr_t idx = (uintptr_t)diff >> segment->page_shift; - mi_assert_internal(idx < segment->capacity); - mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0); - return idx; -} - -// Get the page containing the pointer -static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { - uintptr_t idx = _mi_segment_page_idx_of(segment, p); - return &((mi_segment_t*)segment)->pages[idx]; -} - -// Quick page start for initialized pages -static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { - const size_t bsize = page->xblock_size; - mi_assert_internal(bsize > 0 && (bsize%sizeof(void*)) == 0); - return _mi_segment_page_start(segment, page, bsize, page_size, NULL); -} - -// Get the page containing the pointer -static inline mi_page_t* _mi_ptr_page(void* p) { - return _mi_segment_page_of(_mi_ptr_segment(p), p); -} - -// Get the block size of a page (special cased for huge objects) -static inline size_t mi_page_block_size(const mi_page_t* page) { - const size_t bsize = page->xblock_size; - mi_assert_internal(bsize > 0); - if (mi_likely(bsize < MI_HUGE_BLOCK_SIZE)) { - return bsize; - } - else { - size_t psize; - _mi_segment_page_start(_mi_page_segment(page), page, bsize, &psize, NULL); - return psize; - } -} - -// Get the usable block size of a page without fixed padding. -// This may still include internal padding due to alignment and rounding up size classes. -static inline size_t mi_page_usable_block_size(const mi_page_t* page) { - return mi_page_block_size(page) - MI_PADDING_SIZE; -} - - -// Thread free access -static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { - return (mi_block_t*)(mi_atomic_read_relaxed(&page->xthread_free) & ~3); -} - -static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) { - return (mi_delayed_t)(mi_atomic_read_relaxed(&page->xthread_free) & 3); -} - -// Heap access -static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { - return (mi_heap_t*)(mi_atomic_read_relaxed(&page->xheap)); -} - -static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) { - mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING); - mi_atomic_write(&page->xheap,(uintptr_t)heap); -} - -// Thread free flag helpers -static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { - return (mi_block_t*)(tf & ~0x03); -} -static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) { - return (mi_delayed_t)(tf & 0x03); -} -static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) { - return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed); -} -static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) { - return mi_tf_make(mi_tf_block(tf),delayed); -} -static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) { - return mi_tf_make(block, mi_tf_delayed(tf)); -} - -// are all blocks in a page freed? -// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. -static inline bool mi_page_all_free(const mi_page_t* page) { - mi_assert_internal(page != NULL); - return (page->used == 0); -} - -// are there any available blocks? -static inline bool mi_page_has_any_available(const mi_page_t* page) { - mi_assert_internal(page != NULL && page->reserved > 0); - return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); -} - -// are there immediately available blocks, i.e. blocks available on the free list. -static inline bool mi_page_immediate_available(const mi_page_t* page) { - mi_assert_internal(page != NULL); - return (page->free != NULL); -} - -// is more than 7/8th of a page in use? -static inline bool mi_page_mostly_used(const mi_page_t* page) { - if (page==NULL) return true; - uint16_t frac = page->reserved / 8U; - return (page->reserved - page->used <= frac); -} - -static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { - return &((mi_heap_t*)heap)->pages[_mi_bin(size)]; -} - - - -//----------------------------------------------------------- -// Page flags -//----------------------------------------------------------- -static inline bool mi_page_is_in_full(const mi_page_t* page) { - return page->flags.x.in_full; -} - -static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { - page->flags.x.in_full = in_full; -} - -static inline bool mi_page_has_aligned(const mi_page_t* page) { - return page->flags.x.has_aligned; -} - -static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { - page->flags.x.has_aligned = has_aligned; -} - - -/* ------------------------------------------------------------------- -Encoding/Decoding the free list next pointers - -This is to protect against buffer overflow exploits where the -free list is mutated. Many hardened allocators xor the next pointer `p` -with a secret key `k1`, as `p^k1`. This prevents overwriting with known -values but might be still too weak: if the attacker can guess -the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). -Moreover, if multiple blocks can be read as well, the attacker can -xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot -about the pointers (and subsequently `k1`). - -Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<> (MI_INTPTR_BITS - shift))); -} -static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) { - shift %= MI_INTPTR_BITS; - return ((x >> shift) | (x << (MI_INTPTR_BITS - shift))); -} - -static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { - void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); - return (mi_unlikely(p==null) ? NULL : p); -} - -static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { - uintptr_t x = (uintptr_t)(mi_unlikely(p==NULL) ? null : p); - return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; -} - -static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { - #ifdef MI_ENCODE_FREELIST - return (mi_block_t*)mi_ptr_decode(null, block->next, keys); - #else - UNUSED(keys); UNUSED(null); - return (mi_block_t*)block->next; - #endif -} - -static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { - #ifdef MI_ENCODE_FREELIST - block->next = mi_ptr_encode(null, next, keys); - #else - UNUSED(keys); UNUSED(null); - block->next = (mi_encoded_t)next; - #endif -} - -static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { - #ifdef MI_ENCODE_FREELIST - mi_block_t* next = mi_block_nextx(page,block,page->keys); - // check for free list corruption: is `next` at least in the same page? - // TODO: check if `next` is `page->block_size` aligned? - if (mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next))) { - _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); - next = NULL; - } - return next; - #else - UNUSED(page); - return mi_block_nextx(page,block,NULL); - #endif -} - -static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { - #ifdef MI_ENCODE_FREELIST - mi_block_set_nextx(page,block,next, page->keys); - #else - UNUSED(page); - mi_block_set_nextx(page,block,next,NULL); - #endif -} - -// ------------------------------------------------------------------- -// Fast "random" shuffle -// ------------------------------------------------------------------- - -static inline uintptr_t _mi_random_shuffle(uintptr_t x) { - if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros -#if (MI_INTPTR_SIZE==8) - // by Sebastiano Vigna, see: - x ^= x >> 30; - x *= 0xbf58476d1ce4e5b9UL; - x ^= x >> 27; - x *= 0x94d049bb133111ebUL; - x ^= x >> 31; -#elif (MI_INTPTR_SIZE==4) - // by Chris Wellons, see: - x ^= x >> 16; - x *= 0x7feb352dUL; - x ^= x >> 15; - x *= 0x846ca68bUL; - x ^= x >> 16; -#endif - return x; -} - -// ------------------------------------------------------------------- -// Optimize numa node access for the common case (= one node) -// ------------------------------------------------------------------- - -int _mi_os_numa_node_get(mi_os_tld_t* tld); -size_t _mi_os_numa_node_count_get(void); - -extern size_t _mi_numa_node_count; -static inline int _mi_os_numa_node(mi_os_tld_t* tld) { - if (mi_likely(_mi_numa_node_count == 1)) return 0; - else return _mi_os_numa_node_get(tld); -} -static inline size_t _mi_os_numa_node_count(void) { - if (mi_likely(_mi_numa_node_count>0)) return _mi_numa_node_count; - else return _mi_os_numa_node_count_get(); -} - - -// ------------------------------------------------------------------- -// Getting the thread id should be performant -// as it is called in the fast path of `_mi_free`, -// so we specialize for various platforms. -// ------------------------------------------------------------------- -#if defined(_WIN32) -#define WIN32_LEAN_AND_MEAN -#include -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { - // Windows: works on Intel and ARM in both 32- and 64-bit - return (uintptr_t)NtCurrentTeb(); -} -#elif (defined(__GNUC__) || defined(__clang__)) && \ - (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__)) -// TLS register on x86 is in the FS or GS register -// see: https://akkadia.org/drepper/tls.pdf -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { - uintptr_t tid; - #if defined(__i386__) - __asm__("movl %%gs:0, %0" : "=r" (tid) : : ); // 32-bit always uses GS - #elif defined(__MACH__) - __asm__("movq %%gs:0, %0" : "=r" (tid) : : ); // x86_64 macOS uses GS - #elif defined(__x86_64__) - __asm__("movq %%fs:0, %0" : "=r" (tid) : : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - asm volatile ("mrc p15, 0, %0, c13, c0, 3" : "=r" (tid)); - #elif defined(__aarch64__) - asm volatile ("mrs %0, tpidr_el0" : "=r" (tid)); - #endif - return tid; -} -#else -// otherwise use standard C -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { - return (uintptr_t)&_mi_heap_default; -} -#endif - - -#endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 39ae2df6..cf94809b 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -57,11 +57,10 @@ terms of the MIT license. A copy of the license can be found in the file // Encoded free lists allow detection of corrupted free lists // and can detect buffer overflows, modify after free, and double `free`s. -#if (MI_SECURE>=3 || MI_DEBUG>=1 || defined(MI_PADDING)) +#if (MI_SECURE>=3 || MI_DEBUG>=1 || MI_PADDING > 0) #define MI_ENCODE_FREELIST 1 #endif - // ------------------------------------------------------ // Platform specific values // ------------------------------------------------------ @@ -101,7 +100,7 @@ terms of the MIT license. A copy of the license can be found in the file // Main tuning parameters for segment and page sizes // Sizes for 64-bit, divide by two for 32-bit #define MI_SEGMENT_SLICE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb -#define MI_SEGMENT_SHIFT ( 8 + MI_SEGMENT_SLICE_SHIFT) // 16mb +#define MI_SEGMENT_SHIFT ( 7 + MI_SEGMENT_SLICE_SHIFT) // 8mb #define MI_SMALL_PAGE_SHIFT (MI_SEGMENT_SLICE_SHIFT) // 64kb #define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb @@ -332,7 +331,7 @@ typedef struct mi_random_cxt_s { // In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows -#if defined(MI_PADDING) +#if (MI_PADDING) typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) diff --git a/include/mimalloc.h b/include/mimalloc.h index 5bcb26ef..fd9df26d 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 161 // major + 2 digits minor +#define MI_MALLOC_VERSION 163 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes @@ -28,13 +28,13 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_nodiscard [[nodiscard]] #elif (__GNUC__ >= 4) || defined(__clang__) // includes clang, icc, and clang-cl #define mi_decl_nodiscard __attribute__((warn_unused_result)) -#elif (_MSC_VER >= 1700) +#elif (_MSC_VER >= 1700) #define mi_decl_nodiscard _Check_return_ -#else - #define mi_decl_nodiscard -#endif +#else + #define mi_decl_nodiscard +#endif -#ifdef _MSC_VER +#if defined(_MSC_VER) || defined(__MINGW32__) #if !defined(MI_SHARED_LIB) #define mi_decl_export #elif defined(MI_SHARED_LIB_EXPORT) @@ -42,13 +42,18 @@ terms of the MIT license. A copy of the license can be found in the file #else #define mi_decl_export __declspec(dllimport) #endif - #if (_MSC_VER >= 1900) && !defined(__EDG__) - #define mi_decl_restrict __declspec(allocator) __declspec(restrict) + #if defined(__MINGW32__) + #define mi_decl_restrict + #define mi_attr_malloc __attribute__((malloc)) #else - #define mi_decl_restrict __declspec(restrict) + #if (_MSC_VER >= 1900) && !defined(__EDG__) + #define mi_decl_restrict __declspec(allocator) __declspec(restrict) + #else + #define mi_decl_restrict __declspec(restrict) + #endif + #define mi_attr_malloc #endif #define mi_cdecl __cdecl - #define mi_attr_malloc #define mi_attr_alloc_size(s) #define mi_attr_alloc_size2(s1,s2) #define mi_attr_alloc_align(p) diff --git a/readme.md b/readme.md index 423c91b9..583d54ed 100644 --- a/readme.md +++ b/readme.md @@ -11,7 +11,7 @@ mimalloc (pronounced "me-malloc") is a general purpose allocator with excellent [performance](#performance) characteristics. Initially developed by Daan Leijen for the run-time systems of the [Koka](https://github.com/koka-lang/koka) and [Lean](https://github.com/leanprover/lean) languages. -Latest release:`v1.6.1` (2020-02-17). +Latest release:`v1.6.2` (2020-04-20). It is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -57,6 +57,9 @@ Enjoy! ### Releases +* 2020-04-20, `v1.6.2`: stable release 1.6: fix compilation on Android, MingW, Raspberry, and Conda, + stability fix for Windows 7, fix multiple mimalloc instances in one executable, fix `strnlen` overload, + fix aligned debug padding. * 2020-02-17, `v1.6.1`: stable release 1.6: minor updates (build with clang-cl, fix alignment issue for small objects). * 2020-02-09, `v1.6.0`: stable release 1.6: fixed potential memory leak, improved overriding and thread local support on FreeBSD, NetBSD, DragonFly, and macOSX. New byte-precise @@ -75,6 +78,11 @@ free list encoding](https://github.com/microsoft/mimalloc/blob/783e3377f79ee82af * 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. * 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. +Special thanks to: + +* Jason Gibson (@jasongibson) for exhaustive testing on large workloads and server environments and finding complex bugs in (early versions of) `mimalloc`. +* Manuel Pöter (@mpoeter) and Sam Gross (@colesbury) for finding an ABA concurrency issue in abandoned segment reclamation. + # Building ## Windows @@ -208,31 +216,42 @@ or via environment variables. - `MIMALLOC_SHOW_STATS=1`: show statistics when the program terminates. - `MIMALLOC_VERBOSE=1`: show verbose messages. - `MIMALLOC_SHOW_ERRORS=1`: show error and warning messages. -- `MIMALLOC_PAGE_RESET=1`: reset (or purge) OS pages when not in use. This can reduce - memory fragmentation in long running (server) programs. If performance is impacted, - `MIMALLOC_RESET_DELAY=` can be set higher (100ms by default) to make the page - reset occur less frequently. -- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages when available; for some workloads this can significantly +- `MIMALLOC_PAGE_RESET=0`: by default, mimalloc will reset (or purge) OS pages that are not in use, to signal to the OS + that the underlying physical memory can be reused. This can reduce memory fragmentation in long running (server) + programs. By setting it to `0` this will no longer be done which can improve performance for batch-like programs. + As an alternative, the `MIMALLOC_RESET_DELAY=` can be set higher (100ms by default) to make the page + reset occur less frequently instead of turning it off completely. +- `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected + at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than + the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA + nodes (but this can happen in any case as NUMA local allocation is always a best effort but not guaranteed). +- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages (2MiB) when available; for some workloads this can significantly improve performance. Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs to explicitly allow large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that - can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead when possible). + can have fragmented memory (for that reason, we generally recommend to use `MIMALLOC_RESERVE_HUGE_OS_PAGES` instead whenever possible). -- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where N is the number of 1GiB huge OS pages. This reserves the huge pages at - startup and can give quite a (latency) performance improvement on long running workloads. Usually it is better to not use +- `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where N is the number of 1GiB _huge_ OS pages. This reserves the huge pages at + startup and sometimes this can give a large (latency) performance improvement on big workloads. + Usually it is better to not use `MIMALLOC_LARGE_OS_PAGES` in combination with this setting. Just like large OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at startup only once). - Note that we usually need to explicitly enable huge OS pages (as on [Windows][windows-huge] and [Linux][linux-huge])). With huge OS pages, it may be beneficial to set the setting - `MIMALLOC_EAGER_COMMIT_DELAY=N` (with usually `N` as 1) to delay the initial `N` segments + Note that we usually need to explicitly enable huge OS pages (as on [Windows][windows-huge] and [Linux][linux-huge])). + With huge OS pages, it may be beneficial to set the setting + `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write +for all pages in the original process including the huge OS pages. When any memory is now written in that area, the +OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in big increments. + [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 @@ -280,9 +299,13 @@ the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-i ### Override on Windows -Overriding on Windows is robust but requires that you link your program explicitly with +Overriding on Windows is robust and has the +particular advantage to be able to redirect all malloc/free calls that go through +the (dynamic) C runtime allocator, including those from other DLL's or libraries. + +The overriding on Windows requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -Moreover, you need to ensure the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) is available +Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be available in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in `mimalloc-override.dll`). @@ -297,8 +320,9 @@ is also recommended to also override the `new`/`delete` operations (by including The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. -(Note: in principle, it is possible to patch existing executables -that are linked with the dynamic C runtime (`ucrtbase.dll`) by just putting the `mimalloc-override.dll` into the import table (and putting `mimalloc-redirect.dll` in the same folder) +(Note: in principle, it is possible to even patch existing executables without any recompilation +if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the `mimalloc-override.dll` +into the import table (and put `mimalloc-redirect.dll` in the same folder) Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388)). diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index cc03f5e2..c1c880ca 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -41,8 +41,11 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im ------------------------------------------------------ */ static size_t zone_size(malloc_zone_t* zone, const void* p) { - UNUSED(zone); UNUSED(p); - return 0; // as we cannot guarantee that `p` comes from us, just return 0 + UNUSED(zone); + if (!mi_is_in_heap_region(p)) + return 0; // not our pointer, bail out + + return mi_usable_size(p); } static void* zone_malloc(malloc_zone_t* zone, size_t size) { diff --git a/src/alloc-override.c b/src/alloc-override.c index c0e7bc2b..75c7990f 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -163,18 +163,30 @@ extern "C" { // Posix & Unix functions definitions // ------------------------------------------------------ -void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); -size_t malloc_size(void* p) MI_FORWARD1(mi_usable_size,p); -size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); void cfree(void* p) MI_FORWARD0(mi_free, p); +void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); +size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p); +#if !defined(__ANDROID__) +size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); +#else +size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p); +#endif // no forwarding here due to aliasing/name mangling issues -void* valloc(size_t size) { return mi_valloc(size); } -void* pvalloc(size_t size) { return mi_pvalloc(size); } -void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } -void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } +void* valloc(size_t size) { return mi_valloc(size); } +void* pvalloc(size_t size) { return mi_pvalloc(size); } +void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } +void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } + +// on some glibc `aligned_alloc` is declared `static inline` so we cannot override it (e.g. Conda). This happens +// when _GLIBCXX_HAVE_ALIGNED_ALLOC is not defined. However, in those cases it will use `memalign`, `posix_memalign`, +// or `_aligned_malloc` and we can avoid overriding it ourselves. +#if _GLIBCXX_HAVE_ALIGNED_ALLOC +void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } +#endif + #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) @@ -184,10 +196,10 @@ int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_me void __libc_free(void* p) MI_FORWARD0(mi_free,p); void __libc_cfree(void* p) MI_FORWARD0(mi_free,p); - void* __libc_valloc(size_t size) { return mi_valloc(size); } - void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } - void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } - int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } + void* __libc_valloc(size_t size) { return mi_valloc(size); } + void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } + int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } #endif #ifdef __cplusplus diff --git a/src/alloc.c b/src/alloc.c index 9afb5b6a..2ee8b720 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -44,7 +44,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz mi_heap_stat_increase(heap, normal[bin], 1); } #endif -#if defined(MI_PADDING) && defined(MI_ENCODE_FREELIST) +#if (MI_PADDING > 0) && defined(MI_ENCODE_FREELIST) mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - MI_PADDING_SIZE + delta)); @@ -203,7 +203,7 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block // Check for heap block overflow by setting up padding at the end of the block // --------------------------------------------------------------------------- -#if defined(MI_PADDING) && defined(MI_ENCODE_FREELIST) +#if (MI_PADDING>0) && defined(MI_ENCODE_FREELIST) static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { *bsize = mi_page_usable_block_size(page); const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); @@ -506,15 +506,16 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept { if (p==NULL) return 0; const mi_segment_t* const segment = _mi_ptr_segment(p); const mi_page_t* const page = _mi_segment_page_of(segment, p); - const mi_block_t* const block = (const mi_block_t*)p; - const size_t size = mi_page_usable_size_of(page, block); + const mi_block_t* block = (const mi_block_t*)p; if (mi_unlikely(mi_page_has_aligned(page))) { - ptrdiff_t const adjust = (uint8_t*)p - (uint8_t*)_mi_page_ptr_unalign(segment,page,p); + block = _mi_page_ptr_unalign(segment, page, p); + size_t size = mi_page_usable_size_of(page, block); + ptrdiff_t const adjust = (uint8_t*)p - (uint8_t*)block; mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); return (size - adjust); } else { - return size; + return mi_page_usable_size_of(page, block); } } @@ -677,12 +678,13 @@ mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept { // `strndup` using mi_malloc mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { if (s == NULL) return NULL; - size_t m = strlen(s); - if (n > m) n = m; - char* t = (char*)mi_heap_malloc(heap, n+1); + const char* end = (const char*)memchr(s, 0, n); // find end of string in the first `n` characters (returns NULL if not found) + const size_t m = (end != NULL ? (size_t)(end - s) : n); // `m` is the minimum of `n` or the end-of-string + mi_assert_internal(m <= n); + char* t = (char*)mi_heap_malloc(heap, m+1); if (t == NULL) return NULL; - memcpy(t, s, n); - t[n] = 0; + memcpy(t, s, m); + t[m] = 0; return t; } diff --git a/src/init.c b/src/init.c index b440c689..7a18bc5a 100644 --- a/src/init.c +++ b/src/init.c @@ -35,9 +35,9 @@ const mi_page_t _mi_page_empty = { #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) -#if defined(MI_PADDING) && (MI_INTPTR_SIZE >= 8) +#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } -#elif defined(MI_PADDING) +#elif (MI_PADDING>0) #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #else #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } @@ -312,6 +312,12 @@ static void _mi_thread_done(mi_heap_t* default_heap); // use thread local storage keys to detect thread ending #include #include + #if (_WIN32_WINNT < 0x600) // before Windows Vista + WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback ); + WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex ); + WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData ); + WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex); + #endif static DWORD mi_fls_key = (DWORD)(-1); static void NTAPI mi_fls_done(PVOID value) { if (value!=NULL) _mi_thread_done((mi_heap_t*)value); diff --git a/src/options.c b/src/options.c index d2c1fa26..c9c92003 100644 --- a/src/options.c +++ b/src/options.c @@ -51,7 +51,11 @@ typedef struct mi_option_desc_s { static mi_option_desc_t options[_mi_option_last] = { // stable options - { MI_DEBUG, UNINIT, MI_OPTION(show_errors) }, +#if MI_DEBUG || defined(MI_SHOW_ERRORS) + { 1, UNINIT, MI_OPTION(show_errors) }, +#else + { 0, UNINIT, MI_OPTION(show_errors) }, +#endif { 0, UNINIT, MI_OPTION(show_stats) }, { 0, UNINIT, MI_OPTION(verbose) }, @@ -262,13 +266,17 @@ static void mi_recurse_exit(void) { } void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message) { - if (!mi_recurse_enter()) return; if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) { // TODO: use mi_out_stderr for stderr? + if (!mi_recurse_enter()) return; out = mi_out_get_default(&arg); + if (prefix != NULL) out(prefix, arg); + out(message, arg); + mi_recurse_exit(); + } + else { + if (prefix != NULL) out(prefix, arg); + out(message, arg); } - if (prefix != NULL) out(prefix,arg); - out(message,arg); - mi_recurse_exit(); } // Define our own limited `fprintf` that avoids memory allocation. @@ -350,6 +358,11 @@ static void mi_error_default(int err) { abort(); } #endif +#if defined(MI_XMALLOC) + if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode + abort(); + } +#endif } void mi_register_error(mi_error_fun* fun, void* arg) { diff --git a/src/os.c b/src/os.c index e3dc8cda..83d73043 100644 --- a/src/os.c +++ b/src/os.c @@ -211,10 +211,12 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); if (p != NULL) return p; DWORD err = GetLastError(); - if (err != ERROR_INVALID_ADDRESS) { // if linked with multiple instances, we may have tried to allocate at an already allocated area + if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) + err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) return NULL; } - } + // fall through + } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) // on modern Windows try use VirtualAlloc2 for aligned allocation @@ -227,6 +229,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); } #endif + // last resort return VirtualAlloc(addr, size, flags, PAGE_READWRITE); } @@ -614,7 +617,7 @@ static void mi_mprotect_hint(int err) { } // Commit/Decommit memory. -// Usuelly commit is aligned liberal, while decommit is aligned conservative. +// Usually commit is aligned liberal, while decommit is aligned conservative. // (but not for the reset version where we want commit to be conservative as well) static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, bool* is_zero, mi_stats_t* stats) { // page align in the range, commit liberally, decommit conservative @@ -822,7 +825,7 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`) -----------------------------------------------------------------------------*/ #define MI_HUGE_OS_PAGE_SIZE (GiB) -#if defined(WIN32) && (MI_INTPTR_SIZE >= 8) +#if defined(_WIN32) && (MI_INTPTR_SIZE >= 8) static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { mi_assert_internal(size%GiB == 0); @@ -865,6 +868,8 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) params[0].ULong = (unsigned)numa_node; return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); } + #else + UNUSED(numa_node); #endif // otherwise use regular virtual alloc on older windows return VirtualAlloc(addr, size, flags, PAGE_READWRITE); @@ -904,6 +909,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) } #else static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { + UNUSED(addr); UNUSED(size); UNUSED(numa_node); return NULL; } #endif @@ -939,6 +945,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { } #else static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { + UNUSED(pages); if (total_size != NULL) *total_size = 0; return NULL; } @@ -1011,7 +1018,12 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) { /* ---------------------------------------------------------------------------- Support NUMA aware allocation -----------------------------------------------------------------------------*/ -#ifdef WIN32 +#ifdef _WIN32 + #if (_WIN32_WINNT < 0x601) // before Win7 + typedef struct _PROCESSOR_NUMBER { WORD Group; BYTE Number; BYTE Reserved; } PROCESSOR_NUMBER, *PPROCESSOR_NUMBER; + WINBASEAPI VOID WINAPI GetCurrentProcessorNumberEx(_Out_ PPROCESSOR_NUMBER ProcNumber); + WINBASEAPI BOOL WINAPI GetNumaProcessorNodeEx(_In_ PPROCESSOR_NUMBER Processor, _Out_ PUSHORT NodeNumber); + #endif static size_t mi_os_numa_nodex() { PROCESSOR_NUMBER pnum; USHORT numa_node = 0; diff --git a/src/region.c b/src/region.c index fd7d4544..9db11e0c 100644 --- a/src/region.c +++ b/src/region.c @@ -49,7 +49,7 @@ bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); bool _mi_os_unreset(void* p, size_t size, bool* is_zero, mi_stats_t* stats); // arena.c -void _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats); +void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_stats_t* stats); void* _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld); @@ -187,7 +187,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, const uintptr_t idx = mi_atomic_increment(®ions_count); if (idx >= MI_REGION_MAX) { mi_atomic_decrement(®ions_count); - _mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats); + _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats); _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, GiB)); return false; } @@ -391,7 +391,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re mem_region_t* region; if (mi_memid_is_arena(id,®ion,&bit_idx,&arena_memid)) { // was a direct arena allocation, pass through - _mi_arena_free(p, size, arena_memid, tld->stats); + _mi_arena_free(p, size, arena_memid, full_commit, tld->stats); } else { // allocated in a region @@ -454,12 +454,13 @@ void _mi_mem_collect(mi_os_tld_t* tld) { // on success, free the whole region uint8_t* start = mi_atomic_read_ptr(uint8_t,®ions[i].start); size_t arena_memid = mi_atomic_read_relaxed(®ions[i].arena_memid); + uintptr_t commit = mi_atomic_read_relaxed(®ions[i].commit); memset(®ions[i], 0, sizeof(mem_region_t)); // and release the whole region mi_atomic_write(®ion->info, 0); if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { _mi_abandoned_await_readers(); // ensure no pending reads - _mi_arena_free(start, MI_REGION_SIZE, arena_memid, tld->stats); + _mi_arena_free(start, MI_REGION_SIZE, arena_memid, (~commit == 0), tld->stats); } } } diff --git a/src/segment.c b/src/segment.c index 9b105468..b8e5f2ec 100644 --- a/src/segment.c +++ b/src/segment.c @@ -202,7 +202,6 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa uint8_t* p = (uint8_t*)segment + (idx*MI_SEGMENT_SLICE_SIZE); /* if (idx == 0) { - // the first page starts after the segment info (and possible guard page) p += segment->segment_info_size; psize -= segment->segment_info_size; @@ -1300,7 +1299,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block mi_assert_internal(mi_atomic_read_relaxed(&segment->thread_id)==0); // claim it and free - mi_heap_t* heap = mi_get_default_heap(); + mi_heap_t* heap = mi_heap_get_default(); // issue #221; don't use the internal get_default_heap as we need to ensure the thread is initialized. // paranoia: if this it the last reference, the cas should always succeed if (mi_atomic_cas_strong(&segment->thread_id, heap->thread_id, 0)) { mi_block_set_next(page, block, page->free); @@ -1308,16 +1307,16 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block page->used--; page->is_zero = false; mi_assert(page->used == 0); - mi_segments_tld_t* tld = &heap->tld->segments; + mi_tld_t* tld = heap->tld; const size_t bsize = mi_page_usable_block_size(page); if (bsize <= MI_LARGE_OBJ_SIZE_MAX) { - _mi_stat_decrease(&tld->stats->large, bsize); + _mi_stat_decrease(&tld->stats.large, bsize); } else { - _mi_stat_decrease(&tld->stats->huge, bsize); + _mi_stat_decrease(&tld->stats.huge, bsize); } // mi_segments_track_size((long)segment->segment_size, tld); - _mi_segment_page_free(page, true, tld); + _mi_segment_page_free(page, true, &tld->segments); } } diff --git a/src/static.c b/src/static.c index b3c71e02..9f8cb899 100644 --- a/src/static.c +++ b/src/static.c @@ -24,5 +24,8 @@ terms of the MIT license. A copy of the license can be found in the file #include "alloc.c" #include "alloc-aligned.c" #include "alloc-posix.c" +#if MI_OSX_ZONE +#include "alloc-override-osx.c" +#endif #include "init.c" #include "options.c" diff --git a/src/stats.c b/src/stats.c index e568ec91..89a8e42f 100644 --- a/src/stats.c +++ b/src/stats.c @@ -237,9 +237,51 @@ static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bin #endif + +//------------------------------------------------------------ +// Use an output wrapper for line-buffered output +// (which is nice when using loggers etc.) +//------------------------------------------------------------ +typedef struct buffered_s { + mi_output_fun* out; // original output function + void* arg; // and state + char* buf; // local buffer of at least size `count+1` + size_t used; // currently used chars `used <= count` + size_t count; // total chars available for output +} buffered_t; + +static void mi_buffered_flush(buffered_t* buf) { + buf->buf[buf->used] = 0; + _mi_fputs(buf->out, buf->arg, NULL, buf->buf); + buf->used = 0; +} + +static void mi_buffered_out(const char* msg, void* arg) { + buffered_t* buf = (buffered_t*)arg; + if (msg==NULL || buf==NULL) return; + for (const char* src = msg; *src != 0; src++) { + char c = *src; + if (buf->used >= buf->count) mi_buffered_flush(buf); + mi_assert_internal(buf->used < buf->count); + buf->buf[buf->used++] = c; + if (c == '\n') mi_buffered_flush(buf); + } +} + +//------------------------------------------------------------ +// Print statistics +//------------------------------------------------------------ + static void mi_process_info(mi_msecs_t* utime, mi_msecs_t* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit); -static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out, void* arg) mi_attr_noexcept { +static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun* out0, void* arg0) mi_attr_noexcept { + // wrap the output function to be line buffered + char buf[256]; + buffered_t buffer = { out0, arg0, buf, 0, 255 }; + mi_output_fun* out = &mi_buffered_out; + void* arg = &buffer; + + // and print using that mi_print_header(out,arg); #if MI_STAT>1 mi_stat_count_t normal = { 0,0,0,0 }; @@ -287,7 +329,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_msecs_t elapsed, mi_output_fun _mi_fprintf(out, arg, ", commit charge: "); mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); } - _mi_fprintf(out, arg, "\n"); + _mi_fprintf(out, arg, "\n"); } static mi_msecs_t mi_time_start; // = 0 diff --git a/test/main-override-static.c b/test/main-override-static.c index 9ce943bb..1ab11385 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -184,7 +184,7 @@ int main() { // double_free1(); // double_free2(); // corrupt_free(); - // block_overflow1(); + block_overflow1(); void* p1 = malloc(78); void* p2 = malloc(24); diff --git a/test/main-override.cpp b/test/main-override.cpp index 18d49df3..734e4c94 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -20,16 +20,18 @@ static void msleep(unsigned long msecs) { Sleep(msecs); } static void msleep(unsigned long msecs) { usleep(msecs * 1000UL); } #endif -void heap_no_delete(); -void heap_late_free(); -void padding_shrink(); +void heap_thread_free_large(); // issue #221 +void heap_no_delete(); // issue #202 +void heap_late_free(); // issue #204 +void padding_shrink(); // issue #209 void various_tests(); int main() { mi_stats_reset(); // ignore earlier allocations - // heap_no_delete(); // issue #202 - // heap_late_free(); // issue #204 - padding_shrink(); // issue #209 + heap_thread_free_large(); + heap_no_delete(); + heap_late_free(); + padding_shrink(); various_tests(); mi_stats_print(NULL); return 0; @@ -157,3 +159,17 @@ void padding_shrink(void) t1.join(); mi_free(shared_p); } + + +// Issue #221 +void heap_thread_free_large_worker() { + mi_free(shared_p); +} + +void heap_thread_free_large() { + for (int i = 0; i < 100; i++) { + shared_p = mi_malloc_aligned(2*1024*1024 + 1, 8); + auto t1 = std::thread(heap_thread_free_large_worker); + t1.join(); + } +} diff --git a/test/test-api.c b/test/test-api.c index 166cfca6..e5827a93 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -152,6 +152,9 @@ int main() { } result = ok; }); + CHECK_BODY("malloc-aligned5", { + void* p = mi_malloc_aligned(4097,4096); size_t usable = mi_usable_size(p); result = usable >= 4097 && usable < 10000; mi_free(p); + }); CHECK_BODY("malloc-aligned-at1", { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p); });