From 0f57425f808ea900ff26875f04dc5b25f6dbd204 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 30 May 2021 13:09:02 +0800 Subject: [PATCH 001/180] Distinguish SI and Binary Prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SI prefixes [the decimal prefixes] refer strictly to powers of 10. They should not be used to indicate powers of 2. e.g., one kilobit represents 1000 bits instead of 1024 bits. IEC 60027‐2 symbols are formed adding a "i" to the SI symbol (e.g. G + i = Gi). --- include/mimalloc-types.h | 22 +++++++++++----------- src/arena.c | 6 +++--- src/os.c | 6 +++--- src/segment.c | 8 ++++---- src/stats.c | 10 +++++----- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 18f1623c..502e41f8 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -105,10 +105,10 @@ terms of the MIT license. A copy of the license can be found in the file // Main tuning parameters for segment and page sizes // Sizes for 64-bit, divide by two for 32-bit -#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64kb -#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512kb -#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4mb -#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4mb +#define MI_SMALL_PAGE_SHIFT (13 + MI_INTPTR_SHIFT) // 64KiB +#define MI_MEDIUM_PAGE_SHIFT ( 3 + MI_SMALL_PAGE_SHIFT) // 512KiB +#define MI_LARGE_PAGE_SHIFT ( 3 + MI_MEDIUM_PAGE_SHIFT) // 4MiB +#define MI_SEGMENT_SHIFT ( MI_LARGE_PAGE_SHIFT) // 4MiB // Derived constants #define MI_SEGMENT_SIZE (1UL<512kb) are put into a single page in a segment of the exact size (but still 2mb aligned) + MI_PAGE_HUGE // huge blocks (>512KiB) are put into a single page in a segment of the exact size (but still 2MiB aligned) } mi_page_kind_t; -// Segments are large allocated memory blocks (2mb on 64 bit) from +// Segments are large allocated memory blocks (2MiB on 64 bit) from // the OS. Inside segments we allocated fixed size _pages_ that // contain blocks. typedef struct mi_segment_s { diff --git a/src/arena.c b/src/arena.c index 0e6615a4..fb25a89d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -330,7 +330,7 @@ int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noe _mi_verbose_message("failed to reserve %zu k memory\n", _mi_divide_up(size,1024)); return ENOMEM; } - _mi_verbose_message("reserved %zu kb memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); + _mi_verbose_message("reserved %zu KiB memory%s\n", _mi_divide_up(size,1024), large ? " (in large os pages)" : ""); return 0; } @@ -347,10 +347,10 @@ int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msec size_t pages_reserved = 0; void* p = _mi_os_alloc_huge_os_pages(pages, numa_node, timeout_msecs, &pages_reserved, &hsize); if (p==NULL || pages_reserved==0) { - _mi_warning_message("failed to reserve %zu gb huge pages\n", pages); + _mi_warning_message("failed to reserve %zu GiB huge pages\n", pages); return ENOMEM; } - _mi_verbose_message("numa node %i: reserved %zu gb huge pages (of the %zu gb requested)\n", numa_node, pages_reserved, pages); + _mi_verbose_message("numa node %i: reserved %zu GiB huge pages (of the %zu GiB requested)\n", numa_node, pages_reserved, pages); if (!mi_manage_os_memory(p, hsize, true, true, true, numa_node)) { _mi_os_free_huge_pages(p, hsize, &_mi_stats_main); diff --git a/src/os.c b/src/os.c index 2e2ec88d..4b87054e 100644 --- a/src/os.c +++ b/src/os.c @@ -210,7 +210,7 @@ void _mi_os_init(void) { } #elif defined(__wasi__) void _mi_os_init() { - os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KB + os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } #else @@ -964,7 +964,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) else { // fall back to regular large pages mi_huge_pages_available = false; // don't try further huge pages - _mi_warning_message("unable to allocate using huge (1gb) pages, trying large (2mb) pages instead (status 0x%lx)\n", err); + _mi_warning_message("unable to allocate using huge (1GiB) pages, trying large (2MiB) pages instead (status 0x%lx)\n", err); } } // on modern Windows try use VirtualAlloc2 for numa aware large OS page allocation @@ -1007,7 +1007,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) // see: long err = mi_os_mbind(p, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); if (err != 0) { - _mi_warning_message("failed to bind huge (1gb) pages to numa node %d: %s\n", numa_node, strerror(errno)); + _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d: %s\n", numa_node, strerror(errno)); } } return p; diff --git a/src/segment.c b/src/segment.c index 1d59be9d..bdf97019 100644 --- a/src/segment.c +++ b/src/segment.c @@ -17,14 +17,14 @@ static uint8_t* mi_segment_raw_page_start(const mi_segment_t* segment, const mi_ /* -------------------------------------------------------------------------------- Segment allocation - We allocate pages inside bigger "segments" (4mb on 64-bit). This is to avoid + We allocate pages inside bigger "segments" (4MiB on 64-bit). This is to avoid splitting VMA's on Linux and reduce fragmentation on other OS's. Each thread owns its own segments. Currently we have: - - small pages (64kb), 64 in one segment - - medium pages (512kb), 8 in one segment - - large pages (4mb), 1 in one segment + - small pages (64KiB), 64 in one segment + - medium pages (512KiB), 8 in one segment + - large pages (4MiB), 1 in one segment - huge blocks > MI_LARGE_OBJ_SIZE_MAX become large segment with 1 page In any case the memory for a segment is virtual and usually committed on demand. diff --git a/src/stats.c b/src/stats.c index c94fbde9..07fce5e2 100644 --- a/src/stats.c +++ b/src/stats.c @@ -135,7 +135,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { char buf[32]; int len = 32; - const char* suffix = (unit <= 0 ? " " : "b"); + const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); if (unit>0) n *= unit; @@ -145,13 +145,13 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* } else { int64_t divider = base; - const char* magnitude = "k"; - if (pos >= divider*base) { divider *= base; magnitude = "m"; } - if (pos >= divider*base) { divider *= base; magnitude = "g"; } + const char* magnitude = "K"; + if (pos >= divider*base) { divider *= base; magnitude = "M"; } + if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, suffix); + snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } From a35a7d4f19b23787b67af8a49d17e18e2fc36df9 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 19 Jun 2021 09:14:43 +0000 Subject: [PATCH 002/180] haiku biuld fix proposal, warning suppression. --- src/random.c | 4 +++- src/stats.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/random.c b/src/random.c index 255bede4..723d3507 100644 --- a/src/random.c +++ b/src/random.c @@ -194,8 +194,10 @@ static bool os_random_buf(void* buf, size_t buf_len) { arc4random_buf(buf, buf_len); return true; } -#elif defined(__linux__) +#elif defined(__linux__) || defined(__HAIKU__) +#if defined(__linux__) #include +#endif #include #include #include diff --git a/src/stats.c b/src/stats.c index c94fbde9..74b783e9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -520,6 +520,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec while (get_next_area_info(tid.team, &c, &mem) == B_OK) { *peak_rss += mem.ram_size; } + *page_faults = 0; #elif defined(__APPLE__) *peak_rss = rusage.ru_maxrss; // BSD reports in bytes struct mach_task_basic_info info; From 1c1571742d24b9cfbdbef4a3c537018f41756c64 Mon Sep 17 00:00:00 2001 From: hank Date: Mon, 21 Jun 2021 22:36:47 +0800 Subject: [PATCH 003/180] fix typo --- include/mimalloc-internal.h | 2 +- src/page.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..825b0ed0 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -427,7 +427,7 @@ static inline mi_page_t* _mi_ptr_page(void* p) { return _mi_segment_page_of(_mi_ptr_segment(p), p); } -// Get the block size of a page (special cased for huge objects) +// Get the block size of a page (special case for huge objects) static inline size_t mi_page_block_size(const mi_page_t* page) { const size_t bsize = page->xblock_size; mi_assert_internal(bsize > 0); diff --git a/src/page.c b/src/page.c index c08be9c0..033976dc 100644 --- a/src/page.c +++ b/src/page.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ----------------------------------------------------------- The core of the allocator. Every segment contains - pages of a {certain block size. The main function + pages of a certain block size. The main function exported is `mi_malloc_generic`. ----------------------------------------------------------- */ From 4369fe43239c56ad017911ef1b704b666a3e35e8 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Thu, 24 Jun 2021 17:29:06 +0800 Subject: [PATCH 004/180] Eliminate preprocessor warnings due to undefined "__GNUC__" with ClangCL When building some code against mimalloc with C inside Visual Studio with ClangCL, the compiler complains about __GNUC__ being undefined. Reported by Mojca Miklavec. Close #422 --- include/mimalloc-internal.h | 6 +++--- include/mimalloc.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..3be872af 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_noinline __declspec(noinline) #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) -#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc +#elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) @@ -236,7 +236,7 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) } // Overflow detecting multiply -#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 +#if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include // UINT_MAX, ULONG_MAX #if defined(_CLOCK_T) // for Illumos #undef _CLOCK_T @@ -903,7 +903,7 @@ static inline void _mi_memcpy(void* dst, const void* src, size_t n) { // This is used for example in `mi_realloc`. // ------------------------------------------------------------------------------- -#if (__GNUC__ >= 4) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // On GCC/CLang we provide a hint that the pointers are word aligned. #include static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { diff --git a/include/mimalloc.h b/include/mimalloc.h index fe5aa8f3..7ebf3e60 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -26,7 +26,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__cplusplus) && (__cplusplus >= 201703) #define mi_decl_nodiscard [[nodiscard]] -#elif (__GNUC__ >= 4) || defined(__clang__) // includes clang, icc, and clang-cl +#elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl #define mi_decl_nodiscard __attribute__((warn_unused_result)) #elif (_MSC_VER >= 1700) #define mi_decl_nodiscard _Check_return_ From 71b2c441bb236fd43a1695e80af68ef750cddd0c Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 25 Jun 2021 10:09:19 +0800 Subject: [PATCH 005/180] CI: Update the macOS image to version 10.15 --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aeb2908b..65ceb30b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,7 +115,7 @@ jobs: displayName: macOS pool: vmImage: - macOS-10.14 + macOS-10.15 strategy: matrix: Debug: From c4947c887903ab216589a763013d2b22cf6457ac Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 24 Apr 2021 06:24:06 +0800 Subject: [PATCH 006/180] Use secure random generator on macOS The implementation of arc4random_buf differs from its documentation. It is documented as "always successful, and no return value is reserved to indicate an error" for the sake of FreeBSD compatibility [1]. However, the actual implementation on macOS invokes function "ccrng_generate" [2] without validating the error cases. It might fail silently[3], which leads to unexpected source of entropy. The original arc4random used the RC4 a.k.a. ARC4 algorithm, and ChaCha20 based implementation was introduced in FreeBSD 12.0. Since macOS 10.12, it was replaced with the NIST-approved AES cipher, and it may be replaced again in the future as cryptographic techniques advance. Therefore, we should not assume that arc4random never fails. On the contrary, CCRandomGenerateBytes(), part of Cryptographic Services [4], returns cryptographically strong random bits with explicit status code. This patch properly calls CCRandomGenerateBytes() and checks the status. [1] https://www.freebsd.org/cgi/man.cgi?query=arc4random_buf [2] https://opensource.apple.com/source/CommonCrypto/CommonCrypto-60178.40.2/lib/CommonRandom.c.auto.html [3] https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html [4] https://developer.apple.com/documentation/security --- src/random.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/random.c b/src/random.c index 255bede4..910a3e89 100644 --- a/src/random.c +++ b/src/random.c @@ -156,7 +156,8 @@ uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { /* ---------------------------------------------------------------------------- To initialize a fresh random context we rely on the OS: - Windows : BCryptGenRandom (or RtlGenRandom) -- osX,bsd,wasi: arc4random_buf +- macOS : CCRandomGenerateBytes +- bsd,wasi : arc4random_buf - Linux : getrandom,/dev/urandom If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ @@ -186,7 +187,36 @@ static bool os_random_buf(void* buf, size_t buf_len) { } #endif -#elif defined(ANDROID) || defined(XP_DARWIN) || defined(__APPLE__) || defined(__DragonFly__) || \ +#elif defined(__APPLE__) +#include +#if defined(MAC_OS_X_VERSION_10_10) && \ + MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 +#include +#endif +static bool os_random_buf(void* buf, size_t buf_len) { + /* The implementation of arc4random_buf(3) differs from its documentation. + * It is documented as "always successful, and no return value is reserved + * to indicate an error." However, the actual implementation invokes the + * function "ccrng_generate" without validating the error cases. It might + * fail silently, which leads to unexpected source of entropy. + * See: + * https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html + * + * CCRandomGenerateBytes(), on the contrary, returns cryptographically strong + * random bits with explicit status code. + */ +#if defined(MAC_OS_X_VERSION_10_15) && \ + MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 + return CCRandomGenerateBytes(buf, buf_len) == kCCSuccess; +#else + /* Prior to macOS 10.15, CCRandomGenerateBytes() might take a bit longer time + * to complete, so failback to arc4random_buf(). + */ + arc4random_buf(buf, buf_len); + return true; +#endif +} +#elif defined(ANDROID) || defined(__DragonFly__) || \ defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ defined(__sun) || defined(__wasi__) #include From edb0b93c6fdd2edbe4da520c1d632010e91dedcb Mon Sep 17 00:00:00 2001 From: Artur Sinila Date: Tue, 29 Jun 2021 21:07:13 +0300 Subject: [PATCH 007/180] Fix 'malloc-nomem1' test for 32-bit architectures --- test/test-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-api.c b/test/test-api.c index d3344928..739b1d18 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -83,7 +83,7 @@ int main() { void* p = mi_malloc(0); mi_free(p); }); CHECK_BODY("malloc-nomem1",{ - result = (mi_malloc(SIZE_MAX/2) == NULL); + result = (mi_malloc((size_t)PTRDIFF_MAX + (size_t)1) == NULL); }); CHECK_BODY("malloc-null",{ mi_free(NULL); From a3cf23c19fff6fd34492bee85788f2e52e41531b Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 26 Jul 2021 19:10:21 -0700 Subject: [PATCH 008/180] add test for #445 --- test/main-override.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index 5fe3f9a3..d7663330 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -34,6 +34,7 @@ void various_tests(); void test_mt_shutdown(); void fail_aslr(); // issue #372 void tsan_numa_test(); // issue #414 +void strdup_test(); // issue #445 int main() { mi_stats_reset(); // ignore earlier allocations @@ -43,6 +44,7 @@ int main() { padding_shrink(); various_tests(); tsan_numa_test(); + strdup_test(); //test_mt_shutdown(); //fail_aslr(); @@ -74,18 +76,13 @@ void various_tests() { p1 = malloc(8); char* s = mi_strdup("hello\n"); - //char* s = _strdup("hello\n"); - //char* buf = NULL; - //size_t len; - //_dupenv_s(&buf,&len,"MIMALLOC_VERBOSE"); - //mi_free(buf); - mi_free(p2); p2 = malloc(16); p1 = realloc(p1, 32); free(p1); free(p2); mi_free(s); + Test* t = new Test(42); delete t; t = new (std::nothrow) Test(42); @@ -125,7 +122,17 @@ bool test_stl_allocator2() { return vec.size() == 0; } - +// issue 445 +static void strdup_test() { +#ifdef _MSC_VER + char* s = _strdup("hello\n"); + char* buf = NULL; + size_t len; + _dupenv_s(&buf, &len, "MIMALLOC_VERBOSE"); + mi_free(buf); + mi_free(s); +#endif +} // Issue #202 void heap_no_delete_worker() { From 32c5e4774febc4f50348bd08af5b80cfc8745857 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 26 Jul 2021 19:14:29 -0700 Subject: [PATCH 009/180] update tests with static --- test/main-override.cpp | 48 +++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index d7663330..e1795ecb 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -26,15 +26,15 @@ static void msleep(unsigned long msecs) { Sleep(msecs); } static void msleep(unsigned long msecs) { usleep(msecs * 1000UL); } #endif -void heap_thread_free_large(); // issue #221 -void heap_no_delete(); // issue #202 -void heap_late_free(); // issue #204 -void padding_shrink(); // issue #209 -void various_tests(); -void test_mt_shutdown(); -void fail_aslr(); // issue #372 -void tsan_numa_test(); // issue #414 -void strdup_test(); // issue #445 +static void heap_thread_free_large(); // issue #221 +static void heap_no_delete(); // issue #202 +static void heap_late_free(); // issue #204 +static void padding_shrink(); // issue #209 +static void various_tests(); +static void test_mt_shutdown(); +static void fail_aslr(); // issue #372 +static void tsan_numa_test(); // issue #414 +static void strdup_test(); // issue #445 int main() { mi_stats_reset(); // ignore earlier allocations @@ -68,7 +68,7 @@ public: }; -void various_tests() { +static void various_tests() { atexit(free_p); void* p1 = malloc(78); void* p2 = mi_malloc_aligned(16, 24); @@ -106,7 +106,7 @@ public: static Static s = Static(); -bool test_stl_allocator1() { +static bool test_stl_allocator1() { std::vector > vec; vec.push_back(1); vec.pop_back(); @@ -115,7 +115,7 @@ bool test_stl_allocator1() { struct some_struct { int i; int j; double z; }; -bool test_stl_allocator2() { +static bool test_stl_allocator2() { std::vector > vec; vec.push_back(some_struct()); vec.pop_back(); @@ -135,28 +135,28 @@ static void strdup_test() { } // Issue #202 -void heap_no_delete_worker() { +static void heap_no_delete_worker() { mi_heap_t* heap = mi_heap_new(); void* q = mi_heap_malloc(heap, 1024); // mi_heap_delete(heap); // uncomment to prevent assertion } -void heap_no_delete() { +static void heap_no_delete() { auto t1 = std::thread(heap_no_delete_worker); t1.join(); } // Issue #204 -volatile void* global_p; +static volatile void* global_p; -void t1main() { +static void t1main() { mi_heap_t* heap = mi_heap_new(); global_p = mi_heap_malloc(heap, 1024); mi_heap_delete(heap); } -void heap_late_free() { +static void heap_late_free() { auto t1 = std::thread(t1main); msleep(2000); @@ -173,7 +173,7 @@ static void alloc0(/* void* arg */) shared_p = mi_malloc(8); } -void padding_shrink(void) +static void padding_shrink(void) { auto t1 = std::thread(alloc0); t1.join(); @@ -182,11 +182,11 @@ void padding_shrink(void) // Issue #221 -void heap_thread_free_large_worker() { +static void heap_thread_free_large_worker() { mi_free(shared_p); } -void heap_thread_free_large() { +static void heap_thread_free_large() { for (int i = 0; i < 100; i++) { shared_p = mi_malloc_aligned(2*1024*1024 + 1, 8); auto t1 = std::thread(heap_thread_free_large_worker); @@ -196,7 +196,7 @@ void heap_thread_free_large() { -void test_mt_shutdown() +static void test_mt_shutdown() { const int threads = 5; std::vector< std::future< std::vector< char* > > > ts; @@ -221,7 +221,7 @@ void test_mt_shutdown() } // issue #372 -void fail_aslr() { +static void fail_aslr() { size_t sz = (4ULL << 40); // 4TiB void* p = malloc(sz); printf("pointer p: %p: area up to %p\n", p, (uint8_t*)p + sz); @@ -229,12 +229,12 @@ void fail_aslr() { } // issues #414 -void dummy_worker() { +static void dummy_worker() { void* p = mi_malloc(0); mi_free(p); } -void tsan_numa_test() { +static void tsan_numa_test() { auto t1 = std::thread(dummy_worker); dummy_worker(); t1.join(); From 30be78d97a4a1a25c81755fb2671428a5893a0d9 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:31:48 -0400 Subject: [PATCH 010/180] Fixed typo in headers --- include/mimalloc-types.h | 2 +- src/heap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index caf161d6..66b311c1 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -319,7 +319,7 @@ typedef struct mi_random_cxt_s { } mi_random_ctx_t; -// In debug mode there is a padding stucture at the end of the blocks to check for buffer overflows +// In debug mode there is a padding structure at the end of the blocks to check for buffer overflows #if (MI_PADDING) typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) diff --git a/src/heap.c b/src/heap.c index bda10699..bbf4cef9 100644 --- a/src/heap.c +++ b/src/heap.c @@ -333,7 +333,7 @@ void mi_heap_destroy(mi_heap_t* heap) { Safe Heap delete ----------------------------------------------------------- */ -// Tranfer the pages from one heap to the other +// Transfer the pages from one heap to the other static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { mi_assert_internal(heap!=NULL); if (from==NULL || from->page_count == 0) return; From ee0b01c84dbaf8b65f505fbbb2debb64bfc489c6 Mon Sep 17 00:00:00 2001 From: bmalrat Date: Wed, 4 Aug 2021 17:46:58 -0400 Subject: [PATCH 011/180] Fixed stats in mi_os_mem_alloc_aligned on windows when re-allocate In the last try the previous mi_os_mem_free decrease stat and mi_win_virtual_alloc doesn't increase it --- src/os.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/os.c b/src/os.c index 85415232..89d166ef 100644 --- a/src/os.c +++ b/src/os.c @@ -593,6 +593,11 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); + if (p != NULL) + { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } + } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? mi_os_mem_free(p, size, commit, stats); From a2c3b0f8af5d43cea6398be8e7162a6fbce2ec95 Mon Sep 17 00:00:00 2001 From: paulip1792 Date: Tue, 10 Aug 2021 16:30:44 +0800 Subject: [PATCH 012/180] add option to reserve huge os pages at a specific numa node. --- doc/mimalloc-doc.h | 3 +++ include/mimalloc.h | 1 + readme.md | 2 ++ src/init.c | 7 ++++++- src/options.c | 1 + 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index b448f14a..e2287fe9 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -782,6 +782,7 @@ typedef enum mi_option_e { mi_option_eager_region_commit, ///< Eagerly commit large (256MiB) memory regions (enabled by default, except on Windows) mi_option_large_os_pages, ///< Use large OS pages (2MiB in size) if possible mi_option_reserve_huge_os_pages, ///< The number of huge OS pages (1GiB in size) to reserve at the start of the program. + mi_option_reserve_huge_os_pages_at, ///< Reserve huge OS pages at node N. mi_option_segment_cache, ///< The number of segments per thread to keep cached. mi_option_page_reset, ///< Reset page memory after \a mi_option_reset_delay milliseconds when it becomes free. mi_option_segment_reset, ///< Experimental @@ -1053,6 +1054,8 @@ or via environment variables. `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. + (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the diff --git a/include/mimalloc.h b/include/mimalloc.h index fe5aa8f3..db5bff40 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -306,6 +306,7 @@ typedef enum mi_option_e { mi_option_reset_decommits, mi_option_large_os_pages, // implies eager commit mi_option_reserve_huge_os_pages, + mi_option_reserve_huge_os_pages_at, mi_option_reserve_os_memory, mi_option_segment_cache, mi_option_page_reset, diff --git a/readme.md b/readme.md index cdb1b82a..2c02f665 100644 --- a/readme.md +++ b/readme.md @@ -302,6 +302,8 @@ or via environment variables: `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). +- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. + (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the diff --git a/src/init.c b/src/init.c index c0f09b5e..587e6b1c 100644 --- a/src/init.c +++ b/src/init.c @@ -494,7 +494,12 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { size_t pages = mi_option_get(mi_option_reserve_huge_os_pages); - mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at); + if (reserve_at != -1) { + mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); + } else { + mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); + } } if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); diff --git a/src/options.c b/src/options.c index 30025db2..aa4601fe 100644 --- a/src/options.c +++ b/src/options.c @@ -76,6 +76,7 @@ static mi_option_desc_t options[_mi_option_last] = #endif { 0, UNINIT, MI_OPTION(large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages + { -1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N { 0, UNINIT, MI_OPTION(reserve_os_memory) }, { 0, UNINIT, MI_OPTION(segment_cache) }, // cache N segments per thread { 1, UNINIT, MI_OPTION(page_reset) }, // reset page memory on free From 9ee780894a18c3892a1c64bb2950fa862a3b7588 Mon Sep 17 00:00:00 2001 From: Masashi Fujita Date: Thu, 16 Sep 2021 03:35:56 +0900 Subject: [PATCH 013/180] fix: Fix a typo --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..385d2071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ set(mi_sources src/init.c) # ----------------------------------------------------------------------------- -# Converience: set default build type depending on the build directory +# Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- if (NOT CMAKE_BUILD_TYPE) From 679aad065908ccf40fcf535705c6e2d4a1fbdb82 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 1 Oct 2021 15:05:01 -0700 Subject: [PATCH 014/180] update wasm support with emscripten compilation; now using sbrk instead of wasm_memory_grow --- include/mimalloc-internal.h | 10 +++++--- src/os.c | 49 ++++++++++++++++++++++++++++++++++--- src/random.c | 4 ++- src/stats.c | 2 +- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..ecdcf860 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -32,6 +32,10 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif +#if defined(__EMSCRIPTEN__) && !defined(__wasi__) +#define __wasi__ +#endif + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -242,10 +246,10 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) #undef _CLOCK_T #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #if (SIZE_MAX == UINT_MAX) - return __builtin_umul_overflow(count, size, total); - #elif (SIZE_MAX == ULONG_MAX) + #if (SIZE_MAX == ULONG_MAX) return __builtin_umull_overflow(count, size, total); + #elif (SIZE_MAX == UINT_MAX) + return __builtin_umul_overflow(count, size, total); #else return __builtin_umulll_overflow(count, size, total); #endif diff --git a/src/os.c b/src/os.c index 85415232..5b6b0c7e 100644 --- a/src/os.c +++ b/src/os.c @@ -26,11 +26,14 @@ terms of the MIT license. A copy of the license can be found in the file #pragma warning(disable:4996) // strerror #endif +#if defined(__wasi__) +#define MI_USE_SBRK +#endif #if defined(_WIN32) #include #elif defined(__wasi__) -// stdlib.h is all we need, and has already been included in mimalloc.h +#include // sbrk #else #include // mmap #include // sysconf @@ -99,11 +102,13 @@ size_t _mi_os_large_page_size() { return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); } +#if !defined(MI_USE_SBRK) && !defined(__wasi__) static bool use_large_os_page(size_t size, size_t alignment) { // if we have access, check the size and alignment requirements if (large_os_page_size == 0 || !mi_option_is_enabled(mi_option_large_os_pages)) return false; return ((size % large_os_page_size) == 0 && (alignment % large_os_page_size) == 0); } +#endif // round to a good OS allocation size (bounded by max 12.5% waste) size_t _mi_os_good_alloc_size(size_t size) { @@ -236,8 +241,8 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats bool err = false; #if defined(_WIN32) err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); -#elif defined(__wasi__) - err = 0; // WebAssembly's heap cannot be shrunk +#elif defined(MI_USE_SBRK) + err = 0; // sbrk heap cannot be shrunk #else err = (munmap(addr, size) == -1); #endif @@ -252,7 +257,9 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats } } +#if !defined(MI_USE_SBRK) && !defined(__wasi__) static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); +#endif #ifdef _WIN32 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { @@ -318,7 +325,32 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, return p; } -#elif defined(__wasi__) +#elif defined(MI_USE_SBRK) +#define MI_SBRK_FAIL ((void*)(-1)) +static void* mi_sbrk_heap_grow(size_t size, size_t try_alignment) { + void* pbase0 = sbrk(0); + if (pbase0 == MI_SBRK_FAIL) { + _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); + errno = ENOMEM; + return NULL; + } + uintptr_t base = (uintptr_t)pbase0; + uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); + size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); + mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); + if (alloc_size < size) return NULL; + void* pbase1 = sbrk(alloc_size); + if (pbase1 == MI_SBRK_FAIL) { + _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); + errno = ENOMEM; + return NULL; + } + mi_assert(pbase0 == pbase1); + return (void*)aligned_base; +} + +#elif defined(__wasi__) + // currently unused as we use sbrk() on wasm static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size(); uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); @@ -326,11 +358,13 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); if (alloc_size < size) return NULL; if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) { + _mi_warning_message("unable to allocate wasm_memory_grow() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); errno = ENOMEM; return NULL; } return (void*)aligned_base; } + #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { @@ -506,6 +540,8 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) if (hint%try_alignment != 0) return NULL; return (void*)hint; } +#elif defined(__wasi__) || defined(MI_USE_SBRK) +// no need for mi_os_get_aligned_hint #else static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { UNUSED(try_alignment); UNUSED(size); @@ -536,7 +572,12 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo int flags = MEM_RESERVE; if (commit) flags |= MEM_COMMIT; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); + #elif defined(MI_USE_SBRK) + KK_UNUSED(allow_large); + *is_large = false; + p = mi_sbrk_heap_grow(size, try_alignment); #elif defined(__wasi__) + KK_UNUSED(allow_large); *is_large = false; p = mi_wasm_heap_grow(size, try_alignment); #else diff --git a/src/random.c b/src/random.c index 255bede4..d5f75bf8 100644 --- a/src/random.c +++ b/src/random.c @@ -188,7 +188,7 @@ static bool os_random_buf(void* buf, size_t buf_len) { #elif defined(ANDROID) || defined(XP_DARWIN) || defined(__APPLE__) || defined(__DragonFly__) || \ defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__sun) || defined(__wasi__) + defined(__sun) // todo: what to use with __wasi__? #include static bool os_random_buf(void* buf, size_t buf_len) { arc4random_buf(buf, buf_len); @@ -280,7 +280,9 @@ void _mi_random_init(mi_random_ctx_t* ctx) { if (!os_random_buf(key, sizeof(key))) { // if we fail to get random data from the OS, we fall back to a // weak random source based on the current time + #if !defined(__wasi__) _mi_warning_message("unable to use secure randomness\n"); + #endif uintptr_t x = _os_random_weak(0); for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words. x = _mi_random_shuffle(x); diff --git a/src/stats.c b/src/stats.c index c94fbde9..ac728a1c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -479,7 +479,7 @@ static void mi_stat_process_info(mi_msecs_t* elapsed, mi_msecs_t* utime, mi_msec *page_faults = (size_t)info.PageFaultCount; } -#elif defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__) +#elif !defined(__wasi__) && (defined(__unix__) || defined(__unix) || defined(unix) || defined(__APPLE__) || defined(__HAIKU__)) #include #include #include From c21b6df51e8ff558b852e088ec9d0a99a15bff94 Mon Sep 17 00:00:00 2001 From: Jeremy Lorelli Date: Tue, 5 Oct 2021 08:41:03 -0700 Subject: [PATCH 015/180] Fix missing parameter in mi_free error message --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..1154e70a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -465,7 +465,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #endif #if (MI_DEBUG>0 || MI_SECURE>=4) if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { - _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", p); + _mi_error_message(EINVAL, "%s: pointer does not point to a valid heap space: %p\n", msg, p); } #endif return segment; From 3c058f07a98fe1bc35390099e9674509a74a8d3f Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Fri, 8 Oct 2021 23:59:35 -0700 Subject: [PATCH 016/180] Add an option to disable automatic use of `getenv` inside options.c --- src/options.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 30025db2..2dc7bfee 100644 --- a/src/options.c +++ b/src/options.c @@ -409,6 +409,14 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { dest[dest_size - 1] = 0; } +#ifdef MI_NO_GETENV +static bool mi_getenv(const char* name, char* result, size_t result_size) { + UNUSED(name); + UNUSED(result); + UNUSED(result_size); + return false; +} +#else static inline int mi_strnicmp(const char* s, const char* t, size_t n) { if (n==0) return 0; for (; *s != 0 && *t != 0 && n > 0; s++, t++, n--) { @@ -416,7 +424,6 @@ static inline int mi_strnicmp(const char* s, const char* t, size_t n) { } return (n==0 ? 0 : *s - *t); } - #if defined _WIN32 // On Windows use GetEnvironmentVariable instead of getenv to work // reliably even when this is invoked before the C runtime is initialized. @@ -485,6 +492,7 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { } } #endif +#endif static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 0fb61c9eaa36cc86ca58c99e9522334b9e5fbde6 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 16:25:10 -0700 Subject: [PATCH 017/180] do not call exit if try handler fails but use abort instead --- src/alloc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..f692f495 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -826,7 +826,10 @@ static std_new_handler_t mi_get_new_handler() { static bool mi_try_new_handler(bool nothrow) { std_new_handler_t h = mi_get_new_handler(); if (h==NULL) { - if (!nothrow) exit(ENOMEM); // cannot throw in plain C, use exit as we are out of memory anyway. + if (!nothrow) { + _mi_error_message(EFAULT, "out of memory in 'new' call"); // cannot throw in plain C, use EFAULT to abort + abort(); + } return false; } else { From bd8e3fd8e1aa00f8e2dd42f169cb72d59e8eccba Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 16:46:06 -0700 Subject: [PATCH 018/180] increase robustness of primitive windows allocation by always using a fallback to VirtualAlloc --- src/os.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index 5b6b0c7e..a818bf48 100644 --- a/src/os.c +++ b/src/os.c @@ -269,12 +269,16 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); if (p != NULL) return p; + // for robustness always fall through in case of an error + /* DWORD err = GetLastError(); if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) return NULL; } - // fall through + */ + _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); + // fall through on error } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) @@ -285,7 +289,10 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; param.Type = MemExtendedParameterAddressRequirements; param.Pointer = &reqs; - return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1); + if (p != NULL) return p; + _mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), addr, try_alignment, flags); + // fall through on error } #endif // last resort From 6ef15943cc6ba50beb044daa6baae7baae55b615 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 16:59:19 -0700 Subject: [PATCH 019/180] fix comments --- src/os.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index a818bf48..889ec0c8 100644 --- a/src/os.c +++ b/src/os.c @@ -264,7 +264,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); #ifdef _WIN32 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) - // on 64-bit systems, try to use the virtual address area after 4TiB for 4MiB aligned allocations + // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations void* hint; if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); @@ -377,7 +377,7 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { void* p = NULL; #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) - // on 64-bit systems, use the virtual address area after 4TiB for 4MiB aligned allocations + // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations void* hint; if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { p = mmap(hint,size,protect_flags,flags,fd,0); @@ -509,7 +509,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif // On 64-bit systems, we can do efficient aligned allocation by using -// the 4TiB to 30TiB area to allocate them. +// the 2TiB to 30TiB area to allocate them. #if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; From d6bbc0811923cd8560f636224be71fbbd739b576 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 18 Oct 2021 18:24:59 -0700 Subject: [PATCH 020/180] prefer monotonic clock for stats (issue #457) --- src/stats.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/stats.c b/src/stats.c index ac728a1c..662da53e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -412,10 +412,14 @@ mi_msecs_t _mi_clock_now(void) { } #else #include -#ifdef CLOCK_REALTIME +#if defined(CLOCK_REALTIME) || defined(CLOCK_MONOTONIC) mi_msecs_t _mi_clock_now(void) { struct timespec t; + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &t); + #else clock_gettime(CLOCK_REALTIME, &t); + #endif return ((mi_msecs_t)t.tv_sec * 1000) + ((mi_msecs_t)t.tv_nsec / 1000000); } #else From bcce4d52bf1494746f4dd7f5569e542cbfdf5d36 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 20:39:39 -0700 Subject: [PATCH 021/180] fix bug in determination of block size in pre-reserved arena memory --- src/arena.c | 2 +- src/init.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index 0e6615a4..a4e473a3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -287,7 +287,7 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la is_committed = true; } - const size_t bcount = mi_block_count_of_size(size); + const size_t bcount = size / MI_ARENA_BLOCK_SIZE; const size_t fields = _mi_divide_up(bcount, MI_BITMAP_FIELD_BITS); const size_t bitmaps = (is_committed ? 2 : 3); const size_t asize = sizeof(mi_arena_t) + (bitmaps*fields*sizeof(mi_bitmap_field_t)); diff --git a/src/init.c b/src/init.c index c0f09b5e..d899f6ad 100644 --- a/src/init.c +++ b/src/init.c @@ -498,7 +498,9 @@ void mi_process_init(void) mi_attr_noexcept { } if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); - if (ksize > 0) mi_reserve_os_memory((size_t)ksize*KiB, true, true); + if (ksize > 0) { + mi_reserve_os_memory(_mi_divide_up((size_t)ksize*KiB, MI_SEGMENT_SIZE), true, true); + } } } From 22c2fd82cc6e339ecdf34bf28accf1f6658f5509 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Mon, 18 Oct 2021 20:44:19 -0700 Subject: [PATCH 022/180] ensure managed os memory is at least one arena block in size --- src/arena.c | 4 +++- src/init.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index a4e473a3..b786ffb4 100644 --- a/src/arena.c +++ b/src/arena.c @@ -282,6 +282,8 @@ static bool mi_arena_add(mi_arena_t* arena) { bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept { + if (size < MI_ARENA_BLOCK_SIZE) return false; + if (is_large) { mi_assert_internal(is_committed); is_committed = true; @@ -321,7 +323,7 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la // Reserve a range of regular OS memory int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept { - size = _mi_os_good_alloc_size(size); + size = _mi_align_up(size, MI_ARENA_BLOCK_SIZE); // at least one block bool large = allow_large; void* start = _mi_os_alloc_aligned(size, MI_SEGMENT_ALIGN, commit, &large, &_mi_stats_main); if (start==NULL) return ENOMEM; diff --git a/src/init.c b/src/init.c index d899f6ad..c088cada 100644 --- a/src/init.c +++ b/src/init.c @@ -499,7 +499,7 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); if (ksize > 0) { - mi_reserve_os_memory(_mi_divide_up((size_t)ksize*KiB, MI_SEGMENT_SIZE), true, true); + mi_reserve_os_memory((size_t)ksize*KiB, true, true); } } } From 7c73e3996d5d729c1baec2e833985166a3f5db2e Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 10:38:57 +0200 Subject: [PATCH 023/180] Fix strict function prototype warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix warning ``warning: function declaration isn’t a prototype`` when building mimalloc with ``-Wstrict-prototypes`` flag. In C argumentless functions should be declared as ``func(void)``. Reproducer: ```shell $ cmake ../.. -DCMAKE_C_FLAGS="-Wstrict-prototypes" $ make VERBOSE=1 ``` Co-authored-by: Sam Gross Co-authored-by: Neil Schemenauer Signed-off-by: Christian Heimes --- include/mimalloc-internal.h | 2 +- src/alloc.c | 8 ++++---- src/options.c | 2 +- src/os.c | 2 +- src/region.c | 2 +- test/test-api.c | 10 +++++----- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1e1a7966..9b097b8e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -53,7 +53,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); -bool _mi_preloading(); // true while the C runtime is not ready +bool _mi_preloading(void); // true while the C runtime is not ready // os.c size_t _mi_os_page_size(void); diff --git a/src/alloc.c b/src/alloc.c index 8acff783..d2ba5c68 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -747,7 +747,7 @@ mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char } #else #include // pathconf -static size_t mi_path_max() { +static size_t mi_path_max(void) { static size_t path_max = 0; if (path_max <= 0) { long m = pathconf("/",_PC_PATH_MAX); @@ -807,13 +807,13 @@ static bool mi_try_new_handler(bool nothrow) { } } #else -typedef void (*std_new_handler_t)(); +typedef void (*std_new_handler_t)(void); #if (defined(__GNUC__) || defined(__clang__)) -std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() { +std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv(void) { return NULL; } -static std_new_handler_t mi_get_new_handler() { +static std_new_handler_t mi_get_new_handler(void) { return _ZSt15get_new_handlerv(); } #else diff --git a/src/options.c b/src/options.c index 30025db2..4274af97 100644 --- a/src/options.c +++ b/src/options.c @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file static uintptr_t mi_max_error_count = 16; // stop outputting errors after this static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this -static void mi_add_stderr_output(); +static void mi_add_stderr_output(void); int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; diff --git a/src/os.c b/src/os.c index 85415232..16659b33 100644 --- a/src/os.c +++ b/src/os.c @@ -95,7 +95,7 @@ size_t _mi_os_page_size() { } // if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) -size_t _mi_os_large_page_size() { +size_t _mi_os_large_page_size(void) { return (large_os_page_size != 0 ? large_os_page_size : _mi_os_page_size()); } diff --git a/src/region.c b/src/region.c index 79540730..2f68b140 100644 --- a/src/region.c +++ b/src/region.c @@ -40,7 +40,7 @@ Possible issues: #include "bitmap.h" // Internal raw OS interface -size_t _mi_os_large_page_size(); +size_t _mi_os_large_page_size(void); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); diff --git a/test/test-api.c b/test/test-api.c index d3344928..53e7f3d8 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -64,15 +64,15 @@ static int failed = 0; // --------------------------------------------------------------------------- // Test functions // --------------------------------------------------------------------------- -bool test_heap1(); -bool test_heap2(); -bool test_stl_allocator1(); -bool test_stl_allocator2(); +bool test_heap1(void); +bool test_heap2(void); +bool test_stl_allocator1(void); +bool test_stl_allocator2(void); // --------------------------------------------------------------------------- // Main testing // --------------------------------------------------------------------------- -int main() { +int main(void) { mi_option_disable(mi_option_verbose); // --------------------------------------------------- From afbcf20f24729e9bf2e87a72efadeb48ddcdd6f7 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 19 Oct 2021 15:07:52 +0200 Subject: [PATCH 024/180] Define _DEFAULT_SOURCE for syscall and realpath Define ``_DEFAULT_SOURCE`` in ``random.c`` and ``alloc.c``. The macro is required for ``syscall()`` and ``realpath()``. Other files like ``os.c`` already define the macro. Signed-off-by: Christian Heimes --- src/alloc.c | 4 ++++ src/random.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/alloc.c b/src/alloc.c index 8acff783..5e45e76f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for realpath() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" #include "mimalloc-atomic.h" diff --git a/src/random.c b/src/random.c index 255bede4..0e2331bd 100644 --- a/src/random.c +++ b/src/random.c @@ -4,6 +4,10 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // for syscall() on Linux +#endif + #include "mimalloc.h" #include "mimalloc-internal.h" From 60937b5bc8ac142ebc0ed2df5aca429fabd01c60 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:39:33 -0700 Subject: [PATCH 025/180] add -Wstrict-prototypes flag during compilation --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..ac4c0569 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,7 +165,7 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -Wstrict-prototypes -fvisibility=hidden) if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() From f0f9aecfe49a1a14e6247770e6b677016902fab4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 09:52:20 -0700 Subject: [PATCH 026/180] add comment on #if ending --- src/options.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/options.c b/src/options.c index 8d5f1556..c40a187b 100644 --- a/src/options.c +++ b/src/options.c @@ -491,8 +491,8 @@ static bool mi_getenv(const char* name, char* result, size_t result_size) { return false; } } -#endif -#endif +#endif // !MI_USE_ENVIRON +#endif // !MI_NO_GETENV static void mi_option_init(mi_option_desc_t* desc) { // Read option value from the environment From 5b9409f4d6c1b010c79eaa81117ba9b7e57672e3 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:17:30 -0700 Subject: [PATCH 027/180] add space after _Atomic to prevent errors on msvc without /TP (see PR #452) --- include/mimalloc-atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index dc48f0a2..e07df84d 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -25,7 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_memory_order(name) std::memory_order_##name #elif defined(_MSC_VER) // Use MSVC C wrapper for C11 atomics -#define _Atomic(tp) tp +#define _Atomic(tp) tp #define ATOMIC_VAR_INIT(x) x #define mi_atomic(name) mi_atomic_##name #define mi_memory_order(name) mi_memory_order_##name @@ -173,7 +173,7 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt } static inline void mi_atomic_thread_fence(mi_memory_order mo) { (void)(mo); - _Atomic(uintptr_t)x = 0; + _Atomic(uintptr_t) x = 0; mi_atomic_exchange_explicit(&x, 1, mo); } static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { From adeef8d4035c91b34a97fb83acf1c6bde1f1e4c9 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:31:49 -0700 Subject: [PATCH 028/180] formatting --- src/os.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index d7d3d33d..ba6cc5e3 100644 --- a/src/os.c +++ b/src/os.c @@ -641,10 +641,9 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_mem_free(p, over_size, commit, stats); void* aligned_p = mi_align_up_ptr(p, alignment); p = mi_win_virtual_alloc(aligned_p, size, alignment, flags, false, allow_large, is_large); - if (p != NULL) - { - _mi_stat_increase(&stats->reserved, size); - if (commit) { _mi_stat_increase(&stats->committed, size); } + if (p != NULL) { + _mi_stat_increase(&stats->reserved, size); + if (commit) { _mi_stat_increase(&stats->committed, size); } } if (p == aligned_p) break; // success! if (p != NULL) { // should not happen? From 2d821003b07b4ccdfbf5259592f7831684f996ee Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 10:56:13 -0700 Subject: [PATCH 029/180] don't add prefix if build type is None (PR #427) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d1d9191..abab5a05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,7 +224,7 @@ else() endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel)$")) +if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$")) set(mi_basename "${mi_basename}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version endif() if(MI_BUILD_SHARED) From 9a724889ead0ab9ad04f66b91b3e22064ccd8d52 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 12:50:16 -0700 Subject: [PATCH 030/180] refine stat output with new SI units --- src/stats.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/stats.c b/src/stats.c index f85285e1..dda55cda 100644 --- a/src/stats.c +++ b/src/stats.c @@ -133,7 +133,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // unit == 0: count as decimal // unit < 0 : count in binary static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { - char buf[32]; + char buf[32]; buf[0] = 0; int len = 32; const char* suffix = (unit <= 0 ? " " : "B"); const int64_t base = (unit == 0 ? 1000 : 1024); @@ -141,17 +141,21 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t pos = (n < 0 ? -n : n); if (pos < base) { - snprintf(buf, len, "%d %s ", (int)n, suffix); + if (n!=1 || suffix[0] != 'B') { // skip printing 1 B for the unit column + snprintf(buf, len, "%d %-3s", (int)n, (n==0 ? "" : suffix)); + } } else { - int64_t divider = base; + int64_t divider = base; const char* magnitude = "K"; if (pos >= divider*base) { divider *= base; magnitude = "M"; } if (pos >= divider*base) { divider *= base; magnitude = "G"; } const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - snprintf(buf, len, "%ld.%ld %s%s%s", whole, (frac1 < 0 ? -frac1 : frac1), magnitude, (base == 1024 ? "i" : ""), suffix); + char unitdesc[16]; + snprintf(unitdesc, 16, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); } @@ -221,7 +225,7 @@ static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* static void mi_print_header(mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); + _mi_fprintf(out, arg, "%10s: %10s %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "current ", "unit ", "count "); } #if MI_STAT>1 From bb00a5a32e53067e6e23215d883fbece5ec0a74f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 13:09:43 -0700 Subject: [PATCH 031/180] remove MADV_FREE_REUSABLE from mi_os_reset as it needs MADV_FREE_REUSE to reuse the memory again. --- src/os.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/os.c b/src/os.c index 08068c9d..bf1d4716 100644 --- a/src/os.c +++ b/src/os.c @@ -796,9 +796,9 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ // for commit, just change the protection err = mprotect(start, csize, (PROT_READ | PROT_WRITE)); if (err != 0) { err = errno; } - #if defined(MADV_FREE_REUSE) - while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } - #endif + //#if defined(MADV_FREE_REUSE) + // while ((err = madvise(start, csize, MADV_FREE_REUSE)) != 0 && errno == EAGAIN) { errno = 0; } + //#endif } #else err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE)); @@ -860,17 +860,12 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - #if defined(MADV_FREE_REUSABLE) - #define KK_MADV_FREE_INITIAL MADV_FREE_REUSABLE - #else - #define KK_MADV_FREE_INITIAL MADV_FREE - #endif - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(KK_MADV_FREE_INITIAL); + static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; - if (err != 0 && errno == EINVAL && oadvice == KK_MADV_FREE_INITIAL) { - // if MADV_FREE/MADV_FREE_REUSABLE is not supported, fall back to MADV_DONTNEED from now on + if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { + // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } From ae5aae7e105eaccef37be0cda601b005264d2d29 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 19 Oct 2021 14:00:40 -0700 Subject: [PATCH 032/180] allow decommit in arena separate from initial commit --- src/arena.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/arena.c b/src/arena.c index fb25a89d..2f5a349f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -62,11 +62,11 @@ typedef struct mi_arena_s { size_t field_count; // number of bitmap fields (where `field_count * MI_BITMAP_FIELD_BITS >= block_count`) int numa_node; // associated NUMA node bool is_zero_init; // is the arena zero initialized? - bool is_committed; // is the memory fully committed? (if so, block_committed == NULL) + bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? - mi_bitmap_field_t* blocks_committed; // if `!is_committed`, are the blocks committed? + mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) } mi_arena_t; @@ -129,8 +129,8 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n *memid = mi_arena_id_create(arena_index, bitmap_index); *is_zero = _mi_bitmap_claim_across(arena->blocks_dirty, arena->field_count, needed_bcount, bitmap_index, NULL); *large = arena->is_large; - *is_pinned = (arena->is_large || arena->is_committed); - if (arena->is_committed) { + *is_pinned = (arena->is_large || !arena->allow_decommit); + if (arena->blocks_committed == NULL) { // always committed *commit = true; } @@ -245,12 +245,13 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool all_committed, mi_s return; } // potentially decommit - if (arena->is_committed) { - mi_assert_internal(all_committed); + if (!arena->allow_decommit || arena->blocks_committed == NULL) { + mi_assert_internal(all_committed); // note: may be not true as we may "pretend" to be not committed (in segment.c) } else { mi_assert_internal(arena->blocks_committed != NULL); _mi_os_decommit(p, blocks * MI_ARENA_BLOCK_SIZE, stats); // ok if this fails + // todo: use reset instead of decommit on windows? _mi_bitmap_unclaim_across(arena->blocks_committed, arena->field_count, blocks, bitmap_idx); } // and make it available to others again @@ -300,12 +301,16 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la arena->numa_node = numa_node; // TODO: or get the current numa node if -1? (now it allows anyone to allocate on -1) arena->is_large = is_large; arena->is_zero_init = is_zero; - arena->is_committed = is_committed; + arena->allow_decommit = !is_large && !is_committed; // only allow decommit for initially uncommitted memory arena->search_idx = 0; arena->blocks_dirty = &arena->blocks_inuse[fields]; // just after inuse bitmap - arena->blocks_committed = (is_committed ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap + arena->blocks_committed = (!arena->allow_decommit ? NULL : &arena->blocks_inuse[2*fields]); // just after dirty bitmap // the bitmaps are already zero initialized due to os_alloc - // just claim leftover blocks if needed + // initialize committed bitmap? + if (arena->blocks_committed != NULL && is_committed) { + memset(arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); + } + // and claim leftover blocks if needed (so we never allocate there) ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; mi_assert_internal(post >= 0); if (post > 0) { From 10c31f9b41f5559e459dc30cf463cc750bfa67ac Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Tue, 19 Oct 2021 15:13:01 -0700 Subject: [PATCH 033/180] fix warnings --- src/stats.c | 4 ++-- test/test-api.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/stats.c b/src/stats.c index dda55cda..115d938e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -153,8 +153,8 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* const int64_t tens = (n / (divider/10)); const long whole = (long)(tens/10); const long frac1 = (long)(tens%10); - char unitdesc[16]; - snprintf(unitdesc, 16, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); + char unitdesc[8]; + snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } _mi_fprintf(out, arg, (fmt==NULL ? "%11s" : fmt), buf); diff --git a/test/test-api.c b/test/test-api.c index 55c80431..f72cfaf5 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -4,6 +4,7 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#pragma GCC diagnostic ignored "-Walloc-size-larger-than=" /* Testing allocators is difficult as bugs may only surface after particular From 5834751ca6184e0cdebc5477ddaf98e290962318 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Wed, 20 Oct 2021 16:17:13 +0100 Subject: [PATCH 034/180] api test conceal gcc pragma --- test/test-api.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test-api.c b/test/test-api.c index f72cfaf5..8ddbf7cf 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -4,7 +4,9 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic ignored "-Walloc-size-larger-than=" +#endif /* Testing allocators is difficult as bugs may only surface after particular From 4d89176eb4584c16a2b3141a97c1970cc1664014 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 20 Oct 2021 09:35:58 -0700 Subject: [PATCH 035/180] add MI_USE_LIBATOMIC to link with -latomic on older systems (see also PR #429); rename MI_INTERPOSE to MI_OSX_INTERPOSE --- CMakeLists.txt | 13 +++++++++---- src/alloc-override-osx.c | 2 +- src/alloc-override.c | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index abab5a05..b872e375 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) +option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) @@ -23,6 +23,7 @@ option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems)" OFF) include("cmake/mimalloc-config-version.cmake") @@ -78,11 +79,11 @@ if(MI_OVERRIDE) list(APPEND mi_sources src/alloc-override-osx.c) list(APPEND mi_defines MI_OSX_ZONE=1) endif() - if(MI_INTERPOSE) + if(MI_OSX_INTERPOSE) # use interpose on macOS - message(STATUS " Use interpose to override malloc (MI_INTERPOSE=ON)") + message(STATUS " Use interpose to override malloc (MI_OSX_INTERPOSE=ON)") message(STATUS " WARNING: interpose does not seem to work reliably on the M1; use -DMI_OSX_ZONE=ON instead") - list(APPEND mi_defines MI_INTERPOSE) + list(APPEND mi_defines MI_OSX_INTERPOSE) endif() endif() endif() @@ -203,6 +204,10 @@ else() endif() endif() +if (MI_USE_LIBATOMIC) + list(APPEND mi_libraries atomic) +endif() + # ----------------------------------------------------------------------------- # Install and output names # ----------------------------------------------------------------------------- diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index f506d30a..a2e341bb 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -229,7 +229,7 @@ static malloc_zone_t mi_malloc_zone = { }; -#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) +#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) static malloc_zone_t *mi_malloc_default_zone(void) { return &mi_malloc_zone; diff --git a/src/alloc-override.c b/src/alloc-override.c index 6a87e7bd..f97b6e78 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" #endif -#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_INTERPOSE))) +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_OSX_INTERPOSE))) // ------------------------------------------------------ // Override system malloc @@ -40,7 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif -#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: struct mi_interpose_s { From e02f88a11c31f33b0bdde1f0a46086b7901087b7 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 20 Oct 2021 09:55:03 -0700 Subject: [PATCH 036/180] Fix warnings with g++-11 compilation --- CMakeLists.txt | 5 ++++- src/arena.c | 2 +- src/region.c | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b872e375..c46ffdac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,7 +166,10 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -Wstrict-prototypes -fvisibility=hidden) + list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + if(NOT MI_USE_CXX) + list(APPEND mi_cflags -Wstrict-prototypes) + endif() if(CMAKE_C_COMPILER_ID MATCHES "GNU") list(APPEND mi_cflags -Wno-invalid-memory-model) endif() diff --git a/src/arena.c b/src/arena.c index 9d717e9e..5e883b28 100644 --- a/src/arena.c +++ b/src/arena.c @@ -310,7 +310,7 @@ bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_la // the bitmaps are already zero initialized due to os_alloc // initialize committed bitmap? if (arena->blocks_committed != NULL && is_committed) { - memset(arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); + memset((void*)arena->blocks_committed, 0xFF, fields*sizeof(mi_bitmap_field_t)); // cast to void* to avoid atomic warning } // and claim leftover blocks if needed (so we never allocate there) ptrdiff_t post = (fields * MI_BITMAP_FIELD_BITS) - bcount; diff --git a/src/region.c b/src/region.c index 2f68b140..d9232450 100644 --- a/src/region.c +++ b/src/region.c @@ -463,7 +463,7 @@ void _mi_mem_collect(mi_os_tld_t* tld) { uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ions[i].start); size_t arena_memid = mi_atomic_load_relaxed(®ions[i].arena_memid); uintptr_t commit = mi_atomic_load_relaxed(®ions[i].commit); - memset(®ions[i], 0, sizeof(mem_region_t)); + memset((void*)®ions[i], 0, sizeof(mem_region_t)); // cast to void* to avoid atomic warning // and release the whole region mi_atomic_store_release(®ion->info, (uintptr_t)0); if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { From d482555675fb80a60ebf194a605661062a390091 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 20 Oct 2021 09:55:21 -0700 Subject: [PATCH 037/180] Fix warnings on osx with g++ compilation --- src/alloc-override-osx.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index a2e341bb..723ef226 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -189,6 +189,10 @@ static malloc_zone_t* mi_get_default_zone() } } +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + static malloc_introspection_t mi_introspect = { .enumerator = &intro_enumerator, .good_size = &intro_good_size, @@ -199,23 +203,23 @@ static malloc_introspection_t mi_introspect = { .force_unlock = &intro_force_unlock, #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - .zone_locked = &intro_zone_locked, .statistics = &intro_statistics, + .zone_locked = &intro_zone_locked, #endif }; static malloc_zone_t mi_malloc_zone = { .size = &zone_size, - .zone_name = "mimalloc", - .introspect = &mi_introspect, .malloc = &zone_malloc, .calloc = &zone_calloc, .valloc = &zone_valloc, .free = &zone_free, .realloc = &zone_realloc, .destroy = &zone_destroy, + .zone_name = "mimalloc", .batch_malloc = &zone_batch_malloc, .batch_free = &zone_batch_free, + .introspect = &mi_introspect, #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 // switch to version 9 on OSX 10.6 to support memalign. @@ -243,16 +247,22 @@ __attribute__((used)) static struct { }; #endif -static void __attribute__((constructor(0))) _mi_macos_override_malloc() { + +#if defined(__clang__) +__attribute__((constructor(0))) +#else +__attribute__((constructor)) // seems not supported by g++-11 on the M1 +#endif +static void _mi_macos_override_malloc() { malloc_zone_t* purgeable_zone = NULL; -#if defined(MAC_OS_X_VERSION_10_6) && \ + #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 // force the purgeable zone to exist to avoid strange bugs if (malloc_default_purgeable_zone) { purgeable_zone = malloc_default_purgeable_zone(); } -#endif + #endif // Register our zone. // thomcc: I think this is still needed to put us in the zone list. From b5b52b8879a0574bdd5b8c2b416a26c48080ef41 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Wed, 20 Oct 2021 18:33:12 +0000 Subject: [PATCH 038/180] Haiku build update, since the beta3 few more posix functions are available e.g. madvise --- src/os.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/os.c b/src/os.c index bf1d4716..3066dad5 100644 --- a/src/os.c +++ b/src/os.c @@ -51,10 +51,6 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif -#if defined(__HAIKU__) -#define madvise posix_madvise -#define MADV_DONTNEED POSIX_MADV_DONTNEED -#endif #endif /* ----------------------------------------------------------- From 45321237b53435e75b1d8545b6b37c40cddef898 Mon Sep 17 00:00:00 2001 From: LongYinan Date: Thu, 21 Oct 2021 21:15:08 +0800 Subject: [PATCH 039/180] Fix ARM64 MSVC linker problem Close https://github.com/microsoft/mimalloc/issues/426 --- src/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index c088cada..a712f9f1 100644 --- a/src/init.c +++ b/src/init.c @@ -571,7 +571,7 @@ static void mi_process_done(void) { return 0; } typedef int(*_crt_cb)(void); - #ifdef _M_X64 + #if defined(_M_X64) || defined(_M_ARM64) __pragma(comment(linker, "/include:" "_mi_msvc_initu")) #pragma section(".CRT$XIU", long, read) #else From 13de1920ae7bf39021d6c131222b6b71330ed0c2 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 21 Oct 2021 21:39:28 +0200 Subject: [PATCH 040/180] Rename _os_random_weak to _mi_os_random_weak The ``_os_random_weak`` function is the only non-static function besides ``_ZSt15get_new_handlerv`` that is not prefixed with ``mi`` or ``_mi``. The discrepancy was discovered by CPython's smelly script. The checker looks for exported symbols that don't have well-defined prefixes. Signed-off-by: Christian Heimes --- include/mimalloc-internal.h | 2 +- src/init.c | 2 +- src/random.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 5a29c54d..0563d3de 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -50,7 +50,7 @@ void _mi_random_init(mi_random_ctx_t* ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); uintptr_t _mi_random_next(mi_random_ctx_t* ctx); uintptr_t _mi_heap_random_next(mi_heap_t* heap); -uintptr_t _os_random_weak(uintptr_t extra_seed); +uintptr_t _mi_os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c diff --git a/src/init.c b/src/init.c index c088cada..a41bcdd1 100644 --- a/src/init.c +++ b/src/init.c @@ -141,7 +141,7 @@ mi_stats_t _mi_stats_main = { MI_STATS_NULL }; static void mi_heap_main_init(void) { if (_mi_heap_main.cookie == 0) { _mi_heap_main.thread_id = _mi_thread_id(); - _mi_heap_main.cookie = _os_random_weak((uintptr_t)&mi_heap_main_init); + _mi_heap_main.cookie = _mi_os_random_weak((uintptr_t)&mi_heap_main_init); _mi_random_init(&_mi_heap_main.random); _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); diff --git a/src/random.c b/src/random.c index ce6db7c6..05c5c99c 100644 --- a/src/random.c +++ b/src/random.c @@ -257,8 +257,8 @@ static bool os_random_buf(void* buf, size_t buf_len) { #include #endif -uintptr_t _os_random_weak(uintptr_t extra_seed) { - uintptr_t x = (uintptr_t)&_os_random_weak ^ extra_seed; // ASLR makes the address random +uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { + uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random #if defined(_WIN32) LARGE_INTEGER pcount; @@ -289,7 +289,7 @@ void _mi_random_init(mi_random_ctx_t* ctx) { #if !defined(__wasi__) _mi_warning_message("unable to use secure randomness\n"); #endif - uintptr_t x = _os_random_weak(0); + uintptr_t x = _mi_os_random_weak(0); for (size_t i = 0; i < 8; i++) { // key is eight 32-bit words. x = _mi_random_shuffle(x); ((uint32_t*)key)[i] = (uint32_t)x; From 898a23ec2b4c7974d9381312f5ff01b78747a8d5 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 22 Oct 2021 08:12:14 +0100 Subject: [PATCH 041/180] fixes malloc_usable_size signature on FreeBSD. --- src/alloc-override.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index f97b6e78..af9035dd 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -173,7 +173,7 @@ extern "C" { void cfree(void* p) MI_FORWARD0(mi_free, p) void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) -#if !defined(__ANDROID__) +#if !defined(__ANDROID__) && !defined(__FreeBSD__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) From fc7777ee8c7538fa181ae52c621c09edc11d46b7 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 22 Oct 2021 10:08:16 +0100 Subject: [PATCH 042/180] NUMA base detection on FreeBSD. --- src/os.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/os.c b/src/os.c index 3066dad5..f15a2197 100644 --- a/src/os.c +++ b/src/os.c @@ -51,6 +51,14 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif +#if defined(__FreeBSD__) +#include +#if __FreeBSD_version >= 1200000 +#include +#include +#include +#endif +#endif #endif /* ----------------------------------------------------------- @@ -1236,6 +1244,29 @@ static size_t mi_os_numa_node_countx(void) { } return (node+1); } +#elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 +static size_t mi_os_numa_nodex(void) { + domainset_t dom; + size_t node; + int policy; + + if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; + + for (node = 0; node < MAXMEMDOM; node++) { + if (DOMAINSET_ISSET(node, &dom)) return node; + } + + return 0ul; +} + +static size_t mi_os_numa_node_countx(void) { + size_t ndomains = 0; + size_t len = sizeof(ndomains); + + if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; + + return ndomains; +} #else static size_t mi_os_numa_nodex(void) { return 0; From 2d2d9af5c6e2473b8aa6b651b0dad7d67b8957f8 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 24 Oct 2021 10:57:40 +0100 Subject: [PATCH 043/180] while at it, doing dragonflybsd too --- src/os.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/os.c b/src/os.c index f15a2197..a4b9602f 100644 --- a/src/os.c +++ b/src/os.c @@ -51,13 +51,13 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__DragonFly__) #include #if __FreeBSD_version >= 1200000 #include #include -#include #endif +#include #endif #endif @@ -1267,6 +1267,24 @@ static size_t mi_os_numa_node_countx(void) { return ndomains; } +#elif defined(__DragonFly__) +static size_t mi_os_numa_nodex(void) { + // TODO DragonFlyBSD does not seem to provide any userland mean to + // check this information, even less the possibility to control + // the allocation to a logical core level's granularity, only the kernel + // is fully NUMA aware at the moment. + return 0ul; +} + +static size_t mi_os_numa_node_countx(void) { + size_t ncpus = 0, nvirtcoresperphys = 0; + size_t len = sizeof(size_t); + + if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; + if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; + + return nvirtcoresperphys * ncpus; +} #else static size_t mi_os_numa_nodex(void) { return 0; From 00edd9201bad722eabc5066c9b8ee8622c0f0d8c Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 24 Oct 2021 19:49:11 +0100 Subject: [PATCH 044/180] proposal to not overcommit page unconditionally on Linux and FreeBSD, respecting userland settings. --- src/os.c | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 3066dad5..e40d57a9 100644 --- a/src/os.c +++ b/src/os.c @@ -39,6 +39,8 @@ terms of the MIT license. A copy of the license can be found in the file #include // sysconf #if defined(__linux__) #include +#include +#include #if defined(__GLIBC__) #include // linux mmap flags #else @@ -51,6 +53,9 @@ terms of the MIT license. A copy of the license can be found in the file #include #endif #endif +#if defined(__FreeBSD__) +#include +#endif #endif /* ----------------------------------------------------------- @@ -88,6 +93,9 @@ static size_t os_alloc_granularity = 4096; // if non-zero, use large page allocation static size_t large_os_page_size = 0; +// if non-zero, enable page overcommit +static int os_overcommit = 0; + // OS (small) page size size_t _mi_os_page_size() { return os_page_size; @@ -223,6 +231,31 @@ void _mi_os_init() { os_alloc_granularity = os_page_size; } large_os_page_size = 2*MiB; // TODO: can we query the OS for this? +#if defined(__linux__) + int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd != -1) { + char buf[1] = {0}; + if (read(fd, buf, sizeof(buf)) == sizeof(buf)) { + switch (buf[0]) { + case '0': // heuristic mode + case '1': // always + os_overcommit = 1; + break; + default: // never regardless + os_overcommit = 0; + } + } + close(fd); + } +#elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = val; + } +#elif defined(__NetBSD__) || defined(__HAIKU__) + os_overcommit = 1; +#endif } #endif @@ -398,8 +431,11 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #if !defined(MAP_NORESERVE) #define MAP_NORESERVE 0 #endif - int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + int flags = MAP_PRIVATE | MAP_ANONYMOUS; int fd = -1; + if (os_overcommit) { + flags |= MAP_NORESERVE; + } #if defined(MAP_ALIGNED) // BSD if (try_alignment > 0) { size_t n = mi_bsr(try_alignment); From ca9785d40e8c225f5176f2dfba502ab351da8d9c Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 10:06:34 -0700 Subject: [PATCH 045/180] fix compilation with MI_DEBUG>3, issue #480 --- src/page.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/page.c b/src/page.c index 033976dc..7efbcb8d 100644 --- a/src/page.c +++ b/src/page.c @@ -84,9 +84,10 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_list_is_valid(page,page->local_free)); #if MI_DEBUG>3 // generally too expensive to check this - if (page->flags.is_zero) { - for(mi_block_t* block = page->free; block != NULL; mi_block_next(page,block)) { - mi_assert_expensive(mi_mem_is_zero(block + 1, page->block_size - sizeof(mi_block_t))); + if (page->is_zero) { + const size_t ubsize = mi_page_usable_block_size(page); + for(mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { + mi_assert_expensive(mi_mem_is_zero(block + 1, ubsize - sizeof(mi_block_t))); } } #endif From f38956568c07d0fe31111f141652954ccbb3635c Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:01:12 -0700 Subject: [PATCH 046/180] only set visibility attribute when compiling for a shared library --- include/mimalloc.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 7ebf3e60..fe0fd746 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -58,8 +58,12 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #define mi_attr_alloc_align(p) #elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif #define mi_cdecl // leads to warnings... __attribute__((cdecl)) - #define mi_decl_export __attribute__((visibility("default"))) #define mi_decl_restrict #define mi_attr_malloc __attribute__((malloc)) #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) From 6ccf7e87c0aa62154f63054a905842bb93d52e3f Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:06:10 -0700 Subject: [PATCH 047/180] minor edits --- src/os.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/os.c b/src/os.c index a4b9602f..73ef462e 100644 --- a/src/os.c +++ b/src/os.c @@ -1249,40 +1249,28 @@ static size_t mi_os_numa_nodex(void) { domainset_t dom; size_t node; int policy; - if (cpuset_getdomain(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1, sizeof(dom), &dom, &policy) == -1) return 0ul; - for (node = 0; node < MAXMEMDOM; node++) { - if (DOMAINSET_ISSET(node, &dom)) return node; + if (DOMAINSET_ISSET(node, &dom)) return node; } - return 0ul; } - static size_t mi_os_numa_node_countx(void) { size_t ndomains = 0; size_t len = sizeof(ndomains); - if (sysctlbyname("vm.ndomains", &ndomains, &len, NULL, 0) == -1) return 0ul; - return ndomains; } #elif defined(__DragonFly__) static size_t mi_os_numa_nodex(void) { - // TODO DragonFlyBSD does not seem to provide any userland mean to - // check this information, even less the possibility to control - // the allocation to a logical core level's granularity, only the kernel - // is fully NUMA aware at the moment. + // TODO: DragonFly does not seem to provide any userland means to get this information. return 0ul; } - static size_t mi_os_numa_node_countx(void) { size_t ncpus = 0, nvirtcoresperphys = 0; size_t len = sizeof(size_t); - if (sysctlbyname("hw.ncpu", &ncpus, &len, NULL, 0) == -1) return 0ul; if (sysctlbyname("hw.cpu_topology_ht_ids", &nvirtcoresperphys, &len, NULL, 0) == -1) return 0ul; - return nvirtcoresperphys * ncpus; } #else From fdcdc4cf2aadefce1f7e768cc933508ac88420c4 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:01:12 -0700 Subject: [PATCH 048/180] only set visibility attribute when compiling for a shared library (issue #475) --- include/mimalloc.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 7ebf3e60..fe0fd746 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -58,8 +58,12 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_attr_alloc_size2(s1,s2) #define mi_attr_alloc_align(p) #elif defined(__GNUC__) // includes clang and icc + #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) + #define mi_decl_export __attribute__((visibility("default"))) + #else + #define mi_decl_export + #endif #define mi_cdecl // leads to warnings... __attribute__((cdecl)) - #define mi_decl_export __attribute__((visibility("default"))) #define mi_decl_restrict #define mi_attr_malloc __attribute__((malloc)) #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) From 5fd77aa1986305a30680f6b62a27b58940dfbfd0 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 27 Oct 2021 17:39:11 -0700 Subject: [PATCH 049/180] refactor os_overcommit detection --- src/os.c | 62 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/src/os.c b/src/os.c index b8c3f603..507d9971 100644 --- a/src/os.c +++ b/src/os.c @@ -40,7 +40,6 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__linux__) #include #include -#include #if defined(__GLIBC__) #include // linux mmap flags #else @@ -98,8 +97,9 @@ static size_t os_alloc_granularity = 4096; // if non-zero, use large page allocation static size_t large_os_page_size = 0; -// if non-zero, enable page overcommit -static int os_overcommit = 0; +// is memory overcommit allowed? +// set dynamically in _mi_os_init (and if true we use MAP_NORESERVE) +static bool os_overcommit = true; // OS (small) page size size_t _mi_os_page_size() { @@ -189,7 +189,9 @@ static bool mi_win_enable_large_os_pages() return (ok!=0); } -void _mi_os_init(void) { +void _mi_os_init(void) +{ + os_overcommit = false; // get the page size SYSTEM_INFO si; GetSystemInfo(&si); @@ -224,10 +226,36 @@ void _mi_os_init(void) { } #elif defined(__wasi__) void _mi_os_init() { + os_overcommit = false; os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } + +#else // generic unix + +static void os_detect_overcommit(void) { +#if defined(__linux__) + int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd < 0) return; + char buf[128]; + ssize_t nread = read(fd, &buf, sizeof(buf)); + close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } +#elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } #else + // default: overcommit is true +#endif +} + void _mi_os_init() { // get the page size long result = sysconf(_SC_PAGESIZE); @@ -236,31 +264,7 @@ void _mi_os_init() { os_alloc_granularity = os_page_size; } large_os_page_size = 2*MiB; // TODO: can we query the OS for this? -#if defined(__linux__) - int fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd != -1) { - char buf[1] = {0}; - if (read(fd, buf, sizeof(buf)) == sizeof(buf)) { - switch (buf[0]) { - case '0': // heuristic mode - case '1': // always - os_overcommit = 1; - break; - default: // never regardless - os_overcommit = 0; - } - } - close(fd); - } -#elif defined(__FreeBSD__) - int val = 0; - size_t olen = sizeof(val); - if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { - os_overcommit = val; - } -#elif defined(__NetBSD__) || defined(__HAIKU__) - os_overcommit = 1; -#endif + os_detect_overcommit(); } #endif From 1b1c9186a4a62c959140f437fa5e745dd159e854 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 27 Oct 2021 19:06:27 -0700 Subject: [PATCH 050/180] suppress warning on discard attribute on forwards --- src/alloc-override.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/alloc-override.c b/src/alloc-override.c index af9035dd..6c42ae00 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -22,6 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file #if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) // use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) + #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); #else #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))); From 0ead911b6b1e1b5a531a156587c5d011f52f47e8 Mon Sep 17 00:00:00 2001 From: Yaroslav Syrytsia Date: Tue, 28 Sep 2021 20:34:56 +0300 Subject: [PATCH 051/180] cmake: removed hardcoded names for top level configuration --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b56953c4..9ba14a38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,7 @@ option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clan option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") set(mi_sources @@ -208,9 +209,9 @@ endif() # ----------------------------------------------------------------------------- if (MI_INSTALL_TOPLEVEL) - set(mi_install_libdir "lib") - set(mi_install_incdir "include") - set(mi_install_cmakedir "cmake") + set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") + set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") + set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc") else() set(mi_install_libdir "lib/mimalloc-${mi_version}") set(mi_install_incdir "include/mimalloc-${mi_version}") From 5a5e4e303629d49bc09a2ef2614c1e0374c23dfb Mon Sep 17 00:00:00 2001 From: sblondon Date: Thu, 28 Oct 2021 15:01:36 +0200 Subject: [PATCH 052/180] Fix preload path in documentation example The .so files are usually in `/usr/lib`, not `/usr/bin`. The updated path is the same as below in the text. --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index cdb1b82a..cf984c6b 100644 --- a/readme.md +++ b/readme.md @@ -18,7 +18,7 @@ Latest stable tag: `v1.7.2` (2021-06-17). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: ``` -> LD_PRELOAD=/usr/bin/libmimalloc.so myprogram +> LD_PRELOAD=/usr/lib/libmimalloc.so myprogram ``` It also has an easy way to override the default allocator in [Windows](#override_on_windows). Notable aspects of the design include: From 5b0e73281fd4d98b6f211cdd1b6b29453c3e420e Mon Sep 17 00:00:00 2001 From: dc Date: Thu, 28 Oct 2021 22:43:21 +0100 Subject: [PATCH 053/180] fix spurious build warning with overflow builtins --- include/mimalloc-internal.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 0563d3de..982928cb 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -247,11 +247,11 @@ static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #if (SIZE_MAX == ULONG_MAX) - return __builtin_umull_overflow(count, size, total); + return __builtin_umull_overflow(count, size, (unsigned long *)total); #elif (SIZE_MAX == UINT_MAX) - return __builtin_umul_overflow(count, size, total); + return __builtin_umul_overflow(count, size, (unsigned int *)total); #else - return __builtin_umulll_overflow(count, size, total); + return __builtin_umulll_overflow(count, size, (unsigned long long *)total); #endif } #else /* __builtin_umul_overflow is unavailable */ From d0d47e6489dd5e9214091db9aed293310f7c8670 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:49:05 -0700 Subject: [PATCH 054/180] avoid accessing the thread_id after process initialized is set --- src/init.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index ef824e3a..ef44d273 100644 --- a/src/init.c +++ b/src/init.c @@ -102,6 +102,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = { false }; + // the thread-local default heap for allocation mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; @@ -478,10 +479,11 @@ static void mi_detect_cpu_features(void) { void mi_process_init(void) mi_attr_noexcept { // ensure we are called once if (_mi_process_is_initialized) return; - _mi_process_is_initialized = true; mi_process_setup_auto_thread_done(); _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + _mi_process_is_initialized = true; + mi_detect_cpu_features(); _mi_os_init(); mi_heap_main_init(); From dbf8bafc3ae66ebb6e356b3e8a7485848e12b5ef Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:50:15 -0700 Subject: [PATCH 055/180] avoid accessing the thread_id after process initialized is set --- src/init.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/init.c b/src/init.c index ef44d273..cb9d123f 100644 --- a/src/init.c +++ b/src/init.c @@ -479,10 +479,10 @@ static void mi_detect_cpu_features(void) { void mi_process_init(void) mi_attr_noexcept { // ensure we are called once if (_mi_process_is_initialized) return; - mi_process_setup_auto_thread_done(); - _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); _mi_process_is_initialized = true; + mi_process_setup_auto_thread_done(); + mi_detect_cpu_features(); _mi_os_init(); From 74efd6ddac0215c3b4d54c13563765a97b4e971e Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:50:54 -0700 Subject: [PATCH 056/180] use load relaxed in mi_free for better code on M1 --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 1af6650a..c91dcd92 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -486,7 +486,7 @@ void mi_free(void* p) mi_attr_noexcept mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; - if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks + if (mi_likely(tid == mi_atomic_load_relaxed(&segment->thread_id) && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned if (mi_unlikely(mi_check_is_double_free(page,block))) return; mi_check_padding(page, block); From 5360639748d8bfc7f7e97728e859a6380ac7db9f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:52:00 -0700 Subject: [PATCH 057/180] add mi_malloc_good_size --- include/mimalloc.h | 1 + src/alloc-posix.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/include/mimalloc.h b/include/mimalloc.h index fe0fd746..a77f1dc5 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -346,6 +346,7 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 43931e56..cff01b5c 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -33,13 +33,19 @@ terms of the MIT license. A copy of the license can be found in the file size_t mi_malloc_size(const void* p) mi_attr_noexcept { + //if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); } size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + //if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); } +size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { + return mi_good_size(size); +} + void mi_cfree(void* p) mi_attr_noexcept { if (mi_is_in_heap_region(p)) { mi_free(p); From 751a2249c809735629491d07f1994522e2bd6a84 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:53:20 -0700 Subject: [PATCH 058/180] add mi_decl_externc, nice layout --- include/mimalloc-internal.h | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 0563d3de..e8046528 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -36,6 +36,12 @@ terms of the MIT license. A copy of the license can be found in the file #define __wasi__ #endif +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#else +#define mi_decl_externc +#endif + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -299,6 +305,7 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #if defined(MI_MALLOC_OVERRIDE) #if defined(__APPLE__) // macOS #define MI_TLS_SLOT 89 // seems unused? +// #define MI_TLS_RECURSE_GUARD 1 // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) // see #elif defined(__OpenBSD__) @@ -336,10 +343,12 @@ extern pthread_key_t _mi_heap_default_key; // However, on the Apple M1 we do use the address of this variable as the unique thread-id (issue #356). extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from + static inline mi_heap_t* mi_get_default_heap(void) { #if defined(MI_TLS_SLOT) mi_heap_t* heap = (mi_heap_t*)mi_tls_slot(MI_TLS_SLOT); - return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); + if (mi_unlikely(heap == NULL)) { heap = (mi_heap_t*)&_mi_heap_empty; } //_mi_heap_empty_get(); } + return heap; #elif defined(MI_TLS_PTHREAD_SLOT_OFS) mi_heap_t* heap = *mi_tls_pthread_heap_slot(); return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); @@ -347,7 +356,7 @@ static inline mi_heap_t* mi_get_default_heap(void) { mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? _mi_heap_main_get() : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key)); return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap); #else - #if defined(MI_TLS_RECURSE_GUARD) + #if defined(MI_TLS_RECURSE_GUARD) if (mi_unlikely(!_mi_process_is_initialized)) return _mi_heap_main_get(); #endif return _mi_heap_default; @@ -716,12 +725,12 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { res = tcb[slot]; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); -#if defined(__APPLE__) // M1, issue #343 + #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits -#else + #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); -#endif + #endif res = tcb[slot]; #endif return res; @@ -744,12 +753,12 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { tcb[slot] = value; #elif defined(__aarch64__) void** tcb; UNUSED(ofs); -#if defined(__APPLE__) // M1, issue #343 + #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits -#else + #else __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); -#endif + #endif tcb[slot] = value; #endif } From 32ee13f41eafb7de541438279100c28ed9df437f Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 2 Nov 2021 21:54:44 -0700 Subject: [PATCH 059/180] improve macOS M1 performance; use interpose in combination with zone's; add -fno-builtin-malloc flag in building with MI_OVERRIDE --- CMakeLists.txt | 14 ++- src/alloc-override-osx.c | 234 ++++++++++++++++++++++++++++++++------- src/alloc-override.c | 25 +++-- 3 files changed, 223 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c46ffdac..f07d892f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" OFF) +option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) @@ -78,12 +78,17 @@ if(MI_OVERRIDE) message(STATUS " Use malloc zone to override malloc (MI_OSX_ZONE=ON)") list(APPEND mi_sources src/alloc-override-osx.c) list(APPEND mi_defines MI_OSX_ZONE=1) + if (NOT MI_OSX_INTERPOSE) + message(STATUS " WARNING: zone overriding usually also needs interpose (use -DMI_OSX_INTERPOSE=ON)") + endif() endif() if(MI_OSX_INTERPOSE) # use interpose on macOS message(STATUS " Use interpose to override malloc (MI_OSX_INTERPOSE=ON)") - message(STATUS " WARNING: interpose does not seem to work reliably on the M1; use -DMI_OSX_ZONE=ON instead") list(APPEND mi_defines MI_OSX_INTERPOSE) + if (NOT MI_OSX_ZONE) + message(STATUS " WARNING: interpose usually also needs zone overriding (use -DMI_OSX_INTERPOSE=ON)") + endif() endif() endif() endif() @@ -188,6 +193,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM else() list(APPEND mi_cflags -ftls-model=initial-exec) endif() + if(MI_OVERRIDE) + list(APPEND -fno-builtin-malloc) + endif() endif() if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) @@ -364,7 +372,7 @@ if (MI_BUILD_TESTS) target_compile_definitions(mimalloc-test-api PRIVATE ${mi_defines}) target_compile_options(mimalloc-test-api PRIVATE ${mi_cflags}) target_include_directories(mimalloc-test-api PRIVATE include) - target_link_libraries(mimalloc-test-api PRIVATE mimalloc-static ${mi_libraries}) + target_link_libraries(mimalloc-test-api PRIVATE mimalloc ${mi_libraries}) add_executable(mimalloc-test-stress test/test-stress.c) target_compile_definitions(mimalloc-test-stress PRIVATE ${mi_defines}) diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 723ef226..0759f785 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -17,17 +17,20 @@ terms of the MIT license. A copy of the license can be found in the file /* ------------------------------------------------------ Override system malloc on macOS This is done through the malloc zone interface. - It seems we also need to interpose (see `alloc-override.c`) - or otherwise we get zone errors as there are usually - already allocations done by the time we take over the - zone. Unfortunately, that means we need to replace - the `free` with a checked free (`cfree`) impacting - performance. + It seems to be most robust in combination with interposing + though or otherwise we may get zone errors as there are could + be allocations done by the time we take over the + zone. ------------------------------------------------------ */ #include #include #include // memset +#include + +#ifdef __cplusplus +extern "C" { +#endif #if defined(MAC_OS_X_VERSION_10_6) && \ MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 @@ -41,9 +44,7 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im static size_t zone_size(malloc_zone_t* zone, const void* p) { UNUSED(zone); - if (!mi_is_in_heap_region(p)) - return 0; // not our pointer, bail out - + //if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out return mi_usable_size(p); } @@ -109,6 +110,11 @@ static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { zone_free(zone,p); } +static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { + UNUSED(zone); + return mi_is_in_heap_region(p); +} + /* ------------------------------------------------------ Introspection members @@ -174,21 +180,6 @@ static boolean_t intro_zone_locked(malloc_zone_t* zone) { At process start, override the default allocator ------------------------------------------------------ */ -static malloc_zone_t* mi_get_default_zone() -{ - // The first returned zone is the real default - malloc_zone_t** zones = NULL; - unsigned count = 0; - kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); - if (ret == KERN_SUCCESS && count > 0) { - return zones[0]; - } - else { - // fallback - return malloc_default_zone(); - } -} - #if defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic ignored "-Wmissing-field-initializers" #endif @@ -220,33 +211,197 @@ static malloc_zone_t mi_malloc_zone = { .batch_malloc = &zone_batch_malloc, .batch_free = &zone_batch_free, .introspect = &mi_introspect, -#if defined(MAC_OS_X_VERSION_10_6) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 - // switch to version 9 on OSX 10.6 to support memalign. - .version = 9, +#if defined(MAC_OS_X_VERSION_10_6) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 + // switch to version 9+ on OSX 10.6 to support memalign. .memalign = &zone_memalign, .free_definite_size = &zone_free_definite_size, .pressure_relief = &zone_pressure_relief, + #if defined(MAC_OS_X_VERSION_10_7) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7 + .claimed_address = &zone_claimed_address, + .version = 10 + #else + .version = 9 + #endif #else - .version = 4, + .version = 4 #endif }; +#ifdef __cplusplus +} +#endif -#if defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) -static malloc_zone_t *mi_malloc_default_zone(void) { +#if defined(MI_OSX_INTERPOSE) + +// ------------------------------------------------------ +// Override malloc_xxx and zone_xxx api's to use only +// our mimalloc zone. Since even the loader uses malloc +// on macOS, this ensures that all allocations go through +// mimalloc (as all calls are interposed). +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + static bool init; + if (mi_unlikely(!init)) { + init = true; + malloc_zone_register(&mi_malloc_zone); // by calling register we avoid a zone error on free (see ) + } return &mi_malloc_zone; } -// TODO: should use the macros in alloc-override but they aren't available here. -__attribute__((used)) static struct { - const void *replacement; - const void *target; -} replace_malloc_default_zone[] __attribute__((section("__DATA, __interpose"))) = { - { (const void*)mi_malloc_default_zone, (const void*)malloc_default_zone }, -}; -#endif +mi_decl_externc int malloc_jumpstart(uintptr_t cookie); +mi_decl_externc void _malloc_fork_prepare(void); +mi_decl_externc void _malloc_fork_parent(void); +mi_decl_externc void _malloc_fork_child(void); + + +static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { + UNUSED(size); UNUSED(flags); + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_zone (void) { + return mi_get_default_zone(); +} + +static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { + return mi_get_default_zone(); +} + +static void mi_malloc_destroy_zone(malloc_zone_t* zone) { + UNUSED(zone); + // nothing. +} + +static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { + UNUSED(task); UNUSED(mr); + if (addresses != NULL) *addresses = NULL; + if (count != NULL) *count = 0; + return KERN_SUCCESS; +} + +static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { + return (zone == NULL ? mi_malloc_zone.zone_name : zone->zone_name); +} + +static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { + UNUSED(zone); UNUSED(name); +} + +static int mi_malloc_jumpstart(uintptr_t cookie) { + UNUSED(cookie); + return 1; // or 0 for no error? +} + +static void mi__malloc_fork_prepare(void) { + // nothing +} +static void mi__malloc_fork_parent(void) { + // nothing +} +static void mi__malloc_fork_child(void) { + // nothing +} + +static void mi_malloc_printf(const char* fmt, ...) { + UNUSED(fmt); +} + +static bool zone_check(malloc_zone_t* zone) { + UNUSED(zone); + return true; +} + +static malloc_zone_t* zone_from_ptr(const void* p) { + UNUSED(p); + return mi_get_default_zone(); +} + +static void zone_log(malloc_zone_t* zone, void* p) { + UNUSED(zone); UNUSED(p); +} + +static void zone_print(malloc_zone_t* zone, bool b) { + UNUSED(zone); UNUSED(b); +} + +static void zone_print_ptr_info(void* p) { + UNUSED(p); +} + +static void zone_register(malloc_zone_t* zone) { + UNUSED(zone); +} + +static void zone_unregister(malloc_zone_t* zone) { + UNUSED(zone); +} + +// use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` +// See: +struct mi_interpose_s { + const void* replacement; + const void* target; +}; +#define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } +#define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) +#define MI_INTERPOSE_ZONE(fun) MI_INTERPOSE_FUN(malloc_##fun,fun) +__attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] __attribute__((section("__DATA, __interpose"))) = +{ + + MI_INTERPOSE_MI(malloc_create_zone), + MI_INTERPOSE_MI(malloc_default_purgeable_zone), + MI_INTERPOSE_MI(malloc_default_zone), + MI_INTERPOSE_MI(malloc_destroy_zone), + MI_INTERPOSE_MI(malloc_get_all_zones), + MI_INTERPOSE_MI(malloc_get_zone_name), + MI_INTERPOSE_MI(malloc_jumpstart), + MI_INTERPOSE_MI(malloc_printf), + MI_INTERPOSE_MI(malloc_set_zone_name), + MI_INTERPOSE_MI(_malloc_fork_child), + MI_INTERPOSE_MI(_malloc_fork_parent), + MI_INTERPOSE_MI(_malloc_fork_prepare), + + MI_INTERPOSE_ZONE(zone_batch_free), + MI_INTERPOSE_ZONE(zone_batch_malloc), + MI_INTERPOSE_ZONE(zone_calloc), + MI_INTERPOSE_ZONE(zone_check), + MI_INTERPOSE_ZONE(zone_free), + MI_INTERPOSE_ZONE(zone_from_ptr), + MI_INTERPOSE_ZONE(zone_log), + MI_INTERPOSE_ZONE(zone_malloc), + MI_INTERPOSE_ZONE(zone_memalign), + MI_INTERPOSE_ZONE(zone_print), + MI_INTERPOSE_ZONE(zone_print_ptr_info), + MI_INTERPOSE_ZONE(zone_realloc), + MI_INTERPOSE_ZONE(zone_register), + MI_INTERPOSE_ZONE(zone_unregister), + MI_INTERPOSE_ZONE(zone_valloc) +}; + + +#else + +// ------------------------------------------------------ +// hook into the zone api's without interposing +// ------------------------------------------------------ + +static inline malloc_zone_t* mi_get_default_zone(void) +{ + // The first returned zone is the real default + malloc_zone_t** zones = NULL; + unsigned count = 0; + kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); + if (ret == KERN_SUCCESS && count > 0) { + return zones[0]; + } + else { + // fallback + return malloc_default_zone(); + } +} #if defined(__clang__) __attribute__((constructor(0))) @@ -287,5 +442,6 @@ static void _mi_macos_override_malloc() { } } +#endif // MI_OSX_INTERPOSE #endif // MI_MALLOC_OVERRIDE diff --git a/src/alloc-override.c b/src/alloc-override.c index 6c42ae00..123f5e07 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file #error "It is only possible to override "malloc" on Windows when building as a DLL (and linking the C runtime as a DLL)" #endif -#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // || (defined(__APPLE__) && !defined(MI_OSX_INTERPOSE))) +#if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) // ------------------------------------------------------ // Override system malloc @@ -41,7 +41,12 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif -#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) + #include + mi_decl_externc void vfree(void* p); + mi_decl_externc size_t malloc_size(const void* p); + mi_decl_externc size_t malloc_good_size(size_t size); + // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: struct mi_interpose_s { @@ -50,6 +55,7 @@ terms of the MIT license. A copy of the license can be found in the file }; #define MI_INTERPOSE_FUN(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } #define MI_INTERPOSE_MI(fun) MI_INTERPOSE_FUN(fun,mi_##fun) + __attribute__((used)) static struct mi_interpose_s _mi_interposes[] __attribute__((section("__DATA, __interpose"))) = { MI_INTERPOSE_MI(malloc), @@ -61,21 +67,24 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(posix_memalign), MI_INTERPOSE_MI(reallocf), MI_INTERPOSE_MI(valloc), + MI_INTERPOSE_MI(malloc_size), + MI_INTERPOSE_MI(malloc_good_size), + MI_INTERPOSE_MI(aligned_alloc), #ifndef MI_OSX_ZONE - // some code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + MI_INTERPOSE_FUN(vfree,mi_cfree), #else - // We interpose malloc_default_zone in alloc-override-osx.c + // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely MI_INTERPOSE_MI(free), + MI_INTERPOSE_FUN(vfree,mi_free), #endif - // some code allocates from a zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) - MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us }; #elif defined(_MSC_VER) // cannot override malloc unless using a dll. // we just override new/delete which does work in a static library. #else - // On all other systems forward to our API + // On all other systems forward to our API void* malloc(size_t size) MI_FORWARD1(mi_malloc, size) void* calloc(size_t size, size_t n) MI_FORWARD2(mi_calloc, size, n) void* realloc(void* p, size_t newsize) MI_FORWARD2(mi_realloc, p, newsize) @@ -123,7 +132,7 @@ terms of the MIT license. A copy of the license can be found in the file void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } #endif -#elif (defined(__GNUC__) || defined(__clang__)) +#elif (defined(__GNUC__) || defined(__clang__)) && !defined(MI_OSX_ZONE) // ------------------------------------------------------ // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). From e853f530a0d90ae49caf17c994a61febbe80230a Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 4 Nov 2021 18:54:57 -0700 Subject: [PATCH 060/180] add noexcept attributes to improve mi_free codegen --- include/mimalloc-internal.h | 6 +++--- src/alloc.c | 2 +- src/page.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index e8046528..5c716a79 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -37,9 +37,9 @@ terms of the MIT license. A copy of the license can be found in the file #endif #if defined(__cplusplus) -#define mi_decl_externc extern "C" +#define mi_decl_externc extern "C" #else -#define mi_decl_externc +#define mi_decl_externc #endif // "options.c" @@ -100,7 +100,7 @@ void _mi_abandoned_await_readers(void); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc; -void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... diff --git a/src/alloc.c b/src/alloc.c index c91dcd92..d03207ce 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -435,7 +435,7 @@ mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* p } -static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) { +static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool local, void* p) mi_attr_noexcept { mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_page_has_aligned(page) ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); mi_stat_free(page, block); diff --git a/src/page.c b/src/page.c index 7efbcb8d..61f3245a 100644 --- a/src/page.c +++ b/src/page.c @@ -386,7 +386,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { // Note: called from `mi_free` and benchmarks often // trigger this due to freeing everything and then // allocating again so careful when changing this. -void _mi_page_retire(mi_page_t* page) { +void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_assert_internal(page != NULL); mi_assert_expensive(_mi_page_is_valid(page)); mi_assert_internal(mi_page_all_free(page)); From c47de7eb9090cf8c0a5f307a74437cf927a361be Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 4 Nov 2021 18:55:12 -0700 Subject: [PATCH 061/180] improve macOS overriding --- CMakeLists.txt | 11 ++- src/alloc-override-osx.c | 9 ++- src/alloc-override.c | 145 +++++++++++++++++++++++++-------------- 3 files changed, 108 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f07d892f..ed5fb951 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ set(mi_sources src/options.c src/init.c) + # ----------------------------------------------------------------------------- # Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- @@ -62,6 +63,7 @@ if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") set(MI_SECURE "ON") endif() + # ----------------------------------------------------------------------------- # Process options # ----------------------------------------------------------------------------- @@ -85,11 +87,14 @@ if(MI_OVERRIDE) if(MI_OSX_INTERPOSE) # use interpose on macOS message(STATUS " Use interpose to override malloc (MI_OSX_INTERPOSE=ON)") - list(APPEND mi_defines MI_OSX_INTERPOSE) + list(APPEND mi_defines MI_OSX_INTERPOSE=1) if (NOT MI_OSX_ZONE) message(STATUS " WARNING: interpose usually also needs zone overriding (use -DMI_OSX_INTERPOSE=ON)") endif() endif() + if((NOT MI_USE_CXX) AND MI_OVERRIDE) + message(STATUS " WARNING: if overriding C++ new/delete, it is best to build mimalloc with a C++ compiler (use -DMI_USE_CXX=ON)") + endif() endif() endif() @@ -260,9 +265,9 @@ message(STATUS "") message(STATUS "Library base name: ${mi_basename}") message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") if(MI_USE_CXX) - message(STATUS "Compiler : ${CMAKE_CXX_COMPILER}") + message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() - message(STATUS "Compiler : ${CMAKE_C_COMPILER}") + message(STATUS "C Compiler : ${CMAKE_C_COMPILER}") endif() message(STATUS "Version : ${mi_version}") message(STATUS "Build targets : ${mi_build_targets}") diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 0759f785..35b95a24 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -232,13 +232,16 @@ static malloc_zone_t mi_malloc_zone = { #endif -#if defined(MI_OSX_INTERPOSE) +#if defined(MI_OSX_INTERPOSE) && defined(MI_SHARED_LIB_EXPORT) // ------------------------------------------------------ -// Override malloc_xxx and zone_xxx api's to use only +// Override malloc_xxx and malloc_zone_xxx api's to use only // our mimalloc zone. Since even the loader uses malloc // on macOS, this ensures that all allocations go through // mimalloc (as all calls are interposed). +// The main `malloc`, `free`, etc calls are interposed in `alloc-override.c`, +// Here, we also override macOS specific API's like +// `malloc_zone_calloc` etc. see // ------------------------------------------------------ static inline malloc_zone_t* mi_get_default_zone(void) @@ -386,6 +389,8 @@ __attribute__((used)) static const struct mi_interpose_s _mi_zone_interposes[] // ------------------------------------------------------ // hook into the zone api's without interposing +// This is the official way of adding an allocator but +// it seems less robust than using interpose. // ------------------------------------------------------ static inline malloc_zone_t* mi_get_default_zone(void) diff --git a/src/alloc-override.c b/src/alloc-override.c index 123f5e07..9581a185 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -15,12 +15,21 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(MI_MALLOC_OVERRIDE) && !(defined(_WIN32)) +#if defined(__APPLE__) +mi_decl_externc void vfree(void* p); +mi_decl_externc size_t malloc_size(const void* p); +mi_decl_externc size_t malloc_good_size(size_t size); +#endif + +// helper definition for C override of C++ new +typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; + // ------------------------------------------------------ // Override system malloc // ------------------------------------------------------ -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) - // use aliasing to alias the exported function to one of our `mi_` functions +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__) && !defined(MI_VALGRIND) + // gcc, clang: use aliasing to alias the exported function to one of our `mi_` functions #if (defined(__GNUC__) && __GNUC__ >= 9) #pragma GCC diagnostic ignored "-Wattributes" // or we get warnings that nodiscard is ignored on a forward #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))); @@ -33,7 +42,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD0(fun,x) MI_FORWARD(fun) #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) #else - // use forwarding by calling our `mi_` function + // otherwise use forwarding by calling our `mi_` function #define MI_FORWARD1(fun,x) { return fun(x); } #define MI_FORWARD2(fun,x,y) { return fun(x,y); } #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } @@ -41,11 +50,10 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_FORWARD02(fun,x,y) { fun(x,y); } #endif -#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) - #include - mi_decl_externc void vfree(void* p); - mi_decl_externc size_t malloc_size(const void* p); - mi_decl_externc size_t malloc_good_size(size_t size); +#if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_OSX_INTERPOSE) + // define MI_OSX_IS_INTERPOSED as we should not provide forwarding definitions for + // functions that are interposed (or the interposing does not work) + #define MI_OSX_IS_INTERPOSED // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` // See: @@ -70,16 +78,41 @@ terms of the MIT license. A copy of the license can be found in the file MI_INTERPOSE_MI(malloc_size), MI_INTERPOSE_MI(malloc_good_size), MI_INTERPOSE_MI(aligned_alloc), - #ifndef MI_OSX_ZONE - // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) - MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us - MI_INTERPOSE_FUN(vfree,mi_cfree), - #else + #ifdef MI_OSX_ZONE // we interpose malloc_default_zone in alloc-override-osx.c so we can use mi_free safely MI_INTERPOSE_MI(free), MI_INTERPOSE_FUN(vfree,mi_free), + #else + // sometimes code allocates from default zone but deallocates using plain free :-( (like NxHashResizeToCapacity ) + MI_INTERPOSE_FUN(free,mi_cfree), // use safe free that checks if pointers are from us + MI_INTERPOSE_FUN(vfree,mi_cfree), #endif }; + + #ifdef __cplusplus + extern "C" { + void _ZdlPv(void* p); // delete + void _ZdaPv(void* p); // delete[] + void _ZdlPvm(void* p, size_t n); // delete + void _ZdaPvm(void* p, size_t n); // delete[] + void* _Znwm(size_t n); // new + void* _Znam(size_t n); // new[] + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new nothrow + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag); // new[] nothrow + } + __attribute__((used)) static struct mi_interpose_s _mi_cxx_interposes[] __attribute__((section("__DATA, __interpose"))) = + { + MI_INTERPOSE_FUN(_ZdlPv,mi_free), + MI_INTERPOSE_FUN(_ZdaPv,mi_free), + MI_INTERPOSE_FUN(_ZdlPvm,mi_free_size), + MI_INTERPOSE_FUN(_ZdaPvm,mi_free_size), + MI_INTERPOSE_FUN(_Znwm,mi_new), + MI_INTERPOSE_FUN(_Znam,mi_new), + MI_INTERPOSE_FUN(_ZnwmRKSt9nothrow_t,mi_new_nothrow), + MI_INTERPOSE_FUN(_ZnamRKSt9nothrow_t,mi_new_nothrow), + }; + #endif // __cplusplus + #elif defined(_MSC_VER) // cannot override malloc unless using a dll. // we just override new/delete which does work in a static library. @@ -106,18 +139,21 @@ terms of the MIT license. A copy of the license can be found in the file // see // ------------------------------------------------------ #include - void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) - void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) - void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) - void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + #ifndef MI_OSX_IS_INTERPOSED + void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p) + void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p) - void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } - void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) - #if (__cplusplus >= 201402L || _MSC_VER >= 1916) - void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) - void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + + #if (__cplusplus >= 201402L || _MSC_VER >= 1916) + void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + void operator delete[](void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) + #endif #endif #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) && (!defined(__GNUC__) || (__GNUC__ > 5)) @@ -132,12 +168,13 @@ terms of the MIT license. A copy of the license can be found in the file void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } #endif -#elif (defined(__GNUC__) || defined(__clang__)) && !defined(MI_OSX_ZONE) +#elif (defined(__GNUC__) || defined(__clang__)) // ------------------------------------------------------ // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). // See // ------------------------------------------------------ + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p) // delete void _ZdaPv(void* p) MI_FORWARD0(mi_free,p) // delete[] void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n) @@ -146,67 +183,71 @@ terms of the MIT license. A copy of the license can be found in the file void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } - - typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; + #if (MI_INTPTR_SIZE==8) void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #else - #error "define overloads for new/delete for this platform (just for performance, can be skipped)" + #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif #endif // __cplusplus +// ------------------------------------------------------ +// Further Posix & Unix functions definitions +// ------------------------------------------------------ #ifdef __cplusplus extern "C" { #endif -// ------------------------------------------------------ -// Posix & Unix functions definitions -// ------------------------------------------------------ +#ifndef MI_OSX_IS_INTERPOSED + // Forward Posix/Unix calls as well + void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) + size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_good_size(size_t size) MI_FORWARD1(mi_malloc_good_size,size) + #if !defined(__ANDROID__) && !defined(__FreeBSD__) + size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) + #else + size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + #endif -void cfree(void* p) MI_FORWARD0(mi_free, p) -void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) -size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) -#if !defined(__ANDROID__) && !defined(__FreeBSD__) -size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) -#else -size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + // No forwarding here due to aliasing/name mangling issues + void* valloc(size_t size) { return mi_valloc(size); } + void vfree(void* p) { mi_free(p); } + int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + + // `aligned_alloc` is only available when __USE_ISOC11 is defined. + // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot + // override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. + // Fortunately, in the case where `aligned_alloc` is declared as `static inline` it + // uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. + #if __USE_ISOC11 + void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } + #endif #endif // no forwarding here due to aliasing/name mangling issues -void* valloc(size_t size) { return mi_valloc(size); } +void cfree(void* p) { mi_free(p); } void* pvalloc(size_t size) { return mi_pvalloc(size); } void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -// `aligned_alloc` is only available when __USE_ISOC11 is defined. -// Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot -// override it, but both _ISOC11_SOURCE and __USE_ISOC11 are undefined in Conda GCC7 or GCC9. -// Fortunately, in the case where `aligned_alloc` is declared as `static inline` it -// uses internally `memalign`, `posix_memalign`, or `_aligned_malloc` so we can avoid overriding it ourselves. -#if __USE_ISOC11 -void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } -#endif - - #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) From f2e3cca2136a422024df4c525902cdad33bbcc4c Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 4 Nov 2021 19:10:15 -0700 Subject: [PATCH 062/180] fix build for missing malloc_good_size --- src/alloc-override.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index 9581a185..fa04b460 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -219,7 +219,6 @@ extern "C" { // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) - size_t malloc_good_size(size_t size) MI_FORWARD1(mi_malloc_good_size,size) #if !defined(__ANDROID__) && !defined(__FreeBSD__) size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) #else @@ -227,9 +226,11 @@ extern "C" { #endif // No forwarding here due to aliasing/name mangling issues - void* valloc(size_t size) { return mi_valloc(size); } - void vfree(void* p) { mi_free(p); } - int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + void* valloc(size_t size) { return mi_valloc(size); } + void vfree(void* p) { mi_free(p); } + size_t malloc_good_size(size_t size) { return mi_malloc_good_size(size); } + int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } + // `aligned_alloc` is only available when __USE_ISOC11 is defined. // Note: Conda has a custom glibc where `aligned_alloc` is declared `static inline` and we cannot @@ -242,23 +243,23 @@ extern "C" { #endif // no forwarding here due to aliasing/name mangling issues -void cfree(void* p) { mi_free(p); } -void* pvalloc(size_t size) { return mi_pvalloc(size); } -void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } -void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } -void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } +void cfree(void* p) { mi_free(p); } +void* pvalloc(size_t size) { return mi_pvalloc(size); } +void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } +void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } #if defined(__GLIBC__) && defined(__linux__) // forward __libc interface (needed for glibc-based Linux distributions) - void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) - void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) - void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) - void __libc_free(void* p) MI_FORWARD0(mi_free,p) - void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) + void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size) + void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size) + void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size) + void __libc_free(void* p) MI_FORWARD0(mi_free,p) + void __libc_cfree(void* p) MI_FORWARD0(mi_free,p) - void* __libc_valloc(size_t size) { return mi_valloc(size); } - void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } - void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } + void* __libc_valloc(size_t size) { return mi_valloc(size); } + void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } + void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } #endif From e96614961f1a6b2a3ab01e4b21fc600c23d0ba1b Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 6 Nov 2021 14:19:14 -0700 Subject: [PATCH 063/180] fix printf format type mismatches (issue #486) --- src/os.c | 6 +++--- src/stats.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/os.c b/src/os.c index 507d9971..26b8bcd2 100644 --- a/src/os.c +++ b/src/os.c @@ -846,7 +846,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ if (err != 0) { err = errno; } #endif if (err != 0) { - _mi_warning_message("%s error: start: %p, csize: 0x%x, err: %i\n", commit ? "commit" : "decommit", start, csize, err); + _mi_warning_message("%s error: start: %p, csize: 0x%zx, err: %i\n", commit ? "commit" : "decommit", start, csize, err); mi_mprotect_hint(err); } mi_assert_internal(err == 0); @@ -916,7 +916,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) int err = madvise(start, csize, MADV_DONTNEED); #endif if (err != 0) { - _mi_warning_message("madvise reset error: start: %p, csize: 0x%x, errno: %i\n", start, csize, errno); + _mi_warning_message("madvise reset error: start: %p, csize: 0x%zx, errno: %i\n", start, csize, errno); } //mi_assert(err == 0); if (err != 0) return false; @@ -975,7 +975,7 @@ static bool mi_os_protectx(void* addr, size_t size, bool protect) { if (err != 0) { err = errno; } #endif if (err != 0) { - _mi_warning_message("mprotect error: start: %p, csize: 0x%x, err: %i\n", start, csize, err); + _mi_warning_message("mprotect error: start: %p, csize: 0x%zx, err: %i\n", start, csize, err); mi_mprotect_hint(err); } return (err == 0); diff --git a/src/stats.c b/src/stats.c index 115d938e..6d486f42 100644 --- a/src/stats.c +++ b/src/stats.c @@ -327,7 +327,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print(&stats->commit_calls, "commits", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); mi_stat_counter_print_avg(&stats->searches, "searches", out, arg); - _mi_fprintf(out, arg, "%10s: %7i\n", "numa nodes", _mi_os_numa_node_count()); + _mi_fprintf(out, arg, "%10s: %7zu\n", "numa nodes", _mi_os_numa_node_count()); mi_msecs_t elapsed; mi_msecs_t user_time; From 8c9ccea2f576394b69d0489b6f8883b95c4ddc1a Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 10:46:06 -0800 Subject: [PATCH 064/180] fix huge page madvise in case mmap failed --- src/os.c | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/os.c b/src/os.c index 26b8bcd2..77e8216e 100644 --- a/src/os.c +++ b/src/os.c @@ -510,7 +510,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, (uintptr_t)10); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (uintptr_t)8); // on error, don't try again for the next N allocations } } } @@ -518,29 +518,30 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro if (p == NULL) { *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); - #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). It is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (allow_large && use_large_os_page(size, try_alignment)) { - if (madvise(p, size, MADV_HUGEPAGE) == 0) { - *is_large = true; // possibly - }; - } - #endif - #if defined(__sun) - if (allow_large && use_large_os_page(size, try_alignment)) { - struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = large_os_page_size; - cmd.mha_cmd = MHA_MAPSIZE_VA; - if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - *is_large = true; + if (p != NULL) { + #if defined(MADV_HUGEPAGE) + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (allow_large && use_large_os_page(size, try_alignment)) { + if (madvise(p, size, MADV_HUGEPAGE) == 0) { + *is_large = true; // possibly + }; } + #elif defined(__sun) + if (allow_large && use_large_os_page(size, try_alignment)) { + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = large_os_page_size; + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl(p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + *is_large = true; + } + } + #endif } - #endif } if (p == NULL) { _mi_warning_message("unable to allocate OS memory (%zu bytes, error code: %i, address: %p, large only: %d, allow large: %d)\n", size, errno, addr, large_only, allow_large); From 89abbe75d81ed4ed4858a3319a3ed938bc751ca1 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:23:11 -0800 Subject: [PATCH 065/180] improve aligned support on BSD and MAP_ALIGN systems --- src/os.c | 56 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/os.c b/src/os.c index 77e8216e..8334771c 100644 --- a/src/os.c +++ b/src/os.c @@ -413,23 +413,41 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { + UNUSED(try_alignment); void* p = NULL; + #if defined(MAP_ALIGNED) // BSD + if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + size_t n = mi_bsr(try_alignment); + if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB + flags |= MAP_ALIGNED(n); + p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + } + #elif defined(MAP_ALIGN) // Solaris + if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } + #endif #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations - void* hint; - if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment, size)) != NULL) { - p = mmap(hint,size,protect_flags,flags,fd,0); - if (p==MAP_FAILED) p = NULL; // fall back to regular mmap + if (addr == NULL) { + void* hint = mi_os_get_aligned_hint(try_alignment, size); + if (hint != NULL) { + p = mmap(hint, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // fall back to regular mmap + } } - #else - UNUSED(try_alignment); - UNUSED(mi_os_get_aligned_hint); #endif - if (p==NULL) { - p = mmap(addr,size,protect_flags,flags,fd,0); - if (p==MAP_FAILED) p = NULL; - } - return p; + // regular mmap + p = mmap(addr, size, protect_flags, flags, fd, 0); + if (p!=MAP_FAILED) return p; + // failed to allocate + return NULL; } static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { @@ -444,24 +462,17 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro int fd = -1; if (os_overcommit) { flags |= MAP_NORESERVE; - } - #if defined(MAP_ALIGNED) // BSD - if (try_alignment > 0) { - size_t n = mi_bsr(try_alignment); - if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB - flags |= MAP_ALIGNED(n); - } - } - #endif + } #if defined(PROT_MAX) protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif #if defined(VM_MAKE_TAG) // macOS: tracking anonymous page with a specific ID. (All up to 98 are taken officially but LLVM sanitizers had taken 99) int os_tag = (int)mi_option_get(mi_option_os_tag); - if (os_tag < 100 || os_tag > 255) os_tag = 100; + if (os_tag < 100 || os_tag > 255) { os_tag = 100; } fd = VM_MAKE_TAG(os_tag); #endif + // huge page allocation if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { static _Atomic(uintptr_t) large_page_try_ok; // = 0; uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); @@ -515,6 +526,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro } } } + // regular allocation if (p == NULL) { *is_large = false; p = mi_unix_mmapx(addr, size, try_alignment, protect_flags, flags, fd); From fd61997cef6af8eab86fc40da5636f95c6b6fb47 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:26:36 -0800 Subject: [PATCH 066/180] improve aligned support on BSD and MAP_ALIGN systems --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index 8334771c..d43f9afd 100644 --- a/src/os.c +++ b/src/os.c @@ -564,7 +564,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro // On 64-bit systems, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate them. -#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED))) +#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || defined(MI_OS_USE_MMAP)) static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // Return a 4MiB aligned address that is probably available. From f72e5688f5b6bbbe6a61289f0ed73541518de08e Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 10 Nov 2021 11:58:04 -0800 Subject: [PATCH 067/180] remove assign in while condition --- src/os.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/os.c b/src/os.c index d43f9afd..426ead4a 100644 --- a/src/os.c +++ b/src/os.c @@ -303,20 +303,21 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations - void* hint; - if (addr == NULL && (hint = mi_os_get_aligned_hint(try_alignment,size)) != NULL) { - void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); - if (p != NULL) return p; - // for robustness always fall through in case of an error - /* - DWORD err = GetLastError(); - if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) - err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) - return NULL; + if (addr == NULL) { + void* hint = mi_os_get_aligned_hint(try_alignment,size); + if (hint != NULL) { + void* p = VirtualAlloc(hint, size, flags, PAGE_READWRITE); + if (p != NULL) return p; + // for robustness always fall through in case of an error + /* + DWORD err = GetLastError(); + if (err != ERROR_INVALID_ADDRESS && // If linked with multiple instances, we may have tried to allocate at an already allocated area (#210) + err != ERROR_INVALID_PARAMETER) { // Windows7 instability (#230) + return NULL; + } + */ + _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); } - */ - _mi_warning_message("unable to allocate hinted aligned OS memory (%zu bytes, error code: %x, address: %p, alignment: %d, flags: %x)\n", size, GetLastError(), hint, try_alignment, flags); - // fall through on error } #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) @@ -414,20 +415,19 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { UNUSED(try_alignment); - void* p = NULL; #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); - p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); + void* p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } } #elif defined(MAP_ALIGN) // Solaris if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { - p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); + void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } @@ -437,14 +437,14 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr if (addr == NULL) { void* hint = mi_os_get_aligned_hint(try_alignment, size); if (hint != NULL) { - p = mmap(hint, size, protect_flags, flags, fd, 0); + void* p = mmap(hint, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap } } #endif // regular mmap - p = mmap(addr, size, protect_flags, flags, fd, 0); + void* p = mmap(addr, size, protect_flags, flags, fd, 0); if (p!=MAP_FAILED) return p; // failed to allocate return NULL; From d2de66b8fabc95fcba2dfd5e4f8a9c3004eb78c7 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 12 Nov 2021 18:44:33 -0800 Subject: [PATCH 068/180] do not delay eager commit for the main thread --- src/segment.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index bdf97019..4c2fa32e 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,8 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && !_mi_is_main_thread() && + tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); bool pages_still_good = false; From f9ac60a90ffcaacf576920508431d6fa7f3c92e4 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 12 Nov 2021 19:03:20 -0800 Subject: [PATCH 069/180] suppres eager commit delay for the first 4 threads --- include/mimalloc-internal.h | 1 + src/init.c | 8 ++++++++ src/segment.c | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 5c716a79..a2549f7e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -63,6 +63,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x); extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); +size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not ready // os.c diff --git a/src/init.c b/src/init.c index cb9d123f..db3d24ba 100644 --- a/src/init.c +++ b/src/init.c @@ -332,6 +332,12 @@ bool _mi_is_main_thread(void) { return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); } +static _Atomic(uintptr_t) thread_count = ATOMIC_VAR_INIT(1); + +size_t _mi_current_thread_count(void) { + return mi_atomic_load_relaxed(&thread_count); +} + // This is called from the `mi_malloc_generic` void mi_thread_init(void) mi_attr_noexcept { @@ -344,6 +350,7 @@ void mi_thread_init(void) mi_attr_noexcept if (_mi_heap_init()) return; // returns true if already initialized _mi_stat_increase(&_mi_stats_main.threads, 1); + mi_atomic_increment_relaxed(&thread_count); //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); } @@ -352,6 +359,7 @@ void mi_thread_done(void) mi_attr_noexcept { } static void _mi_thread_done(mi_heap_t* heap) { + mi_atomic_decrement_relaxed(&thread_count); _mi_stat_decrease(&_mi_stats_main.threads, 1); // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... diff --git a/src/segment.c b/src/segment.c index 4c2fa32e..1fffd745 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && !_mi_is_main_thread() && + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 4 && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From d52b559b571a14bb190a7cd7de15c2d23f0804a3 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 12 Nov 2021 19:59:59 -0800 Subject: [PATCH 070/180] use eager delay after 2 threads --- src/segment.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/segment.c b/src/segment.c index 1fffd745..64b4720c 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 4 && + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 2 && tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From c56be7ac5ae682733ddeb14288fd432a68951d5c Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 13:30:03 -0800 Subject: [PATCH 071/180] show cflags in cmake; fix -fno-builtin-malloc flag --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed5fb951..65549e47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,7 @@ set(mi_sources # Convenience: set default build type depending on the build directory # ----------------------------------------------------------------------------- +message(STATUS "") if (NOT CMAKE_BUILD_TYPE) if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$" OR MI_DEBUG_FULL) message(STATUS "No build type selected, default to: Debug") @@ -199,7 +200,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM list(APPEND mi_cflags -ftls-model=initial-exec) endif() if(MI_OVERRIDE) - list(APPEND -fno-builtin-malloc) + list(APPEND mi_cflags -fno-builtin-malloc) endif() endif() @@ -263,13 +264,14 @@ endif() message(STATUS "") message(STATUS "Library base name: ${mi_basename}") +message(STATUS "Version : ${mi_version}") message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") if(MI_USE_CXX) message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() message(STATUS "C Compiler : ${CMAKE_C_COMPILER}") endif() -message(STATUS "Version : ${mi_version}") +message(STATUS "Compiler flags : ${mi_cflags}") message(STATUS "Build targets : ${mi_build_targets}") message(STATUS "") From 959845540d8263ed08fea321f78313d9e89090ae Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 14:13:03 -0800 Subject: [PATCH 072/180] use W4 for msvc compilation --- ide/vs2019/mimalloc.vcxproj | 6 +++--- src/options.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 6c7e276c..c12955c3 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -92,7 +92,7 @@ - Level3 + Level4 Disabled true true @@ -138,7 +138,7 @@ - Level3 + Level4 MaxSpeed true true @@ -166,7 +166,7 @@ - Level3 + Level4 MaxSpeed true true diff --git a/src/options.c b/src/options.c index c40a187b..e3d6c8c9 100644 --- a/src/options.c +++ b/src/options.c @@ -113,7 +113,7 @@ void _mi_options_init(void) { mi_max_warning_count = mi_option_get(mi_option_max_warnings); } -long mi_option_get(mi_option_t option) { +mi_decl_nodiscard long mi_option_get(mi_option_t option) { mi_assert(option >= 0 && option < _mi_option_last); mi_option_desc_t* desc = &options[option]; mi_assert(desc->option == option); // index should match the option @@ -139,7 +139,7 @@ void mi_option_set_default(mi_option_t option, long value) { } } -bool mi_option_is_enabled(mi_option_t option) { +mi_decl_nodiscard bool mi_option_is_enabled(mi_option_t option) { return (mi_option_get(option) != 0); } From 28896e5b19c5fd947626fdb924acde3de6425913 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 14:46:03 -0800 Subject: [PATCH 073/180] prefix UNUSED,KiB,MiB,GiB; add mi_threadid_t type; add mi_ssize_t --- include/mimalloc-internal.h | 38 ++++++++++----------- include/mimalloc-types.h | 64 +++++++++++++++++++++++++--------- src/alloc-override-osx.c | 68 ++++++++++++++++++------------------- src/alloc-override.c | 20 +++++------ src/alloc.c | 34 +++++++++---------- src/arena.c | 2 +- src/bitmap.c | 8 ++--- src/heap.c | 34 +++++++++---------- src/init.c | 8 ++--- src/options.c | 20 +++++------ src/os.c | 52 ++++++++++++++-------------- src/page.c | 6 ++-- src/region.c | 8 ++--- src/segment.c | 2 +- 14 files changed, 198 insertions(+), 166 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index a2549f7e..23078330 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -187,11 +187,11 @@ bool _mi_page_is_valid(mi_page_t* page); /* ----------------------------------------------------------- Inlined definitions ----------------------------------------------------------- */ -#define UNUSED(x) (void)(x) +#define MI_UNUSED(x) (void)(x) #if (MI_DEBUG>0) -#define UNUSED_RELEASE(x) +#define MI_UNUSED_RELEASE(x) #else -#define UNUSED_RELEASE(x) UNUSED(x) +#define MI_UNUSED_RELEASE(x) MI_UNUSED(x) #endif #define MI_INIT4(x) x(),x(),x(),x() @@ -413,11 +413,11 @@ static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { } // used internally -static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { +static inline size_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; mi_assert_internal(diff >= 0 && (size_t)diff < MI_SEGMENT_SIZE); - uintptr_t idx = (uintptr_t)diff >> segment->page_shift; + size_t idx = (size_t)diff >> segment->page_shift; mi_assert_internal(idx < segment->capacity); mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0); return idx; @@ -425,7 +425,7 @@ static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, con // Get the page containing the pointer static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { - uintptr_t idx = _mi_segment_page_idx_of(segment, p); + size_t idx = _mi_segment_page_idx_of(segment, p); return &((mi_segment_t*)segment)->pages[idx]; } @@ -583,8 +583,8 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { mi_segment_t* segmentp = _mi_ptr_segment(p); mi_segment_t* segmentq = _mi_ptr_segment(q); if (segmentp != segmentq) return false; - uintptr_t idxp = _mi_segment_page_idx_of(segmentp, p); - uintptr_t idxq = _mi_segment_page_idx_of(segmentq, q); + size_t idxp = _mi_segment_page_idx_of(segmentp, p); + size_t idxq = _mi_segment_page_idx_of(segmentq, q); return (idxp == idxq); } @@ -611,7 +611,7 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl #ifdef MI_ENCODE_FREELIST return (mi_block_t*)mi_ptr_decode(null, block->next, keys); #else - UNUSED(keys); UNUSED(null); + MI_UNUSED(keys); MI_UNUSED(null); return (mi_block_t*)block->next; #endif } @@ -620,7 +620,7 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const #ifdef MI_ENCODE_FREELIST block->next = mi_ptr_encode(null, next, keys); #else - UNUSED(keys); UNUSED(null); + MI_UNUSED(keys); MI_UNUSED(null); block->next = (mi_encoded_t)next; #endif } @@ -636,7 +636,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* } return next; #else - UNUSED(page); + MI_UNUSED(page); return mi_block_nextx(page,block,NULL); #endif } @@ -645,7 +645,7 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c #ifdef MI_ENCODE_FREELIST mi_block_set_nextx(page,block,next, page->keys); #else - UNUSED(page); + MI_UNUSED(page); mi_block_set_nextx(page,block,next,NULL); #endif } @@ -700,7 +700,7 @@ static inline size_t _mi_os_numa_node_count(void) { #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #include -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { +static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { // Windows: works on Intel and ARM in both 32- and 64-bit return (uintptr_t)NtCurrentTeb(); } @@ -721,11 +721,11 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { #elif defined(__x86_64__) __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); res = tcb[slot]; #elif defined(__aarch64__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits @@ -749,11 +749,11 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { #elif defined(__x86_64__) __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); tcb[slot] = value; #elif defined(__aarch64__) - void** tcb; UNUSED(ofs); + void** tcb; MI_UNUSED(ofs); #if defined(__APPLE__) // M1, issue #343 __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits @@ -764,7 +764,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { #endif } -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { +static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { #if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) // on Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); @@ -775,7 +775,7 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { } #else // otherwise use standard C -static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { +static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { return (uintptr_t)&_mi_heap_default; } #endif diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 3ddd4e8a..c9f399df 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -17,7 +17,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Minimal alignment necessary. On most platforms 16 bytes are needed -// due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` +// due to SSE registers for example. This must be at least `sizeof(void*)` #ifndef MI_MAX_ALIGN_SIZE #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif @@ -67,6 +67,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif + // ------------------------------------------------------ // Platform specific values // ------------------------------------------------------ @@ -83,20 +84,43 @@ terms of the MIT license. A copy of the license can be found in the file // or otherwise one might define an intptr_t type that is larger than a pointer... // ------------------------------------------------------ -#if INTPTR_MAX == 9223372036854775807LL +#if INTPTR_MAX > INT64_MAX +# define MI_INTPTR_SHIFT (4) // assume 128-bit (as on arm CHERI for example) +#elif INTPTR_MAX == INT64_MAX # define MI_INTPTR_SHIFT (3) -#elif INTPTR_MAX == 2147483647LL +#elif INTPTR_MAX == INT32_MAX # define MI_INTPTR_SHIFT (2) #else -#error platform must be 32 or 64 bits +#error platform pointers must be 32, 64, or 128 bits +#endif + +#if SIZE_MAX == UINT64_MAX +# define MI_SIZE_SHIFT (3) +typedef int64_t mi_ssize_t; +#elif SIZE_MAX == UINT32_MAX +# define MI_SIZE_SHIFT (2) +typedef int32_t mi_ssize_t; +#else +#error platform objects must be 32 or 64 bits +#endif + +#if (SIZE_MAX/2) > LONG_MAX +# define MI_ZU(x) x##ULL +# define MI_ZI(x) x##LL +#else +# define MI_ZU(x) x##UL +# define MI_ZI(x) x##L #endif #define MI_INTPTR_SIZE (1<cookie` // layout like this to optimize access in `mi_free` - size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). - _Atomic(uintptr_t) thread_id; // unique id of the thread owning this segment + size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). + _Atomic(mi_threadid_t) thread_id; // unique id of the thread owning this segment mi_page_kind_t page_kind; // kind of pages: small, large, or huge mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages } mi_segment_t; @@ -341,7 +373,7 @@ struct mi_heap_s { mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") _Atomic(mi_block_t*) thread_delayed_free; - uintptr_t thread_id; // thread this heap belongs too + mi_threadid_t thread_id; // thread this heap belongs too uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) uintptr_t keys[2]; // two random keys used to encode the `thread_delayed_free` list mi_random_ctx_t random; // random number context used for secure allocation diff --git a/src/alloc-override-osx.c b/src/alloc-override-osx.c index 35b95a24..63297c4c 100644 --- a/src/alloc-override-osx.c +++ b/src/alloc-override-osx.c @@ -43,43 +43,43 @@ extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_im ------------------------------------------------------ */ static size_t zone_size(malloc_zone_t* zone, const void* p) { - UNUSED(zone); + MI_UNUSED(zone); //if (!mi_is_in_heap_region(p)){ return 0; } // not our pointer, bail out return mi_usable_size(p); } static void* zone_malloc(malloc_zone_t* zone, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_malloc(size); } static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_calloc(count, size); } static void* zone_valloc(malloc_zone_t* zone, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_malloc_aligned(size, _mi_os_page_size()); } static void zone_free(malloc_zone_t* zone, void* p) { - UNUSED(zone); + MI_UNUSED(zone); mi_free(p); } static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { - UNUSED(zone); + MI_UNUSED(zone); return mi_realloc(p, newsize); } static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_malloc_aligned(size,alignment); } static void zone_destroy(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // todo: ignore for now? } @@ -100,18 +100,18 @@ static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { } static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { - UNUSED(zone); UNUSED(size); + MI_UNUSED(zone); MI_UNUSED(size); mi_collect(false); return 0; } static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { - UNUSED(size); + MI_UNUSED(size); zone_free(zone,p); } static boolean_t zone_claimed_address(malloc_zone_t* zone, void* p) { - UNUSED(zone); + MI_UNUSED(zone); return mi_is_in_heap_region(p); } @@ -126,43 +126,43 @@ static kern_return_t intro_enumerator(task_t task, void* p, vm_range_recorder_t recorder) { // todo: enumerate all memory - UNUSED(task); UNUSED(p); UNUSED(type_mask); UNUSED(zone_address); - UNUSED(reader); UNUSED(recorder); + MI_UNUSED(task); MI_UNUSED(p); MI_UNUSED(type_mask); MI_UNUSED(zone_address); + MI_UNUSED(reader); MI_UNUSED(recorder); return KERN_SUCCESS; } static size_t intro_good_size(malloc_zone_t* zone, size_t size) { - UNUSED(zone); + MI_UNUSED(zone); return mi_good_size(size); } static boolean_t intro_check(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); return true; } static void intro_print(malloc_zone_t* zone, boolean_t verbose) { - UNUSED(zone); UNUSED(verbose); + MI_UNUSED(zone); MI_UNUSED(verbose); mi_stats_print(NULL); } static void intro_log(malloc_zone_t* zone, void* p) { - UNUSED(zone); UNUSED(p); + MI_UNUSED(zone); MI_UNUSED(p); // todo? } static void intro_force_lock(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // todo? } static void intro_force_unlock(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // todo? } static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { - UNUSED(zone); + MI_UNUSED(zone); // todo... stats->blocks_in_use = 0; stats->size_in_use = 0; @@ -171,7 +171,7 @@ static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { } static boolean_t intro_zone_locked(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); return false; } @@ -261,7 +261,7 @@ mi_decl_externc void _malloc_fork_child(void); static malloc_zone_t* mi_malloc_create_zone(vm_size_t size, unsigned flags) { - UNUSED(size); UNUSED(flags); + MI_UNUSED(size); MI_UNUSED(flags); return mi_get_default_zone(); } @@ -274,12 +274,12 @@ static malloc_zone_t* mi_malloc_default_purgeable_zone(void) { } static void mi_malloc_destroy_zone(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); // nothing. } static kern_return_t mi_malloc_get_all_zones (task_t task, memory_reader_t mr, vm_address_t** addresses, unsigned* count) { - UNUSED(task); UNUSED(mr); + MI_UNUSED(task); MI_UNUSED(mr); if (addresses != NULL) *addresses = NULL; if (count != NULL) *count = 0; return KERN_SUCCESS; @@ -290,11 +290,11 @@ static const char* mi_malloc_get_zone_name(malloc_zone_t* zone) { } static void mi_malloc_set_zone_name(malloc_zone_t* zone, const char* name) { - UNUSED(zone); UNUSED(name); + MI_UNUSED(zone); MI_UNUSED(name); } static int mi_malloc_jumpstart(uintptr_t cookie) { - UNUSED(cookie); + MI_UNUSED(cookie); return 1; // or 0 for no error? } @@ -309,37 +309,37 @@ static void mi__malloc_fork_child(void) { } static void mi_malloc_printf(const char* fmt, ...) { - UNUSED(fmt); + MI_UNUSED(fmt); } static bool zone_check(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); return true; } static malloc_zone_t* zone_from_ptr(const void* p) { - UNUSED(p); + MI_UNUSED(p); return mi_get_default_zone(); } static void zone_log(malloc_zone_t* zone, void* p) { - UNUSED(zone); UNUSED(p); + MI_UNUSED(zone); MI_UNUSED(p); } static void zone_print(malloc_zone_t* zone, bool b) { - UNUSED(zone); UNUSED(b); + MI_UNUSED(zone); MI_UNUSED(b); } static void zone_print_ptr_info(void* p) { - UNUSED(p); + MI_UNUSED(p); } static void zone_register(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); } static void zone_unregister(malloc_zone_t* zone) { - UNUSED(zone); + MI_UNUSED(zone); } // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` diff --git a/src/alloc-override.c b/src/alloc-override.c index fa04b460..42fecbb3 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -147,8 +147,8 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n) - void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } - void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { MI_UNUSED(tag); return mi_new_nothrow(n); } #if (__cplusplus >= 201402L || _MSC_VER >= 1916) void operator delete (void* p, std::size_t n) noexcept MI_FORWARD02(mi_free_size,p,n) @@ -187,21 +187,21 @@ typedef struct mi_nothrow_s { int _tag; } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit - void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit - void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } - void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) - void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #else #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif diff --git a/src/alloc.c b/src/alloc.c index d03207ce..1dbd31a7 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -123,7 +123,7 @@ extern inline mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) { // note: we need to initialize the whole usable block size to zero, not just the requested size, // or the recalloc/rezalloc functions cannot safely expand in place (see issue #63) - UNUSED(size); + MI_UNUSED(size); mi_assert_internal(p != NULL); mi_assert_internal(mi_usable_size(p) >= size); // size can be zero mi_assert_internal(_mi_ptr_page(p)==page); @@ -205,8 +205,8 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block } #else static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { - UNUSED(page); - UNUSED(block); + MI_UNUSED(page); + MI_UNUSED(block); return false; } #endif @@ -278,19 +278,19 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co } #else static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - UNUSED(page); - UNUSED(block); + MI_UNUSED(page); + MI_UNUSED(block); } static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) { - UNUSED(block); + MI_UNUSED(block); return mi_page_usable_block_size(page); } static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { - UNUSED(page); - UNUSED(block); - UNUSED(min_size); + MI_UNUSED(page); + MI_UNUSED(block); + MI_UNUSED(min_size); } #endif @@ -298,7 +298,7 @@ static void mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, co #if (MI_STAT>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { #if (MI_STAT < 2) - UNUSED(block); + MI_UNUSED(block); #endif mi_heap_t* const heap = mi_heap_get_default(); const size_t bsize = mi_page_usable_block_size(page); @@ -315,7 +315,7 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { } #else static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { - UNUSED(page); UNUSED(block); + MI_UNUSED(page); MI_UNUSED(block); } #endif @@ -333,7 +333,7 @@ static void mi_stat_huge_free(const mi_page_t* page) { } #else static void mi_stat_huge_free(const mi_page_t* page) { - UNUSED(page); + MI_UNUSED(page); } #endif @@ -447,7 +447,7 @@ static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, bool l // (and secure mode) if this was a valid pointer. static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg) { - UNUSED(msg); + MI_UNUSED(msg); #if (MI_DEBUG>0) if (mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); @@ -482,7 +482,7 @@ void mi_free(void* p) mi_attr_noexcept const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; - const uintptr_t tid = _mi_thread_id(); + const mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; @@ -574,19 +574,19 @@ void* _mi_externs[] = { // ------------------------------------------------------ void mi_free_size(void* p, size_t size) mi_attr_noexcept { - UNUSED_RELEASE(size); + MI_UNUSED_RELEASE(size); mi_assert(p == NULL || size <= _mi_usable_size(p,"mi_free_size")); mi_free(p); } void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - UNUSED_RELEASE(alignment); + MI_UNUSED_RELEASE(alignment); mi_assert(((uintptr_t)p % alignment) == 0); mi_free_size(p,size); } void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { - UNUSED_RELEASE(alignment); + MI_UNUSED_RELEASE(alignment); mi_assert(((uintptr_t)p % alignment) == 0); mi_free(p); } diff --git a/src/arena.c b/src/arena.c index 5e883b28..a87a53a3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -396,7 +396,7 @@ int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t } int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept { - UNUSED(max_secs); + MI_UNUSED(max_secs); _mi_warning_message("mi_reserve_huge_os_pages is deprecated: use mi_reserve_huge_os_pages_interleave/at instead\n"); if (pages_reserved != NULL) *pages_reserved = 0; int err = mi_reserve_huge_os_pages_interleave(pages, 0, (size_t)(max_secs * 1000.0)); diff --git a/src/bitmap.c b/src/bitmap.c index 3b5c8199..7936e07f 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -122,7 +122,7 @@ bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, m const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); const uintptr_t mask = mi_bitmap_mask_(count, bitidx); - mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); // mi_assert_internal((bitmap[idx] & mask) == mask); uintptr_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); return ((prev & mask) == mask); @@ -135,7 +135,7 @@ bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); const uintptr_t mask = mi_bitmap_mask_(count, bitidx); - mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); uintptr_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); if (any_zero != NULL) *any_zero = ((prev & mask) != mask); @@ -147,7 +147,7 @@ static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); const uintptr_t mask = mi_bitmap_mask_(count, bitidx); - mi_assert_internal(bitmap_fields > idx); UNUSED(bitmap_fields); + mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); uintptr_t field = mi_atomic_load_relaxed(&bitmap[idx]); if (any_ones != NULL) *any_ones = ((field & mask) != 0); return ((field & mask) == mask); @@ -281,7 +281,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm // Helper for masks across fields; returns the mid count, post_mask may be 0 static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, uintptr_t* pre_mask, uintptr_t* mid_mask, uintptr_t* post_mask) { - UNUSED_RELEASE(bitmap_fields); + MI_UNUSED_RELEASE(bitmap_fields); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) { *pre_mask = mi_bitmap_mask_(count, bitidx); diff --git a/src/heap.c b/src/heap.c index bbf4cef9..d560fbc6 100644 --- a/src/heap.c +++ b/src/heap.c @@ -50,9 +50,9 @@ static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void #if MI_DEBUG>=2 static bool mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - UNUSED(arg1); - UNUSED(arg2); - UNUSED(pq); + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(pq); mi_assert_internal(mi_page_heap(page) == heap); mi_segment_t* segment = _mi_page_segment(page); mi_assert_internal(segment->thread_id == heap->thread_id); @@ -86,8 +86,8 @@ typedef enum mi_collect_e { static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { - UNUSED(arg2); - UNUSED(heap); + MI_UNUSED(arg2); + MI_UNUSED(heap); mi_assert_internal(mi_heap_page_is_valid(heap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); _mi_page_free_collect(page, collect >= MI_FORCE); @@ -104,10 +104,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t } static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - UNUSED(arg1); - UNUSED(arg2); - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(heap); + MI_UNUSED(pq); _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); return true; // don't break } @@ -262,10 +262,10 @@ static void mi_heap_free(mi_heap_t* heap) { ----------------------------------------------------------- */ static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - UNUSED(arg1); - UNUSED(arg2); - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(arg1); + MI_UNUSED(arg2); + MI_UNUSED(heap); + MI_UNUSED(pq); // ensure no more thread_delayed_free will be added _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE, false); @@ -422,8 +422,8 @@ bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) { - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(heap); + MI_UNUSED(pq); bool* found = (bool*)vfound; mi_segment_t* segment = _mi_page_segment(page); void* start = _mi_page_start(segment, page, NULL); @@ -521,8 +521,8 @@ typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { - UNUSED(heap); - UNUSED(pq); + MI_UNUSED(heap); + MI_UNUSED(pq); mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; mi_heap_area_ex_t xarea; const size_t bsize = mi_page_block_size(page); diff --git a/src/init.c b/src/init.c index db3d24ba..77b0c07f 100644 --- a/src/init.c +++ b/src/init.c @@ -450,7 +450,7 @@ static void mi_process_load(void) { mi_heap_main_init(); #if defined(MI_TLS_RECURSE_GUARD) volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; - UNUSED(dummy); + MI_UNUSED(dummy); #endif os_preloading = false; atexit(&mi_process_done); @@ -509,7 +509,7 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_os_memory)) { long ksize = mi_option_get(mi_option_reserve_os_memory); if (ksize > 0) { - mi_reserve_os_memory((size_t)ksize*KiB, true, true); + mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } } @@ -548,8 +548,8 @@ static void mi_process_done(void) { #if defined(_WIN32) && defined(MI_SHARED_LIB) // Windows DLL: easy to hook into process_init and thread_done __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { - UNUSED(reserved); - UNUSED(inst); + MI_UNUSED(reserved); + MI_UNUSED(inst); if (reason==DLL_PROCESS_ATTACH) { mi_process_load(); } diff --git a/src/options.c b/src/options.c index e3d6c8c9..66da8372 100644 --- a/src/options.c +++ b/src/options.c @@ -103,7 +103,7 @@ void _mi_options_init(void) { mi_add_stderr_output(); // now it safe to use stderr for output for(int i = 0; i < _mi_option_last; i++ ) { mi_option_t option = (mi_option_t)i; - long l = mi_option_get(option); UNUSED(l); // initialize + long l = mi_option_get(option); MI_UNUSED(l); // initialize if (option != mi_option_verbose) { mi_option_desc_t* desc = &options[option]; _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value); @@ -161,7 +161,7 @@ void mi_option_disable(mi_option_t option) { static void mi_out_stderr(const char* msg, void* arg) { - UNUSED(arg); + MI_UNUSED(arg); #ifdef _WIN32 // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. @@ -182,7 +182,7 @@ static char out_buf[MI_MAX_DELAY_OUTPUT+1]; static _Atomic(uintptr_t) out_len; static void mi_out_buf(const char* msg, void* arg) { - UNUSED(arg); + MI_UNUSED(arg); if (msg==NULL) return; if (mi_atomic_load_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; size_t n = strlen(msg); @@ -353,7 +353,7 @@ static mi_error_fun* volatile mi_error_handler; // = NULL static _Atomic(void*) mi_error_arg; // = NULL static void mi_error_default(int err) { - UNUSED(err); + MI_UNUSED(err); #if (MI_DEBUG>0) if (err==EFAULT) { #ifdef _MSC_VER @@ -411,9 +411,9 @@ static void mi_strlcat(char* dest, const char* src, size_t dest_size) { #ifdef MI_NO_GETENV static bool mi_getenv(const char* name, char* result, size_t result_size) { - UNUSED(name); - UNUSED(result); - UNUSED(result_size); + MI_UNUSED(name); + MI_UNUSED(result); + MI_UNUSED(result_size); return false; } #else @@ -521,9 +521,9 @@ static void mi_option_init(mi_option_desc_t* desc) { if (desc->option == mi_option_reserve_os_memory) { // this option is interpreted in KiB to prevent overflow of `long` if (*end == 'K') { end++; } - else if (*end == 'M') { value *= KiB; end++; } - else if (*end == 'G') { value *= MiB; end++; } - else { value = (value + KiB - 1) / KiB; } + else if (*end == 'M') { value *= MI_KiB; end++; } + else if (*end == 'G') { value *= MI_MiB; end++; } + else { value = (value + MI_KiB - 1) / MI_KiB; } if (*end == 'B') { end++; } } if (*end == 0) { diff --git a/src/os.c b/src/os.c index 426ead4a..b4853689 100644 --- a/src/os.c +++ b/src/os.c @@ -122,11 +122,11 @@ static bool use_large_os_page(size_t size, size_t alignment) { // round to a good OS allocation size (bounded by max 12.5% waste) size_t _mi_os_good_alloc_size(size_t size) { size_t align_size; - if (size < 512*KiB) align_size = _mi_os_page_size(); - else if (size < 2*MiB) align_size = 64*KiB; - else if (size < 8*MiB) align_size = 256*KiB; - else if (size < 32*MiB) align_size = 1*MiB; - else align_size = 4*MiB; + if (size < 512*MI_KiB) align_size = _mi_os_page_size(); + else if (size < 2*MI_MiB) align_size = 64*MI_KiB; + else if (size < 8*MI_MiB) align_size = 256*MI_KiB; + else if (size < 32*MI_MiB) align_size = 1*MI_MiB; + else align_size = 4*MI_MiB; if (mi_unlikely(size >= (SIZE_MAX - align_size))) return size; // possible overflow? return _mi_align_up(size, align_size); } @@ -263,7 +263,7 @@ void _mi_os_init() { os_page_size = (size_t)result; os_alloc_granularity = os_page_size; } - large_os_page_size = 2*MiB; // TODO: can we query the OS for this? + large_os_page_size = 2*MI_MiB; // TODO: can we query the OS for this? os_detect_overcommit(); } #endif @@ -414,7 +414,7 @@ static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { - UNUSED(try_alignment); + MI_UNUSED(try_alignment); #if defined(MAP_ALIGNED) // BSD if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); @@ -494,7 +494,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif #ifdef MAP_HUGE_1GB static bool mi_huge_pages_available = true; - if ((size % GiB) == 0 && mi_huge_pages_available) { + if ((size % MI_GiB) == 0 && mi_huge_pages_available) { lflags |= MAP_HUGE_1GB; } else @@ -582,7 +582,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; if ((size%MI_SEGMENT_SIZE) != 0) return NULL; - if (size > 1*GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. + if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. #endif @@ -605,7 +605,7 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) // no need for mi_os_get_aligned_hint #else static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { - UNUSED(try_alignment); UNUSED(size); + MI_UNUSED(try_alignment); MI_UNUSED(size); return NULL; } #endif @@ -732,7 +732,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, ----------------------------------------------------------- */ void* _mi_os_alloc(size_t size, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); @@ -741,7 +741,7 @@ void* _mi_os_alloc(size_t size, mi_stats_t* tld_stats) { } void _mi_os_free_ex(void* p, size_t size, bool was_committed, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (size == 0 || p == NULL) return; size = _mi_os_good_alloc_size(size); @@ -754,7 +754,7 @@ void _mi_os_free(void* p, size_t size, mi_stats_t* stats) { void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* large, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); if (size == 0) return NULL; size = _mi_os_good_alloc_size(size); alignment = _mi_align_up(alignment, _mi_os_page_size()); @@ -805,7 +805,7 @@ static void mi_mprotect_hint(int err) { " > sudo sysctl -w vm.max_map_count=262144\n"); } #else - UNUSED(err); + MI_UNUSED(err); #endif } @@ -867,13 +867,13 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ } bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; return mi_os_commitx(addr, size, true, false /* liberal */, is_zero, stats); } bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; bool is_zero; return mi_os_commitx(addr, size, false, true /* conservative */, &is_zero, stats); @@ -942,7 +942,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) // pages and reduce swapping while keeping the memory committed. // We page align to a conservative area inside the range to reset. bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (mi_option_is_enabled(mi_option_reset_decommits)) { return _mi_os_decommit(addr, size, stats); @@ -953,7 +953,7 @@ bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats) { } bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { - UNUSED(tld_stats); + MI_UNUSED(tld_stats); mi_stats_t* stats = &_mi_stats_main; if (mi_option_is_enabled(mi_option_reset_decommits)) { return mi_os_commit_unreset(addr, size, is_zero, stats); // re-commit it (conservatively!) @@ -1029,12 +1029,12 @@ bool _mi_os_shrink(void* p, size_t oldsize, size_t newsize, mi_stats_t* stats) { Support for allocating huge OS pages (1Gib) that are reserved up-front and possibly associated with a specific NUMA node. (use `numa_node>=0`) -----------------------------------------------------------------------------*/ -#define MI_HUGE_OS_PAGE_SIZE (GiB) +#define MI_HUGE_OS_PAGE_SIZE (MI_GiB) #if defined(_WIN32) && (MI_INTPTR_SIZE >= 8) static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - mi_assert_internal(size%GiB == 0); + mi_assert_internal(size%MI_GiB == 0); mi_assert_internal(addr != NULL); const DWORD flags = MEM_LARGE_PAGES | MEM_COMMIT | MEM_RESERVE; @@ -1075,7 +1075,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1); } #else - UNUSED(numa_node); + MI_UNUSED(numa_node); #endif // otherwise use regular virtual alloc on older windows return VirtualAlloc(addr, size, flags, PAGE_READWRITE); @@ -1092,12 +1092,12 @@ static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, cons } #else static long mi_os_mbind(void* start, unsigned long len, unsigned long mode, const unsigned long* nmask, unsigned long maxnode, unsigned flags) { - UNUSED(start); UNUSED(len); UNUSED(mode); UNUSED(nmask); UNUSED(maxnode); UNUSED(flags); + MI_UNUSED(start); MI_UNUSED(len); MI_UNUSED(mode); MI_UNUSED(nmask); MI_UNUSED(maxnode); MI_UNUSED(flags); return 0; } #endif static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - mi_assert_internal(size%GiB == 0); + mi_assert_internal(size%MI_GiB == 0); bool is_large = true; void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); if (p == NULL) return NULL; @@ -1115,7 +1115,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) } #else static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) { - UNUSED(addr); UNUSED(size); UNUSED(numa_node); + MI_UNUSED(addr); MI_UNUSED(size); MI_UNUSED(numa_node); return NULL; } #endif @@ -1151,7 +1151,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { } #else static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { - UNUSED(pages); + MI_UNUSED(pages); if (total_size != NULL) *total_size = 0; return NULL; } @@ -1352,7 +1352,7 @@ size_t _mi_os_numa_node_count_get(void) { } int _mi_os_numa_node_get(mi_os_tld_t* tld) { - UNUSED(tld); + MI_UNUSED(tld); size_t numa_count = _mi_os_numa_node_count(); if (numa_count<=1) return 0; // optimize on single numa node systems: always node 0 // never more than the node count and >= 0 diff --git a/src/page.c b/src/page.c index 61f3245a..c0659395 100644 --- a/src/page.c +++ b/src/page.c @@ -30,7 +30,7 @@ terms of the MIT license. A copy of the license can be found in the file // Index a block in a page static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_start, size_t block_size, size_t i) { - UNUSED(page); + MI_UNUSED(page); mi_assert_internal(page != NULL); mi_assert_internal(i <= page->reserved); return (mi_block_t*)((uint8_t*)page_start + (i * block_size)); @@ -459,7 +459,7 @@ void _mi_heap_collect_retired(mi_heap_t* heap, bool force) { #define MI_MIN_SLICES (2) static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) { - UNUSED(stats); + MI_UNUSED(stats); #if (MI_SECURE<=2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); @@ -517,7 +517,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* co static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend, mi_stats_t* const stats) { - UNUSED(stats); + MI_UNUSED(stats); #if (MI_SECURE <= 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); diff --git a/src/region.c b/src/region.c index d9232450..d99b74af 100644 --- a/src/region.c +++ b/src/region.c @@ -57,9 +57,9 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo // Constants #if (MI_INTPTR_SIZE==8) -#define MI_HEAP_REGION_MAX_SIZE (256 * GiB) // 64KiB for the region map +#define MI_HEAP_REGION_MAX_SIZE (256 * MI_GiB) // 64KiB for the region map #elif (MI_INTPTR_SIZE==4) -#define MI_HEAP_REGION_MAX_SIZE (3 * GiB) // ~ KiB for the region map +#define MI_HEAP_REGION_MAX_SIZE (3 * MI_GiB) // ~ KiB for the region map #else #error "define the maximum heap space allowed for regions on this platform" #endif @@ -190,7 +190,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, if (idx >= MI_REGION_MAX) { mi_atomic_decrement_acq_rel(®ions_count); _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats); - _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, GiB)); + _mi_warning_message("maximum regions used: %zu GiB (perhaps recompile with a larger setting for MI_HEAP_REGION_MAX_SIZE)", _mi_divide_up(MI_HEAP_REGION_MAX_SIZE, MI_GiB)); return false; } @@ -441,7 +441,7 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re // and unclaim bool all_unclaimed = mi_bitmap_unclaim(®ion->in_use, 1, blocks, bit_idx); - mi_assert_internal(all_unclaimed); UNUSED(all_unclaimed); + mi_assert_internal(all_unclaimed); MI_UNUSED(all_unclaimed); } } diff --git a/src/segment.c b/src/segment.c index 64b4720c..a853bf41 100644 --- a/src/segment.c +++ b/src/segment.c @@ -696,7 +696,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, mi_page_kind_t page_kind, } static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { - UNUSED(force); + MI_UNUSED(force); mi_assert(segment != NULL); // note: don't reset pages even on abandon as the whole segment is freed? (and ready for reuse) bool force_reset = (force && mi_option_is_enabled(mi_option_abandoned_page_reset)); From 09e59e06104f800d2a154df4ba4aa908e5f26945 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 13 Nov 2021 15:13:16 -0800 Subject: [PATCH 074/180] use size_t instead of uintptr_t where appropiate --- src/arena.c | 8 ++--- src/bitmap.c | 82 +++++++++++++++++++++++----------------------- src/bitmap.h | 10 +++--- src/init.c | 2 +- src/options.c | 14 ++++---- src/os.c | 16 ++++----- src/region.c | 24 +++++++------- src/segment.c | 14 ++++---- test/test-stress.c | 2 +- 9 files changed, 86 insertions(+), 86 deletions(-) diff --git a/src/arena.c b/src/arena.c index a87a53a3..133f1546 100644 --- a/src/arena.c +++ b/src/arena.c @@ -20,7 +20,7 @@ which is sometimes needed for embedded devices or shared memory for example. The arena allocation needs to be thread safe and we use an atomic bitmap to allocate. The current implementation of the bitmap can -only do this within a field (`uintptr_t`) so we can allocate at most +only do this within a field (`size_t`) so we can allocate at most blocks of 2GiB (64*32MiB) and no object can cross the boundary. This can lead to fragmentation but fortunately most objects will be regions of 256MiB in practice. @@ -64,7 +64,7 @@ typedef struct mi_arena_s { bool is_zero_init; // is the arena zero initialized? bool allow_decommit; // is decommit allowed? if true, is_large should be false and blocks_committed != NULL bool is_large; // large- or huge OS pages (always committed) - _Atomic(uintptr_t) search_idx; // optimization to start the search for free blocks + _Atomic(size_t) search_idx; // optimization to start the search for free blocks mi_bitmap_field_t* blocks_dirty; // are the blocks potentially non-zero? mi_bitmap_field_t* blocks_committed; // are the blocks committed? (can be NULL for memory that cannot be decommitted) mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`) @@ -73,7 +73,7 @@ typedef struct mi_arena_s { // The available arenas static mi_decl_cache_align _Atomic(mi_arena_t*) mi_arenas[MI_MAX_ARENAS]; -static mi_decl_cache_align _Atomic(uintptr_t) mi_arena_count; // = 0 +static mi_decl_cache_align _Atomic(size_t) mi_arena_count; // = 0 /* ----------------------------------------------------------- @@ -272,7 +272,7 @@ static bool mi_arena_add(mi_arena_t* arena) { mi_assert_internal((uintptr_t)mi_atomic_load_ptr_relaxed(uint8_t,&arena->start) % MI_SEGMENT_ALIGN == 0); mi_assert_internal(arena->block_count > 0); - uintptr_t i = mi_atomic_increment_acq_rel(&mi_arena_count); + size_t i = mi_atomic_increment_acq_rel(&mi_arena_count); if (i >= MI_MAX_ARENAS) { mi_atomic_decrement_acq_rel(&mi_arena_count); return false; diff --git a/src/bitmap.c b/src/bitmap.c index 7936e07f..51926bbd 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`uintptr_t`) +represeted as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -26,12 +26,12 @@ between the fields. (This is used in arena allocation) ----------------------------------------------------------- */ // The bit mask for a given number of blocks at a specified bit index. -static inline uintptr_t mi_bitmap_mask_(size_t count, size_t bitidx) { +static inline size_t mi_bitmap_mask_(size_t count, size_t bitidx) { mi_assert_internal(count + bitidx <= MI_BITMAP_FIELD_BITS); mi_assert_internal(count > 0); if (count >= MI_BITMAP_FIELD_BITS) return MI_BITMAP_FIELD_FULL; if (count == 0) return 0; - return ((((uintptr_t)1 << count) - 1) << bitidx); + return ((((size_t)1 << count) - 1) << bitidx); } @@ -46,27 +46,27 @@ bool _mi_bitmap_try_find_claim_field(mi_bitmap_t bitmap, size_t idx, const size_ { mi_assert_internal(bitmap_idx != NULL); mi_assert_internal(count <= MI_BITMAP_FIELD_BITS); - _Atomic(uintptr_t)* field = &bitmap[idx]; - uintptr_t map = mi_atomic_load_relaxed(field); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t map = mi_atomic_load_relaxed(field); if (map==MI_BITMAP_FIELD_FULL) return false; // short cut // search for 0-bit sequence of length count - const uintptr_t mask = mi_bitmap_mask_(count, 0); - const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; + const size_t mask = mi_bitmap_mask_(count, 0); + const size_t bitidx_max = MI_BITMAP_FIELD_BITS - count; #ifdef MI_HAVE_FAST_BITSCAN size_t bitidx = mi_ctz(~map); // quickly find the first zero bit if possible #else size_t bitidx = 0; // otherwise start at 0 #endif - uintptr_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx + size_t m = (mask << bitidx); // invariant: m == mask shifted by bitidx // scan linearly for a free range of zero bits while (bitidx <= bitidx_max) { - const uintptr_t mapm = map & m; + const size_t mapm = map & m; if (mapm == 0) { // are the mask bits free at bitidx? mi_assert_internal((m >> bitidx) == mask); // no overflow? - const uintptr_t newmap = map | m; + const size_t newmap = map | m; mi_assert_internal((newmap^map) >> bitidx == mask); if (!mi_atomic_cas_weak_acq_rel(field, &map, newmap)) { // TODO: use strong cas here? // no success, another thread claimed concurrently.. keep going (with updated `map`) @@ -121,10 +121,10 @@ bool _mi_bitmap_try_find_claim(mi_bitmap_t bitmap, const size_t bitmap_fields, c bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + const size_t mask = mi_bitmap_mask_(count, bitidx); mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); // mi_assert_internal((bitmap[idx] & mask) == mask); - uintptr_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); + size_t prev = mi_atomic_and_acq_rel(&bitmap[idx], ~mask); return ((prev & mask) == mask); } @@ -134,10 +134,10 @@ bool mi_bitmap_unclaim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, m bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_zero) { const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + const size_t mask = mi_bitmap_mask_(count, bitidx); mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); //mi_assert_internal(any_zero != NULL || (bitmap[idx] & mask) == 0); - uintptr_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); + size_t prev = mi_atomic_or_acq_rel(&bitmap[idx], mask); if (any_zero != NULL) *any_zero = ((prev & mask) != mask); return ((prev & mask) == 0); } @@ -146,9 +146,9 @@ bool _mi_bitmap_claim(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi static bool mi_bitmap_is_claimedx(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* any_ones) { const size_t idx = mi_bitmap_index_field(bitmap_idx); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); - const uintptr_t mask = mi_bitmap_mask_(count, bitidx); + const size_t mask = mi_bitmap_mask_(count, bitidx); mi_assert_internal(bitmap_fields > idx); MI_UNUSED(bitmap_fields); - uintptr_t field = mi_atomic_load_relaxed(&bitmap[idx]); + size_t field = mi_atomic_load_relaxed(&bitmap[idx]); if (any_ones != NULL) *any_ones = ((field & mask) != 0); return ((field & mask) == mask); } @@ -176,8 +176,8 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit mi_assert_internal(bitmap_idx != NULL); // check initial trailing zeros - _Atomic(uintptr_t)* field = &bitmap[idx]; - uintptr_t map = mi_atomic_load_relaxed(field); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t map = mi_atomic_load_relaxed(field); const size_t initial = mi_clz(map); // count of initial zeros starting at idx mi_assert_internal(initial <= MI_BITMAP_FIELD_BITS); if (initial == 0) return false; @@ -186,11 +186,11 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit // scan ahead size_t found = initial; - uintptr_t mask = 0; // mask bits for the final field + size_t mask = 0; // mask bits for the final field while(found < count) { field++; map = mi_atomic_load_relaxed(field); - const uintptr_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); + const size_t mask_bits = (found + MI_BITMAP_FIELD_BITS <= count ? MI_BITMAP_FIELD_BITS : (count - found)); mask = mi_bitmap_mask_(mask_bits, 0); if ((map & mask) != 0) return false; found += mask_bits; @@ -199,13 +199,13 @@ static bool mi_bitmap_try_find_claim_field_across(mi_bitmap_t bitmap, size_t bit // found range of zeros up to the final field; mask contains mask in the final field // now claim it atomically - _Atomic(uintptr_t)* const final_field = field; - const uintptr_t final_mask = mask; - _Atomic(uintptr_t)* const initial_field = &bitmap[idx]; - const uintptr_t initial_mask = mi_bitmap_mask_(initial, MI_BITMAP_FIELD_BITS - initial); + mi_bitmap_field_t* const final_field = field; + const size_t final_mask = mask; + mi_bitmap_field_t* const initial_field = &bitmap[idx]; + const size_t initial_mask = mi_bitmap_mask_(initial, MI_BITMAP_FIELD_BITS - initial); // initial field - uintptr_t newmap; + size_t newmap; field = initial_field; map = mi_atomic_load_relaxed(field); do { @@ -280,7 +280,7 @@ bool _mi_bitmap_try_find_from_claim_across(mi_bitmap_t bitmap, const size_t bitm } // Helper for masks across fields; returns the mid count, post_mask may be 0 -static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, uintptr_t* pre_mask, uintptr_t* mid_mask, uintptr_t* post_mask) { +static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_fields, size_t count, size_t* pre_mask, size_t* mid_mask, size_t* post_mask) { MI_UNUSED_RELEASE(bitmap_fields); const size_t bitidx = mi_bitmap_index_bit_in_field(bitmap_idx); if (mi_likely(bitidx + count <= MI_BITMAP_FIELD_BITS)) { @@ -308,13 +308,13 @@ static size_t mi_bitmap_mask_across(mi_bitmap_index_t bitmap_idx, size_t bitmap_ // Returns `true` if all `count` bits were 1 previously. bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx) { size_t idx = mi_bitmap_index_field(bitmap_idx); - uintptr_t pre_mask; - uintptr_t mid_mask; - uintptr_t post_mask; + size_t pre_mask; + size_t mid_mask; + size_t post_mask; size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); bool all_one = true; - _Atomic(uintptr_t)*field = &bitmap[idx]; - uintptr_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t prev = mi_atomic_and_acq_rel(field++, ~pre_mask); if ((prev & pre_mask) != pre_mask) all_one = false; while(mid_count-- > 0) { prev = mi_atomic_and_acq_rel(field++, ~mid_mask); @@ -331,14 +331,14 @@ bool _mi_bitmap_unclaim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t // Returns `true` if all `count` bits were 0 previously. `any_zero` is `true` if there was at least one zero bit. bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_zero) { size_t idx = mi_bitmap_index_field(bitmap_idx); - uintptr_t pre_mask; - uintptr_t mid_mask; - uintptr_t post_mask; + size_t pre_mask; + size_t mid_mask; + size_t post_mask; size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); bool all_zero = true; bool any_zero = false; - _Atomic(uintptr_t)*field = &bitmap[idx]; - uintptr_t prev = mi_atomic_or_acq_rel(field++, pre_mask); + _Atomic(size_t)*field = &bitmap[idx]; + size_t prev = mi_atomic_or_acq_rel(field++, pre_mask); if ((prev & pre_mask) != 0) all_zero = false; if ((prev & pre_mask) != pre_mask) any_zero = true; while (mid_count-- > 0) { @@ -360,14 +360,14 @@ bool _mi_bitmap_claim_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t co // `any_ones` is `true` if there was at least one bit set to one. static bool mi_bitmap_is_claimedx_across(mi_bitmap_t bitmap, size_t bitmap_fields, size_t count, mi_bitmap_index_t bitmap_idx, bool* pany_ones) { size_t idx = mi_bitmap_index_field(bitmap_idx); - uintptr_t pre_mask; - uintptr_t mid_mask; - uintptr_t post_mask; + size_t pre_mask; + size_t mid_mask; + size_t post_mask; size_t mid_count = mi_bitmap_mask_across(bitmap_idx, bitmap_fields, count, &pre_mask, &mid_mask, &post_mask); bool all_ones = true; bool any_ones = false; - _Atomic(uintptr_t)* field = &bitmap[idx]; - uintptr_t prev = mi_atomic_load_relaxed(field++); + mi_bitmap_field_t* field = &bitmap[idx]; + size_t prev = mi_atomic_load_relaxed(field++); if ((prev & pre_mask) != pre_mask) all_ones = false; if ((prev & pre_mask) != 0) any_ones = true; while (mid_count-- > 0) { diff --git a/src/bitmap.h b/src/bitmap.h index 21fd4e13..39ca55b2 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file /* ---------------------------------------------------------------------------- Concurrent bitmap that can set/reset sequences of bits atomically, -represeted as an array of fields where each field is a machine word (`uintptr_t`) +represeted as an array of fields where each field is a machine word (`size_t`) There are two api's; the standard one cannot have sequences that cross between the bitmap fields (and a sequence must be <= MI_BITMAP_FIELD_BITS). @@ -24,11 +24,11 @@ between the fields. (This is used in arena allocation) Bitmap definition ----------------------------------------------------------- */ -#define MI_BITMAP_FIELD_BITS (8*MI_INTPTR_SIZE) -#define MI_BITMAP_FIELD_FULL (~((uintptr_t)0)) // all bits set +#define MI_BITMAP_FIELD_BITS (8*MI_SIZE_SIZE) +#define MI_BITMAP_FIELD_FULL (~((size_t)0)) // all bits set -// An atomic bitmap of `uintptr_t` fields -typedef _Atomic(uintptr_t) mi_bitmap_field_t; +// An atomic bitmap of `size_t` fields +typedef _Atomic(size_t) mi_bitmap_field_t; typedef mi_bitmap_field_t* mi_bitmap_t; // A bitmap index is the index of the bit in a bitmap. diff --git a/src/init.c b/src/init.c index 77b0c07f..f9c73a3f 100644 --- a/src/init.c +++ b/src/init.c @@ -332,7 +332,7 @@ bool _mi_is_main_thread(void) { return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); } -static _Atomic(uintptr_t) thread_count = ATOMIC_VAR_INIT(1); +static _Atomic(size_t) thread_count = ATOMIC_VAR_INIT(1); size_t _mi_current_thread_count(void) { return mi_atomic_load_relaxed(&thread_count); diff --git a/src/options.c b/src/options.c index 66da8372..9549e3ad 100644 --- a/src/options.c +++ b/src/options.c @@ -19,8 +19,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif -static uintptr_t mi_max_error_count = 16; // stop outputting errors after this -static uintptr_t mi_max_warning_count = 16; // stop outputting warnings after this +static size_t mi_max_error_count = 16; // stop outputting errors after this +static size_t mi_max_warning_count = 16; // stop outputting warnings after this static void mi_add_stderr_output(void); @@ -176,10 +176,10 @@ static void mi_out_stderr(const char* msg, void* arg) { // an output function is registered it is called immediately with // the output up to that point. #ifndef MI_MAX_DELAY_OUTPUT -#define MI_MAX_DELAY_OUTPUT ((uintptr_t)(32*1024)) +#define MI_MAX_DELAY_OUTPUT ((size_t)(32*1024)) #endif static char out_buf[MI_MAX_DELAY_OUTPUT+1]; -static _Atomic(uintptr_t) out_len; +static _Atomic(size_t) out_len; static void mi_out_buf(const char* msg, void* arg) { MI_UNUSED(arg); @@ -188,7 +188,7 @@ static void mi_out_buf(const char* msg, void* arg) { size_t n = strlen(msg); if (n==0) return; // claim space - uintptr_t start = mi_atomic_add_acq_rel(&out_len, n); + size_t start = mi_atomic_add_acq_rel(&out_len, n); if (start >= MI_MAX_DELAY_OUTPUT) return; // check bound if (start+n >= MI_MAX_DELAY_OUTPUT) { @@ -251,8 +251,8 @@ static void mi_add_stderr_output() { // -------------------------------------------------------- // Messages, all end up calling `_mi_fputs`. // -------------------------------------------------------- -static _Atomic(uintptr_t) error_count; // = 0; // when >= max_error_count stop emitting errors -static _Atomic(uintptr_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings +static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors +static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings // When overriding malloc, we may recurse into mi_vfprintf if an allocation // inside the C runtime causes another message. diff --git a/src/os.c b/src/os.c index b4853689..67b6c6db 100644 --- a/src/os.c +++ b/src/os.c @@ -340,11 +340,11 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) { mi_assert_internal(!(large_only && !allow_large)); - static _Atomic(uintptr_t) large_page_try_ok; // = 0; + static _Atomic(size_t) large_page_try_ok; // = 0; void* p = NULL; if ((large_only || use_large_os_page(size, try_alignment)) && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { - uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. // therefore, once a large page allocation failed, we don't try again for `large_page_try_ok` times. @@ -474,8 +474,8 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif // huge page allocation if ((large_only || use_large_os_page(size, try_alignment)) && allow_large) { - static _Atomic(uintptr_t) large_page_try_ok; // = 0; - uintptr_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); + static _Atomic(size_t) large_page_try_ok; // = 0; + size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { // If the OS is not configured for large OS pages, or the user does not have // enough permission, the `mmap` will always fail (but it might also fail for other reasons). @@ -521,7 +521,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif if (large_only) return p; if (p == NULL) { - mi_atomic_store_release(&large_page_try_ok, (uintptr_t)8); // on error, don't try again for the next N allocations + mi_atomic_store_release(&large_page_try_ok, (size_t)8); // on error, don't try again for the next N allocations } } } @@ -914,13 +914,13 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) if (p != start) return false; #else #if defined(MADV_FREE) - static _Atomic(uintptr_t) advice = ATOMIC_VAR_INIT(MADV_FREE); + static _Atomic(size_t) advice = ATOMIC_VAR_INIT(MADV_FREE); int oadvice = (int)mi_atomic_load_relaxed(&advice); int err; while ((err = madvise(start, csize, oadvice)) != 0 && errno == EAGAIN) { errno = 0; }; if (err != 0 && errno == EINVAL && oadvice == MADV_FREE) { // if MADV_FREE is not supported, fall back to MADV_DONTNEED from now on - mi_atomic_store_release(&advice, (uintptr_t)MADV_DONTNEED); + mi_atomic_store_release(&advice, (size_t)MADV_DONTNEED); err = madvise(start, csize, MADV_DONTNEED); } #elif defined(__wasi__) @@ -1102,7 +1102,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) void* p = mi_unix_mmap(addr, size, MI_SEGMENT_SIZE, PROT_READ | PROT_WRITE, true, true, &is_large); if (p == NULL) return NULL; if (numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes - uintptr_t numa_mask = (1UL << numa_node); + unsigned long numa_mask = (1UL << numa_node); // TODO: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: diff --git a/src/region.c b/src/region.c index d99b74af..f864f73b 100644 --- a/src/region.c +++ b/src/region.c @@ -74,7 +74,7 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, boo // Region info typedef union mi_region_info_u { - uintptr_t value; + size_t value; struct { bool valid; // initialized? bool is_large:1; // allocated in fixed large/huge OS pages @@ -87,21 +87,21 @@ typedef union mi_region_info_u { // A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with // a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block. typedef struct mem_region_s { - _Atomic(uintptr_t) info; // mi_region_info_t.value + _Atomic(size_t) info; // mi_region_info_t.value _Atomic(void*) start; // start of the memory area mi_bitmap_field_t in_use; // bit per in-use block mi_bitmap_field_t dirty; // track if non-zero per block mi_bitmap_field_t commit; // track if committed per block mi_bitmap_field_t reset; // track if reset per block - _Atomic(uintptr_t) arena_memid; // if allocated from a (huge page) arena - uintptr_t padding; // round to 8 fields + _Atomic(size_t) arena_memid; // if allocated from a (huge page) arena + size_t padding; // round to 8 fields } mem_region_t; // The region map static mem_region_t regions[MI_REGION_MAX]; // Allocated regions -static _Atomic(uintptr_t) regions_count; // = 0; +static _Atomic(size_t) regions_count; // = 0; /* ---------------------------------------------------------------------------- @@ -186,7 +186,7 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, mi_assert_internal(!region_large || region_commit); // claim a fresh slot - const uintptr_t idx = mi_atomic_increment_acq_rel(®ions_count); + const size_t idx = mi_atomic_increment_acq_rel(®ions_count); if (idx >= MI_REGION_MAX) { mi_atomic_decrement_acq_rel(®ions_count); _mi_arena_free(start, MI_REGION_SIZE, arena_memid, region_commit, tld->stats); @@ -197,10 +197,10 @@ static bool mi_region_try_alloc_os(size_t blocks, bool commit, bool allow_large, // allocated, initialize and claim the initial blocks mem_region_t* r = ®ions[idx]; r->arena_memid = arena_memid; - mi_atomic_store_release(&r->in_use, (uintptr_t)0); + mi_atomic_store_release(&r->in_use, (size_t)0); mi_atomic_store_release(&r->dirty, (is_zero ? 0 : MI_BITMAP_FIELD_FULL)); mi_atomic_store_release(&r->commit, (region_commit ? MI_BITMAP_FIELD_FULL : 0)); - mi_atomic_store_release(&r->reset, (uintptr_t)0); + mi_atomic_store_release(&r->reset, (size_t)0); *bit_idx = 0; _mi_bitmap_claim(&r->in_use, 1, blocks, *bit_idx, NULL); mi_atomic_store_ptr_release(void,&r->start, start); @@ -451,21 +451,21 @@ void _mi_mem_free(void* p, size_t size, size_t id, bool full_commit, bool any_re -----------------------------------------------------------------------------*/ void _mi_mem_collect(mi_os_tld_t* tld) { // free every region that has no segments in use. - uintptr_t rcount = mi_atomic_load_relaxed(®ions_count); + size_t rcount = mi_atomic_load_relaxed(®ions_count); for (size_t i = 0; i < rcount; i++) { mem_region_t* region = ®ions[i]; if (mi_atomic_load_relaxed(®ion->info) != 0) { // if no segments used, try to claim the whole region - uintptr_t m = mi_atomic_load_relaxed(®ion->in_use); + size_t m = mi_atomic_load_relaxed(®ion->in_use); while (m == 0 && !mi_atomic_cas_weak_release(®ion->in_use, &m, MI_BITMAP_FIELD_FULL)) { /* nothing */ }; if (m == 0) { // on success, free the whole region uint8_t* start = (uint8_t*)mi_atomic_load_ptr_acquire(uint8_t,®ions[i].start); size_t arena_memid = mi_atomic_load_relaxed(®ions[i].arena_memid); - uintptr_t commit = mi_atomic_load_relaxed(®ions[i].commit); + size_t commit = mi_atomic_load_relaxed(®ions[i].commit); memset((void*)®ions[i], 0, sizeof(mem_region_t)); // cast to void* to avoid atomic warning // and release the whole region - mi_atomic_store_release(®ion->info, (uintptr_t)0); + mi_atomic_store_release(®ion->info, (size_t)0); if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) { _mi_abandoned_await_readers(); // ensure no pending reads _mi_arena_free(start, MI_REGION_SIZE, arena_memid, (~commit == 0), tld->stats); diff --git a/src/segment.c b/src/segment.c index a853bf41..6de3a8ae 100644 --- a/src/segment.c +++ b/src/segment.c @@ -897,13 +897,13 @@ static mi_decl_cache_align _Atomic(mi_segment_t*) abandoned_visited; // = static mi_decl_cache_align _Atomic(mi_tagged_segment_t) abandoned; // = NULL // Maintain these for debug purposes (these counts may be a bit off) -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_count; -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_visited_count; +static mi_decl_cache_align _Atomic(size_t) abandoned_count; +static mi_decl_cache_align _Atomic(size_t) abandoned_visited_count; // We also maintain a count of current readers of the abandoned list // in order to prevent resetting/decommitting segment memory if it might // still be read. -static mi_decl_cache_align _Atomic(uintptr_t) abandoned_readers; // = 0 +static mi_decl_cache_align _Atomic(size_t) abandoned_readers; // = 0 // Push on the visited list static void mi_abandoned_visited_push(mi_segment_t* segment) { @@ -932,7 +932,7 @@ static bool mi_abandoned_visited_revisit(void) mi_tagged_segment_t afirst; mi_tagged_segment_t ts = mi_atomic_load_relaxed(&abandoned); if (mi_tagged_segment_ptr(ts)==NULL) { - uintptr_t count = mi_atomic_load_relaxed(&abandoned_visited_count); + size_t count = mi_atomic_load_relaxed(&abandoned_visited_count); afirst = mi_tagged_segment(first, ts); if (mi_atomic_cas_strong_acq_rel(&abandoned, &ts, afirst)) { mi_atomic_add_relaxed(&abandoned_count, count); @@ -951,7 +951,7 @@ static bool mi_abandoned_visited_revisit(void) // and atomically prepend to the abandoned list // (no need to increase the readers as we don't access the abandoned segments) mi_tagged_segment_t anext = mi_atomic_load_relaxed(&abandoned); - uintptr_t count; + size_t count; do { count = mi_atomic_load_relaxed(&abandoned_visited_count); mi_atomic_store_ptr_release(mi_segment_t, &last->abandoned_next, mi_tagged_segment_ptr(anext)); @@ -979,7 +979,7 @@ static void mi_abandoned_push(mi_segment_t* segment) { // Wait until there are no more pending reads on segments that used to be in the abandoned list void _mi_abandoned_await_readers(void) { - uintptr_t n; + size_t n; do { n = mi_atomic_load_acquire(&abandoned_readers); if (n != 0) mi_atomic_yield(); @@ -1327,7 +1327,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block // claim it and free mi_heap_t* heap = mi_heap_get_default(); // issue #221; don't use the internal get_default_heap as we need to ensure the thread is initialized. // paranoia: if this it the last reference, the cas should always succeed - uintptr_t expected_tid = 0; + size_t expected_tid = 0; if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected_tid, heap->thread_id)) { mi_block_set_next(page, block, page->free); page->free = block; diff --git a/test/test-stress.c b/test/test-stress.c index d45e9899..498b7ec6 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -25,7 +25,7 @@ terms of the MIT license. // // argument defaults static int THREADS = 32; // more repeatable if THREADS <= #processors -static int SCALE = 10; // scaling factor +static int SCALE = 25; // scaling factor static int ITER = 50; // N full iterations destructing and re-creating all threads // static int THREADS = 8; // more repeatable if THREADS <= #processors From 4b0cda2012ba0088639afc899d024f4e5e0514e4 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Nov 2021 11:23:11 -0800 Subject: [PATCH 075/180] take overcommit into account for eager commit delay --- include/mimalloc-internal.h | 1 + src/os.c | 6 +++++- src/segment.c | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 23078330..058acf80 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -72,6 +72,7 @@ void _mi_os_init(void); // called fro void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data size_t _mi_os_good_alloc_size(size_t size); +bool _mi_os_has_overcommit(void); // memory.c void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, size_t* id, mi_os_tld_t* tld); diff --git a/src/os.c b/src/os.c index 67b6c6db..ee6e92c0 100644 --- a/src/os.c +++ b/src/os.c @@ -101,6 +101,10 @@ static size_t large_os_page_size = 0; // set dynamically in _mi_os_init (and if true we use MAP_NORESERVE) static bool os_overcommit = true; +bool _mi_os_has_overcommit(void) { + return os_overcommit; +} + // OS (small) page size size_t _mi_os_page_size() { return os_page_size; @@ -460,7 +464,7 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro #endif int flags = MAP_PRIVATE | MAP_ANONYMOUS; int fd = -1; - if (os_overcommit) { + if (_mi_os_has_overcommit()) { flags |= MAP_NORESERVE; } #if defined(PROT_MAX) diff --git a/src/segment.c b/src/segment.c index 6de3a8ae..424495ca 100644 --- a/src/segment.c +++ b/src/segment.c @@ -579,7 +579,9 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_ mi_assert_internal(segment_size >= required); // Initialize parameters - const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && _mi_current_thread_count() > 2 && + const bool eager_delayed = (page_kind <= MI_PAGE_MEDIUM && // don't delay for large objects + !_mi_os_has_overcommit() && // never delay on overcommit systems + _mi_current_thread_count() > 2 && // do not delay for the first N threads tld->count < (size_t)mi_option_get(mi_option_eager_commit_delay)); const bool eager = !eager_delayed && mi_option_is_enabled(mi_option_eager_commit); bool commit = eager; // || (page_kind >= MI_PAGE_LARGE); From 4ce6821c09506d29feabdb8cfc4ac1d94b6574b6 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:32:21 -0800 Subject: [PATCH 076/180] update cmake install for MI_INSTALL_TOPLEVEL --- cmake/mimalloc-config.cmake | 6 +++--- test/CMakeLists.txt | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 024c97d9..9b0a56d7 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -4,8 +4,8 @@ if (MIMALLOC_SHARE_DIR MATCHES "/share/") string(REPLACE "/share/" "/lib/" MIMALLOC_LIBRARY_DIR ${MIMALLOC_SHARE_DIR}) string(REPLACE "/share/" "/include/" MIMALLOC_INCLUDE_DIR ${MIMALLOC_SHARE_DIR}) else() - # if MI_INSTALL_TOPLEVEL==ON - set(MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}/lib") - set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}/include") + # installed with -DMI_INSTALL_TOPLEVEL=ON + string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}") + string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}") endif() set(MIMALLOC_TARGET_DIR "${MIMALLOC_LIBRARY_DIR}") # legacy diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7986d2da..13bf32de 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,9 @@ cmake_minimum_required(VERSION 3.0) project(mimalloc-test C CXX) +set(CMAKE_C_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) + # Set default build type if (NOT CMAKE_BUILD_TYPE) if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$") From d67ff1ca9fbffee067f5bbfa8b0c6e6e90437ec2 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:32:43 -0800 Subject: [PATCH 077/180] add include cstdef for std::size_t in C++ --- include/mimalloc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/mimalloc.h b/include/mimalloc.h index a77f1dc5..0a4ae689 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -388,6 +388,7 @@ mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, // --------------------------------------------------------------------------------------------- #ifdef __cplusplus +#include // std::size_t #include // PTRDIFF_MAX #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 #include // std::true_type From 07253fb44c3c220c93dbb64c6dd12c5d7608fb4a Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:33:04 -0800 Subject: [PATCH 078/180] emit error message if out-of-memory in C++ --- src/alloc.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 1dbd31a7..d9b6dd60 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -802,7 +802,10 @@ static bool mi_try_new_handler(bool nothrow) { std::set_new_handler(h); #endif if (h==NULL) { - if (!nothrow) throw std::bad_alloc(); + _mi_error_message(ENOMEM, "out of memory in 'new'"); + if (!nothrow) { + throw std::bad_alloc(); + } return false; } else { @@ -830,9 +833,9 @@ static std_new_handler_t mi_get_new_handler() { static bool mi_try_new_handler(bool nothrow) { std_new_handler_t h = mi_get_new_handler(); if (h==NULL) { + _mi_error_message(ENOMEM, "out of memory in 'new'"); if (!nothrow) { - _mi_error_message(EFAULT, "out of memory in 'new' call"); // cannot throw in plain C, use EFAULT to abort - abort(); + abort(); // cannot throw in plain C, use abort } return false; } From 4a590b1447e9d27e00dcd5bc9ac62eab6d7e51bb Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 14 Nov 2021 15:33:56 -0800 Subject: [PATCH 079/180] bump version number to 1.7.3 --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 0a4ae689..4467173e 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 171 // major + 2 digits minor +#define MI_MALLOC_VERSION 173 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes From 6ca454a04a272a2a2f1309db3b8a050049734a14 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Nov 2021 16:41:24 -0800 Subject: [PATCH 080/180] update readme --- readme.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/readme.md b/readme.md index cdb1b82a..2369b9c1 100644 --- a/readme.md +++ b/readme.md @@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the run-time systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release tag: `v2.0.2` (beta, 2021-06-17). -Latest stable tag: `v1.7.2` (2021-06-17). +Latest release tag: `v2.0.3` (beta, 2021-11-14). +Latest stable tag: `v1.7.3` (2021-11-14). mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -77,6 +77,10 @@ Note: the `v2.x` beta has a new algorithm for managing internal mimalloc pages t and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance (see [below](#performance)); please report if you observe any significant performance regression. +* 2021-11-14, `v1.7.3`, `v2.0.3` (beta): improved WASM support, improved macOS support and performance (including + M1), improved performance for v2 for large objects, Python integration improvements, more standard + installation directories, various small fixes. + * 2021-06-17, `v1.7.2`, `v2.0.2` (beta): support M1, better installation layout on Linux, fix thread_id on Android, prefer 2-6TiB area for aligned allocation to work better on pre-windows 8, various small fixes. @@ -142,7 +146,7 @@ mimalloc is used in various large scale low-latency services and programs, for e ## Windows -Open `ide/vs2019/mimalloc.sln` in Visual Studio 2019 and build (or `ide/vs2017/mimalloc.sln`). +Open `ide/vs2019/mimalloc.sln` in Visual Studio 2019 and build. The `mimalloc` project builds a static library (in `out/msvc-x64`), while the `mimalloc-override` project builds a DLL for overriding malloc in the entire program. @@ -191,6 +195,11 @@ Notes: 2. Install CCMake: `sudo apt-get install cmake-curses-gui` +## Single source + +You can also directly build the single `src/static.c` file as part of your project without +needing `cmake` at all. Make sure to also add the mimalloc `include` directory to the include path. + # Using the library @@ -337,9 +346,9 @@ When _mimalloc_ is built using debug mode, various checks are done at runtime to - Corrupted free-lists and some forms of use-after-free are detected. -# Overriding Malloc +# Overriding Standard Malloc -Overriding the standard `malloc` can be done either _dynamically_ or _statically_. +Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. ## Dynamic override @@ -370,13 +379,12 @@ On macOS we can also preload the mimalloc shared library so all calls to the standard `malloc` interface are resolved to the _mimalloc_ library. ``` -> env DYLD_FORCE_FLAT_NAMESPACE=1 DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram +> env DYLD_INSERT_LIBRARIES=/usr/lib/libmimalloc.dylib myprogram ``` Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -(Note: macOS support for dynamic overriding is recent, please report any issues.) ### Override on Windows @@ -386,7 +394,7 @@ the (dynamic) C runtime allocator, including those from other DLL's or libraries The overriding on Windows requires that you link your program explicitly with the mimalloc DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be available +Also, the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) must be put in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency). The redirection DLL ensures that all calls to the C runtime malloc API get redirected to mimalloc (in `mimalloc-override.dll`). From 0560fc27c08d28d523b7f741a42deb26cd01c0c6 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Nov 2021 16:47:50 -0800 Subject: [PATCH 081/180] update readme --- readme.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 1586bddc..dfef82d5 100644 --- a/readme.md +++ b/readme.md @@ -311,8 +311,9 @@ or via environment variables: `MIMALLOC_EAGER_COMMIT_DELAY=N` (`N` is 1 by default) to delay the initial `N` segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset). -- `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N`: where N is the numa node. This reserves the huge pages at a specific numa node. - (`N` is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected)) + The huge pages are usually allocated evenly among NUMA nodes. + We can use `MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N` where `N` is the numa node (starting at 0) to allocate all + the huge pages at a specific numa node instead. Use caution when using `fork` in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the From 09c658dd401a9a4a7a2c807f6bd274d0420e109d Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 15 Nov 2021 10:10:47 -0800 Subject: [PATCH 082/180] try to allocate within our hint space (partially addresses issue #360) --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index ee6e92c0..90773fc6 100644 --- a/src/os.c +++ b/src/os.c @@ -585,7 +585,7 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; - if ((size%MI_SEGMENT_SIZE) != 0) return NULL; + size = _mi_align_up(size, MI_SEGMENT_SIZE); if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. From 1c22650719cc09cdeba10b9add7af4b1e9b2ef2d Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 15 Nov 2021 10:52:09 -0800 Subject: [PATCH 083/180] remove no-invalid-memory-model supression --- CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 87f4cddd..7c2c6cfc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -181,10 +181,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) if(NOT MI_USE_CXX) list(APPEND mi_cflags -Wstrict-prototypes) - endif() - if(CMAKE_C_COMPILER_ID MATCHES "GNU") - list(APPEND mi_cflags -Wno-invalid-memory-model) - endif() + endif() if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wpedantic -Wno-static-in-inline) endif() From 8b60a5ab70b342ca1ff404a4ee6c5340ccfb47a6 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Nov 2021 17:59:27 -0800 Subject: [PATCH 084/180] add mi_unsafe_free_with_threadid and mi_get_current_threadid() --- include/mimalloc-internal.h | 3 +++ include/mimalloc.h | 2 ++ src/alloc.c | 25 +++++++++++++++++++++---- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf5b6783..1333c80f 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -20,14 +20,17 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) #define mi_decl_noinline __declspec(noinline) +#define mi_decl_always_inline __forceinline #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) +#define mi_decl_always_inline __attribute__((always_inline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) #else #define mi_decl_noinline +#define mi_decl_always_inline inline #define mi_decl_thread __thread // hope for the best :-) #define mi_decl_cache_align #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index 756e516c..f90af4a3 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -271,6 +271,8 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; +mi_decl_export size_t mi_get_current_threadid(void) mi_attr_noexcept; +mi_decl_export void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept; // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; diff --git a/src/alloc.c b/src/alloc.c index d9b6dd60..4d0d9e59 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -475,14 +475,12 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms return segment; } - -// Free a block -void mi_free(void* p) mi_attr_noexcept +// Free a block with a known threadid +static mi_decl_always_inline void _mi_free_with_threadid(void* p, mi_threadid_t tid) mi_attr_noexcept { const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; - const mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; @@ -507,6 +505,25 @@ void mi_free(void* p) mi_attr_noexcept } } +// Get the current thread id +size_t mi_get_current_threadid(void) mi_attr_noexcept { + return _mi_thread_id(); +} + +// Free a block passing the current thread id explicitly +void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept +{ + mi_assert(current_tid == _mi_thread_id()); + _mi_free_with_threadid(p,current_tid); +} + + +// Free a block +void mi_free(void* p) mi_attr_noexcept { + _mi_free_with_threadid(p, _mi_thread_id()); +} + + bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); From 3548d8d716569f96967af4da0fdc57d2f09e7c38 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Nov 2021 18:39:03 -0800 Subject: [PATCH 085/180] add noinline to avoid warnings --- src/alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 4d0d9e59..e9167848 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -511,7 +511,7 @@ size_t mi_get_current_threadid(void) mi_attr_noexcept { } // Free a block passing the current thread id explicitly -void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept +void mi_decl_noinline mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept { mi_assert(current_tid == _mi_thread_id()); _mi_free_with_threadid(p,current_tid); @@ -519,7 +519,7 @@ void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept // Free a block -void mi_free(void* p) mi_attr_noexcept { +void mi_decl_noinline mi_free(void* p) mi_attr_noexcept { _mi_free_with_threadid(p, _mi_thread_id()); } From 9183b1eec005be9863e58d51ba0f96b97721d48c Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 23 Nov 2021 19:04:41 -0800 Subject: [PATCH 086/180] remove experiment with unsafe_free_with_threadid --- include/mimalloc-internal.h | 3 --- include/mimalloc.h | 3 --- src/alloc.c | 24 +++--------------------- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1333c80f..cf5b6783 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -20,17 +20,14 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(_MSC_VER) #pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths) #define mi_decl_noinline __declspec(noinline) -#define mi_decl_always_inline __forceinline #define mi_decl_thread __declspec(thread) #define mi_decl_cache_align __declspec(align(MI_CACHE_LINE)) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #define mi_decl_noinline __attribute__((noinline)) -#define mi_decl_always_inline __attribute__((always_inline)) #define mi_decl_thread __thread #define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE))) #else #define mi_decl_noinline -#define mi_decl_always_inline inline #define mi_decl_thread __thread // hope for the best :-) #define mi_decl_cache_align #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index f90af4a3..06a16703 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -271,9 +271,6 @@ mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; -mi_decl_export size_t mi_get_current_threadid(void) mi_attr_noexcept; -mi_decl_export void mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept; - // deprecated mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; diff --git a/src/alloc.c b/src/alloc.c index e9167848..ca32caba 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -475,12 +475,13 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms return segment; } -// Free a block with a known threadid -static mi_decl_always_inline void _mi_free_with_threadid(void* p, mi_threadid_t tid) mi_attr_noexcept +// Free a block +void mi_free(void* p) mi_attr_noexcept { const mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if (mi_unlikely(segment == NULL)) return; + mi_threadid_t tid = _mi_thread_id(); mi_page_t* const page = _mi_segment_page_of(segment, p); mi_block_t* const block = (mi_block_t*)p; @@ -505,25 +506,6 @@ static mi_decl_always_inline void _mi_free_with_threadid(void* p, mi_threadid_t } } -// Get the current thread id -size_t mi_get_current_threadid(void) mi_attr_noexcept { - return _mi_thread_id(); -} - -// Free a block passing the current thread id explicitly -void mi_decl_noinline mi_unsafe_free_with_threadid(void* p, size_t current_tid ) mi_attr_noexcept -{ - mi_assert(current_tid == _mi_thread_id()); - _mi_free_with_threadid(p,current_tid); -} - - -// Free a block -void mi_decl_noinline mi_free(void* p) mi_attr_noexcept { - _mi_free_with_threadid(p, _mi_thread_id()); -} - - bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page const mi_segment_t* const segment = _mi_ptr_segment(block); From 0be71a2cac17062bd8913cbd272c472a44331b7f Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 24 Nov 2021 12:54:54 -0800 Subject: [PATCH 087/180] fix prefix --- src/os.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/os.c b/src/os.c index 90773fc6..603ec04a 100644 --- a/src/os.c +++ b/src/os.c @@ -578,29 +578,29 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) -#define KK_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start -#define KK_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) -#define KK_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) +#define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start +#define MI_HINT_AREA ((uintptr_t)4 << 40) // upto 6TiB (since before win8 there is "only" 8TiB available to processes) +#define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; size = _mi_align_up(size, MI_SEGMENT_SIZE); - if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(KK_HINT_AREA / 1<<30) = 1/4096. + if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) size += MI_SEGMENT_SIZE; // put in `MI_SEGMENT_SIZE` virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. #endif uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); - if (hint == 0 || hint > KK_HINT_MAX) { // wrap or initialize - uintptr_t init = KK_HINT_BASE; + if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize + uintptr_t init = MI_HINT_BASE; #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode uintptr_t r = _mi_heap_random_next(mi_get_default_heap()); - init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % KK_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > KK_HINT_MAX but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all } if (hint%try_alignment != 0) return NULL; return (void*)hint; @@ -638,11 +638,11 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo if (commit) flags |= MEM_COMMIT; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); #elif defined(MI_USE_SBRK) - KK_UNUSED(allow_large); + MI_UNUSED(allow_large); *is_large = false; p = mi_sbrk_heap_grow(size, try_alignment); #elif defined(__wasi__) - KK_UNUSED(allow_large); + MI_UNUSED(allow_large); *is_large = false; p = mi_wasm_heap_grow(size, try_alignment); #else From 20e37bf7c1ce89515b38bb82aa6b8b3838b9c456 Mon Sep 17 00:00:00 2001 From: DC Date: Sun, 28 Nov 2021 08:14:21 +0000 Subject: [PATCH 088/180] removes dragonflybsd warning since the api and library overriding works, also for the os doc report purpose. --- include/mimalloc-internal.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf5b6783..e22953d4 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -314,9 +314,6 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea // use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) // see #define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) -#elif defined(__DragonFly__) -#warning "mimalloc is not working correctly on DragonFly yet." -//#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) #endif #endif From acc64ee5cd58277490a89d4c4f97e3f3839beff5 Mon Sep 17 00:00:00 2001 From: Igor Vlasenko Date: Sun, 28 Nov 2021 19:14:31 +0200 Subject: [PATCH 089/180] added SOVERSION to mimalloc shared lib (issue #490) This is a portability improvement. A cross-platform library needs SOVERSION field for Unix platforms. With SOVERSION field cmake itself will do proper management of libmimalloc.so.SOVERSION -> libmimalloc.so.VERSION symlink on Unix, so a piece of code that tried to emulate this behavior manually is no more needed and is removed here too. --- CMakeLists.txt | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c2c6cfc..8eb5c958 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -280,7 +280,7 @@ message(STATUS "") # shared library if(MI_BUILD_SHARED) add_library(mimalloc SHARED ${mi_sources}) - set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} OUTPUT_NAME ${mi_basename} ) + set_target_properties(mimalloc PROPERTIES VERSION ${mi_version} SOVERSION ${mi_version_major} OUTPUT_NAME ${mi_basename} ) target_compile_definitions(mimalloc PRIVATE ${mi_defines} MI_SHARED_LIB MI_SHARED_LIB_EXPORT) target_compile_options(mimalloc PRIVATE ${mi_cflags}) target_link_libraries(mimalloc PUBLIC ${mi_libraries}) @@ -336,16 +336,6 @@ install(FILES include/mimalloc-new-delete.h DESTINATION ${mi_install_incdir}) install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_cmakedir}) install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_cmakedir}) -if(NOT WIN32 AND MI_BUILD_SHARED AND NOT MI_INSTALL_TOPLEVEL) - # install a symlink in the /usr/local/lib to the versioned library - # note: use delayed prefix expansion as \${CMAKE_INSTALL_PREFIX} - set(mi_symlink "${CMAKE_SHARED_MODULE_PREFIX}${mi_basename}${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(mi_soname "mimalloc-${mi_version}/${mi_symlink}.${mi_version}") - install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") - install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink} -> ${mi_soname}\")") - install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${mi_soname} ${mi_symlink}.${mi_version} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib)") - install(CODE "MESSAGE(\"-- Symbolic link: \${CMAKE_INSTALL_PREFIX}/lib/${mi_symlink}.${mi_version} -> ${mi_soname}\")") -endif() # single object file for more predictable static overriding if (MI_BUILD_OBJECT) From f82e13ac91716ff60d4e656e1f825ce727b7fd7d Mon Sep 17 00:00:00 2001 From: Igor Vlasenko Date: Sun, 28 Nov 2021 19:20:12 +0200 Subject: [PATCH 090/180] let the library VERSION = the project's one (issue #490) This is a cross-platform usability improvement. On Unix platforms it is customary for library to have VERSION and SOVERSION, where SOVERSION changes on major API changes and VERSION is the same as project's version, so library users always know what vesion this library belongs to just by name. With this patch we have a proper libmimalloc.so.VERSION on Unix. --- cmake/mimalloc-config-version.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index ed95c19e..3768d7de 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,7 @@ set(mi_version_major 1) set(mi_version_minor 7) -set(mi_version ${mi_version_major}.${mi_version_minor}) +set(mi_version_patch 3) +set(mi_version ${mi_version_major}.${mi_version_minor}.${mi_version_patch}) set(PACKAGE_VERSION ${mi_version}) if(PACKAGE_FIND_VERSION_MAJOR) From d6a56dd99d5abe4e1f6af555976ee8dc64e1ec8a Mon Sep 17 00:00:00 2001 From: David Carlier Date: Wed, 1 Dec 2021 22:43:13 +0000 Subject: [PATCH 091/180] introduces netbsd's reallocarr overriding. and couple of tests. --- include/mimalloc-override.h | 1 + include/mimalloc.h | 1 + src/alloc-override.c | 1 + src/alloc-posix.c | 14 ++++++++++++++ test/main-override.c | 8 +++++++- 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-override.h b/include/mimalloc-override.h index 2362bfbc..c00fc594 100644 --- a/include/mimalloc-override.h +++ b/include/mimalloc-override.h @@ -48,6 +48,7 @@ not accidentally mix pointers from different allocators). #define valloc(n) mi_valloc(n) #define pvalloc(n) mi_pvalloc(n) #define reallocarray(p,s,n) mi_reallocarray(p,s,n) +#define reallocarr(p,s,n) mi_reallocarr(p,s,n) #define memalign(a,n) mi_memalign(a,n) #define aligned_alloc(a,n) mi_aligned_alloc(a,n) #define posix_memalign(p,a,n) mi_posix_memalign(p,a,n) diff --git a/include/mimalloc.h b/include/mimalloc.h index a2835d02..414d628d 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -351,6 +351,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; diff --git a/src/alloc-override.c b/src/alloc-override.c index 5906bd20..d24cd8bc 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -183,6 +183,7 @@ size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p); void* valloc(size_t size) { return mi_valloc(size); } void* pvalloc(size_t size) { return mi_pvalloc(size); } void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } +int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(&p, count, size); } void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } diff --git a/src/alloc-posix.c b/src/alloc-posix.c index eef70ab5..6465304c 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -90,6 +90,20 @@ void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { return newp; } +int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD + void** op = (void** )p; + int serrno = errno; + void* newp = mi_reallocn(p,count,size); + if (mi_unlikely(newp == NULL)) { + errno = ENOMEM; + return errno; + } else { + *op = newp; + errno = serrno; + return 0; + } +} + void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft void* res = mi_expand(p, newsize); if (res == NULL) errno = ENOMEM; diff --git a/test/main-override.c b/test/main-override.c index 1bec1179..284fdd20 100644 --- a/test/main-override.c +++ b/test/main-override.c @@ -3,7 +3,7 @@ #include #include -#include +#include int main() { mi_version(); // ensure mimalloc library is linked @@ -25,6 +25,12 @@ int main() { //free(p1); //p2 = malloc(32); //mi_free(p2); + p1 = malloc(24); + p2 = reallocarray(p1, 16, 16); + free(p2); + p1 = malloc(24); + assert(reallocarr(&p1, 16, 16) == 0); + free(p1); mi_stats_print(NULL); return 0; } From 36edfbc70ae4610d06b9fce245f35d92e6be92a5 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 9 Dec 2021 16:18:17 -0800 Subject: [PATCH 092/180] use rtlgenrandom by default on windows --- src/random.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/random.c b/src/random.c index 05c5c99c..1220401a 100644 --- a/src/random.c +++ b/src/random.c @@ -167,8 +167,9 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim #if defined(_WIN32) -#if !defined(MI_USE_RTLGENRANDOM) -// We prefer BCryptGenRandom over RtlGenRandom +#if defined(MI_USE_BCRYPTGENRANDOM) +// We would like to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock +// under the VS debugger when using dynamic overriding. #pragma comment (lib,"bcrypt.lib") #include static bool os_random_buf(void* buf, size_t buf_len) { From 3a212d1895c8722a31a78d354793fbd6b2f3192f Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 12 Dec 2021 10:35:13 -0800 Subject: [PATCH 093/180] fix assembly for mi_tls_slot_set on x32 and x64. Issue #488 --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf5b6783..4713a75e 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -746,9 +746,9 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { #elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI + __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI #elif defined(__x86_64__) - __asm__("movq %1,%%fs:%1" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS + __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS #elif defined(__arm__) void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); From d575aacfde579c241580d61297ec2430c4994109 Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 13 Dec 2021 13:10:33 -0800 Subject: [PATCH 094/180] use find_library for pthread (issue #496) --- CMakeLists.txt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c2c6cfc..47b43ce5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,13 +210,14 @@ endif() if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() - if(NOT ${CMAKE_C_COMPILER} MATCHES "android") - list(APPEND mi_libraries pthread) - find_library(LIBRT rt) - if(LIBRT) - list(APPEND mi_libraries ${LIBRT}) - endif() + find_library(LIBPTHREAD pthread) + if (LIBPTHREAD) + list(APPEND mi_libraries ${LIBPTHREAD}) endif() + find_library(LIBRT rt) + if(LIBRT) + list(APPEND mi_libraries ${LIBRT}) + endif() endif() if (MI_USE_LIBATOMIC) @@ -270,6 +271,8 @@ else() message(STATUS "C Compiler : ${CMAKE_C_COMPILER}") endif() message(STATUS "Compiler flags : ${mi_cflags}") +message(STATUS "Compiler defines : ${mi_defines}") +message(STATUS "Link libraries : ${mi_libraries}") message(STATUS "Build targets : ${mi_build_targets}") message(STATUS "") From 69b6b246880d04e911416ffce8e8ccf6539ae2d7 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 14 Dec 2021 18:29:14 -0800 Subject: [PATCH 095/180] further improvements to installation directories --- CMakeLists.txt | 19 +++++++++++-------- cmake/mimalloc-config-version.cmake | 4 ++-- cmake/mimalloc-config.cmake | 19 +++++++++++-------- include/mimalloc.h | 2 +- test/CMakeLists.txt | 2 +- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3790b09..0ceadffd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -228,14 +228,17 @@ endif() # Install and output names # ----------------------------------------------------------------------------- +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") # for dynamic/shared library and symlinks + +# install at top level or use versioned directories for side-by-side installation? if (MI_INSTALL_TOPLEVEL) - set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") - set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") - set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc") + set(mi_install_objdir "${CMAKE_INSTALL_LIBDIR}") + set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") + set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc") else() - set(mi_install_libdir "lib/mimalloc-${mi_version}") - set(mi_install_incdir "include/mimalloc-${mi_version}") - set(mi_install_cmakedir "share/mimalloc-${mi_version}/cmake") + set(mi_install_objdir "${CMAKE_INSTALL_LIBDIR}/mimalloc-${mi_version}") # for static library and object files + set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}/mimalloc-${mi_version}") # for includes + set(mi_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/mimalloc-${mi_version}") # for cmake package info endif() if(MI_SECURE) @@ -329,7 +332,7 @@ if (MI_BUILD_STATIC) set_target_properties(mimalloc-static PROPERTIES OUTPUT_NAME ${mi_basename}) endif() - install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY) + install(TARGETS mimalloc-static EXPORT mimalloc DESTINATION ${mi_install_objdir} LIBRARY) endif() # install include files @@ -357,7 +360,7 @@ if (MI_BUILD_OBJECT) # the FILES expression can also be: $ # but that fails cmake versions less than 3.10 so we leave it as is for now install(FILES ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION} - DESTINATION ${mi_install_libdir} + DESTINATION ${mi_install_objdir} RENAME ${mi_basename}${CMAKE_C_OUTPUT_EXTENSION} ) endif() diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 3768d7de..7bbd7313 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,7 +1,7 @@ set(mi_version_major 1) set(mi_version_minor 7) -set(mi_version_patch 3) -set(mi_version ${mi_version_major}.${mi_version_minor}.${mi_version_patch}) +set(mi_version_patch 4) +set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) if(PACKAGE_FIND_VERSION_MAJOR) diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 9b0a56d7..45d9692f 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -1,11 +1,14 @@ include(${CMAKE_CURRENT_LIST_DIR}/mimalloc.cmake) -get_filename_component(MIMALLOC_SHARE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/share/mimalloc-2.0 -if (MIMALLOC_SHARE_DIR MATCHES "/share/") - string(REPLACE "/share/" "/lib/" MIMALLOC_LIBRARY_DIR ${MIMALLOC_SHARE_DIR}) - string(REPLACE "/share/" "/include/" MIMALLOC_INCLUDE_DIR ${MIMALLOC_SHARE_DIR}) -else() - # installed with -DMI_INSTALL_TOPLEVEL=ON - string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_SHARE_DIR}") - string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_SHARE_DIR}") +get_filename_component(MIMALLOC_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/lib/cmake/mimalloc-2.0 +get_filename_component(MIMALLOC_VER_DIR "${CMAKE_CURRENT_LIST_DIR}" NAME) +string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_CMAKE_DIR}") +if("${MIMALLOC_VER_DIR}" EQUAL "mimalloc") + # top level install + string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_CMAKE_DIR}") + set(MIMALLOC_OBJECT_DIR "${MIMALLOC_LIBRARY_DIR}") +else() + # versioned + string(REPLACE "/lib/cmake/" "/include/" MIMALLOC_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}") + string(REPLACE "/lib/cmake/" "/lib/" MIMALLOC_OBJECT_DIR "${CMAKE_CURRENT_LIST_DIR}") endif() set(MIMALLOC_TARGET_DIR "${MIMALLOC_LIBRARY_DIR}") # legacy diff --git a/include/mimalloc.h b/include/mimalloc.h index 06a16703..a955e35c 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 173 // major + 2 digits minor +#define MI_MALLOC_VERSION 174 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 13bf32de..407dbee9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -29,7 +29,7 @@ target_link_libraries(dynamic-override-cxx PUBLIC mimalloc) # overriding with a static object file works reliable as the symbols in the # object file have priority over those in library files -add_executable(static-override-obj main-override.c ${MIMALLOC_LIBRARY_DIR}/mimalloc.o) +add_executable(static-override-obj main-override.c ${MIMALLOC_OBJECT_DIR}/mimalloc.o) target_include_directories(static-override-obj PUBLIC ${MIMALLOC_INCLUDE_DIR}) target_link_libraries(static-override-obj PUBLIC pthread) From 73ced777dd11feeafc4664e8fe1814fa942481af Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 14 Dec 2021 18:42:10 -0800 Subject: [PATCH 096/180] upgrade macos to latest --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aeb2908b..7261fc13 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,7 +115,7 @@ jobs: displayName: macOS pool: vmImage: - macOS-10.14 + macOS-latest strategy: matrix: Debug: From 6503ad7a76dacfae142ed4f8f23d1039a18dede8 Mon Sep 17 00:00:00 2001 From: daan Date: Tue, 14 Dec 2021 18:45:44 -0800 Subject: [PATCH 097/180] check if using bcryptgenrandom fixes windows pipeline --- src/random.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/random.c b/src/random.c index 1220401a..fabd6c6e 100644 --- a/src/random.c +++ b/src/random.c @@ -167,8 +167,8 @@ If we cannot get good randomness, we fall back to weak randomness based on a tim #if defined(_WIN32) -#if defined(MI_USE_BCRYPTGENRANDOM) -// We would like to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock +#if !defined(MI_USE_RTLGENRANDOM) +// We prefer to use BCryptGenRandom instead of RtlGenRandom but it can lead to a deadlock // under the VS debugger when using dynamic overriding. #pragma comment (lib,"bcrypt.lib") #include From 2d9b8aa6b56241ff3cc88ed5d5e3d862d91fd096 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:33:14 -0800 Subject: [PATCH 098/180] rename VER_DIR to VERSION_DIR --- cmake/mimalloc-config.cmake | 4 ++-- test/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/mimalloc-config.cmake b/cmake/mimalloc-config.cmake index 45d9692f..8a28e37e 100644 --- a/cmake/mimalloc-config.cmake +++ b/cmake/mimalloc-config.cmake @@ -1,8 +1,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/mimalloc.cmake) get_filename_component(MIMALLOC_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}" PATH) # one up from the cmake dir, e.g. /usr/local/lib/cmake/mimalloc-2.0 -get_filename_component(MIMALLOC_VER_DIR "${CMAKE_CURRENT_LIST_DIR}" NAME) +get_filename_component(MIMALLOC_VERSION_DIR "${CMAKE_CURRENT_LIST_DIR}" NAME) string(REPLACE "/lib/cmake" "/lib" MIMALLOC_LIBRARY_DIR "${MIMALLOC_CMAKE_DIR}") -if("${MIMALLOC_VER_DIR}" EQUAL "mimalloc") +if("${MIMALLOC_VERSION_DIR}" EQUAL "mimalloc") # top level install string(REPLACE "/lib/cmake" "/include" MIMALLOC_INCLUDE_DIR "${MIMALLOC_CMAKE_DIR}") set(MIMALLOC_OBJECT_DIR "${MIMALLOC_LIBRARY_DIR}") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 407dbee9..054d9409 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,7 +17,7 @@ endif() # Import mimalloc (if installed) find_package(mimalloc 1.7 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) -message(STATUS "Found mimalloc installed at: ${MIMALLOC_LIBRARY_DIR}") +message(STATUS "Found mimalloc installed at: ${MIMALLOC_LIBRARY_DIR} (${MIMALLOC_VERSION_DIR})") # overriding with a dynamic library add_executable(dynamic-override main-override.c) From b7d33c2c33aa4475014f80602a09e5d348d5821e Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:41:52 -0800 Subject: [PATCH 099/180] fix MI_ prefix for libraries --- CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ceadffd..48ecd315 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,13 +210,13 @@ endif() if(WIN32) list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt) else() - find_library(LIBPTHREAD pthread) - if (LIBPTHREAD) - list(APPEND mi_libraries ${LIBPTHREAD}) + find_library(MI_LIBPTHREAD pthread) + if (MI_LIBPTHREAD) + list(APPEND mi_libraries ${MI_LIBPTHREAD}) endif() - find_library(LIBRT rt) - if(LIBRT) - list(APPEND mi_libraries ${LIBRT}) + find_library(MI_LIBRT rt) + if(MI_LIBRT) + list(APPEND mi_libraries ${MI_LIBRT}) endif() endif() @@ -228,9 +228,11 @@ endif() # Install and output names # ----------------------------------------------------------------------------- -set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") # for dynamic/shared library and symlinks +# dynamic/shared library and symlinks always go to /usr/local/lib equivalent +set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}") -# install at top level or use versioned directories for side-by-side installation? +# static libraries and object files, includes, and cmake config files +# are either installed at top level, or use versioned directories for side-by-side installation (default) if (MI_INSTALL_TOPLEVEL) set(mi_install_objdir "${CMAKE_INSTALL_LIBDIR}") set(mi_install_incdir "${CMAKE_INSTALL_INCLUDEDIR}") From 144b4a2d3e1535634b5e8a5d12fa9e64e44dadbd Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:42:58 -0800 Subject: [PATCH 100/180] link with libatomic automatically if found --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 48ecd315..6b2e9b83 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,10 +218,10 @@ else() if(MI_LIBRT) list(APPEND mi_libraries ${MI_LIBRT}) endif() -endif() - -if (MI_USE_LIBATOMIC) - list(APPEND mi_libraries atomic) + find_library(MI_LIBATOMIC atomic) + if (MI_LIBATOMIC OR MI_USE_LIBATOMIC) + list(APPEND mi_libraries atomic) + endif() endif() # ----------------------------------------------------------------------------- From ee3ae2a425c900cb3e2c7a71740c8657cdd109b0 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 08:53:55 -0800 Subject: [PATCH 101/180] add deprecated notice --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b2e9b83..4d52a72d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,9 +21,11 @@ option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) + +# deprecated options option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) -option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) -option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems)" OFF) +option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version (deprecated)" OFF) +option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) include(GNUInstallDirs) include("cmake/mimalloc-config-version.cmake") From 523e6d5c9b251bc8d969a685b2b6feda6fd1f8ab Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 16:05:07 -0800 Subject: [PATCH 102/180] fix thread slot on arm32 (issue #495) --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 4713a75e..324ad4a3 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -766,8 +766,8 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__BIONIC__) && (defined(__arm__) || defined(__aarch64__)) - // on Android, slot 1 is the thread ID (pointer to pthread internal struct) +#if defined(__arm__) || (defined(__BIONIC__) && defined(__aarch64__)) + // issue #384, #495: on arm32 and arm32/arm64 Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); #else // in all our other targets, slot 0 is the pointer to the thread control block From 9fbf83c4332da1cd1b788a7592d0f80ca52c6b1b Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 16:32:26 -0800 Subject: [PATCH 103/180] fix reallocarr indirection (see #492) --- src/alloc-posix.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 6627d2b7..2eaede07 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -92,27 +92,23 @@ mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_n void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD void* newp = mi_reallocn(p,count,size); - if (newp==NULL) errno = ENOMEM; + if (newp==NULL) { errno = ENOMEM; } return newp; } int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD - void** op = (void** )p; - int serrno = errno; - void* newp = mi_reallocn(p,count,size); - if (mi_unlikely(newp == NULL)) { - errno = ENOMEM; - return errno; - } else { - *op = newp; - errno = serrno; - return 0; - } + mi_assert(p != NULL); + if (p == NULL) return EINVAL; // should we set errno as well? + void** op = (void**)p; + void* newp = mi_reallocarray(*op, count, size); + if (mi_unlikely(newp == NULL)) return errno; + *op = newp; + return 0; } void* mi__expand(void* p, size_t newsize) mi_attr_noexcept { // Microsoft void* res = mi_expand(p, newsize); - if (res == NULL) errno = ENOMEM; + if (res == NULL) { errno = ENOMEM; } return res; } From a8b5106ce21bd8ac957249c5a67f80b6eee52187 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 16:33:49 -0800 Subject: [PATCH 104/180] fix address dereference in mi_reallocarr override --- src/alloc-override.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc-override.c b/src/alloc-override.c index eab202aa..6bbe4aac 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -245,7 +245,7 @@ extern "C" { void cfree(void* p) { mi_free(p); } void* pvalloc(size_t size) { return mi_pvalloc(size); } void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } -int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(&p, count, size); } +int reallocarr(void* p, size_t count, size_t size) { return mi_reallocarr(p, count, size); } void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } void* _aligned_malloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } From 317093d78b9dea347ceeaf15775c74e78e0e5fc7 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 16:48:57 -0800 Subject: [PATCH 105/180] fix Android overriding (issue #381 and PR #487) --- include/mimalloc-internal.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 324ad4a3..b8fc754f 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -317,6 +317,9 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #elif defined(__DragonFly__) #warning "mimalloc is not working correctly on DragonFly yet." //#define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) +#elif defined(__ANDROID__) +// See issue #381 +#define MI_TLS_PTHREAD #endif #endif @@ -766,7 +769,7 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__arm__) || (defined(__BIONIC__) && defined(__aarch64__)) +#if defined(__arm__) || (defined(__ANDROID__) && defined(__aarch64__)) // issue #384, #495: on arm32 and arm32/arm64 Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); #else From 0e851de2babfc32cfe85963f75c20f17dba6f4c5 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 17:06:09 -0800 Subject: [PATCH 106/180] shorten comments for pr #390 --- src/random.c | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/src/random.c b/src/random.c index ff1ecced..e47946a6 100644 --- a/src/random.c +++ b/src/random.c @@ -160,7 +160,7 @@ uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { /* ---------------------------------------------------------------------------- To initialize a fresh random context we rely on the OS: - Windows : BCryptGenRandom (or RtlGenRandom) -- macOS : CCRandomGenerateBytes +- macOS : CCRandomGenerateBytes, arc4random_buf - bsd,wasi : arc4random_buf - Linux : getrandom,/dev/urandom If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. @@ -194,34 +194,22 @@ static bool os_random_buf(void* buf, size_t buf_len) { #elif defined(__APPLE__) #include -#if defined(MAC_OS_X_VERSION_10_10) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 +#if defined(MAC_OS_X_VERSION_10_10) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_10 #include #endif static bool os_random_buf(void* buf, size_t buf_len) { - /* The implementation of arc4random_buf(3) differs from its documentation. - * It is documented as "always successful, and no return value is reserved - * to indicate an error." However, the actual implementation invokes the - * function "ccrng_generate" without validating the error cases. It might - * fail silently, which leads to unexpected source of entropy. - * See: - * https://opensource.apple.com/source/Libc/Libc-1439.40.11/gen/FreeBSD/arc4random.c.auto.html - * - * CCRandomGenerateBytes(), on the contrary, returns cryptographically strong - * random bits with explicit status code. - */ -#if defined(MAC_OS_X_VERSION_10_15) && \ - MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 - return CCRandomGenerateBytes(buf, buf_len) == kCCSuccess; -#else - /* Prior to macOS 10.15, CCRandomGenerateBytes() might take a bit longer time - * to complete, so failback to arc4random_buf(). - */ - arc4random_buf(buf, buf_len); - return true; -#endif + #if defined(MAC_OS_X_VERSION_10_15) && MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_15 + // We prefere CCRandomGenerateBytes as it returns an error code while arc4random_buf + // may fail silently on macOS. See PR #390, and + return (CCRandomGenerateBytes(buf, buf_len) == kCCSuccess); + #else + // fall back on older macOS + arc4random_buf(buf, buf_len); + return true; + #endif } -#elif defined(ANDROID) || defined(__DragonFly__) || \ + +#elif defined(__ANDROID__) || defined(__DragonFly__) || \ defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ defined(__sun) // todo: what to use with __wasi__? #include From 8612d1228a76c306898e784fe0a1829ab67d4cd2 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 15 Dec 2021 18:04:02 -0800 Subject: [PATCH 107/180] wip: initial fix for race condition on sbrk/wasm_memory_grow; issue #497 --- src/os.c | 87 +++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 32 deletions(-) diff --git a/src/os.c b/src/os.c index 603ec04a..dc2afc78 100644 --- a/src/os.c +++ b/src/os.c @@ -375,42 +375,69 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, return p; } -#elif defined(MI_USE_SBRK) +#elif defined(MI_USE_SBRK) || defined(__wasi__) + +#if defined(MI_USE_SBRK) #define MI_SBRK_FAIL ((void*)(-1)) -static void* mi_sbrk_heap_grow(size_t size, size_t try_alignment) { - void* pbase0 = sbrk(0); - if (pbase0 == MI_SBRK_FAIL) { +static void* mi_memory_grow( size_t size ) { + void* p = sbrk(size); + if (p == MI_SBRK_FAIL) { _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); errno = ENOMEM; return NULL; } - uintptr_t base = (uintptr_t)pbase0; - uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); - size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); - mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); - if (alloc_size < size) return NULL; - void* pbase1 = sbrk(alloc_size); - if (pbase1 == MI_SBRK_FAIL) { - _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); - errno = ENOMEM; - return NULL; - } - mi_assert(pbase0 == pbase1); - return (void*)aligned_base; + return p; } +#elif defined(__wasi__) +static void* mi_memory_grow( size_t size ) { + if (size > 0) { + size_t base = __builtin_wasm_memory_grow( 0, _mi_divide_up(size, _mi_os_page_size()) ); + if (base == SIZE_MAX) { + _mi_warning_message("unable to allocate wasm_memory_grow OS memory (%zu bytes)\n", size); + errno = ENOMEM; + return NULL; + } + return (void*)(base * _mi_os_page_size()); + } + else { + size_t base = __builtin_wasm_memory_size(0); + if (base == SIZE_MAX) { + errno = ENOMEM; + return NULL; + } + return (void*)(base * _mi_os_page_size()); + } +} +#endif -#elif defined(__wasi__) - // currently unused as we use sbrk() on wasm -static void* mi_wasm_heap_grow(size_t size, size_t try_alignment) { - uintptr_t base = __builtin_wasm_memory_size(0) * _mi_os_page_size(); - uintptr_t aligned_base = _mi_align_up(base, (uintptr_t) try_alignment); +static void* mi_heap_grow(size_t size, size_t try_alignment) { + if (try_alignment == 0) { try_alignment = _mi_os_page_size(); }; + void* pbase0 = mi_memory_grow(0); + if (pbase0 == NULL) { return NULL; } + uintptr_t base = (uintptr_t)pbase0; + uintptr_t aligned_base = _mi_align_up(base, try_alignment); size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); if (alloc_size < size) return NULL; - if (__builtin_wasm_memory_grow(0, alloc_size / _mi_os_page_size()) == SIZE_MAX) { - _mi_warning_message("unable to allocate wasm_memory_grow() OS memory (%zu bytes, %zu requested)\n", size, alloc_size); - errno = ENOMEM; - return NULL; + void* pbase1 = mi_memory_grow(alloc_size); + if (pbase1 == NULL) { return NULL; } + if (pbase0 != pbase1) { + // another thread allocated in-between; now we may not be able to align correctly + base = (uintptr_t)pbase1; + aligned_base = _mi_align_up(base, try_alignment); + if (aligned_base + size > base + alloc_size) { + // we do not have enough space after alignment; since we cannot shrink safely, + // we waste the space :-( and allocate fresh with guaranteed enough overallocation + alloc_size = _mi_align_up( size + try_alignment, _mi_os_page_size() ); + errno = 0; + void* pbase2 = mi_memory_grow( alloc_size ); + if (pbase2 == NULL) { return NULL; } + aligned_base = _mi_align_up(base, try_alignment); + mi_assert_internal(aligned_base + size <= (uintptr_t)pbase2 + alloc_size); + } + } + else { + mi_assert_internal(aligned_base + size <= (uintptr_t)pbase1 + alloc_size); } return (void*)aligned_base; } @@ -637,14 +664,10 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo int flags = MEM_RESERVE; if (commit) flags |= MEM_COMMIT; p = mi_win_virtual_alloc(NULL, size, try_alignment, flags, false, allow_large, is_large); - #elif defined(MI_USE_SBRK) + #elif defined(MI_USE_SBRK) || defined(__wasi__) MI_UNUSED(allow_large); *is_large = false; - p = mi_sbrk_heap_grow(size, try_alignment); - #elif defined(__wasi__) - MI_UNUSED(allow_large); - *is_large = false; - p = mi_wasm_heap_grow(size, try_alignment); + p = mi_heap_grow(size, try_alignment); #else int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); From 82a0e56f35fcd30be09343f924a57714b2eb4744 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 18:54:57 -0800 Subject: [PATCH 108/180] fix warning on mi_reallocarr --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 87d0e7d2..9c3e2dea 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -356,7 +356,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); -mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); +mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; From 55ccc947077ac4cfee808d17463253cfcbd0856b Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 15 Dec 2021 19:28:34 -0800 Subject: [PATCH 109/180] wip: fix zero init for sbrk; add lock around sbrk --- include/mimalloc-internal.h | 7 +++- src/init.c | 7 ---- src/os.c | 73 +++++++++++++++++++++---------------- 3 files changed, 47 insertions(+), 40 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 44631f8b..68239573 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -42,6 +42,11 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_externc #endif +#if !defined(_WIN32) && !defined(__wasi__) +#define MI_USE_PTHREADS +#include +#endif + // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); @@ -326,7 +331,6 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #if defined(MI_TLS_SLOT) static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept; // forward declaration #elif defined(MI_TLS_PTHREAD_SLOT_OFS) -#include static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { pthread_t self = pthread_self(); #if defined(__DragonFly__) @@ -338,7 +342,6 @@ static inline mi_heap_t** mi_tls_pthread_heap_slot(void) { return (mi_heap_t**)((uint8_t*)self + MI_TLS_PTHREAD_SLOT_OFS); } #elif defined(MI_TLS_PTHREAD) -#include extern pthread_key_t _mi_heap_default_key; #endif diff --git a/src/init.c b/src/init.c index f2619000..e06fa6b9 100644 --- a/src/init.c +++ b/src/init.c @@ -275,12 +275,6 @@ static bool _mi_heap_done(mi_heap_t* heap) { static void _mi_thread_done(mi_heap_t* default_heap); -#ifdef __wasi__ -// no pthreads in the WebAssembly Standard Interface -#elif !defined(_WIN32) -#define MI_USE_PTHREADS -#endif - #if defined(_WIN32) && defined(MI_SHARED_LIB) // nothing to do as it is done in DllMain #elif defined(_WIN32) && !defined(MI_SHARED_LIB) @@ -300,7 +294,6 @@ static void _mi_thread_done(mi_heap_t* default_heap); #elif defined(MI_USE_PTHREADS) // use pthread local storage keys to detect thread ending // (and used with MI_TLS_PTHREADS for the default heap) - #include pthread_key_t _mi_heap_default_key = (pthread_key_t)(-1); static void mi_pthread_done(void* value) { if (value!=NULL) _mi_thread_done((mi_heap_t*)value); diff --git a/src/os.c b/src/os.c index dc2afc78..afa94a1c 100644 --- a/src/os.c +++ b/src/os.c @@ -231,7 +231,7 @@ void _mi_os_init(void) #elif defined(__wasi__) void _mi_os_init() { os_overcommit = false; - os_page_size = 0x10000; // WebAssembly has a fixed page size: 64KiB + os_page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB os_alloc_granularity = 16; } @@ -283,7 +283,7 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats bool err = false; #if defined(_WIN32) err = (VirtualFree(addr, 0, MEM_RELEASE) == 0); -#elif defined(MI_USE_SBRK) +#elif defined(MI_USE_SBRK) || defined(__wasi__) err = 0; // sbrk heap cannot be shrunk #else err = (munmap(addr, size) == -1); @@ -376,38 +376,47 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, } #elif defined(MI_USE_SBRK) || defined(__wasi__) - #if defined(MI_USE_SBRK) -#define MI_SBRK_FAIL ((void*)(-1)) -static void* mi_memory_grow( size_t size ) { - void* p = sbrk(size); - if (p == MI_SBRK_FAIL) { - _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); - errno = ENOMEM; - return NULL; +// unfortunately sbrk is usually not safe to call from multiple threads +#if defined(MI_USE_PTHREADS) + static pthread_mutex_t mi_sbrk_mutex = PTHREAD_MUTEX_INITIALIZER; + static void* mi_sbrk(size_t size) { + pthread_mutex_lock(&mi_sbrk_mutex); + void* p = sbrk(size); + pthread_mutex_unlock(&mi_sbrk_mutex); + return p; + } + #else + static void* mi_sbrk(size_t size) { + return sbrk(size); + } + #endif + static void* mi_memory_grow( size_t size ) { + void* p = mi_sbrk(size); + if (p == (void*)(-1)) { + _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); + errno = ENOMEM; + return NULL; + } + if (size > 0) { memset(p,0,size); } + return p; } - return p; -} #elif defined(__wasi__) -static void* mi_memory_grow( size_t size ) { - if (size > 0) { - size_t base = __builtin_wasm_memory_grow( 0, _mi_divide_up(size, _mi_os_page_size()) ); + static void* mi_memory_grow( size_t size ) { + size_t base; + if (size > 0) { + base = __builtin_wasm_memory_grow( 0, _mi_divide_up(size, _mi_os_page_size()) ); + } + else { + base = __builtin_wasm_memory_size(0); + } if (base == SIZE_MAX) { _mi_warning_message("unable to allocate wasm_memory_grow OS memory (%zu bytes)\n", size); errno = ENOMEM; return NULL; } - return (void*)(base * _mi_os_page_size()); + return (void*)(base * _mi_os_page_size()); } - else { - size_t base = __builtin_wasm_memory_size(0); - if (base == SIZE_MAX) { - errno = ENOMEM; - return NULL; - } - return (void*)(base * _mi_os_page_size()); - } -} #endif static void* mi_heap_grow(size_t size, size_t try_alignment) { @@ -432,17 +441,19 @@ static void* mi_heap_grow(size_t size, size_t try_alignment) { errno = 0; void* pbase2 = mi_memory_grow( alloc_size ); if (pbase2 == NULL) { return NULL; } + base = (uintptr_t)pbase2; aligned_base = _mi_align_up(base, try_alignment); - mi_assert_internal(aligned_base + size <= (uintptr_t)pbase2 + alloc_size); } - } - else { - mi_assert_internal(aligned_base + size <= (uintptr_t)pbase1 + alloc_size); - } + else { + // it still fits + mi_assert_internal(aligned_base + size <= base + alloc_size); + } + } + mi_assert_internal(aligned_base + size <= base + alloc_size); return (void*)aligned_base; } -#else +#else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { MI_UNUSED(try_alignment); From b0a866685da1ee1ff3da400a880768d1c7774828 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 16 Dec 2021 14:54:10 -0800 Subject: [PATCH 110/180] improve sbrk/wasm_memory_grow implementation --- src/os.c | 157 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 84 insertions(+), 73 deletions(-) diff --git a/src/os.c b/src/os.c index afa94a1c..45182f64 100644 --- a/src/os.c +++ b/src/os.c @@ -88,6 +88,7 @@ static void* mi_align_down_ptr(void* p, size_t alignment) { return (void*)_mi_align_down((uintptr_t)p, alignment); } + // page size (initialized properly in `os_init`) static size_t os_page_size = 4096; @@ -274,8 +275,8 @@ void _mi_os_init() { /* ----------------------------------------------------------- - Raw allocation on Windows (VirtualAlloc) and Unix's (mmap). ------------------------------------------------------------ */ + free memory +-------------------------------------------------------------- */ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats_t* stats) { @@ -303,6 +304,10 @@ static bool mi_os_mem_free(void* addr, size_t size, bool was_committed, mi_stats static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size); #endif +/* ----------------------------------------------------------- + Raw allocation on Windows (VirtualAlloc) +-------------------------------------------------------------- */ + #ifdef _WIN32 static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment, DWORD flags) { #if (MI_INTPTR_SIZE >= 8) @@ -326,7 +331,7 @@ static void* mi_win_virtual_allocx(void* addr, size_t size, size_t try_alignment #endif #if defined(MEM_EXTENDED_PARAMETER_TYPE_BITS) // on modern Windows try use VirtualAlloc2 for aligned allocation - if (try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { + if (try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0 && pVirtualAlloc2 != NULL) { MEM_ADDRESS_REQUIREMENTS reqs = { 0, 0, 0 }; reqs.Alignment = try_alignment; MEM_EXTENDED_PARAMETER param = { {0, 0}, {0} }; @@ -375,90 +380,93 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, return p; } +/* ----------------------------------------------------------- + Raw allocation using `sbrk` or `wasm_memory_grow` +-------------------------------------------------------------- */ + #elif defined(MI_USE_SBRK) || defined(__wasi__) #if defined(MI_USE_SBRK) -// unfortunately sbrk is usually not safe to call from multiple threads -#if defined(MI_USE_PTHREADS) - static pthread_mutex_t mi_sbrk_mutex = PTHREAD_MUTEX_INITIALIZER; - static void* mi_sbrk(size_t size) { - pthread_mutex_lock(&mi_sbrk_mutex); - void* p = sbrk(size); - pthread_mutex_unlock(&mi_sbrk_mutex); - return p; - } - #else - static void* mi_sbrk(size_t size) { - return sbrk(size); - } - #endif static void* mi_memory_grow( size_t size ) { - void* p = mi_sbrk(size); - if (p == (void*)(-1)) { - _mi_warning_message("unable to allocate sbrk() OS memory (%zu bytes)\n", size); - errno = ENOMEM; - return NULL; - } - if (size > 0) { memset(p,0,size); } + void* p = sbrk(size); + if (p == (void*)(-1)) return NULL; + #if !defined(__wasi__) // on wasi this is always zero initialized already (?) + memset(p,0,size); + #endif return p; } #elif defined(__wasi__) static void* mi_memory_grow( size_t size ) { - size_t base; - if (size > 0) { - base = __builtin_wasm_memory_grow( 0, _mi_divide_up(size, _mi_os_page_size()) ); - } - else { - base = __builtin_wasm_memory_size(0); - } - if (base == SIZE_MAX) { - _mi_warning_message("unable to allocate wasm_memory_grow OS memory (%zu bytes)\n", size); - errno = ENOMEM; - return NULL; - } + size_t base = (size > 0 ? __builtin_wasm_memory_grow(0,_mi_divide_up(size, _mi_os_page_size())) + : __builtin_wasm_memory_size(0)); + if (base == SIZE_MAX) return NULL; return (void*)(base * _mi_os_page_size()); } #endif +#if defined(MI_USE_PTHREADS) +static pthread_mutex_t mi_heap_grow_mutex = PTHREAD_MUTEX_INITIALIZER; +#endif + static void* mi_heap_grow(size_t size, size_t try_alignment) { - if (try_alignment == 0) { try_alignment = _mi_os_page_size(); }; - void* pbase0 = mi_memory_grow(0); - if (pbase0 == NULL) { return NULL; } - uintptr_t base = (uintptr_t)pbase0; - uintptr_t aligned_base = _mi_align_up(base, try_alignment); - size_t alloc_size = _mi_align_up( aligned_base - base + size, _mi_os_page_size()); - mi_assert(alloc_size >= size && (alloc_size % _mi_os_page_size()) == 0); - if (alloc_size < size) return NULL; - void* pbase1 = mi_memory_grow(alloc_size); - if (pbase1 == NULL) { return NULL; } - if (pbase0 != pbase1) { - // another thread allocated in-between; now we may not be able to align correctly - base = (uintptr_t)pbase1; - aligned_base = _mi_align_up(base, try_alignment); - if (aligned_base + size > base + alloc_size) { - // we do not have enough space after alignment; since we cannot shrink safely, - // we waste the space :-( and allocate fresh with guaranteed enough overallocation - alloc_size = _mi_align_up( size + try_alignment, _mi_os_page_size() ); - errno = 0; - void* pbase2 = mi_memory_grow( alloc_size ); - if (pbase2 == NULL) { return NULL; } - base = (uintptr_t)pbase2; - aligned_base = _mi_align_up(base, try_alignment); - } - else { - // it still fits - mi_assert_internal(aligned_base + size <= base + alloc_size); + void* p = NULL; + if (try_alignment <= 1) { + // `sbrk` is not thread safe in general so try to protect it (we could skip this on WASM but leave it in for now) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_heap_grow_mutex); + #endif + p = mi_memory_grow(size); + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_heap_grow_mutex); + #endif + } + else { + void* base = NULL; + size_t alloc_size = 0; + // to allocate aligned use a lock to try to avoid thread interaction + // between getting the current size and actual allocation + // (also, `sbrk` is not thread safe in general) + #if defined(MI_USE_PTHREADS) + pthread_mutex_lock(&mi_heap_grow_mutex); + #endif + { + void* current = mi_memory_grow(0); // get current size + if (current != NULL) { + void* aligned_current = mi_align_up_ptr(current, try_alignment); // and align from there to minimize wasted space + alloc_size = _mi_align_up( ((uint8_t*)aligned_current - (uint8_t*)current) + size, _mi_os_page_size()); + base = mi_memory_grow(alloc_size); + } } - } - mi_assert_internal(aligned_base + size <= base + alloc_size); - return (void*)aligned_base; + #if defined(MI_USE_PTHREADS) + pthread_mutex_unlock(&mi_heap_grow_mutex); + #endif + if (base != NULL) { + p = mi_align_up_ptr(base, try_alignment); + if ((uint8_t*)p + size > (uint8_t*)base + alloc_size) { + // another thread used wasm_memory_grow/sbrk in-between and we do not have enough + // space after alignment. Give up (and waste the space as we cannot shrink :-( ) + // (in `mi_os_mem_alloc_aligned` this will fall back to overallocation to align) + p = NULL; + } + } + } + if (p == NULL) { + _mi_warning_message("unable to allocate sbrk/wasm_memory_grow OS memory (%zu bytes, %zu alignment)\n", size, try_alignment); + errno = ENOMEM; + return NULL; + } + mi_assert_internal( try_alignment == 0 || (uintptr_t)p % try_alignment == 0 ); + return p; } +/* ----------------------------------------------------------- + Raw allocation on Unix's (mmap) +-------------------------------------------------------------- */ #else #define MI_OS_USE_MMAP static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int protect_flags, int flags, int fd) { MI_UNUSED(try_alignment); #if defined(MAP_ALIGNED) // BSD - if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { size_t n = mi_bsr(try_alignment); if (((size_t)1 << n) == try_alignment && n >= 12 && n <= 30) { // alignment is a power of 2 and 4096 <= alignment <= 1GiB flags |= MAP_ALIGNED(n); @@ -468,7 +476,7 @@ static void* mi_unix_mmapx(void* addr, size_t size, size_t try_alignment, int pr } } #elif defined(MAP_ALIGN) // Solaris - if (addr == NULL && try_alignment > 0 && (try_alignment % _mi_os_page_size()) == 0) { + if (addr == NULL && try_alignment > 1 && (try_alignment % _mi_os_page_size()) == 0) { void* p = mmap(try_alignment, size, protect_flags, flags | MAP_ALIGN, fd, 0); if (p!=MAP_FAILED) return p; // fall back to regular mmap @@ -622,7 +630,7 @@ static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { - if (try_alignment == 0 || try_alignment > MI_SEGMENT_SIZE) return NULL; + if (try_alignment <= 1 || try_alignment > MI_SEGMENT_SIZE) return NULL; size = _mi_align_up(size, MI_SEGMENT_SIZE); if (size > 1*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<30) = 1/4096. #if (MI_SECURE>0) @@ -652,13 +660,16 @@ static void* mi_os_get_aligned_hint(size_t try_alignment, size_t size) { } #endif +/* ----------------------------------------------------------- + Primitive allocation from the OS. +-------------------------------------------------------------- */ -// Primitive allocation from the OS. // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, mi_stats_t* stats) { mi_assert_internal(size > 0 && (size % _mi_os_page_size()) == 0); if (size == 0) return NULL; if (!commit) allow_large = false; + if (try_alignment == 0) try_alignment = 1; // avoid 0 to ensure there will be no divide by zero when aligning void* p = NULL; /* @@ -746,9 +757,9 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, } #else // overallocate... - p = mi_os_mem_alloc(over_size, alignment, commit, false, is_large, stats); + p = mi_os_mem_alloc(over_size, 1, commit, false, is_large, stats); if (p == NULL) return NULL; - // and selectively unmap parts around the over-allocated area. + // and selectively unmap parts around the over-allocated area. (noop on sbrk) void* aligned_p = mi_align_up_ptr(p, alignment); size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p; size_t mid_size = _mi_align_up(size, _mi_os_page_size()); @@ -756,7 +767,7 @@ static void* mi_os_mem_alloc_aligned(size_t size, size_t alignment, bool commit, mi_assert_internal(pre_size < over_size && post_size < over_size && mid_size >= size); if (pre_size > 0) mi_os_mem_free(p, pre_size, commit, stats); if (post_size > 0) mi_os_mem_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); - // we can return the aligned pointer on `mmap` systems + // we can return the aligned pointer on `mmap` (and sbrk) systems p = aligned_p; #endif } From 27e24e619b7ed3fbc20ab74060a10fc1e2f30bf7 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 16 Dec 2021 15:11:47 -0800 Subject: [PATCH 111/180] use wasm_memory_grow instead of sbrk on wasm --- src/os.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/os.c b/src/os.c index 45182f64..71b371d2 100644 --- a/src/os.c +++ b/src/os.c @@ -26,10 +26,6 @@ terms of the MIT license. A copy of the license can be found in the file #pragma warning(disable:4996) // strerror #endif -#if defined(__wasi__) -#define MI_USE_SBRK -#endif - #if defined(_WIN32) #include #elif defined(__wasi__) From 5bbb2ff416ec4d374d49f742dff610028ecd72e4 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 16 Dec 2021 15:34:52 -0800 Subject: [PATCH 112/180] reenable the use of sbrk on wasm as it seems more stable than direct memory_grow --- src/os.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/os.c b/src/os.c index 71b371d2..a799cdf1 100644 --- a/src/os.c +++ b/src/os.c @@ -8,6 +8,8 @@ terms of the MIT license. A copy of the license can be found in the file #define _DEFAULT_SOURCE // ensure mmap flags are defined #endif +#define MI_USE_SBRK + #if defined(__sun) // illumos provides new mman.h api when any of these are defined // otherwise the old api based on caddr_t which predates the void pointers one. @@ -26,6 +28,10 @@ terms of the MIT license. A copy of the license can be found in the file #pragma warning(disable:4996) // strerror #endif +#if defined(__wasi__) +#define MI_USE_SBRK +#endif + #if defined(_WIN32) #include #elif defined(__wasi__) From 60790e9013eb8fe594cf7b3916baeae2cf03b03c Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 16 Dec 2021 15:35:53 -0800 Subject: [PATCH 113/180] remove accidental define --- src/os.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/os.c b/src/os.c index a799cdf1..45182f64 100644 --- a/src/os.c +++ b/src/os.c @@ -8,8 +8,6 @@ terms of the MIT license. A copy of the license can be found in the file #define _DEFAULT_SOURCE // ensure mmap flags are defined #endif -#define MI_USE_SBRK - #if defined(__sun) // illumos provides new mman.h api when any of these are defined // otherwise the old api based on caddr_t which predates the void pointers one. From 684c2c82a795f527cff015ab1c6a32e8680cb165 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 17 Dec 2021 11:40:46 -0800 Subject: [PATCH 114/180] restrict max aligment boundary to prevent bug with segment determination (found by Matthew Parkinson). --- include/mimalloc-types.h | 5 ++++- include/mimalloc.h | 1 + src/alloc-aligned.c | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index c9f399df..d45a1eea 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -158,7 +158,10 @@ typedef int32_t mi_ssize_t; #define MI_BIN_HUGE (73U) #if (MI_LARGE_OBJ_WSIZE_MAX >= 655360) -#error "define more bins" +#error "mimalloc internal: define more bins" +#endif +#if (MI_ALIGNED_MAX > MI_SEGMENT_SIZE/2) +#error "mimalloc internal: the max aligned boundary is too large for the segment size" #endif // Used as a special value to encode block sizes in 32 bits. diff --git a/include/mimalloc.h b/include/mimalloc.h index 9c3e2dea..b65027d3 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -166,6 +166,7 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s // Note that `alignment` always follows `size` for consistency with unaligned // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. // ------------------------------------------------------------------------------------- +#define MI_ALIGNED_MAX (1024*1024UL) // maximum supported alignment is 1MiB mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 724c0a1b..15062389 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -20,6 +20,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t mi_assert(alignment > 0); if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see ) if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see ) + if (mi_unlikely(alignment>MI_ALIGNED_MAX)) return NULL; // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` // try if there is a small block available with just the right alignment From 89f583a69b86a06966fb07b4794b4046faa01fee Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 17 Dec 2021 13:18:05 -0800 Subject: [PATCH 115/180] improve aligned allocation performance --- include/mimalloc-internal.h | 5 -- src/alloc-aligned.c | 111 +++++++++++++++++++++++++++--------- src/alloc-posix.c | 19 +++--- test/main-override-static.c | 2 +- test/test-api.c | 14 ++++- 5 files changed, 108 insertions(+), 43 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 68239573..1df8b5de 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -247,11 +247,6 @@ static inline size_t _mi_wsize_from_size(size_t size) { return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); } -// Does malloc satisfy the alignment constraints already? -static inline bool mi_malloc_satisfies_alignment(size_t alignment, size_t size) { - return (alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2))); -} - // Overflow detecting multiply #if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include // UINT_MAX, ULONG_MAX diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 15062389..12706f7f 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -14,32 +14,14 @@ terms of the MIT license. A copy of the license can be found in the file // Aligned Allocation // ------------------------------------------------------ -static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept { - // note: we don't require `size > offset`, we just guarantee that - // the address at offset is aligned regardless of the allocated size. - mi_assert(alignment > 0); - if (mi_unlikely(size > PTRDIFF_MAX)) return NULL; // we don't allocate more than PTRDIFF_MAX (see ) - if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) return NULL; // require power-of-two (see ) - if (mi_unlikely(alignment>MI_ALIGNED_MAX)) return NULL; // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) +// Fallback primitive aligned allocation -- split out for better codegen +static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +{ + mi_assert_internal(size <= PTRDIFF_MAX); + mi_assert_internal(alignment!=0 && _mi_is_power_of_two(alignment) && alignment <= MI_ALIGNED_MAX); + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - - // try if there is a small block available with just the right alignment const size_t padsize = size + MI_PADDING_SIZE; - if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) { - mi_page_t* page = _mi_heap_get_free_small_page(heap,padsize); - const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0; - if (mi_likely(page->free != NULL && is_aligned)) - { - #if MI_STAT>1 - mi_heap_stat_increase( heap, malloc, size); - #endif - void* p = _mi_page_malloc(heap,page,padsize); // TODO: inline _mi_page_malloc - mi_assert_internal(p != NULL); - mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); - if (zero) _mi_block_zero_init(page,p,size); - return p; - } - } // use regular allocation if it is guaranteed to fit the alignment constraints if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) { @@ -47,7 +29,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); return p; } - + // otherwise over-allocate void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero); if (p == NULL) return NULL; @@ -56,21 +38,90 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t uintptr_t adjust = alignment - (((uintptr_t)p + offset) & align_mask); mi_assert_internal(adjust <= alignment); void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust)); - if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true); + if (aligned_p != p) mi_page_set_has_aligned(_mi_ptr_page(p), true); mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); - mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) ); + mi_assert_internal(p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p), _mi_ptr_page(aligned_p), aligned_p)); return aligned_p; } +// Primitive aligned allocation +static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +{ + // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. + mi_assert(alignment > 0); + if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) { // require power-of-two (see ) + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)", size, alignment); + #endif + return NULL; + } + if (mi_unlikely(alignment > MI_ALIGNED_MAX)) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)", MI_ALIGNED_MAX, size, alignment); + #endif + return NULL; + } + if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see ) + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)", size, alignment); + #endif + return NULL; + } + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; // note: cannot overflow due to earlier size > PTRDIFF_MAX check + + // try first if there happens to be a small block available with just the right alignment + if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) { + mi_page_t* page = _mi_heap_get_free_small_page(heap, padsize); + const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0; + if (mi_likely(page->free != NULL && is_aligned)) + { + #if MI_STAT>1 + mi_heap_stat_increase(heap, malloc, size); + #endif + void* p = _mi_page_malloc(heap, page, padsize); // TODO: inline _mi_page_malloc + mi_assert_internal(p != NULL); + mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + if (zero) { _mi_block_zero_init(page, p, size); } + return p; + } + } + // fallback + return mi_heap_malloc_zero_aligned_at_fallback(heap, size, alignment, offset, zero); +} + + +// ------------------------------------------------------ +// Optimized mi_heap_malloc_aligned / mi_malloc_aligned +// ------------------------------------------------------ mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); } mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { - return mi_heap_malloc_aligned_at(heap, size, alignment, 0); + #if !MI_PADDING + // without padding, any small sized allocation is naturally aligned (see also `_mi_segment_page_start`) + if (!_mi_is_power_of_two(alignment)) return NULL; + if (mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX)) + #else + // with padding, we can only guarantee this for fixed alignments + if (mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIGN_SIZE/2))) + && size <= MI_SMALL_SIZE_MAX)) + #endif + { + // fast path for common alignment and size + return mi_heap_malloc_small(heap, size); + } + else { + return mi_heap_malloc_aligned_at(heap, size, alignment, 0); + } } +// ------------------------------------------------------ +// Aligned Allocation +// ------------------------------------------------------ + mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); } @@ -114,6 +165,10 @@ mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t align } +// ------------------------------------------------------ +// Aligned re-allocation +// ------------------------------------------------------ + static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 2eaede07..efe62817 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -56,9 +56,9 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept // Note: The spec dictates we should not modify `*p` on an error. (issue#27) // if (p == NULL) return EINVAL; - if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment - if (!_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 - void* q = (mi_malloc_satisfies_alignment(alignment, size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment)); + if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment + if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 + void* q = mi_malloc_aligned(size, alignment); if (q==NULL && size != 0) return ENOMEM; mi_assert_internal(((uintptr_t)q % alignment) == 0); *p = q; @@ -66,7 +66,7 @@ int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept } mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept { - void* p = (mi_malloc_satisfies_alignment(alignment,size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment)); + void* p = mi_malloc_aligned(size, alignment); mi_assert_internal(((uintptr_t)p % alignment) == 0); return p; } @@ -83,9 +83,14 @@ mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept { } mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { - if (alignment==0 || !_mi_is_power_of_two(alignment)) return NULL; - if ((size&(alignment-1)) != 0) return NULL; // C11 requires integral multiple, see - void* p = (mi_malloc_satisfies_alignment(alignment, size) ? mi_malloc(size) : mi_malloc_aligned(size, alignment)); + if (mi_unlikely((size&(alignment-1)) != 0)) { // C11 requires alignment>0 && integral multiple, see + #if MI_DEBUG > 0 + _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)", size, alignment); + #endif + return NULL; + } + // C11 also requires alignment to be a power-of-two which is checked in mi_malloc_aligned + void* p = mi_malloc_aligned(size, alignment); mi_assert_internal(((uintptr_t)p % alignment) == 0); return p; } diff --git a/test/main-override-static.c b/test/main-override-static.c index 221db7e8..071e4248 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -42,7 +42,7 @@ int main() { free(p1); free(p2); free(s); - + /* now test if override worked by allocating/freeing across the api's*/ //p1 = mi_malloc(32); //free(p1); diff --git a/test/test-api.c b/test/test-api.c index 8ddbf7cf..1f810d53 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -158,6 +158,16 @@ int main(void) { CHECK_BODY("malloc-aligned5", { void* p = mi_malloc_aligned(4097,4096); size_t usable = mi_usable_size(p); result = usable >= 4097 && usable < 10000; mi_free(p); }); + CHECK_BODY("malloc-aligned6", { + void* p; + bool ok = true; + for (int i = 1; i < 8 && ok; i++) { + size_t align = 1UL << i; + p = mi_malloc_aligned(2*align, align); + ok = (p != NULL && (uintptr_t)(p) % align == 0); mi_free(p); + } + result = ok; + }); CHECK_BODY("malloc-aligned-at1", { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p); }); @@ -172,8 +182,8 @@ int main(void) { ok = (p != NULL && (uintptr_t)(p) % 16 == 0); mi_free(p); } result = ok; - }); - + }); + // --------------------------------------------------- // Heaps // --------------------------------------------------- From faf2a7020c4d3234434b165f5c27f89d7777016b Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 17 Dec 2021 13:25:38 -0800 Subject: [PATCH 116/180] add extra alignment tests --- src/alloc-aligned.c | 6 +++--- src/alloc-posix.c | 2 +- test/test-api.c | 8 +++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 12706f7f..3a608b83 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -51,19 +51,19 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t mi_assert(alignment > 0); if (mi_unlikely(alignment==0 || !_mi_is_power_of_two(alignment))) { // require power-of-two (see ) #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)", size, alignment); + _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } if (mi_unlikely(alignment > MI_ALIGNED_MAX)) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)", MI_ALIGNED_MAX, size, alignment); + _mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)\n", MI_ALIGNED_MAX, size, alignment); #endif return NULL; } if (mi_unlikely(size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see ) #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)", size, alignment); + _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } diff --git a/src/alloc-posix.c b/src/alloc-posix.c index efe62817..03171081 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -85,7 +85,7 @@ mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept { mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { if (mi_unlikely((size&(alignment-1)) != 0)) { // C11 requires alignment>0 && integral multiple, see #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)", size, alignment); + _mi_error_message(EOVERFLOW, "(mi_)aligned_alloc requires the size to be an integral multiple of the alignment (size %zu, alignment %zu)\n", size, alignment); #endif return NULL; } diff --git a/test/test-api.c b/test/test-api.c index 1f810d53..56835d00 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -162,12 +162,18 @@ int main(void) { void* p; bool ok = true; for (int i = 1; i < 8 && ok; i++) { - size_t align = 1UL << i; + size_t align = (size_t)1 << i; p = mi_malloc_aligned(2*align, align); ok = (p != NULL && (uintptr_t)(p) % align == 0); mi_free(p); } result = ok; }); + CHECK_BODY("malloc-aligned7", { + void* p = mi_malloc_aligned(1024,MI_ALIGNED_MAX); mi_free(p); + }); + CHECK_BODY("malloc-aligned8", { + void* p = mi_malloc_aligned(1024,2*MI_ALIGNED_MAX); mi_free(p); + }); CHECK_BODY("malloc-aligned-at1", { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p); }); From f5ea6c70bdaa55e3709858490164752dc8c30945 Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 17 Dec 2021 13:40:39 -0800 Subject: [PATCH 117/180] NetBSD mi_reallocarr should update errno (based on PR #499 by @devexen) --- src/alloc-posix.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 03171081..ee5babe1 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -103,7 +103,10 @@ void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { int mi_reallocarr( void* p, size_t count, size_t size ) mi_attr_noexcept { // NetBSD mi_assert(p != NULL); - if (p == NULL) return EINVAL; // should we set errno as well? + if (p == NULL) { + errno = EINVAL; + return EINVAL; + } void** op = (void**)p; void* newp = mi_reallocarray(*op, count, size); if (mi_unlikely(newp == NULL)) return errno; From 30a99e2c514b7f69b3224181b4d3877deffd824c Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 17 Dec 2021 13:48:39 -0800 Subject: [PATCH 118/180] default to portable C for the thread id on arm 32-bit, issue #495 --- include/mimalloc-internal.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1df8b5de..06c3e95f 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -708,21 +708,21 @@ static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { } #elif defined(__GNUC__) && \ - (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__)) + (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__)) -// TLS register on x86 is in the FS or GS register, see: https://akkadia.org/drepper/tls.pdf +// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { void* res; const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) - __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // 32-bit always uses GS + __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS #elif defined(__APPLE__) && defined(__x86_64__) __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI #elif defined(__x86_64__) __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) +#elif defined(__arm__) // arm32: defined but currently not used (see issue #495) void** tcb; MI_UNUSED(ofs); __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); res = tcb[slot]; @@ -739,7 +739,7 @@ static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { return res; } -// setting is only used on macOSX for now +// setting a tls slot is only used on macOSX for now static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); #if defined(__i386__) @@ -767,8 +767,8 @@ static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { } static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__arm__) || (defined(__ANDROID__) && defined(__aarch64__)) - // issue #384, #495: on arm32 and arm32/arm64 Android, slot 1 is the thread ID (pointer to pthread internal struct) +#if defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__)) + // issue #384, #495: on arm Android, slot 1 is the thread ID (pointer to pthread internal struct) return (uintptr_t)mi_tls_slot(1); #else // in all our other targets, slot 0 is the pointer to the thread control block @@ -776,7 +776,7 @@ static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { #endif } #else -// otherwise use standard C +// otherwise use portable C static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { return (uintptr_t)&_mi_heap_default; } From 89090510bd965ca48a83857eaa6b6e78a951bf7e Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sat, 18 Dec 2021 11:11:44 -0800 Subject: [PATCH 119/180] update alignment tests --- include/mimalloc-types.h | 2 +- include/mimalloc.h | 2 +- src/alloc-aligned.c | 6 +++--- test/test-api.c | 20 +++++++++++++------- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index d45a1eea..b349dfc3 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -160,7 +160,7 @@ typedef int32_t mi_ssize_t; #if (MI_LARGE_OBJ_WSIZE_MAX >= 655360) #error "mimalloc internal: define more bins" #endif -#if (MI_ALIGNED_MAX > MI_SEGMENT_SIZE/2) +#if (MI_ALIGNMENT_MAX > MI_SEGMENT_SIZE/2) #error "mimalloc internal: the max aligned boundary is too large for the segment size" #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index b65027d3..a0b71389 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -166,7 +166,7 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s // Note that `alignment` always follows `size` for consistency with unaligned // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. // ------------------------------------------------------------------------------------- -#define MI_ALIGNED_MAX (1024*1024UL) // maximum supported alignment is 1MiB +#define MI_ALIGNMENT_MAX (1024*1024UL) // maximum supported alignment is 1MiB mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 3a608b83..6b15e653 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -18,7 +18,7 @@ terms of the MIT license. A copy of the license can be found in the file static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept { mi_assert_internal(size <= PTRDIFF_MAX); - mi_assert_internal(alignment!=0 && _mi_is_power_of_two(alignment) && alignment <= MI_ALIGNED_MAX); + mi_assert_internal(alignment!=0 && _mi_is_power_of_two(alignment) && alignment <= MI_ALIGNMENT_MAX); const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; @@ -55,9 +55,9 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t #endif return NULL; } - if (mi_unlikely(alignment > MI_ALIGNED_MAX)) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) + if (mi_unlikely(alignment > MI_ALIGNMENT_MAX)) { // we cannot align at a boundary larger than this (or otherwise we cannot find segment headers) #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)\n", MI_ALIGNED_MAX, size, alignment); + _mi_error_message(EOVERFLOW, "aligned allocation has a maximum alignment of %zu (size %zu, alignment %zu)\n", MI_ALIGNMENT_MAX, size, alignment); #endif return NULL; } diff --git a/test/test-api.c b/test/test-api.c index 56835d00..96817337 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -159,20 +159,26 @@ int main(void) { void* p = mi_malloc_aligned(4097,4096); size_t usable = mi_usable_size(p); result = usable >= 4097 && usable < 10000; mi_free(p); }); CHECK_BODY("malloc-aligned6", { - void* p; bool ok = true; - for (int i = 1; i < 8 && ok; i++) { - size_t align = (size_t)1 << i; - p = mi_malloc_aligned(2*align, align); - ok = (p != NULL && (uintptr_t)(p) % align == 0); mi_free(p); + for (size_t align = 1; align <= MI_ALIGNMENT_MAX && ok; align *= 2) { + void* ps[8]; + for (int i = 0; i < 8 && ok; i++) { + ps[i] = mi_malloc_aligned(align/2 /*size*/, align); + if (ps[i] == NULL || (uintptr_t)(ps[i]) % align != 0) { + ok = false; + } + } + for (int i = 0; i < 8 && ok; i++) { + mi_free(ps[i]); + } } result = ok; }); CHECK_BODY("malloc-aligned7", { - void* p = mi_malloc_aligned(1024,MI_ALIGNED_MAX); mi_free(p); + void* p = mi_malloc_aligned(1024,MI_ALIGNMENT_MAX); mi_free(p); }); CHECK_BODY("malloc-aligned8", { - void* p = mi_malloc_aligned(1024,2*MI_ALIGNED_MAX); mi_free(p); + void* p = mi_malloc_aligned(1024,2*MI_ALIGNMENT_MAX); mi_free(p); }); CHECK_BODY("malloc-aligned-at1", { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p); From f16435447c53d90b61f1bce6da90c4759d5d7f5b Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 18 Dec 2021 11:37:00 -0800 Subject: [PATCH 120/180] fix bug in freeing huge OS page allocation spanning multiple huge OS pages --- src/os.c | 5 +++-- test/test-api.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index 45182f64..c31df757 100644 --- a/src/os.c +++ b/src/os.c @@ -1254,8 +1254,8 @@ void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_mse } } mi_assert_internal(page*MI_HUGE_OS_PAGE_SIZE <= size); - if (pages_reserved != NULL) *pages_reserved = page; - if (psize != NULL) *psize = page * MI_HUGE_OS_PAGE_SIZE; + if (pages_reserved != NULL) { *pages_reserved = page; } + if (psize != NULL) { *psize = page * MI_HUGE_OS_PAGE_SIZE; } return (page == 0 ? NULL : start); } @@ -1267,6 +1267,7 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) { while (size >= MI_HUGE_OS_PAGE_SIZE) { _mi_os_free(base, MI_HUGE_OS_PAGE_SIZE, stats); size -= MI_HUGE_OS_PAGE_SIZE; + base += MI_HUGE_OS_PAGE_SIZE; } } diff --git a/test/test-api.c b/test/test-api.c index 96817337..f057799a 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -163,7 +163,7 @@ int main(void) { for (size_t align = 1; align <= MI_ALIGNMENT_MAX && ok; align *= 2) { void* ps[8]; for (int i = 0; i < 8 && ok; i++) { - ps[i] = mi_malloc_aligned(align/2 /*size*/, align); + ps[i] = mi_malloc_aligned(align*13 /*size*/, align); if (ps[i] == NULL || (uintptr_t)(ps[i]) % align != 0) { ok = false; } From 9f01c661511bfb4636227d46a2ac70cd4442a6a3 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 18 Dec 2021 16:36:37 -0800 Subject: [PATCH 121/180] update documentation --- doc/mimalloc-doc.h | 51 ++- docs/annotated.html | 22 +- docs/bench.html | 28 +- docs/build.html | 47 ++- docs/classes.html | 37 +- docs/doxygen.css | 175 +++++---- docs/dynsections.js | 33 +- docs/environment.html | 27 +- docs/functions.html | 22 +- docs/functions_vars.html | 22 +- docs/group__aligned.html | 52 ++- docs/group__aligned.js | 1 + docs/group__analysis.html | 32 +- docs/group__cpp.html | 41 +-- docs/group__extended.html | 187 ++++++++-- docs/group__extended.js | 2 + docs/group__heap.html | 54 ++- docs/group__malloc.html | 48 ++- docs/group__options.html | 64 ++-- docs/group__options.js | 1 + docs/group__posix.html | 66 +++- docs/group__posix.js | 1 + docs/group__typed.html | 51 ++- docs/group__zeroinit.html | 22 +- docs/index.html | 27 +- docs/jquery.js | 94 ++--- docs/mimalloc-doc_8h_source.html | 611 +++++++++++++++++++++++-------- docs/modules.html | 22 +- docs/navtree.js | 74 ++-- docs/navtreedata.js | 33 +- docs/navtreeindex0.js | 73 ++-- docs/overrides.html | 92 ++++- docs/pages.html | 22 +- docs/resize.js | 38 +- docs/search/all_0.html | 19 +- docs/search/all_0.js | 2 +- docs/search/all_1.html | 19 +- docs/search/all_1.js | 2 +- docs/search/all_2.html | 19 +- docs/search/all_2.js | 8 +- docs/search/all_3.html | 19 +- docs/search/all_3.js | 4 +- docs/search/all_4.html | 19 +- docs/search/all_4.js | 4 +- docs/search/all_5.html | 19 +- docs/search/all_5.js | 4 +- docs/search/all_6.html | 19 +- docs/search/all_6.js | 295 +++++++-------- docs/search/all_7.html | 19 +- docs/search/all_7.js | 2 +- docs/search/all_8.html | 19 +- docs/search/all_8.js | 4 +- docs/search/all_9.html | 19 +- docs/search/all_9.js | 4 +- docs/search/all_a.html | 19 +- docs/search/all_a.js | 2 +- docs/search/all_b.html | 19 +- docs/search/all_b.js | 4 +- docs/search/all_c.html | 19 +- docs/search/all_c.js | 2 +- docs/search/classes_0.html | 19 +- docs/search/classes_0.js | 4 +- docs/search/enums_0.html | 19 +- docs/search/enums_0.js | 2 +- docs/search/enumvalues_0.html | 19 +- docs/search/enumvalues_0.js | 2 +- docs/search/enumvalues_1.html | 19 +- docs/search/enumvalues_1.js | 31 +- docs/search/functions_0.html | 19 +- docs/search/functions_0.js | 223 +++++------ docs/search/groups_0.html | 19 +- docs/search/groups_0.js | 2 +- docs/search/groups_1.html | 19 +- docs/search/groups_1.js | 2 +- docs/search/groups_2.html | 19 +- docs/search/groups_2.js | 2 +- docs/search/groups_3.html | 19 +- docs/search/groups_3.js | 2 +- docs/search/groups_4.html | 19 +- docs/search/groups_4.js | 4 +- docs/search/groups_5.html | 19 +- docs/search/groups_5.js | 2 +- docs/search/groups_6.html | 19 +- docs/search/groups_6.js | 2 +- docs/search/groups_7.html | 19 +- docs/search/groups_7.js | 2 +- docs/search/groups_8.html | 19 +- docs/search/groups_8.js | 2 +- docs/search/nomatches.html | 3 +- docs/search/pages_0.html | 19 +- docs/search/pages_0.js | 2 +- docs/search/pages_1.html | 19 +- docs/search/pages_1.js | 2 +- docs/search/pages_2.html | 19 +- docs/search/pages_2.js | 2 +- docs/search/pages_3.html | 19 +- docs/search/pages_3.js | 2 +- docs/search/pages_4.html | 19 +- docs/search/pages_4.js | 2 +- docs/search/search.js | 52 +-- docs/search/typedefs_0.html | 19 +- docs/search/typedefs_0.js | 10 +- docs/search/variables_0.html | 19 +- docs/search/variables_0.js | 4 +- docs/search/variables_1.html | 19 +- docs/search/variables_1.js | 2 +- docs/search/variables_2.html | 19 +- docs/search/variables_2.js | 2 +- docs/search/variables_3.html | 19 +- docs/search/variables_3.js | 2 +- docs/using.html | 69 +++- 111 files changed, 2220 insertions(+), 1428 deletions(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index e2287fe9..4cf8c2c3 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -51,7 +51,7 @@ Notable aspects of the design include: programs. - __secure__: _mimalloc_ can be build in secure mode, adding guard pages, randomized allocation, encrypted free lists, etc. to protect against various - heap vulnerabilities. The performance penalty is only around 3% on average + heap vulnerabilities. The performance penalty is only around 5% on average over our benchmarks. - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions. A heap can be destroyed at once instead of deallocating each object separately. @@ -413,6 +413,28 @@ void mi_register_error(mi_error_fun* errfun, void* arg); /// This function is relatively fast. bool mi_is_in_heap_region(const void* p); +/// Reserve OS memory for use by mimalloc. Reserved areas are used +/// before allocating from the OS again. By reserving a large area upfront, +/// allocation can be more efficient, and can be better managed on systems +/// without `mmap`/`VirtualAlloc` (like WASM for example). +/// @param size The size to reserve. +/// @param commit Commit the memory upfront. +/// @param allow_large Allow large OS pages (2MiB) to be used? +/// @return \a 0 if successful, and an error code otherwise (e.g. `ENOMEM`). +int mi_reserve_os_memory(size_t size, bool commit, bool allow_large); + +/// Manage a particular memory area for use by mimalloc. +/// This is just like `mi_reserve_os_memory` except that the area should already be +/// allocated in some manner and available for use my mimalloc. +/// @param start Start of the memory area +/// @param size The size of the memory area. +/// @param commit Is the area already committed? +/// @param is_large Does it consist of large OS pages? Set this to \a true as well for memory +/// that should not be decommitted or protected (like rdma etc.) +/// @param is_zero Does the area consists of zero's? +/// @param numa_node Possible associated numa node or `-1`. +/// @return \a true if successful, and \a false on error. +bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node); /// Reserve \a pages of huge OS pages (1GiB) evenly divided over \a numa_nodes nodes, /// but stops after at most `timeout_msecs` seconds. @@ -476,9 +498,12 @@ void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_m /// /// \{ +/// The maximum supported alignment size (currently 1MiB). +#define MI_ALIGNMENT_MAX (1024*1024UL) + /// Allocate \a size bytes aligned by \a alignment. /// @param size number of bytes to allocate. -/// @param alignment the minimal alignment of the allocated memory. +/// @param alignment the minimal alignment of the allocated memory. Must be less than #MI_ALIGNMENT_MAX. /// @returns pointer to the allocated memory or \a NULL if out of memory. /// The returned pointer is aligned by \a alignment, i.e. /// `(uintptr_t)p % alignment == 0`. @@ -829,8 +854,14 @@ void* mi_valloc(size_t size); void* mi_pvalloc(size_t size); void* mi_aligned_alloc(size_t alignment, size_t size); + +/// Correspond s to [reallocarray](https://www.freebsd.org/cgi/man.cgi?query=reallocarray&sektion=3&manpath=freebsd-release-ports) +/// in FreeBSD. void* mi_reallocarray(void* p, size_t count, size_t size); +/// Corresponds to [reallocarr](https://man.netbsd.org/reallocarr.3) in NetBSD. +int mi_reallocarr(void* p, size_t count, size_t size); + void mi_free_size(void* p, size_t size); void mi_free_size_aligned(void* p, size_t size, size_t alignment); void mi_free_aligned(void* p, size_t alignment); @@ -1161,6 +1192,12 @@ void* calloc(size_t size, size_t n); void* realloc(void* p, size_t newsize); void free(void* p); +void* aligned_alloc(size_t alignment, size_t size); +char* strdup(const char* s); +char* strndup(const char* s, size_t n); +char* realpath(const char* fname, char* resolved_name); + + // C++ void operator delete(void* p); void operator delete[](void* p); @@ -1180,16 +1217,24 @@ int posix_memalign(void** p, size_t alignment, size_t size); // Linux void* memalign(size_t alignment, size_t size); -void* aligned_alloc(size_t alignment, size_t size); void* valloc(size_t size); void* pvalloc(size_t size); size_t malloc_usable_size(void *p); +void* reallocf(void* p, size_t newsize); + +// macOS +void vfree(void* p); +size_t malloc_size(const void* p); +size_t malloc_good_size(size_t size); // BSD void* reallocarray( void* p, size_t count, size_t size ); void* reallocf(void* p, size_t newsize); void cfree(void* p); +// NetBSD +int reallocarr(void* p, size_t count, size_t size); + // Windows void* _expand(void* p, size_t newsize); size_t _msize(void* p); diff --git a/docs/annotated.html b/docs/annotated.html index feba2438..f3e392a4 100644 --- a/docs/annotated.html +++ b/docs/annotated.html @@ -3,7 +3,7 @@ - + mi-malloc: Data Structures @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
mi-malloc -  1.6 +  1.7/2.0
- @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
@@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
@@ -113,9 +109,7 @@ $(document).ready(function(){initNavTree('annotated.html','');}); diff --git a/docs/bench.html b/docs/bench.html index f39fade6..6c472895 100644 --- a/docs/bench.html +++ b/docs/bench.html @@ -3,7 +3,7 @@ - + mi-malloc: Performance @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
mi-malloc -  1.6 +  1.7/2.0
- @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
@@ -60,10 +56,10 @@
- + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
@@ -103,17 +99,15 @@ $(document).ready(function(){initNavTree('bench.html','');});

We tested mimalloc against many other top allocators over a wide range of benchmarks, ranging from various real world programs to synthetic benchmarks that see how the allocator behaves under more extreme circumstances.

-

In our benchmarks, mimalloc always outperforms all other leading allocators (jemalloc, tcmalloc, Hoard, etc) (Apr 2019), and usually uses less memory (up to 25% more in the worst case). A nice property is that it does consistently well over the wide range of benchmarks.

+

In our benchmarks, mimalloc always outperforms all other leading allocators (jemalloc, tcmalloc, Hoard, etc) (Jan 2021), and usually uses less memory (up to 25% more in the worst case). A nice property is that it does consistently well over the wide range of benchmarks.

See the Performance section in the mimalloc repository for benchmark results, or the the technical report for detailed benchmark results.

-
- + + diff --git a/docs/build.html b/docs/build.html index 2bd06f13..dbcc0d75 100644 --- a/docs/build.html +++ b/docs/build.html @@ -3,7 +3,7 @@ - + mi-malloc: Building @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
mi-malloc -  1.6 +  1.7/2.0
- @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
@@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
@@ -102,28 +98,39 @@ $(document).ready(function(){initNavTree('build.html','');});
Building
-

Checkout the sources from Github:

git clone https://github.com/microsoft/mimalloc

Windows

+

Checkout the sources from Github:

git clone https://github.com/microsoft/mimalloc
+

Windows

Open ide/vs2019/mimalloc.sln in Visual Studio 2019 and build (or ide/vs2017/mimalloc.sln). The mimalloc project builds a static library (in out/msvc-x64), while the mimalloc-override project builds a DLL for overriding malloc in the entire program.

macOS, Linux, BSD, etc.

We use cmake1 as the build system:

-
> mkdir -p out/release
> cd out/release
> cmake ../..
> make

This builds the library as a shared (dynamic) library (.so or .dylib), a static library (.a), and as a single object file (.o).

+
> mkdir -p out/release
+
> cd out/release
+
> cmake ../..
+
> make
+

This builds the library as a shared (dynamic) library (.so or .dylib), a static library (.a), and as a single object file (.o).

> sudo make install (install the library and header files in /usr/local/lib and /usr/local/include)

You can build the debug version which does many internal checks and maintains detailed statistics as:

-
> mkdir -p out/debug
> cd out/debug
> cmake -DCMAKE_BUILD_TYPE=Debug ../..
> make

This will name the shared library as libmimalloc-debug.so.

-

Finally, you can build a secure version that uses guard pages, encrypted free lists, etc, as:

> mkdir -p out/secure
> cd out/secure
> cmake -DMI_SECURE=ON ../..
> make

This will name the shared library as libmimalloc-secure.so. Use ccmake2 instead of cmake to see and customize all the available build options.

+
> mkdir -p out/debug
+
> cd out/debug
+
> cmake -DCMAKE_BUILD_TYPE=Debug ../..
+
> make
+

This will name the shared library as libmimalloc-debug.so.

+

Finally, you can build a secure version that uses guard pages, encrypted free lists, etc, as:

> mkdir -p out/secure
+
> cd out/secure
+
> cmake -DMI_SECURE=ON ../..
+
> make
+

This will name the shared library as libmimalloc-secure.so. Use ccmake2 instead of cmake to see and customize all the available build options.

Notes:

  1. Install CMake: sudo apt-get install cmake
  2. Install CCMake: sudo apt-get install cmake-curses-gui
-
-
+ + diff --git a/docs/classes.html b/docs/classes.html index e74a0a24..b744c4d9 100644 --- a/docs/classes.html +++ b/docs/classes.html @@ -3,7 +3,7 @@ - + mi-malloc: Data Structure Index @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
mi-malloc -  1.6 +  1.7/2.0
- @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
@@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
@@ -102,23 +98,18 @@ $(document).ready(function(){initNavTree('classes.html','');});
Data Structure Index
-
m
- - - - - -
  m  
-		mi_stl_allocator   
mi_heap_area_t   
-
m
+
M
+
+
+
M
+
mi_heap_area_t
mi_stl_allocator
+
diff --git a/docs/doxygen.css b/docs/doxygen.css index 72384716..f090ef79 100644 --- a/docs/doxygen.css +++ b/docs/doxygen.css @@ -1,4 +1,4 @@ -/* The standard CSS for doxygen 1.8.15 */ +/* The standard CSS for doxygen 1.9.1 */ body, table, div, p, dl { font: 400 14px/22px Roboto,sans-serif; @@ -53,17 +53,24 @@ dt { font-weight: bold; } -div.multicol { +ul.multicol { -moz-column-gap: 1em; -webkit-column-gap: 1em; + column-gap: 1em; -moz-column-count: 3; -webkit-column-count: 3; + column-count: 3; } p.startli, p.startdd { margin-top: 2px; } +th p.starttd, th p.intertd, th p.endtd { + font-size: 100%; + font-weight: 700; +} + p.starttd { margin-top: 0px; } @@ -96,30 +103,96 @@ caption { } span.legend { - font-size: 70%; - text-align: center; -} - -h3.version { - font-size: 90%; - text-align: center; -} - -div.qindex, div.navtab{ - background-color: #D6D9D9; - border: 1px solid #636C6D; + font-size: 70%; text-align: center; } -div.qindex, div.navpath { - width: 100%; - line-height: 140%; +h3.version { + font-size: 90%; + text-align: center; } div.navtab { - margin-right: 15px; + border-right: 1px solid #636C6D; + padding-right: 15px; + text-align: right; + line-height: 110%; } +div.navtab table { + border-spacing: 0; +} + +td.navtab { + padding-right: 6px; + padding-left: 6px; +} +td.navtabHL { + background-image: url('tab_a.png'); + background-repeat:repeat-x; + padding-right: 6px; + padding-left: 6px; +} + +td.navtabHL a, td.navtabHL a:visited { + color: #fff; + text-shadow: 0px 1px 1px rgba(0, 0, 0, 1.0); +} + +a.navtab { + font-weight: bold; +} + +div.qindex{ + text-align: center; + width: 100%; + line-height: 140%; + font-size: 130%; + color: #A0A0A0; +} + +dt.alphachar{ + font-size: 180%; + font-weight: bold; +} + +.alphachar a{ + color: black; +} + +.alphachar a:hover, .alphachar a:visited{ + text-decoration: none; +} + +.classindex dl { + padding: 25px; + column-count:1 +} + +.classindex dd { + display:inline-block; + margin-left: 50px; + width: 90%; + line-height: 1.15em; +} + +.classindex dl.odd { + background-color: #F0F1F1; +} + +@media(min-width: 1120px) { + .classindex dl { + column-count:2 + } +} + +@media(min-width: 1320px) { + .classindex dl { + column-count:3 + } +} + + /* @group Link Styling */ a { @@ -136,17 +209,6 @@ a:hover { text-decoration: underline; } -a.qindex { - font-weight: bold; -} - -a.qindexHL { - font-weight: bold; - background-color: #5B6364; - color: #FFFFFF; - border: 1px double #464C4D; -} - .contents a.qindexHL:visited { color: #FFFFFF; } @@ -533,7 +595,7 @@ table.memberdecls { white-space: nowrap; } -.memItemRight { +.memItemRight, .memTemplItemRight { width: 100%; } @@ -701,7 +763,7 @@ dl.reflist dd { padding-left: 0px; } -.params .paramname, .retval .paramname, .tparams .paramname { +.params .paramname, .retval .paramname, .tparams .paramname, .exception .paramname { font-weight: bold; vertical-align: top; } @@ -1351,10 +1413,12 @@ dl.citelist dt { font-weight:bold; margin-right:10px; padding:5px; + text-align:right; + width:52px; } dl.citelist dd { - margin:2px 0; + margin:2px 0 2px 72px; padding:5px 0; } @@ -1417,6 +1481,12 @@ div.toc li.level4 { margin-left: 45px; } +span.emoji { + /* font family used at the site: https://unicode.org/emoji/charts/full-emoji-list.html + * font-family: "Noto Color Emoji", "Apple Color Emoji", "Segoe UI Emoji", Times, Symbola, Aegyptus, Code2000, Code2001, Code2002, Musica, serif, LastResort; + */ +} + .PageDocRTL-title div.toc li.level1 { margin-left: 0 !important; margin-right: 0; @@ -1654,47 +1724,6 @@ tr.heading h2 { /* @group Markdown */ -/* -table.markdownTable { - border-collapse:collapse; - margin-top: 4px; - margin-bottom: 4px; -} - -table.markdownTable td, table.markdownTable th { - border: 1px solid #060606; - padding: 3px 7px 2px; -} - -table.markdownTableHead tr { -} - -table.markdownTableBodyLeft td, table.markdownTable th { - border: 1px solid #060606; - padding: 3px 7px 2px; -} - -th.markdownTableHeadLeft th.markdownTableHeadRight th.markdownTableHeadCenter th.markdownTableHeadNone { - background-color: #0B0C0C; - color: #FFFFFF; - font-size: 110%; - padding-bottom: 4px; - padding-top: 5px; -} - -th.markdownTableHeadLeft { - text-align: left -} - -th.markdownTableHeadRight { - text-align: right -} - -th.markdownTableHeadCenter { - text-align: center -} -*/ - table.markdownTable { border-collapse:collapse; margin-top: 4px; diff --git a/docs/dynsections.js b/docs/dynsections.js index ea0a7b39..3174bd7b 100644 --- a/docs/dynsections.js +++ b/docs/dynsections.js @@ -1,25 +1,26 @@ /* - @licstart The following is the entire license notice for the - JavaScript code in this file. + @licstart The following is the entire license notice for the JavaScript code in this file. - Copyright (C) 1997-2017 by Dimitri van Heesch + The MIT License (MIT) - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. + Copyright (C) 1997-2020 by Dimitri van Heesch - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. - @licend The above is the entire license notice - for the JavaScript code in this file + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file */ function toggleVisibility(linkObj) { diff --git a/docs/environment.html b/docs/environment.html index 87d67e4a..f571f95f 100644 --- a/docs/environment.html +++ b/docs/environment.html @@ -3,7 +3,7 @@ - + mi-malloc: Environment Options @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
mi-malloc -  1.6 +  1.7/2.0
- @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
@@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
@@ -110,17 +106,16 @@ $(document).ready(function(){initNavTree('environment.html','');});
  • MIMALLOC_PAGE_RESET=0: by default, mimalloc will reset (or purge) OS pages when not in use to signal to the OS that the underlying physical memory can be reused. This can reduce memory fragmentation in long running (server) programs. By setting it to 0 no such page resets will be done which can improve performance for programs that are not long running. As an alternative, the MIMALLOC_RESET_DELAY=<msecs> can be set higher (100ms by default) to make the page reset occur less frequently instead of turning it off completely.
  • MIMALLOC_LARGE_OS_PAGES=1: use large OS pages (2MiB) when available; for some workloads this can significantly improve performance. Use MIMALLOC_VERBOSE to check if the large OS pages are enabled – usually one needs to explicitly allow large OS pages (as on Windows and Linux). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that can have fragmented memory (for that reason, we generally recommend to use MIMALLOC_RESERVE_HUGE_OS_PAGES instead when possible).
  • MIMALLOC_RESERVE_HUGE_OS_PAGES=N: where N is the number of 1GiB huge OS pages. This reserves the huge pages at startup and sometimes this can give a large (latency) performance improvement on big workloads. Usually it is better to not use MIMALLOC_LARGE_OS_PAGES in combination with this setting. Just like large OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented (but reserving the huge pages is done at startup only once). Note that we usually need to explicitly enable huge OS pages (as on Windows and Linux)). With huge OS pages, it may be beneficial to set the setting MIMALLOC_EAGER_COMMIT_DELAY=N (N is 1 by default) to delay the initial N segments (of 4MiB) of a thread to not allocate in the huge OS pages; this prevents threads that are short lived and allocate just a little to take up space in the huge OS page area (which cannot be reset).
    • +
  • MIMALLOC_RESERVE_HUGE_OS_PAGES_AT=N: where N is the numa node. This reserves the huge pages at a specific numa node. (N is -1 by default to reserve huge pages evenly among the given number of numa nodes (or use the available ones as detected))

    • Use caution when using fork in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in big increments.

    -
    - + + diff --git a/docs/functions.html b/docs/functions.html index 62021210..373fafe2 100644 --- a/docs/functions.html +++ b/docs/functions.html @@ -3,7 +3,7 @@ - + mi-malloc: Data Fields @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -120,9 +116,7 @@ $(document).ready(function(){initNavTree('functions.html','');}); diff --git a/docs/functions_vars.html b/docs/functions_vars.html index 7d41d10e..a12ef622 100644 --- a/docs/functions_vars.html +++ b/docs/functions_vars.html @@ -3,7 +3,7 @@ - + mi-malloc: Data Fields - Variables @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -120,9 +116,7 @@ $(document).ready(function(){initNavTree('functions_vars.html','');}); diff --git a/docs/group__aligned.html b/docs/group__aligned.html index a3eaacf0..bd11f30f 100644 --- a/docs/group__aligned.html +++ b/docs/group__aligned.html @@ -3,7 +3,7 @@ - + mi-malloc: Aligned Allocation @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -99,6 +95,7 @@ $(document).ready(function(){initNavTree('group__aligned.html','');});
    +Macros | Functions
    Aligned Allocation
    @@ -108,10 +105,16 @@ $(document).ready(function(){initNavTree('group__aligned.html','');});

    Allocating aligned memory blocks. More...

    + + + + +

    +Macros

    #define MI_ALIGNMENT_MAX
     The maximum supported alignment size (currently 1MiB). More...
     
    - + @@ -120,7 +123,7 @@ Functions - + @@ -131,6 +134,23 @@ Functions

    Functions

    void * mi_malloc_aligned (size_t size, size_t alignment)
     Allocate size bytes aligned by alignment. More...
     Allocate size bytes aligned by alignment. More...
     
    void * mi_zalloc_aligned (size_t size, size_t alignment)
     
    void * mi_realloc_aligned (void *p, size_t newsize, size_t alignment)
     
    void * mi_malloc_aligned_at (size_t size, size_t alignment, size_t offset)
     Allocate size bytes aligned by alignment at a specified offset. More...
     Allocate size bytes aligned by alignment at a specified offset. More...
     
    void * mi_zalloc_aligned_at (size_t size, size_t alignment, size_t offset)
     

    Detailed Description

    Allocating aligned memory blocks.

    +

    Macro Definition Documentation

    + +

    ◆ MI_ALIGNMENT_MAX

    + +
    +
    + + + + +
    #define MI_ALIGNMENT_MAX
    +
    + +

    The maximum supported alignment size (currently 1MiB).

    + +
    +

    Function Documentation

    ◆ mi_calloc_aligned()

    @@ -236,7 +256,7 @@ Functions
    Parameters
    - +
    sizenumber of bytes to allocate.
    alignmentthe minimal alignment of the allocated memory.
    alignmentthe minimal alignment of the allocated memory. Must be less than MI_ALIGNMENT_MAX.
    @@ -438,9 +458,7 @@ Functions diff --git a/docs/group__aligned.js b/docs/group__aligned.js index 0a5aa5c8..06ccb0c3 100644 --- a/docs/group__aligned.js +++ b/docs/group__aligned.js @@ -1,5 +1,6 @@ var group__aligned = [ + [ "MI_ALIGNMENT_MAX", "group__aligned.html#ga83c03016066b438f51a8095e9140be06", null ], [ "mi_calloc_aligned", "group__aligned.html#ga53dddb4724042a90315b94bc268fb4c9", null ], [ "mi_calloc_aligned_at", "group__aligned.html#ga08647c4593f3b2eef24a919a73eba3a3", null ], [ "mi_malloc_aligned", "group__aligned.html#ga68930196751fa2cca9e1fd0d71bade56", null ], diff --git a/docs/group__analysis.html b/docs/group__analysis.html index 2487c240..883099fe 100644 --- a/docs/group__analysis.html +++ b/docs/group__analysis.html @@ -3,7 +3,7 @@ - + mi-malloc: Heap Introspection @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - +
    @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -119,22 +115,22 @@ Data Structures

    Typedefs

    typedef bool() mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg) - Visitor function passed to mi_heap_visit_blocks() More...
    + Visitor function passed to mi_heap_visit_blocks() More...
      - + - + - + - +

    Functions

    bool mi_heap_contains_block (mi_heap_t *heap, const void *p)
     Does a heap contain a pointer to a previously allocated block? More...
     Does a heap contain a pointer to a previously allocated block? More...
     
    bool mi_heap_check_owned (mi_heap_t *heap, const void *p)
     Check safely if any pointer is part of a heap. More...
     Check safely if any pointer is part of a heap. More...
     
    bool mi_check_owned (const void *p)
     Check safely if any pointer is part of the default heap of this thread. More...
     Check safely if any pointer is part of the default heap of this thread. More...
     
    bool mi_heap_visit_blocks (const mi_heap_t *heap, bool visit_all_blocks, mi_block_visit_fun *visitor, void *arg)
     Visit all areas and blocks in a heap. More...
     Visit all areas and blocks in a heap. More...
     

    Detailed Description

    @@ -376,9 +372,7 @@ bytes in use by allocated blocks diff --git a/docs/group__cpp.html b/docs/group__cpp.html index 88c75888..e81179fe 100644 --- a/docs/group__cpp.html +++ b/docs/group__cpp.html @@ -3,7 +3,7 @@ - + mi-malloc: C++ wrappers @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -118,25 +114,25 @@ Data Structures

    Functions

    void * mi_new (std::size_t n) noexcept(false) - like mi_malloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
    + like mi_malloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
      void * mi_new_n (size_t count, size_t size) noexcept(false) - like mi_mallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
    + like mi_mallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
      void * mi_new_aligned (std::size_t n, std::align_val_t alignment) noexcept(false) - like mi_malloc_aligned(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
    + like mi_malloc_aligned(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
      void * mi_new_nothrow (size_t n) - like mi_malloc, but when out of memory, use std::get_new_handler but return NULL on failure. More...
    + like mi_malloc, but when out of memory, use std::get_new_handler but return NULL on failure. More...
      void * mi_new_aligned_nothrow (size_t n, size_t alignment) - like mi_malloc_aligned, but when out of memory, use std::get_new_handler but return NULL on failure. More...
    + like mi_malloc_aligned, but when out of memory, use std::get_new_handler but return NULL on failure. More...
      void * mi_new_realloc (void *p, size_t newsize) - like mi_realloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
    + like mi_realloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
      void * mi_new_reallocn (void *p, size_t newcount, size_t size) - like mi_reallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
    + like mi_reallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception on failure. More...
     

    Detailed Description

    @@ -158,7 +154,10 @@ Functions struct mi_stl_allocator< T >

    std::allocator implementation for mimalloc for use in STL containers.

    -

    For example:

    std::vector<int, mi_stl_allocator<int> > vec;
    vec.push_back(1);
    vec.pop_back();
    +

    For example:

    std::vector<int, mi_stl_allocator<int> > vec;
    +
    vec.push_back(1);
    +
    vec.pop_back();
    +

    Function Documentation

    @@ -387,9 +386,7 @@ struct mi_stl_allocator< T > diff --git a/docs/group__extended.html b/docs/group__extended.html index 12e51cbb..e5499118 100644 --- a/docs/group__extended.html +++ b/docs/group__extended.html @@ -3,7 +3,7 @@ - + mi-malloc: Extended Functions @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -113,82 +109,88 @@ $(document).ready(function(){initNavTree('group__extended.html','');});

    Macros

    #define MI_SMALL_SIZE_MAX - Maximum size allowed for small allocations in mi_malloc_small and mi_zalloc_small (usually 128*sizeof(void*) (= 1KB on 64-bit systems)) More...
    + Maximum size allowed for small allocations in mi_malloc_small and mi_zalloc_small (usually 128*sizeof(void*) (= 1KB on 64-bit systems)) More...
      - + - + - +

    Typedefs

    typedef void() mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
     Type of deferred free functions. More...
     Type of deferred free functions. More...
     
    typedef void() mi_output_fun(const char *msg, void *arg)
     Type of output functions. More...
     Type of output functions. More...
     
    typedef void() mi_error_fun(int err, void *arg)
     Type of error callback functions. More...
     Type of error callback functions. More...
     
    - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - + - +

    Functions

    void * mi_malloc_small (size_t size)
     Allocate a small object. More...
     Allocate a small object. More...
     
    void * mi_zalloc_small (size_t size)
     Allocate a zero initialized small object. More...
     Allocate a zero initialized small object. More...
     
    size_t mi_usable_size (void *p)
     Return the available bytes in a memory block. More...
     Return the available bytes in a memory block. More...
     
    size_t mi_good_size (size_t size)
     Return the used allocation size. More...
     Return the used allocation size. More...
     
    void mi_collect (bool force)
     Eagerly free memory. More...
     Eagerly free memory. More...
     
    void mi_stats_print (void *out)
     Deprecated. More...
     Deprecated. More...
     
    void mi_stats_print_out (mi_output_fun *out, void *arg)
     Print the main statistics. More...
     Print the main statistics. More...
     
    void mi_stats_reset (void)
     Reset statistics. More...
     Reset statistics. More...
     
    void mi_stats_merge (void)
     Merge thread local statistics with the main statistics and reset. More...
     Merge thread local statistics with the main statistics and reset. More...
     
    void mi_thread_init (void)
     Initialize mimalloc on a thread. More...
     Initialize mimalloc on a thread. More...
     
    void mi_thread_done (void)
     Uninitialize mimalloc on a thread. More...
     Uninitialize mimalloc on a thread. More...
     
    void mi_thread_stats_print_out (mi_output_fun *out, void *arg)
     Print out heap statistics for this thread. More...
     Print out heap statistics for this thread. More...
     
    void mi_register_deferred_free (mi_deferred_free_fun *deferred_free, void *arg)
     Register a deferred free function. More...
     Register a deferred free function. More...
     
    void mi_register_output (mi_output_fun *out, void *arg)
     Register an output function. More...
     Register an output function. More...
     
    void mi_register_error (mi_error_fun *errfun, void *arg)
     Register an error callback function. More...
     Register an error callback function. More...
     
    bool mi_is_in_heap_region (const void *p)
     Is a pointer part of our heap? More...
     Is a pointer part of our heap? More...
     
    int mi_reserve_os_memory (size_t size, bool commit, bool allow_large)
     Reserve OS memory for use by mimalloc. More...
     
    bool mi_manage_os_memory (void *start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node)
     Manage a particular memory area for use by mimalloc. More...
     
    int mi_reserve_huge_os_pages_interleave (size_t pages, size_t numa_nodes, size_t timeout_msecs)
     Reserve pages of huge OS pages (1GiB) evenly divided over numa_nodes nodes, but stops after at most timeout_msecs seconds. More...
     Reserve pages of huge OS pages (1GiB) evenly divided over numa_nodes nodes, but stops after at most timeout_msecs seconds. More...
     
    int mi_reserve_huge_os_pages_at (size_t pages, int numa_node, size_t timeout_msecs)
     Reserve pages of huge OS pages (1GiB) at a specific numa_node, but stops after at most timeout_msecs seconds. More...
     Reserve pages of huge OS pages (1GiB) at a specific numa_node, but stops after at most timeout_msecs seconds. More...
     
    bool mi_is_redirected ()
     Is the C runtime malloc API redirected? More...
     Is the C runtime malloc API redirected? More...
     
    void mi_process_info (size_t *elapsed_msecs, size_t *user_msecs, size_t *system_msecs, size_t *current_rss, size_t *peak_rss, size_t *current_commit, size_t *peak_commit, size_t *page_faults)
     Return process information (time and memory usage). More...
     Return process information (time and memory usage). More...
     

    Detailed Description

    @@ -414,6 +416,72 @@ Functions
    Returns
    a pointer to newly allocated memory of at least size bytes, or NULL if out of memory. This function is meant for use in run-time systems for best performance and does not check if size was indeed small – use with care!
    +
    + + +

    ◆ mi_manage_os_memory()

    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    bool mi_manage_os_memory (void * start,
    size_t size,
    bool is_committed,
    bool is_large,
    bool is_zero,
    int numa_node 
    )
    +
    + +

    Manage a particular memory area for use by mimalloc.

    +

    This is just like mi_reserve_os_memory except that the area should already be allocated in some manner and available for use my mimalloc.

    Parameters
    + + + + + + + +
    startStart of the memory area
    sizeThe size of the memory area.
    commitIs the area already committed?
    is_largeDoes it consist of large OS pages? Set this to true as well for memory that should not be decommitted or protected (like rdma etc.)
    is_zeroDoes the area consists of zero's?
    numa_nodePossible associated numa node or -1.
    +
    +
    +
    Returns
    true if successful, and false on error.
    +
    @@ -706,6 +774,51 @@ Functions
    Returns
    0 if successfull, ENOMEM if running out of memory, or ETIMEDOUT if timed out.

    The reserved memory is used by mimalloc to satisfy allocations. May quit before timeout_msecs are expired if it estimates it will take more than 1.5 times timeout_msecs. The time limit is needed because on some operating systems it can take a long time to reserve contiguous memory if the physical memory is fragmented.

    + + + +

    ◆ mi_reserve_os_memory()

    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    int mi_reserve_os_memory (size_t size,
    bool commit,
    bool allow_large 
    )
    +
    + +

    Reserve OS memory for use by mimalloc.

    +

    Reserved areas are used before allocating from the OS again. By reserving a large area upfront, allocation can be more efficient, and can be better managed on systems without mmap/VirtualAlloc (like WASM for example).

    Parameters
    + + + + +
    sizeThe size to reserve.
    commitCommit the memory upfront.
    allow_largeAllow large OS pages (2MiB) to be used?
    +
    +
    +
    Returns
    0 if successful, and an error code otherwise (e.g. ENOMEM).
    +
    @@ -958,9 +1071,7 @@ Functions diff --git a/docs/group__extended.js b/docs/group__extended.js index ed4a8b46..c217aaca 100644 --- a/docs/group__extended.js +++ b/docs/group__extended.js @@ -9,12 +9,14 @@ var group__extended = [ "mi_is_in_heap_region", "group__extended.html#ga5f071b10d4df1c3658e04e7fd67a94e6", null ], [ "mi_is_redirected", "group__extended.html#gaad25050b19f30cd79397b227e0157a3f", null ], [ "mi_malloc_small", "group__extended.html#ga7136c2e55cb22c98ecf95d08d6debb99", null ], + [ "mi_manage_os_memory", "group__extended.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf", null ], [ "mi_process_info", "group__extended.html#ga7d862c2affd5790381da14eb102a364d", null ], [ "mi_register_deferred_free", "group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece", null ], [ "mi_register_error", "group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45", null ], [ "mi_register_output", "group__extended.html#gae5b17ff027cd2150b43a33040250cf3f", null ], [ "mi_reserve_huge_os_pages_at", "group__extended.html#ga7795a13d20087447281858d2c771cca1", null ], [ "mi_reserve_huge_os_pages_interleave", "group__extended.html#ga3132f521fb756fc0e8ec0b74fb58df50", null ], + [ "mi_reserve_os_memory", "group__extended.html#ga00ec3324b6b2591c7fe3677baa30a767", null ], [ "mi_stats_merge", "group__extended.html#ga854b1de8cb067c7316286c28b2fcd3d1", null ], [ "mi_stats_print", "group__extended.html#ga2d126e5c62d3badc35445e5d84166df2", null ], [ "mi_stats_print_out", "group__extended.html#ga537f13b299ddf801e49a5a94fde02c79", null ], diff --git a/docs/group__heap.html b/docs/group__heap.html index 1a38c936..0f21ea42 100644 --- a/docs/group__heap.html +++ b/docs/group__heap.html @@ -3,7 +3,7 @@ - + mi-malloc: Heap Allocation @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -112,55 +108,55 @@ $(document).ready(function(){initNavTree('group__heap.html','');});

    Typedefs

    typedef struct mi_heap_s mi_heap_t - Type of first-class heaps. More...
    + Type of first-class heaps. More...
      - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1071,9 +1067,7 @@ Functions diff --git a/docs/group__malloc.html b/docs/group__malloc.html index 224c4b08..2dc16656 100644 --- a/docs/group__malloc.html +++ b/docs/group__malloc.html @@ -3,7 +3,7 @@ - +mi-malloc: Basic Allocation @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ @@ -60,10 +56,10 @@

    Functions

    mi_heap_t * mi_heap_new ()
     Create a new heap that can be used for allocation. More...
     Create a new heap that can be used for allocation. More...
     
    void mi_heap_delete (mi_heap_t *heap)
     Delete a previously allocated heap. More...
     Delete a previously allocated heap. More...
     
    void mi_heap_destroy (mi_heap_t *heap)
     Destroy a heap, freeing all its still allocated blocks. More...
     Destroy a heap, freeing all its still allocated blocks. More...
     
    mi_heap_t * mi_heap_set_default (mi_heap_t *heap)
     Set the default heap to use for mi_malloc() et al. More...
     Set the default heap to use for mi_malloc() et al. More...
     
    mi_heap_t * mi_heap_get_default ()
     Get the default heap that is used for mi_malloc() et al. More...
     Get the default heap that is used for mi_malloc() et al. More...
     
    mi_heap_t * mi_heap_get_backing ()
     Get the backing heap. More...
     Get the backing heap. More...
     
    void mi_heap_collect (mi_heap_t *heap, bool force)
     Release outstanding resources in a specific heap. More...
     Release outstanding resources in a specific heap. More...
     
    void * mi_heap_malloc (mi_heap_t *heap, size_t size)
     Allocate in a specific heap. More...
     Allocate in a specific heap. More...
     
    void * mi_heap_malloc_small (mi_heap_t *heap, size_t size)
     Allocate a small object in a specific heap. More...
     Allocate a small object in a specific heap. More...
     
    void * mi_heap_zalloc (mi_heap_t *heap, size_t size)
     Allocate zero-initialized in a specific heap. More...
     Allocate zero-initialized in a specific heap. More...
     
    void * mi_heap_calloc (mi_heap_t *heap, size_t count, size_t size)
     Allocate count zero-initialized elements in a specific heap. More...
     Allocate count zero-initialized elements in a specific heap. More...
     
    void * mi_heap_mallocn (mi_heap_t *heap, size_t count, size_t size)
     Allocate count elements in a specific heap. More...
     Allocate count elements in a specific heap. More...
     
    char * mi_heap_strdup (mi_heap_t *heap, const char *s)
     Duplicate a string in a specific heap. More...
     Duplicate a string in a specific heap. More...
     
    char * mi_heap_strndup (mi_heap_t *heap, const char *s, size_t n)
     Duplicate a string of at most length n in a specific heap. More...
     Duplicate a string of at most length n in a specific heap. More...
     
    char * mi_heap_realpath (mi_heap_t *heap, const char *fname, char *resolved_name)
     Resolve a file path name using a specific heap to allocate the result. More...
     Resolve a file path name using a specific heap to allocate the result. More...
     
    void * mi_heap_realloc (mi_heap_t *heap, void *p, size_t newsize)
     
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -111,43 +107,43 @@ $(document).ready(function(){initNavTree('group__malloc.html','');});

    Functions

    void mi_free (void *p) - Free previously allocated memory. More...
    + Free previously allocated memory. More...
      void * mi_malloc (size_t size) - Allocate size bytes. More...
    + Allocate size bytes. More...
      void * mi_zalloc (size_t size) - Allocate zero-initialized size bytes. More...
    + Allocate zero-initialized size bytes. More...
      void * mi_calloc (size_t count, size_t size) - Allocate zero-initialized count elements of size bytes. More...
    + Allocate zero-initialized count elements of size bytes. More...
      void * mi_realloc (void *p, size_t newsize) - Re-allocate memory to newsize bytes. More...
    + Re-allocate memory to newsize bytes. More...
      void * mi_recalloc (void *p, size_t count, size_t size) - Re-allocate memory to count elements of size bytes, with extra memory initialized to zero. More...
    + Re-allocate memory to count elements of size bytes, with extra memory initialized to zero. More...
      void * mi_expand (void *p, size_t newsize) - Try to re-allocate memory to newsize bytes in place. More...
    + Try to re-allocate memory to newsize bytes in place. More...
      void * mi_mallocn (size_t count, size_t size) - Allocate count elements of size bytes. More...
    + Allocate count elements of size bytes. More...
      void * mi_reallocn (void *p, size_t count, size_t size) - Re-allocate memory to count elements of size bytes. More...
    + Re-allocate memory to count elements of size bytes. More...
      void * mi_reallocf (void *p, size_t newsize) - Re-allocate memory to newsize bytes,. More...
    + Re-allocate memory to newsize bytes,. More...
      char * mi_strdup (const char *s) - Allocate and duplicate a string. More...
    + Allocate and duplicate a string. More...
      char * mi_strndup (const char *s, size_t n) - Allocate and duplicate a string up to n bytes. More...
    + Allocate and duplicate a string up to n bytes. More...
      char * mi_realpath (const char *fname, char *resolved_name) - Resolve a file path name. More...
    + Resolve a file path name. More...
     

    Detailed Description

    @@ -635,9 +631,7 @@ mi_zallocn() diff --git a/docs/group__options.html b/docs/group__options.html index 9425765e..f92905d4 100644 --- a/docs/group__options.html +++ b/docs/group__options.html @@ -3,7 +3,7 @@ - + mi-malloc: Runtime Options @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -112,25 +108,27 @@ $(document).ready(function(){initNavTree('group__options.html','');});

    Enumerations

    enum  mi_option_t {
    -  mi_option_show_errors, -mi_option_show_stats, -mi_option_verbose, -mi_option_eager_commit, -
    -  mi_option_eager_region_commit, -mi_option_large_os_pages, -mi_option_reserve_huge_os_pages, -mi_option_segment_cache, -
    -  mi_option_page_reset, -mi_option_segment_reset, -mi_option_reset_delay, -mi_option_use_numa_nodes, -
    -  mi_option_reset_decommits, -mi_option_eager_commit_delay, -mi_option_os_tag, -_mi_option_last +  mi_option_show_errors +, mi_option_show_stats +, mi_option_verbose +, mi_option_eager_commit +,
    +  mi_option_eager_region_commit +, mi_option_large_os_pages +, mi_option_reserve_huge_os_pages +, mi_option_reserve_huge_os_pages_at +,
    +  mi_option_segment_cache +, mi_option_page_reset +, mi_option_segment_reset +, mi_option_reset_delay +,
    +  mi_option_use_numa_nodes +, mi_option_reset_decommits +, mi_option_eager_commit_delay +, mi_option_os_tag +,
    +  _mi_option_last
    }  Runtime options. More...
    @@ -186,6 +184,8 @@ Functions mi_option_reserve_huge_os_pages 

    The number of huge OS pages (1GiB in size) to reserve at the start of the program.

    +mi_option_reserve_huge_os_pages_at 

    Reserve huge OS pages at node N.

    + mi_option_segment_cache 

    The number of segments per thread to keep cached.

    mi_option_page_reset 

    Reset page memory after mi_option_reset_delay milliseconds when it becomes free.

    @@ -397,9 +397,7 @@ Functions diff --git a/docs/group__options.js b/docs/group__options.js index 9aaf2318..c8836cdc 100644 --- a/docs/group__options.js +++ b/docs/group__options.js @@ -8,6 +8,7 @@ var group__options = [ "mi_option_eager_region_commit", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca32ce97ece29f69e82579679cf8a307ad", null ], [ "mi_option_large_os_pages", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4192d491200d0055df0554d4cf65054e", null ], [ "mi_option_reserve_huge_os_pages", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2", null ], + [ "mi_option_reserve_huge_os_pages_at", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c", null ], [ "mi_option_segment_cache", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca2ecbe7ef32f5c84de3739aa4f0b805a1", null ], [ "mi_option_page_reset", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cada854dd272c66342f18a93ee254a2968", null ], [ "mi_option_segment_reset", "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafb121d30d87591850d5410ccc3a95c6d", null ], diff --git a/docs/group__posix.html b/docs/group__posix.html index fe3a88e1..539f7ec6 100644 --- a/docs/group__posix.html +++ b/docs/group__posix.html @@ -3,7 +3,7 @@ - + mi-malloc: Posix @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -115,7 +111,7 @@ Functions size_t mi_malloc_usable_size (const void *p)   void mi_cfree (void *p) - Just as free but also checks if the pointer p belongs to our heap. More...
    + Just as free but also checks if the pointer p belongs to our heap. More...
      int mi_posix_memalign (void **p, size_t alignment, size_t size)   @@ -130,7 +126,11 @@ Functions void * mi_aligned_alloc (size_t alignment, size_t size)   void * mi_reallocarray (void *p, size_t count, size_t size) + Correspond s to reallocarray in FreeBSD. More...
      +int mi_reallocarr (void *p, size_t count, size_t size) + Corresponds to reallocarr in NetBSD. More...
    +  void mi_free_size (void *p, size_t size)   void mi_free_size_aligned (void *p, size_t size, size_t alignment) @@ -428,6 +428,42 @@ Functions
    +
    + + +

    ◆ mi_reallocarr()

    + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    int mi_reallocarr (void * p,
    size_t count,
    size_t size 
    )
    +
    + +

    Corresponds to reallocarr in NetBSD.

    +
    @@ -462,6 +498,8 @@ Functions
    +

    Correspond s to reallocarray in FreeBSD.

    +
    @@ -487,9 +525,7 @@ Functions diff --git a/docs/group__posix.js b/docs/group__posix.js index e43453d9..50c248c8 100644 --- a/docs/group__posix.js +++ b/docs/group__posix.js @@ -11,6 +11,7 @@ var group__posix = [ "mi_memalign", "group__posix.html#gaab7fa71ea93b96873f5d9883db57d40e", null ], [ "mi_posix_memalign", "group__posix.html#gacff84f226ba9feb2031b8992e5579447", null ], [ "mi_pvalloc", "group__posix.html#gaeb325c39b887d3b90d85d1eb1712fb1e", null ], + [ "mi_reallocarr", "group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5", null ], [ "mi_reallocarray", "group__posix.html#ga48fad8648a2f1dab9c87ea9448a52088", null ], [ "mi_valloc", "group__posix.html#ga73baaf5951f5165ba0763d0c06b6a93b", null ] ]; \ No newline at end of file diff --git a/docs/group__typed.html b/docs/group__typed.html index 5cbfbd6b..c19c7f4a 100644 --- a/docs/group__typed.html +++ b/docs/group__typed.html @@ -3,7 +3,7 @@ - + mi-malloc: Typed Macros @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -111,42 +107,44 @@ $(document).ready(function(){initNavTree('group__typed.html','');});

    Macros

    #define mi_malloc_tp(tp) - Allocate a block of type tp. More...
    + Allocate a block of type tp. More...
      #define mi_zalloc_tp(tp) - Allocate a zero-initialized block of type tp. More...
    + Allocate a zero-initialized block of type tp. More...
      #define mi_calloc_tp(tp, count) - Allocate count zero-initialized blocks of type tp. More...
    + Allocate count zero-initialized blocks of type tp. More...
      #define mi_mallocn_tp(tp, count) - Allocate count blocks of type tp. More...
    + Allocate count blocks of type tp. More...
      #define mi_reallocn_tp(p, tp, count) - Re-allocate to count blocks of type tp. More...
    + Re-allocate to count blocks of type tp. More...
      #define mi_heap_malloc_tp(hp, tp) - Allocate a block of type tp in a heap hp. More...
    + Allocate a block of type tp in a heap hp. More...
      #define mi_heap_zalloc_tp(hp, tp) - Allocate a zero-initialized block of type tp in a heap hp. More...
    + Allocate a zero-initialized block of type tp in a heap hp. More...
      #define mi_heap_calloc_tp(hp, tp, count) - Allocate count zero-initialized blocks of type tp in a heap hp. More...
    + Allocate count zero-initialized blocks of type tp in a heap hp. More...
      #define mi_heap_mallocn_tp(hp, tp, count) - Allocate count blocks of type tp in a heap hp. More...
    + Allocate count blocks of type tp in a heap hp. More...
      #define mi_heap_reallocn_tp(hp, p, tp, count) - Re-allocate to count blocks of type tp in a heap hp. More...
    + Re-allocate to count blocks of type tp in a heap hp. More...
      #define mi_heap_recalloc_tp(hp, p, tp, count) - Re-allocate to count zero initialized blocks of type tp in a heap hp. More...
    + Re-allocate to count zero initialized blocks of type tp in a heap hp. More...
     

    Detailed Description

    Typed allocation macros.

    -

    For example:

    int* p = mi_malloc_tp(int)

    Macro Definition Documentation

    +

    For example:

    int* p = mi_malloc_tp(int)
    +
    mi_malloc_tp
    #define mi_malloc_tp(tp)
    Allocate a block of type tp.
    Definition: mimalloc-doc.h:692
    +

    Macro Definition Documentation

    ◆ mi_calloc_tp

    @@ -417,7 +415,8 @@ Macros
    Returns
    A pointer to an object of type tp, or NULL if out of memory.
    -

    Example:

    int* p = mi_malloc_tp(int)
    See also
    mi_malloc()
    +

    Example:

    int* p = mi_malloc_tp(int)
    +
    See also
    mi_malloc()
    @@ -512,9 +511,7 @@ Macros diff --git a/docs/group__zeroinit.html b/docs/group__zeroinit.html index 3c04a5a6..329a7739 100644 --- a/docs/group__zeroinit.html +++ b/docs/group__zeroinit.html @@ -3,7 +3,7 @@ - + mi-malloc: Zero initialized re-allocation @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@ - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -588,9 +584,7 @@ Functions diff --git a/docs/index.html b/docs/index.html index 01af9bec..2ea91215 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3,7 +3,7 @@ - + mi-malloc: Main Page @@ -13,10 +13,6 @@ - @@ -37,12 +33,12 @@ Logo
    mi-malloc -  1.6 +  1.7/2.0
    - @@ -51,7 +47,7 @@ onblur="searchBox.OnSearchFieldFocus(false)" onkeyup="searchBox.OnSearchFieldChange(event)"/> - +
    @@ -60,10 +56,10 @@
    - + @@ -79,7 +75,7 @@ var searchBox = new SearchBox("searchBox", "search",false,'Search');
    @@ -103,13 +99,14 @@ $(document).ready(function(){initNavTree('index.html','');});

    This is the API documentation of the mimalloc allocator (pronounced "me-malloc") – a general purpose allocator with excellent performance characteristics. Initially developed by Daan Leijen for the run-time systems of the Koka and Lean languages.

    -

    It is a drop-in replacement for malloc and can be used in other programs without code changes, for example, on Unix you can use it as:

    > LD_PRELOAD=/usr/bin/libmimalloc.so myprogram

    Notable aspects of the design include:

    +

    It is a drop-in replacement for malloc and can be used in other programs without code changes, for example, on Unix you can use it as:

    > LD_PRELOAD=/usr/bin/libmimalloc.so myprogram
    +

    Notable aspects of the design include:

    • small and consistent: the library is about 8k LOC using simple and consistent data structures. This makes it very suitable to integrate and adapt in other projects. For runtime systems it provides hooks for a monotonic heartbeat and deferred freeing (for bounded worst-case times with reference counting).
    • free list sharding: instead of one big free list (per size class) we have many smaller lists per "mimalloc page" which reduces fragmentation and increases locality – things that are allocated close in time get allocated close in memory. (A mimalloc page contains blocks of one size class and is usually 64KiB on a 64-bit system).
    • free list multi-sharding: the big idea! Not only do we shard the free list per mimalloc page, but for each page we have multiple free lists. In particular, there is one list for thread-local free operations, and another one for concurrent free operations. Free-ing from another thread can now be a single CAS without needing sophisticated coordination between threads. Since there will be thousands of separate free lists, contention is naturally distributed over the heap, and the chance of contending on a single location will be low – this is quite similar to randomized algorithms like skip lists where adding a random oracle removes the need for a more complex algorithm.
    • eager page reset: when a "page" becomes empty (with increased chance due to free list sharding) the memory is marked to the OS as unused ("reset" or "purged") reducing (real) memory pressure and fragmentation, especially in long running programs.
      • -
    • secure: mimalloc can be build in secure mode, adding guard pages, randomized allocation, encrypted free lists, etc. to protect against various heap vulnerabilities. The performance penalty is only around 3% on average over our benchmarks.
      • +
    • secure: mimalloc can be build in secure mode, adding guard pages, randomized allocation, encrypted free lists, etc. to protect against various heap vulnerabilities. The performance penalty is only around 5% on average over our benchmarks.
    • first-class heaps: efficiently create and use multiple heaps to allocate across different regions. A heap can be destroyed at once instead of deallocating each object separately.
    • bounded: it does not suffer from blowup [1], has bounded worst-case allocation times (wcat), bounded space overhead (~0.2% meta-data, with at most 12.5% waste in allocation sizes), and has no internal points of contention using only atomic operations.
    • fast: In our benchmarks (see below), mimalloc outperforms all other leading allocators (jemalloc, tcmalloc, Hoard, etc), and usually uses less memory (up to 25% more in the worst case). A nice property is that it does consistently well over a wide range of benchmarks.
      • @@ -138,9 +135,7 @@ $(document).ready(function(){initNavTree('index.html','');}); diff --git a/docs/jquery.js b/docs/jquery.js index 1ee895ca..103c32d7 100644 --- a/docs/jquery.js +++ b/docs/jquery.js @@ -1,71 +1,26 @@ +/*! jQuery v3.4.1 | (c) JS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],E=C.document,r=Object.getPrototypeOf,s=t.slice,g=t.concat,u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.4.1",k=function(e,t){return new k.fn.init(e,t)},p=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;function d(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp($),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+$),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\([\\da-f]{1,6}"+M+"?|("+M+")|.)","ig"),ne=function(e,t,n){var r="0x"+t-65536;return r!=r||n?t:r<0?String.fromCharCode(r+65536):String.fromCharCode(r>>10|55296,1023&r|56320)},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(m.childNodes),m.childNodes),t[m.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&((e?e.ownerDocument||e:m)!==C&&T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!A[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&U.test(t)){(s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=k),o=(l=h(t)).length;while(o--)l[o]="#"+s+" "+xe(l[o]);c=l.join(","),f=ee.test(t)&&ye(e.parentNode)||e}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){A(t,!0)}finally{s===k&&e.removeAttribute("id")}}}return g(t.replace(B,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[k]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:m;return r!==C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),m!==C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=k,!C.getElementsByName||!C.getElementsByName(k).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+k+"-]").length||v.push("~="),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+k+"+*").length||v.push(".#.+[+~]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",$)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)===(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e===C||e.ownerDocument===m&&y(m,e)?-1:t===C||t.ownerDocument===m&&y(m,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e===C?-1:t===C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]===m?-1:s[r]===m?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if((e.ownerDocument||e)!==C&&T(e),d.matchesSelector&&E&&!A[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){A(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=p[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&p(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?k.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?k.grep(e,function(e){return e===n!==r}):"string"!=typeof n?k.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(k.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||q,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:L.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof k?t[0]:t,k.merge(this,k.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),D.test(r[1])&&k.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(k):k.makeArray(e,this)}).prototype=k.fn,q=k(E);var H=/^(?:parents|prev(?:Until|All))/,O={children:!0,contents:!0,next:!0,prev:!0};function P(e,t){while((e=e[t])&&1!==e.nodeType);return e}k.fn.extend({has:function(e){var t=k(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i,ge={option:[1,""],thead:[1,"","
      "],col:[2,"","
      "],tr:[2,"","
      "],td:[3,"","
      "],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?k.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;nx",y.noCloneChecked=!!me.cloneNode(!0).lastChild.defaultValue;var Te=/^key/,Ce=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,Ee=/^([^.]*)(?:\.(.+)|)/;function ke(){return!0}function Se(){return!1}function Ne(e,t){return e===function(){try{return E.activeElement}catch(e){}}()==("focus"===t)}function Ae(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Ae(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=Se;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return k().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=k.guid++)),e.each(function(){k.event.add(this,t,i,r,n)})}function De(e,i,o){o?(Q.set(e,i,!1),k.event.add(e,i,{namespace:!1,handler:function(e){var t,n,r=Q.get(this,i);if(1&e.isTrigger&&this[i]){if(r.length)(k.event.special[i]||{}).delegateType&&e.stopPropagation();else if(r=s.call(arguments),Q.set(this,i,r),t=o(this,i),this[i](),r!==(n=Q.get(this,i))||t?Q.set(this,i,!1):n={},r!==n)return e.stopImmediatePropagation(),e.preventDefault(),n.value}else r.length&&(Q.set(this,i,{value:k.event.trigger(k.extend(r[0],k.Event.prototype),r.slice(1),this)}),e.stopImmediatePropagation())}})):void 0===Q.get(e,i)&&k.event.add(e,i,ke)}k.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.get(t);if(v){n.handler&&(n=(o=n).handler,i=o.selector),i&&k.find.matchesSelector(ie,i),n.guid||(n.guid=k.guid++),(u=v.events)||(u=v.events={}),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof k&&k.event.triggered!==e.type?k.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(R)||[""]).length;while(l--)d=g=(s=Ee.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=k.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=k.event.special[d]||{},c=k.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&k.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),k.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=Q.hasData(e)&&Q.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(R)||[""]).length;while(l--)if(d=g=(s=Ee.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=k.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||k.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)k.event.remove(e,d+t[l],n,r,!0);k.isEmptyObject(u)&&Q.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=k.event.fix(e),u=new Array(arguments.length),l=(Q.get(this,"events")||{})[s.type]||[],c=k.event.special[s.type]||{};for(u[0]=s,t=1;t\x20\t\r\n\f]*)[^>]*)\/>/gi,qe=/\s*$/g;function Oe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&k(e).children("tbody")[0]||e}function Pe(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function Re(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Me(e,t){var n,r,i,o,a,s,u,l;if(1===t.nodeType){if(Q.hasData(e)&&(o=Q.access(e),a=Q.set(t,o),l=o.events))for(i in delete a.handle,a.events={},l)for(n=0,r=l[i].length;n")},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=oe(e);if(!(y.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||k.isXMLDoc(e)))for(a=ve(c),r=0,i=(o=ve(e)).length;r").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Vt,Gt=[],Yt=/(=)\?(?=&|$)|\?\?/;k.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Gt.pop()||k.expando+"_"+kt++;return this[e]=!0,e}}),k.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Yt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Yt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Yt,"$1"+r):!1!==e.jsonp&&(e.url+=(St.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||k.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?k(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Gt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Vt=E.implementation.createHTMLDocument("").body).innerHTML="
      ",2===Vt.childNodes.length),k.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=D.exec(e))?[t.createElement(i[1])]:(i=we([e],t,o),o&&o.length&&k(o).remove(),k.merge([],i.childNodes)));var r,i,o},k.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(k.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},k.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){k.fn[t]=function(e){return this.on(t,e)}}),k.expr.pseudos.animated=function(t){return k.grep(k.timers,function(e){return t===e.elem}).length},k.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=k.css(e,"position"),c=k(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=k.css(e,"top"),u=k.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,k.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},k.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){k.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===k.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===k.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=k(e).offset()).top+=k.css(e,"borderTopWidth",!0),i.left+=k.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-k.css(r,"marginTop",!0),left:t.left-i.left-k.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===k.css(e,"position"))e=e.offsetParent;return e||ie})}}),k.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;k.fn[t]=function(e){return _(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),k.each(["top","left"],function(e,n){k.cssHooks[n]=ze(y.pixelPosition,function(e,t){if(t)return t=_e(e,n),$e.test(t)?k(e).position()[n]+"px":t})}),k.each({Height:"height",Width:"width"},function(a,s){k.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){k.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return _(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?k.css(e,t,i):k.style(e,t,n,i)},s,n?e:void 0,n)}})}),k.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){k.fn[n]=function(e,t){return 0a;a++)for(i in o[a])n=o[a][i],o[a].hasOwnProperty(i)&&void 0!==n&&(e[i]=t.isPlainObject(n)?t.isPlainObject(e[i])?t.widget.extend({},e[i],n):t.widget.extend({},n):n);return e},t.widget.bridge=function(e,i){var n=i.prototype.widgetFullName||e;t.fn[e]=function(o){var a="string"==typeof o,r=s.call(arguments,1),h=this;return a?this.length||"instance"!==o?this.each(function(){var i,s=t.data(this,n);return"instance"===o?(h=s,!1):s?t.isFunction(s[o])&&"_"!==o.charAt(0)?(i=s[o].apply(s,r),i!==s&&void 0!==i?(h=i&&i.jquery?h.pushStack(i.get()):i,!1):void 0):t.error("no such method '"+o+"' for "+e+" widget instance"):t.error("cannot call methods on "+e+" prior to initialization; "+"attempted to call method '"+o+"'")}):h=void 0:(r.length&&(o=t.widget.extend.apply(null,[o].concat(r))),this.each(function(){var e=t.data(this,n);e?(e.option(o||{}),e._init&&e._init()):t.data(this,n,new i(o,this))})),h}},t.Widget=function(){},t.Widget._childConstructors=[],t.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",defaultElement:"
      ",options:{classes:{},disabled:!1,create:null},_createWidget:function(e,s){s=t(s||this.defaultElement||this)[0],this.element=t(s),this.uuid=i++,this.eventNamespace="."+this.widgetName+this.uuid,this.bindings=t(),this.hoverable=t(),this.focusable=t(),this.classesElementLookup={},s!==this&&(t.data(s,this.widgetFullName,this),this._on(!0,this.element,{remove:function(t){t.target===s&&this.destroy()}}),this.document=t(s.style?s.ownerDocument:s.document||s),this.window=t(this.document[0].defaultView||this.document[0].parentWindow)),this.options=t.widget.extend({},this.options,this._getCreateOptions(),e),this._create(),this.options.disabled&&this._setOptionDisabled(this.options.disabled),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:function(){return{}},_getCreateEventData:t.noop,_create:t.noop,_init:t.noop,destroy:function(){var e=this;this._destroy(),t.each(this.classesElementLookup,function(t,i){e._removeClass(i,t)}),this.element.off(this.eventNamespace).removeData(this.widgetFullName),this.widget().off(this.eventNamespace).removeAttr("aria-disabled"),this.bindings.off(this.eventNamespace)},_destroy:t.noop,widget:function(){return this.element},option:function(e,i){var s,n,o,a=e;if(0===arguments.length)return t.widget.extend({},this.options);if("string"==typeof e)if(a={},s=e.split("."),e=s.shift(),s.length){for(n=a[e]=t.widget.extend({},this.options[e]),o=0;s.length-1>o;o++)n[s[o]]=n[s[o]]||{},n=n[s[o]];if(e=s.pop(),1===arguments.length)return void 0===n[e]?null:n[e];n[e]=i}else{if(1===arguments.length)return void 0===this.options[e]?null:this.options[e];a[e]=i}return this._setOptions(a),this},_setOptions:function(t){var e;for(e in t)this._setOption(e,t[e]);return this},_setOption:function(t,e){return"classes"===t&&this._setOptionClasses(e),this.options[t]=e,"disabled"===t&&this._setOptionDisabled(e),this},_setOptionClasses:function(e){var i,s,n;for(i in e)n=this.classesElementLookup[i],e[i]!==this.options.classes[i]&&n&&n.length&&(s=t(n.get()),this._removeClass(n,i),s.addClass(this._classes({element:s,keys:i,classes:e,add:!0})))},_setOptionDisabled:function(t){this._toggleClass(this.widget(),this.widgetFullName+"-disabled",null,!!t),t&&(this._removeClass(this.hoverable,null,"ui-state-hover"),this._removeClass(this.focusable,null,"ui-state-focus"))},enable:function(){return this._setOptions({disabled:!1})},disable:function(){return this._setOptions({disabled:!0})},_classes:function(e){function i(i,o){var a,r;for(r=0;i.length>r;r++)a=n.classesElementLookup[i[r]]||t(),a=e.add?t(t.unique(a.get().concat(e.element.get()))):t(a.not(e.element).get()),n.classesElementLookup[i[r]]=a,s.push(i[r]),o&&e.classes[i[r]]&&s.push(e.classes[i[r]])}var s=[],n=this;return e=t.extend({element:this.element,classes:this.options.classes||{}},e),this._on(e.element,{remove:"_untrackClassesElement"}),e.keys&&i(e.keys.match(/\S+/g)||[],!0),e.extra&&i(e.extra.match(/\S+/g)||[]),s.join(" ")},_untrackClassesElement:function(e){var i=this;t.each(i.classesElementLookup,function(s,n){-1!==t.inArray(e.target,n)&&(i.classesElementLookup[s]=t(n.not(e.target).get()))})},_removeClass:function(t,e,i){return this._toggleClass(t,e,i,!1)},_addClass:function(t,e,i){return this._toggleClass(t,e,i,!0)},_toggleClass:function(t,e,i,s){s="boolean"==typeof s?s:i;var n="string"==typeof t||null===t,o={extra:n?e:i,keys:n?t:e,element:n?this.element:t,add:s};return o.element.toggleClass(this._classes(o),s),this},_on:function(e,i,s){var n,o=this;"boolean"!=typeof e&&(s=i,i=e,e=!1),s?(i=n=t(i),this.bindings=this.bindings.add(i)):(s=i,i=this.element,n=this.widget()),t.each(s,function(s,a){function r(){return e||o.options.disabled!==!0&&!t(this).hasClass("ui-state-disabled")?("string"==typeof a?o[a]:a).apply(o,arguments):void 0}"string"!=typeof a&&(r.guid=a.guid=a.guid||r.guid||t.guid++);var h=s.match(/^([\w:-]*)\s*(.*)$/),l=h[1]+o.eventNamespace,c=h[2];c?n.on(l,c,r):i.on(l,r)})},_off:function(e,i){i=(i||"").split(" ").join(this.eventNamespace+" ")+this.eventNamespace,e.off(i).off(i),this.bindings=t(this.bindings.not(e).get()),this.focusable=t(this.focusable.not(e).get()),this.hoverable=t(this.hoverable.not(e).get())},_delay:function(t,e){function i(){return("string"==typeof t?s[t]:t).apply(s,arguments)}var s=this;return setTimeout(i,e||0)},_hoverable:function(e){this.hoverable=this.hoverable.add(e),this._on(e,{mouseenter:function(e){this._addClass(t(e.currentTarget),null,"ui-state-hover")},mouseleave:function(e){this._removeClass(t(e.currentTarget),null,"ui-state-hover")}})},_focusable:function(e){this.focusable=this.focusable.add(e),this._on(e,{focusin:function(e){this._addClass(t(e.currentTarget),null,"ui-state-focus")},focusout:function(e){this._removeClass(t(e.currentTarget),null,"ui-state-focus")}})},_trigger:function(e,i,s){var n,o,a=this.options[e];if(s=s||{},i=t.Event(i),i.type=(e===this.widgetEventPrefix?e:this.widgetEventPrefix+e).toLowerCase(),i.target=this.element[0],o=i.originalEvent)for(n in o)n in i||(i[n]=o[n]);return this.element.trigger(i,s),!(t.isFunction(a)&&a.apply(this.element[0],[i].concat(s))===!1||i.isDefaultPrevented())}},t.each({show:"fadeIn",hide:"fadeOut"},function(e,i){t.Widget.prototype["_"+e]=function(s,n,o){"string"==typeof n&&(n={effect:n});var a,r=n?n===!0||"number"==typeof n?i:n.effect||i:e;n=n||{},"number"==typeof n&&(n={duration:n}),a=!t.isEmptyObject(n),n.complete=o,n.delay&&s.delay(n.delay),a&&t.effects&&t.effects.effect[r]?s[e](n):r!==e&&s[r]?s[r](n.duration,n.easing,o):s.queue(function(i){t(this)[e](),o&&o.call(s[0]),i()})}}),t.widget,function(){function e(t,e,i){return[parseFloat(t[0])*(u.test(t[0])?e/100:1),parseFloat(t[1])*(u.test(t[1])?i/100:1)]}function i(e,i){return parseInt(t.css(e,i),10)||0}function s(e){var i=e[0];return 9===i.nodeType?{width:e.width(),height:e.height(),offset:{top:0,left:0}}:t.isWindow(i)?{width:e.width(),height:e.height(),offset:{top:e.scrollTop(),left:e.scrollLeft()}}:i.preventDefault?{width:0,height:0,offset:{top:i.pageY,left:i.pageX}}:{width:e.outerWidth(),height:e.outerHeight(),offset:e.offset()}}var n,o=Math.max,a=Math.abs,r=/left|center|right/,h=/top|center|bottom/,l=/[\+\-]\d+(\.[\d]+)?%?/,c=/^\w+/,u=/%$/,d=t.fn.position;t.position={scrollbarWidth:function(){if(void 0!==n)return n;var e,i,s=t("
      "),o=s.children()[0];return t("body").append(s),e=o.offsetWidth,s.css("overflow","scroll"),i=o.offsetWidth,e===i&&(i=s[0].clientWidth),s.remove(),n=e-i},getScrollInfo:function(e){var i=e.isWindow||e.isDocument?"":e.element.css("overflow-x"),s=e.isWindow||e.isDocument?"":e.element.css("overflow-y"),n="scroll"===i||"auto"===i&&e.widthi?"left":e>0?"right":"center",vertical:0>r?"top":s>0?"bottom":"middle"};l>p&&p>a(e+i)&&(u.horizontal="center"),c>f&&f>a(s+r)&&(u.vertical="middle"),u.important=o(a(e),a(i))>o(a(s),a(r))?"horizontal":"vertical",n.using.call(this,t,u)}),h.offset(t.extend(D,{using:r}))})},t.ui.position={fit:{left:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollLeft:s.offset.left,a=s.width,r=t.left-e.collisionPosition.marginLeft,h=n-r,l=r+e.collisionWidth-a-n;e.collisionWidth>a?h>0&&0>=l?(i=t.left+h+e.collisionWidth-a-n,t.left+=h-i):t.left=l>0&&0>=h?n:h>l?n+a-e.collisionWidth:n:h>0?t.left+=h:l>0?t.left-=l:t.left=o(t.left-r,t.left)},top:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollTop:s.offset.top,a=e.within.height,r=t.top-e.collisionPosition.marginTop,h=n-r,l=r+e.collisionHeight-a-n;e.collisionHeight>a?h>0&&0>=l?(i=t.top+h+e.collisionHeight-a-n,t.top+=h-i):t.top=l>0&&0>=h?n:h>l?n+a-e.collisionHeight:n:h>0?t.top+=h:l>0?t.top-=l:t.top=o(t.top-r,t.top)}},flip:{left:function(t,e){var i,s,n=e.within,o=n.offset.left+n.scrollLeft,r=n.width,h=n.isWindow?n.scrollLeft:n.offset.left,l=t.left-e.collisionPosition.marginLeft,c=l-h,u=l+e.collisionWidth-r-h,d="left"===e.my[0]?-e.elemWidth:"right"===e.my[0]?e.elemWidth:0,p="left"===e.at[0]?e.targetWidth:"right"===e.at[0]?-e.targetWidth:0,f=-2*e.offset[0];0>c?(i=t.left+d+p+f+e.collisionWidth-r-o,(0>i||a(c)>i)&&(t.left+=d+p+f)):u>0&&(s=t.left-e.collisionPosition.marginLeft+d+p+f-h,(s>0||u>a(s))&&(t.left+=d+p+f))},top:function(t,e){var i,s,n=e.within,o=n.offset.top+n.scrollTop,r=n.height,h=n.isWindow?n.scrollTop:n.offset.top,l=t.top-e.collisionPosition.marginTop,c=l-h,u=l+e.collisionHeight-r-h,d="top"===e.my[1],p=d?-e.elemHeight:"bottom"===e.my[1]?e.elemHeight:0,f="top"===e.at[1]?e.targetHeight:"bottom"===e.at[1]?-e.targetHeight:0,m=-2*e.offset[1];0>c?(s=t.top+p+f+m+e.collisionHeight-r-o,(0>s||a(c)>s)&&(t.top+=p+f+m)):u>0&&(i=t.top-e.collisionPosition.marginTop+p+f+m-h,(i>0||u>a(i))&&(t.top+=p+f+m))}},flipfit:{left:function(){t.ui.position.flip.left.apply(this,arguments),t.ui.position.fit.left.apply(this,arguments)},top:function(){t.ui.position.flip.top.apply(this,arguments),t.ui.position.fit.top.apply(this,arguments)}}}}(),t.ui.position,t.extend(t.expr[":"],{data:t.expr.createPseudo?t.expr.createPseudo(function(e){return function(i){return!!t.data(i,e)}}):function(e,i,s){return!!t.data(e,s[3])}}),t.fn.extend({disableSelection:function(){var t="onselectstart"in document.createElement("div")?"selectstart":"mousedown";return function(){return this.on(t+".ui-disableSelection",function(t){t.preventDefault()})}}(),enableSelection:function(){return this.off(".ui-disableSelection")}}),t.ui.focusable=function(i,s){var n,o,a,r,h,l=i.nodeName.toLowerCase();return"area"===l?(n=i.parentNode,o=n.name,i.href&&o&&"map"===n.nodeName.toLowerCase()?(a=t("img[usemap='#"+o+"']"),a.length>0&&a.is(":visible")):!1):(/^(input|select|textarea|button|object)$/.test(l)?(r=!i.disabled,r&&(h=t(i).closest("fieldset")[0],h&&(r=!h.disabled))):r="a"===l?i.href||s:s,r&&t(i).is(":visible")&&e(t(i)))},t.extend(t.expr[":"],{focusable:function(e){return t.ui.focusable(e,null!=t.attr(e,"tabindex"))}}),t.ui.focusable,t.fn.form=function(){return"string"==typeof this[0].form?this.closest("form"):t(this[0].form)},t.ui.formResetMixin={_formResetHandler:function(){var e=t(this);setTimeout(function(){var i=e.data("ui-form-reset-instances");t.each(i,function(){this.refresh()})})},_bindFormResetHandler:function(){if(this.form=this.element.form(),this.form.length){var t=this.form.data("ui-form-reset-instances")||[];t.length||this.form.on("reset.ui-form-reset",this._formResetHandler),t.push(this),this.form.data("ui-form-reset-instances",t)}},_unbindFormResetHandler:function(){if(this.form.length){var e=this.form.data("ui-form-reset-instances");e.splice(t.inArray(this,e),1),e.length?this.form.data("ui-form-reset-instances",e):this.form.removeData("ui-form-reset-instances").off("reset.ui-form-reset")}}},"1.7"===t.fn.jquery.substring(0,3)&&(t.each(["Width","Height"],function(e,i){function s(e,i,s,o){return t.each(n,function(){i-=parseFloat(t.css(e,"padding"+this))||0,s&&(i-=parseFloat(t.css(e,"border"+this+"Width"))||0),o&&(i-=parseFloat(t.css(e,"margin"+this))||0)}),i}var n="Width"===i?["Left","Right"]:["Top","Bottom"],o=i.toLowerCase(),a={innerWidth:t.fn.innerWidth,innerHeight:t.fn.innerHeight,outerWidth:t.fn.outerWidth,outerHeight:t.fn.outerHeight};t.fn["inner"+i]=function(e){return void 0===e?a["inner"+i].call(this):this.each(function(){t(this).css(o,s(this,e)+"px")})},t.fn["outer"+i]=function(e,n){return"number"!=typeof e?a["outer"+i].call(this,e):this.each(function(){t(this).css(o,s(this,e,!0,n)+"px")})}}),t.fn.addBack=function(t){return this.add(null==t?this.prevObject:this.prevObject.filter(t))}),t.ui.keyCode={BACKSPACE:8,COMMA:188,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,LEFT:37,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SPACE:32,TAB:9,UP:38},t.ui.escapeSelector=function(){var t=/([!"#$%&'()*+,./:;<=>?@[\]^`{|}~])/g;return function(e){return e.replace(t,"\\$1")}}(),t.fn.labels=function(){var e,i,s,n,o;return this[0].labels&&this[0].labels.length?this.pushStack(this[0].labels):(n=this.eq(0).parents("label"),s=this.attr("id"),s&&(e=this.eq(0).parents().last(),o=e.add(e.length?e.siblings():this.siblings()),i="label[for='"+t.ui.escapeSelector(s)+"']",n=n.add(o.find(i).addBack(i))),this.pushStack(n))},t.fn.scrollParent=function(e){var i=this.css("position"),s="absolute"===i,n=e?/(auto|scroll|hidden)/:/(auto|scroll)/,o=this.parents().filter(function(){var e=t(this);return s&&"static"===e.css("position")?!1:n.test(e.css("overflow")+e.css("overflow-y")+e.css("overflow-x"))}).eq(0);return"fixed"!==i&&o.length?o:t(this[0].ownerDocument||document)},t.extend(t.expr[":"],{tabbable:function(e){var i=t.attr(e,"tabindex"),s=null!=i;return(!s||i>=0)&&t.ui.focusable(e,s)}}),t.fn.extend({uniqueId:function(){var t=0;return function(){return this.each(function(){this.id||(this.id="ui-id-"+ ++t)})}}(),removeUniqueId:function(){return this.each(function(){/^ui-id-\d+$/.test(this.id)&&t(this).removeAttr("id")})}}),t.ui.ie=!!/msie [\w.]+/.exec(navigator.userAgent.toLowerCase());var n=!1;t(document).on("mouseup",function(){n=!1}),t.widget("ui.mouse",{version:"1.12.1",options:{cancel:"input, textarea, button, select, option",distance:1,delay:0},_mouseInit:function(){var e=this;this.element.on("mousedown."+this.widgetName,function(t){return e._mouseDown(t)}).on("click."+this.widgetName,function(i){return!0===t.data(i.target,e.widgetName+".preventClickEvent")?(t.removeData(i.target,e.widgetName+".preventClickEvent"),i.stopImmediatePropagation(),!1):void 0}),this.started=!1},_mouseDestroy:function(){this.element.off("."+this.widgetName),this._mouseMoveDelegate&&this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate)},_mouseDown:function(e){if(!n){this._mouseMoved=!1,this._mouseStarted&&this._mouseUp(e),this._mouseDownEvent=e;var i=this,s=1===e.which,o="string"==typeof this.options.cancel&&e.target.nodeName?t(e.target).closest(this.options.cancel).length:!1;return s&&!o&&this._mouseCapture(e)?(this.mouseDelayMet=!this.options.delay,this.mouseDelayMet||(this._mouseDelayTimer=setTimeout(function(){i.mouseDelayMet=!0},this.options.delay)),this._mouseDistanceMet(e)&&this._mouseDelayMet(e)&&(this._mouseStarted=this._mouseStart(e)!==!1,!this._mouseStarted)?(e.preventDefault(),!0):(!0===t.data(e.target,this.widgetName+".preventClickEvent")&&t.removeData(e.target,this.widgetName+".preventClickEvent"),this._mouseMoveDelegate=function(t){return i._mouseMove(t)},this._mouseUpDelegate=function(t){return i._mouseUp(t)},this.document.on("mousemove."+this.widgetName,this._mouseMoveDelegate).on("mouseup."+this.widgetName,this._mouseUpDelegate),e.preventDefault(),n=!0,!0)):!0}},_mouseMove:function(e){if(this._mouseMoved){if(t.ui.ie&&(!document.documentMode||9>document.documentMode)&&!e.button)return this._mouseUp(e);if(!e.which)if(e.originalEvent.altKey||e.originalEvent.ctrlKey||e.originalEvent.metaKey||e.originalEvent.shiftKey)this.ignoreMissingWhich=!0;else if(!this.ignoreMissingWhich)return this._mouseUp(e)}return(e.which||e.button)&&(this._mouseMoved=!0),this._mouseStarted?(this._mouseDrag(e),e.preventDefault()):(this._mouseDistanceMet(e)&&this._mouseDelayMet(e)&&(this._mouseStarted=this._mouseStart(this._mouseDownEvent,e)!==!1,this._mouseStarted?this._mouseDrag(e):this._mouseUp(e)),!this._mouseStarted)},_mouseUp:function(e){this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate),this._mouseStarted&&(this._mouseStarted=!1,e.target===this._mouseDownEvent.target&&t.data(e.target,this.widgetName+".preventClickEvent",!0),this._mouseStop(e)),this._mouseDelayTimer&&(clearTimeout(this._mouseDelayTimer),delete this._mouseDelayTimer),this.ignoreMissingWhich=!1,n=!1,e.preventDefault()},_mouseDistanceMet:function(t){return Math.max(Math.abs(this._mouseDownEvent.pageX-t.pageX),Math.abs(this._mouseDownEvent.pageY-t.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return!0}}),t.ui.plugin={add:function(e,i,s){var n,o=t.ui[e].prototype;for(n in s)o.plugins[n]=o.plugins[n]||[],o.plugins[n].push([i,s[n]])},call:function(t,e,i,s){var n,o=t.plugins[e];if(o&&(s||t.element[0].parentNode&&11!==t.element[0].parentNode.nodeType))for(n=0;o.length>n;n++)t.options[o[n][0]]&&o[n][1].apply(t.element,i)}},t.widget("ui.resizable",t.ui.mouse,{version:"1.12.1",widgetEventPrefix:"resize",options:{alsoResize:!1,animate:!1,animateDuration:"slow",animateEasing:"swing",aspectRatio:!1,autoHide:!1,classes:{"ui-resizable-se":"ui-icon ui-icon-gripsmall-diagonal-se"},containment:!1,ghost:!1,grid:!1,handles:"e,s,se",helper:!1,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:90,resize:null,start:null,stop:null},_num:function(t){return parseFloat(t)||0},_isNumber:function(t){return!isNaN(parseFloat(t))},_hasScroll:function(e,i){if("hidden"===t(e).css("overflow"))return!1;var s=i&&"left"===i?"scrollLeft":"scrollTop",n=!1;return e[s]>0?!0:(e[s]=1,n=e[s]>0,e[s]=0,n)},_create:function(){var e,i=this.options,s=this;this._addClass("ui-resizable"),t.extend(this,{_aspectRatio:!!i.aspectRatio,aspectRatio:i.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:i.helper||i.ghost||i.animate?i.helper||"ui-resizable-helper":null}),this.element[0].nodeName.match(/^(canvas|textarea|input|select|button|img)$/i)&&(this.element.wrap(t("
      ").css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("ui-resizable",this.element.resizable("instance")),this.elementIsWrapper=!0,e={marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom"),marginLeft:this.originalElement.css("marginLeft")},this.element.css(e),this.originalElement.css("margin",0),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css(e),this._proportionallyResize()),this._setupHandles(),i.autoHide&&t(this.element).on("mouseenter",function(){i.disabled||(s._removeClass("ui-resizable-autohide"),s._handles.show())}).on("mouseleave",function(){i.disabled||s.resizing||(s._addClass("ui-resizable-autohide"),s._handles.hide())}),this._mouseInit()},_destroy:function(){this._mouseDestroy();var e,i=function(e){t(e).removeData("resizable").removeData("ui-resizable").off(".resizable").find(".ui-resizable-handle").remove()};return this.elementIsWrapper&&(i(this.element),e=this.element,this.originalElement.css({position:e.css("position"),width:e.outerWidth(),height:e.outerHeight(),top:e.css("top"),left:e.css("left")}).insertAfter(e),e.remove()),this.originalElement.css("resize",this.originalResizeStyle),i(this.originalElement),this},_setOption:function(t,e){switch(this._super(t,e),t){case"handles":this._removeHandles(),this._setupHandles();break;default:}},_setupHandles:function(){var e,i,s,n,o,a=this.options,r=this;if(this.handles=a.handles||(t(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se"),this._handles=t(),this.handles.constructor===String)for("all"===this.handles&&(this.handles="n,e,s,w,se,sw,ne,nw"),s=this.handles.split(","),this.handles={},i=0;s.length>i;i++)e=t.trim(s[i]),n="ui-resizable-"+e,o=t("
      "),this._addClass(o,"ui-resizable-handle "+n),o.css({zIndex:a.zIndex}),this.handles[e]=".ui-resizable-"+e,this.element.append(o);this._renderAxis=function(e){var i,s,n,o;e=e||this.element;for(i in this.handles)this.handles[i].constructor===String?this.handles[i]=this.element.children(this.handles[i]).first().show():(this.handles[i].jquery||this.handles[i].nodeType)&&(this.handles[i]=t(this.handles[i]),this._on(this.handles[i],{mousedown:r._mouseDown})),this.elementIsWrapper&&this.originalElement[0].nodeName.match(/^(textarea|input|select|button)$/i)&&(s=t(this.handles[i],this.element),o=/sw|ne|nw|se|n|s/.test(i)?s.outerHeight():s.outerWidth(),n=["padding",/ne|nw|n/.test(i)?"Top":/se|sw|s/.test(i)?"Bottom":/^e$/.test(i)?"Right":"Left"].join(""),e.css(n,o),this._proportionallyResize()),this._handles=this._handles.add(this.handles[i])},this._renderAxis(this.element),this._handles=this._handles.add(this.element.find(".ui-resizable-handle")),this._handles.disableSelection(),this._handles.on("mouseover",function(){r.resizing||(this.className&&(o=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i)),r.axis=o&&o[1]?o[1]:"se")}),a.autoHide&&(this._handles.hide(),this._addClass("ui-resizable-autohide"))},_removeHandles:function(){this._handles.remove()},_mouseCapture:function(e){var i,s,n=!1;for(i in this.handles)s=t(this.handles[i])[0],(s===e.target||t.contains(s,e.target))&&(n=!0);return!this.options.disabled&&n},_mouseStart:function(e){var i,s,n,o=this.options,a=this.element;return this.resizing=!0,this._renderProxy(),i=this._num(this.helper.css("left")),s=this._num(this.helper.css("top")),o.containment&&(i+=t(o.containment).scrollLeft()||0,s+=t(o.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:i,top:s},this.size=this._helper?{width:this.helper.width(),height:this.helper.height()}:{width:a.width(),height:a.height()},this.originalSize=this._helper?{width:a.outerWidth(),height:a.outerHeight()}:{width:a.width(),height:a.height()},this.sizeDiff={width:a.outerWidth()-a.width(),height:a.outerHeight()-a.height()},this.originalPosition={left:i,top:s},this.originalMousePosition={left:e.pageX,top:e.pageY},this.aspectRatio="number"==typeof o.aspectRatio?o.aspectRatio:this.originalSize.width/this.originalSize.height||1,n=t(".ui-resizable-"+this.axis).css("cursor"),t("body").css("cursor","auto"===n?this.axis+"-resize":n),this._addClass("ui-resizable-resizing"),this._propagate("start",e),!0},_mouseDrag:function(e){var i,s,n=this.originalMousePosition,o=this.axis,a=e.pageX-n.left||0,r=e.pageY-n.top||0,h=this._change[o];return this._updatePrevProperties(),h?(i=h.apply(this,[e,a,r]),this._updateVirtualBoundaries(e.shiftKey),(this._aspectRatio||e.shiftKey)&&(i=this._updateRatio(i,e)),i=this._respectSize(i,e),this._updateCache(i),this._propagate("resize",e),s=this._applyChanges(),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),t.isEmptyObject(s)||(this._updatePrevProperties(),this._trigger("resize",e,this.ui()),this._applyChanges()),!1):!1},_mouseStop:function(e){this.resizing=!1;var i,s,n,o,a,r,h,l=this.options,c=this;return this._helper&&(i=this._proportionallyResizeElements,s=i.length&&/textarea/i.test(i[0].nodeName),n=s&&this._hasScroll(i[0],"left")?0:c.sizeDiff.height,o=s?0:c.sizeDiff.width,a={width:c.helper.width()-o,height:c.helper.height()-n},r=parseFloat(c.element.css("left"))+(c.position.left-c.originalPosition.left)||null,h=parseFloat(c.element.css("top"))+(c.position.top-c.originalPosition.top)||null,l.animate||this.element.css(t.extend(a,{top:h,left:r})),c.helper.height(c.size.height),c.helper.width(c.size.width),this._helper&&!l.animate&&this._proportionallyResize()),t("body").css("cursor","auto"),this._removeClass("ui-resizable-resizing"),this._propagate("stop",e),this._helper&&this.helper.remove(),!1},_updatePrevProperties:function(){this.prevPosition={top:this.position.top,left:this.position.left},this.prevSize={width:this.size.width,height:this.size.height}},_applyChanges:function(){var t={};return this.position.top!==this.prevPosition.top&&(t.top=this.position.top+"px"),this.position.left!==this.prevPosition.left&&(t.left=this.position.left+"px"),this.size.width!==this.prevSize.width&&(t.width=this.size.width+"px"),this.size.height!==this.prevSize.height&&(t.height=this.size.height+"px"),this.helper.css(t),t},_updateVirtualBoundaries:function(t){var e,i,s,n,o,a=this.options;o={minWidth:this._isNumber(a.minWidth)?a.minWidth:0,maxWidth:this._isNumber(a.maxWidth)?a.maxWidth:1/0,minHeight:this._isNumber(a.minHeight)?a.minHeight:0,maxHeight:this._isNumber(a.maxHeight)?a.maxHeight:1/0},(this._aspectRatio||t)&&(e=o.minHeight*this.aspectRatio,s=o.minWidth/this.aspectRatio,i=o.maxHeight*this.aspectRatio,n=o.maxWidth/this.aspectRatio,e>o.minWidth&&(o.minWidth=e),s>o.minHeight&&(o.minHeight=s),o.maxWidth>i&&(o.maxWidth=i),o.maxHeight>n&&(o.maxHeight=n)),this._vBoundaries=o},_updateCache:function(t){this.offset=this.helper.offset(),this._isNumber(t.left)&&(this.position.left=t.left),this._isNumber(t.top)&&(this.position.top=t.top),this._isNumber(t.height)&&(this.size.height=t.height),this._isNumber(t.width)&&(this.size.width=t.width)},_updateRatio:function(t){var e=this.position,i=this.size,s=this.axis;return this._isNumber(t.height)?t.width=t.height*this.aspectRatio:this._isNumber(t.width)&&(t.height=t.width/this.aspectRatio),"sw"===s&&(t.left=e.left+(i.width-t.width),t.top=null),"nw"===s&&(t.top=e.top+(i.height-t.height),t.left=e.left+(i.width-t.width)),t},_respectSize:function(t){var e=this._vBoundaries,i=this.axis,s=this._isNumber(t.width)&&e.maxWidth&&e.maxWidtht.width,a=this._isNumber(t.height)&&e.minHeight&&e.minHeight>t.height,r=this.originalPosition.left+this.originalSize.width,h=this.originalPosition.top+this.originalSize.height,l=/sw|nw|w/.test(i),c=/nw|ne|n/.test(i);return o&&(t.width=e.minWidth),a&&(t.height=e.minHeight),s&&(t.width=e.maxWidth),n&&(t.height=e.maxHeight),o&&l&&(t.left=r-e.minWidth),s&&l&&(t.left=r-e.maxWidth),a&&c&&(t.top=h-e.minHeight),n&&c&&(t.top=h-e.maxHeight),t.width||t.height||t.left||!t.top?t.width||t.height||t.top||!t.left||(t.left=null):t.top=null,t},_getPaddingPlusBorderDimensions:function(t){for(var e=0,i=[],s=[t.css("borderTopWidth"),t.css("borderRightWidth"),t.css("borderBottomWidth"),t.css("borderLeftWidth")],n=[t.css("paddingTop"),t.css("paddingRight"),t.css("paddingBottom"),t.css("paddingLeft")];4>e;e++)i[e]=parseFloat(s[e])||0,i[e]+=parseFloat(n[e])||0;return{height:i[0]+i[2],width:i[1]+i[3]}},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var t,e=0,i=this.helper||this.element;this._proportionallyResizeElements.length>e;e++)t=this._proportionallyResizeElements[e],this.outerDimensions||(this.outerDimensions=this._getPaddingPlusBorderDimensions(t)),t.css({height:i.height()-this.outerDimensions.height||0,width:i.width()-this.outerDimensions.width||0})},_renderProxy:function(){var e=this.element,i=this.options;this.elementOffset=e.offset(),this._helper?(this.helper=this.helper||t("
      "),this._addClass(this.helper,this._helper),this.helper.css({width:this.element.outerWidth(),height:this.element.outerHeight(),position:"absolute",left:this.elementOffset.left+"px",top:this.elementOffset.top+"px",zIndex:++i.zIndex}),this.helper.appendTo("body").disableSelection()):this.helper=this.element +},_change:{e:function(t,e){return{width:this.originalSize.width+e}},w:function(t,e){var i=this.originalSize,s=this.originalPosition;return{left:s.left+e,width:i.width-e}},n:function(t,e,i){var s=this.originalSize,n=this.originalPosition;return{top:n.top+i,height:s.height-i}},s:function(t,e,i){return{height:this.originalSize.height+i}},se:function(e,i,s){return t.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[e,i,s]))},sw:function(e,i,s){return t.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[e,i,s]))},ne:function(e,i,s){return t.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[e,i,s]))},nw:function(e,i,s){return t.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[e,i,s]))}},_propagate:function(e,i){t.ui.plugin.call(this,e,[i,this.ui()]),"resize"!==e&&this._trigger(e,i,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),t.ui.plugin.add("resizable","animate",{stop:function(e){var i=t(this).resizable("instance"),s=i.options,n=i._proportionallyResizeElements,o=n.length&&/textarea/i.test(n[0].nodeName),a=o&&i._hasScroll(n[0],"left")?0:i.sizeDiff.height,r=o?0:i.sizeDiff.width,h={width:i.size.width-r,height:i.size.height-a},l=parseFloat(i.element.css("left"))+(i.position.left-i.originalPosition.left)||null,c=parseFloat(i.element.css("top"))+(i.position.top-i.originalPosition.top)||null;i.element.animate(t.extend(h,c&&l?{top:c,left:l}:{}),{duration:s.animateDuration,easing:s.animateEasing,step:function(){var s={width:parseFloat(i.element.css("width")),height:parseFloat(i.element.css("height")),top:parseFloat(i.element.css("top")),left:parseFloat(i.element.css("left"))};n&&n.length&&t(n[0]).css({width:s.width,height:s.height}),i._updateCache(s),i._propagate("resize",e)}})}}),t.ui.plugin.add("resizable","containment",{start:function(){var e,i,s,n,o,a,r,h=t(this).resizable("instance"),l=h.options,c=h.element,u=l.containment,d=u instanceof t?u.get(0):/parent/.test(u)?c.parent().get(0):u;d&&(h.containerElement=t(d),/document/.test(u)||u===document?(h.containerOffset={left:0,top:0},h.containerPosition={left:0,top:0},h.parentData={element:t(document),left:0,top:0,width:t(document).width(),height:t(document).height()||document.body.parentNode.scrollHeight}):(e=t(d),i=[],t(["Top","Right","Left","Bottom"]).each(function(t,s){i[t]=h._num(e.css("padding"+s))}),h.containerOffset=e.offset(),h.containerPosition=e.position(),h.containerSize={height:e.innerHeight()-i[3],width:e.innerWidth()-i[1]},s=h.containerOffset,n=h.containerSize.height,o=h.containerSize.width,a=h._hasScroll(d,"left")?d.scrollWidth:o,r=h._hasScroll(d)?d.scrollHeight:n,h.parentData={element:d,left:s.left,top:s.top,width:a,height:r}))},resize:function(e){var i,s,n,o,a=t(this).resizable("instance"),r=a.options,h=a.containerOffset,l=a.position,c=a._aspectRatio||e.shiftKey,u={top:0,left:0},d=a.containerElement,p=!0;d[0]!==document&&/static/.test(d.css("position"))&&(u=h),l.left<(a._helper?h.left:0)&&(a.size.width=a.size.width+(a._helper?a.position.left-h.left:a.position.left-u.left),c&&(a.size.height=a.size.width/a.aspectRatio,p=!1),a.position.left=r.helper?h.left:0),l.top<(a._helper?h.top:0)&&(a.size.height=a.size.height+(a._helper?a.position.top-h.top:a.position.top),c&&(a.size.width=a.size.height*a.aspectRatio,p=!1),a.position.top=a._helper?h.top:0),n=a.containerElement.get(0)===a.element.parent().get(0),o=/relative|absolute/.test(a.containerElement.css("position")),n&&o?(a.offset.left=a.parentData.left+a.position.left,a.offset.top=a.parentData.top+a.position.top):(a.offset.left=a.element.offset().left,a.offset.top=a.element.offset().top),i=Math.abs(a.sizeDiff.width+(a._helper?a.offset.left-u.left:a.offset.left-h.left)),s=Math.abs(a.sizeDiff.height+(a._helper?a.offset.top-u.top:a.offset.top-h.top)),i+a.size.width>=a.parentData.width&&(a.size.width=a.parentData.width-i,c&&(a.size.height=a.size.width/a.aspectRatio,p=!1)),s+a.size.height>=a.parentData.height&&(a.size.height=a.parentData.height-s,c&&(a.size.width=a.size.height*a.aspectRatio,p=!1)),p||(a.position.left=a.prevPosition.left,a.position.top=a.prevPosition.top,a.size.width=a.prevSize.width,a.size.height=a.prevSize.height)},stop:function(){var e=t(this).resizable("instance"),i=e.options,s=e.containerOffset,n=e.containerPosition,o=e.containerElement,a=t(e.helper),r=a.offset(),h=a.outerWidth()-e.sizeDiff.width,l=a.outerHeight()-e.sizeDiff.height;e._helper&&!i.animate&&/relative/.test(o.css("position"))&&t(this).css({left:r.left-n.left-s.left,width:h,height:l}),e._helper&&!i.animate&&/static/.test(o.css("position"))&&t(this).css({left:r.left-n.left-s.left,width:h,height:l})}}),t.ui.plugin.add("resizable","alsoResize",{start:function(){var e=t(this).resizable("instance"),i=e.options;t(i.alsoResize).each(function(){var e=t(this);e.data("ui-resizable-alsoresize",{width:parseFloat(e.width()),height:parseFloat(e.height()),left:parseFloat(e.css("left")),top:parseFloat(e.css("top"))})})},resize:function(e,i){var s=t(this).resizable("instance"),n=s.options,o=s.originalSize,a=s.originalPosition,r={height:s.size.height-o.height||0,width:s.size.width-o.width||0,top:s.position.top-a.top||0,left:s.position.left-a.left||0};t(n.alsoResize).each(function(){var e=t(this),s=t(this).data("ui-resizable-alsoresize"),n={},o=e.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];t.each(o,function(t,e){var i=(s[e]||0)+(r[e]||0);i&&i>=0&&(n[e]=i||null)}),e.css(n)})},stop:function(){t(this).removeData("ui-resizable-alsoresize")}}),t.ui.plugin.add("resizable","ghost",{start:function(){var e=t(this).resizable("instance"),i=e.size;e.ghost=e.originalElement.clone(),e.ghost.css({opacity:.25,display:"block",position:"relative",height:i.height,width:i.width,margin:0,left:0,top:0}),e._addClass(e.ghost,"ui-resizable-ghost"),t.uiBackCompat!==!1&&"string"==typeof e.options.ghost&&e.ghost.addClass(this.options.ghost),e.ghost.appendTo(e.helper)},resize:function(){var e=t(this).resizable("instance");e.ghost&&e.ghost.css({position:"relative",height:e.size.height,width:e.size.width})},stop:function(){var e=t(this).resizable("instance");e.ghost&&e.helper&&e.helper.get(0).removeChild(e.ghost.get(0))}}),t.ui.plugin.add("resizable","grid",{resize:function(){var e,i=t(this).resizable("instance"),s=i.options,n=i.size,o=i.originalSize,a=i.originalPosition,r=i.axis,h="number"==typeof s.grid?[s.grid,s.grid]:s.grid,l=h[0]||1,c=h[1]||1,u=Math.round((n.width-o.width)/l)*l,d=Math.round((n.height-o.height)/c)*c,p=o.width+u,f=o.height+d,m=s.maxWidth&&p>s.maxWidth,g=s.maxHeight&&f>s.maxHeight,_=s.minWidth&&s.minWidth>p,v=s.minHeight&&s.minHeight>f;s.grid=h,_&&(p+=l),v&&(f+=c),m&&(p-=l),g&&(f-=c),/^(se|s|e)$/.test(r)?(i.size.width=p,i.size.height=f):/^(ne)$/.test(r)?(i.size.width=p,i.size.height=f,i.position.top=a.top-d):/^(sw)$/.test(r)?(i.size.width=p,i.size.height=f,i.position.left=a.left-u):((0>=f-c||0>=p-l)&&(e=i._getPaddingPlusBorderDimensions(this)),f-c>0?(i.size.height=f,i.position.top=a.top-d):(f=c-e.height,i.size.height=f,i.position.top=a.top+o.height-f),p-l>0?(i.size.width=p,i.position.left=a.left-u):(p=l-e.width,i.size.width=p,i.position.left=a.left+o.width-p))}}),t.ui.resizable});/** + * Copyright (c) 2007 Ariel Flesler - aflesler ○ gmail • com | https://github.com/flesler + * Licensed under MIT + * @author Ariel Flesler + * @version 2.1.2 + */ +;(function(f){"use strict";"function"===typeof define&&define.amd?define(["jquery"],f):"undefined"!==typeof module&&module.exports?module.exports=f(require("jquery")):f(jQuery)})(function($){"use strict";function n(a){return!a.nodeName||-1!==$.inArray(a.nodeName.toLowerCase(),["iframe","#document","html","body"])}function h(a){return $.isFunction(a)||$.isPlainObject(a)?a:{top:a,left:a}}var p=$.scrollTo=function(a,d,b){return $(window).scrollTo(a,d,b)};p.defaults={axis:"xy",duration:0,limit:!0};$.fn.scrollTo=function(a,d,b){"object"=== typeof d&&(b=d,d=0);"function"===typeof b&&(b={onAfter:b});"max"===a&&(a=9E9);b=$.extend({},p.defaults,b);d=d||b.duration;var u=b.queue&&1=f[g]?0:Math.min(f[g],n));!a&&1)[^>]*$|#([\w\-]*)$)/,bM=/\S/,bI=/^\s+/,bE=/\s+$/,bA=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,bN=/^[\],:{}\s]*$/,bW=/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,bP=/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,bJ=/(?:^|:|,)(?:\s*\[)+/g,by=/(webkit)[ \/]([\w.]+)/,bR=/(opera)(?:.*version)?[ \/]([\w.]+)/,bQ=/(msie) ([\w.]+)/,bS=/(mozilla)(?:.*? rv:([\w.]+))?/,bB=/-([a-z]|[0-9])/ig,bZ=/^-ms-/,bT=function(b0,b1){return(b1+"").toUpperCase()},bX=bu.userAgent,bV,bC,e,bL=Object.prototype.toString,bG=Object.prototype.hasOwnProperty,bz=Array.prototype.push,bK=Array.prototype.slice,bO=String.prototype.trim,bv=Array.prototype.indexOf,bx={};bF.fn=bF.prototype={constructor:bF,init:function(b0,b4,b3){var b2,b5,b1,b6;if(!b0){return this}if(b0.nodeType){this.context=this[0]=b0;this.length=1;return this}if(b0==="body"&&!b4&&av.body){this.context=av;this[0]=av.body;this.selector=b0;this.length=1;return this}if(typeof b0==="string"){if(b0.charAt(0)==="<"&&b0.charAt(b0.length-1)===">"&&b0.length>=3){b2=[null,b0,null]}else{b2=bY.exec(b0)}if(b2&&(b2[1]||!b4)){if(b2[1]){b4=b4 instanceof bF?b4[0]:b4;b6=(b4?b4.ownerDocument||b4:av);b1=bA.exec(b0);if(b1){if(bF.isPlainObject(b4)){b0=[av.createElement(b1[1])];bF.fn.attr.call(b0,b4,true)}else{b0=[b6.createElement(b1[1])]}}else{b1=bF.buildFragment([b2[1]],[b6]);b0=(b1.cacheable?bF.clone(b1.fragment):b1.fragment).childNodes}return bF.merge(this,b0)}else{b5=av.getElementById(b2[2]);if(b5&&b5.parentNode){if(b5.id!==b2[2]){return b3.find(b0)}this.length=1;this[0]=b5}this.context=av;this.selector=b0;return this}}else{if(!b4||b4.jquery){return(b4||b3).find(b0)}else{return this.constructor(b4).find(b0)}}}else{if(bF.isFunction(b0)){return b3.ready(b0)}}if(b0.selector!==L){this.selector=b0.selector;this.context=b0.context}return bF.makeArray(b0,this)},selector:"",jquery:"1.7.2",length:0,size:function(){return this.length},toArray:function(){return bK.call(this,0)},get:function(b0){return b0==null?this.toArray():(b0<0?this[this.length+b0]:this[b0])},pushStack:function(b1,b3,b0){var b2=this.constructor();if(bF.isArray(b1)){bz.apply(b2,b1)}else{bF.merge(b2,b1)}b2.prevObject=this;b2.context=this.context;if(b3==="find"){b2.selector=this.selector+(this.selector?" ":"")+b0}else{if(b3){b2.selector=this.selector+"."+b3+"("+b0+")"}}return b2},each:function(b1,b0){return bF.each(this,b1,b0)},ready:function(b0){bF.bindReady();bC.add(b0);return this},eq:function(b0){b0=+b0;return b0===-1?this.slice(b0):this.slice(b0,b0+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(bK.apply(this,arguments),"slice",bK.call(arguments).join(","))},map:function(b0){return this.pushStack(bF.map(this,function(b2,b1){return b0.call(b2,b1,b2)}))},end:function(){return this.prevObject||this.constructor(null)},push:bz,sort:[].sort,splice:[].splice};bF.fn.init.prototype=bF.fn;bF.extend=bF.fn.extend=function(){var b9,b2,b0,b1,b6,b7,b5=arguments[0]||{},b4=1,b3=arguments.length,b8=false;if(typeof b5==="boolean"){b8=b5;b5=arguments[1]||{};b4=2}if(typeof b5!=="object"&&!bF.isFunction(b5)){b5={}}if(b3===b4){b5=this;--b4}for(;b40){return}bC.fireWith(av,[bF]);if(bF.fn.trigger){bF(av).trigger("ready").off("ready")}}},bindReady:function(){if(bC){return}bC=bF.Callbacks("once memory");if(av.readyState==="complete"){return setTimeout(bF.ready,1)}if(av.addEventListener){av.addEventListener("DOMContentLoaded",e,false);bd.addEventListener("load",bF.ready,false)}else{if(av.attachEvent){av.attachEvent("onreadystatechange",e);bd.attachEvent("onload",bF.ready);var b0=false;try{b0=bd.frameElement==null}catch(b1){}if(av.documentElement.doScroll&&b0){bw()}}}},isFunction:function(b0){return bF.type(b0)==="function"},isArray:Array.isArray||function(b0){return bF.type(b0)==="array"},isWindow:function(b0){return b0!=null&&b0==b0.window},isNumeric:function(b0){return !isNaN(parseFloat(b0))&&isFinite(b0)},type:function(b0){return b0==null?String(b0):bx[bL.call(b0)]||"object"},isPlainObject:function(b2){if(!b2||bF.type(b2)!=="object"||b2.nodeType||bF.isWindow(b2)){return false}try{if(b2.constructor&&!bG.call(b2,"constructor")&&!bG.call(b2.constructor.prototype,"isPrototypeOf")){return false}}catch(b1){return false}var b0;for(b0 in b2){}return b0===L||bG.call(b2,b0)},isEmptyObject:function(b1){for(var b0 in b1){return false}return true},error:function(b0){throw new Error(b0)},parseJSON:function(b0){if(typeof b0!=="string"||!b0){return null}b0=bF.trim(b0);if(bd.JSON&&bd.JSON.parse){return bd.JSON.parse(b0)}if(bN.test(b0.replace(bW,"@").replace(bP,"]").replace(bJ,""))){return(new Function("return "+b0))()}bF.error("Invalid JSON: "+b0)},parseXML:function(b2){if(typeof b2!=="string"||!b2){return null}var b0,b1;try{if(bd.DOMParser){b1=new DOMParser();b0=b1.parseFromString(b2,"text/xml")}else{b0=new ActiveXObject("Microsoft.XMLDOM");b0.async="false";b0.loadXML(b2)}}catch(b3){b0=L}if(!b0||!b0.documentElement||b0.getElementsByTagName("parsererror").length){bF.error("Invalid XML: "+b2)}return b0},noop:function(){},globalEval:function(b0){if(b0&&bM.test(b0)){(bd.execScript||function(b1){bd["eval"].call(bd,b1)})(b0)}},camelCase:function(b0){return b0.replace(bZ,"ms-").replace(bB,bT)},nodeName:function(b1,b0){return b1.nodeName&&b1.nodeName.toUpperCase()===b0.toUpperCase()},each:function(b3,b6,b2){var b1,b4=0,b5=b3.length,b0=b5===L||bF.isFunction(b3);if(b2){if(b0){for(b1 in b3){if(b6.apply(b3[b1],b2)===false){break}}}else{for(;b40&&b0[0]&&b0[b1-1])||b1===0||bF.isArray(b0));if(b3){for(;b21?aK.call(arguments,0):bG;if(!(--bw)){bC.resolveWith(bC,bx)}}}function bz(bF){return function(bG){bB[bF]=arguments.length>1?aK.call(arguments,0):bG;bC.notifyWith(bE,bB)}}if(e>1){for(;bv
      a";bH=bv.getElementsByTagName("*");bE=bv.getElementsByTagName("a")[0];if(!bH||!bH.length||!bE){return{}}bF=av.createElement("select");bx=bF.appendChild(av.createElement("option"));bD=bv.getElementsByTagName("input")[0];bI={leadingWhitespace:(bv.firstChild.nodeType===3),tbody:!bv.getElementsByTagName("tbody").length,htmlSerialize:!!bv.getElementsByTagName("link").length,style:/top/.test(bE.getAttribute("style")),hrefNormalized:(bE.getAttribute("href")==="/a"),opacity:/^0.55/.test(bE.style.opacity),cssFloat:!!bE.style.cssFloat,checkOn:(bD.value==="on"),optSelected:bx.selected,getSetAttribute:bv.className!=="t",enctype:!!av.createElement("form").enctype,html5Clone:av.createElement("nav").cloneNode(true).outerHTML!=="<:nav>",submitBubbles:true,changeBubbles:true,focusinBubbles:false,deleteExpando:true,noCloneEvent:true,inlineBlockNeedsLayout:false,shrinkWrapBlocks:false,reliableMarginRight:true,pixelMargin:true};b.boxModel=bI.boxModel=(av.compatMode==="CSS1Compat");bD.checked=true;bI.noCloneChecked=bD.cloneNode(true).checked;bF.disabled=true;bI.optDisabled=!bx.disabled;try{delete bv.test}catch(bB){bI.deleteExpando=false}if(!bv.addEventListener&&bv.attachEvent&&bv.fireEvent){bv.attachEvent("onclick",function(){bI.noCloneEvent=false});bv.cloneNode(true).fireEvent("onclick")}bD=av.createElement("input");bD.value="t";bD.setAttribute("type","radio");bI.radioValue=bD.value==="t";bD.setAttribute("checked","checked");bD.setAttribute("name","t");bv.appendChild(bD);bC=av.createDocumentFragment();bC.appendChild(bv.lastChild);bI.checkClone=bC.cloneNode(true).cloneNode(true).lastChild.checked;bI.appendChecked=bD.checked;bC.removeChild(bD);bC.appendChild(bv);if(bv.attachEvent){for(by in {submit:1,change:1,focusin:1}){bA="on"+by;bw=(bA in bv);if(!bw){bv.setAttribute(bA,"return;");bw=(typeof bv[bA]==="function")}bI[by+"Bubbles"]=bw}}bC.removeChild(bv);bC=bF=bx=bv=bD=null;b(function(){var bM,bV,bW,bU,bO,bP,bR,bL,bK,bQ,bN,e,bT,bS=av.getElementsByTagName("body")[0];if(!bS){return}bL=1;bT="padding:0;margin:0;border:";bN="position:absolute;top:0;left:0;width:1px;height:1px;";e=bT+"0;visibility:hidden;";bK="style='"+bN+bT+"5px solid #000;";bQ="
      ";bM=av.createElement("div");bM.style.cssText=e+"width:0;height:0;position:static;top:0;margin-top:"+bL+"px";bS.insertBefore(bM,bS.firstChild);bv=av.createElement("div");bM.appendChild(bv);bv.innerHTML="
      t
      ";bz=bv.getElementsByTagName("td");bw=(bz[0].offsetHeight===0);bz[0].style.display="";bz[1].style.display="none";bI.reliableHiddenOffsets=bw&&(bz[0].offsetHeight===0);if(bd.getComputedStyle){bv.innerHTML="";bR=av.createElement("div");bR.style.width="0";bR.style.marginRight="0";bv.style.width="2px";bv.appendChild(bR);bI.reliableMarginRight=(parseInt((bd.getComputedStyle(bR,null)||{marginRight:0}).marginRight,10)||0)===0}if(typeof bv.style.zoom!=="undefined"){bv.innerHTML="";bv.style.width=bv.style.padding="1px";bv.style.border=0;bv.style.overflow="hidden";bv.style.display="inline";bv.style.zoom=1;bI.inlineBlockNeedsLayout=(bv.offsetWidth===3);bv.style.display="block";bv.style.overflow="visible";bv.innerHTML="
      ";bI.shrinkWrapBlocks=(bv.offsetWidth!==3)}bv.style.cssText=bN+e;bv.innerHTML=bQ;bV=bv.firstChild;bW=bV.firstChild;bO=bV.nextSibling.firstChild.firstChild;bP={doesNotAddBorder:(bW.offsetTop!==5),doesAddBorderForTableAndCells:(bO.offsetTop===5)};bW.style.position="fixed";bW.style.top="20px";bP.fixedPosition=(bW.offsetTop===20||bW.offsetTop===15);bW.style.position=bW.style.top="";bV.style.overflow="hidden";bV.style.position="relative";bP.subtractsBorderForOverflowNotVisible=(bW.offsetTop===-5);bP.doesNotIncludeMarginInBodyOffset=(bS.offsetTop!==bL);if(bd.getComputedStyle){bv.style.marginTop="1%";bI.pixelMargin=(bd.getComputedStyle(bv,null)||{marginTop:0}).marginTop!=="1%"}if(typeof bM.style.zoom!=="undefined"){bM.style.zoom=1}bS.removeChild(bM);bR=bv=bM=null;b.extend(bI,bP)});return bI})();var aT=/^(?:\{.*\}|\[.*\])$/,aA=/([A-Z])/g;b.extend({cache:{},uuid:0,expando:"jQuery"+(b.fn.jquery+Math.random()).replace(/\D/g,""),noData:{embed:true,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:true},hasData:function(e){e=e.nodeType?b.cache[e[b.expando]]:e[b.expando];return !!e&&!S(e)},data:function(bx,bv,bz,by){if(!b.acceptData(bx)){return}var bG,bA,bD,bE=b.expando,bC=typeof bv==="string",bF=bx.nodeType,e=bF?b.cache:bx,bw=bF?bx[bE]:bx[bE]&&bE,bB=bv==="events";if((!bw||!e[bw]||(!bB&&!by&&!e[bw].data))&&bC&&bz===L){return}if(!bw){if(bF){bx[bE]=bw=++b.uuid}else{bw=bE}}if(!e[bw]){e[bw]={};if(!bF){e[bw].toJSON=b.noop}}if(typeof bv==="object"||typeof bv==="function"){if(by){e[bw]=b.extend(e[bw],bv)}else{e[bw].data=b.extend(e[bw].data,bv)}}bG=bA=e[bw];if(!by){if(!bA.data){bA.data={}}bA=bA.data}if(bz!==L){bA[b.camelCase(bv)]=bz}if(bB&&!bA[bv]){return bG.events}if(bC){bD=bA[bv];if(bD==null){bD=bA[b.camelCase(bv)]}}else{bD=bA}return bD},removeData:function(bx,bv,by){if(!b.acceptData(bx)){return}var bB,bA,bz,bC=b.expando,bD=bx.nodeType,e=bD?b.cache:bx,bw=bD?bx[bC]:bC;if(!e[bw]){return}if(bv){bB=by?e[bw]:e[bw].data;if(bB){if(!b.isArray(bv)){if(bv in bB){bv=[bv]}else{bv=b.camelCase(bv);if(bv in bB){bv=[bv]}else{bv=bv.split(" ")}}}for(bA=0,bz=bv.length;bA1,null,false)},removeData:function(e){return this.each(function(){b.removeData(this,e)})}});function a6(bx,bw,by){if(by===L&&bx.nodeType===1){var bv="data-"+bw.replace(aA,"-$1").toLowerCase();by=bx.getAttribute(bv);if(typeof by==="string"){try{by=by==="true"?true:by==="false"?false:by==="null"?null:b.isNumeric(by)?+by:aT.test(by)?b.parseJSON(by):by}catch(bz){}b.data(bx,bw,by)}else{by=L}}return by}function S(bv){for(var e in bv){if(e==="data"&&b.isEmptyObject(bv[e])){continue}if(e!=="toJSON"){return false}}return true}function bj(by,bx,bA){var bw=bx+"defer",bv=bx+"queue",e=bx+"mark",bz=b._data(by,bw);if(bz&&(bA==="queue"||!b._data(by,bv))&&(bA==="mark"||!b._data(by,e))){setTimeout(function(){if(!b._data(by,bv)&&!b._data(by,e)){b.removeData(by,bw,true);bz.fire()}},0)}}b.extend({_mark:function(bv,e){if(bv){e=(e||"fx")+"mark";b._data(bv,e,(b._data(bv,e)||0)+1)}},_unmark:function(by,bx,bv){if(by!==true){bv=bx;bx=by;by=false}if(bx){bv=bv||"fx";var e=bv+"mark",bw=by?0:((b._data(bx,e)||1)-1);if(bw){b._data(bx,e,bw)}else{b.removeData(bx,e,true);bj(bx,bv,"mark")}}},queue:function(bv,e,bx){var bw;if(bv){e=(e||"fx")+"queue";bw=b._data(bv,e);if(bx){if(!bw||b.isArray(bx)){bw=b._data(bv,e,b.makeArray(bx))}else{bw.push(bx)}}return bw||[]}},dequeue:function(by,bx){bx=bx||"fx";var bv=b.queue(by,bx),bw=bv.shift(),e={};if(bw==="inprogress"){bw=bv.shift()}if(bw){if(bx==="fx"){bv.unshift("inprogress")}b._data(by,bx+".run",e);bw.call(by,function(){b.dequeue(by,bx)},e)}if(!bv.length){b.removeData(by,bx+"queue "+bx+".run",true);bj(by,bx,"queue")}}});b.fn.extend({queue:function(e,bv){var bw=2;if(typeof e!=="string"){bv=e;e="fx";bw--}if(arguments.length1)},removeAttr:function(e){return this.each(function(){b.removeAttr(this,e)})},prop:function(e,bv){return b.access(this,b.prop,e,bv,arguments.length>1)},removeProp:function(e){e=b.propFix[e]||e;return this.each(function(){try{this[e]=L;delete this[e]}catch(bv){}})},addClass:function(by){var bA,bw,bv,bx,bz,bB,e;if(b.isFunction(by)){return this.each(function(bC){b(this).addClass(by.call(this,bC,this.className))})}if(by&&typeof by==="string"){bA=by.split(ag);for(bw=0,bv=this.length;bw-1){return true}}return false},val:function(bx){var e,bv,by,bw=this[0];if(!arguments.length){if(bw){e=b.valHooks[bw.type]||b.valHooks[bw.nodeName.toLowerCase()];if(e&&"get" in e&&(bv=e.get(bw,"value"))!==L){return bv}bv=bw.value;return typeof bv==="string"?bv.replace(aV,""):bv==null?"":bv}return}by=b.isFunction(bx);return this.each(function(bA){var bz=b(this),bB;if(this.nodeType!==1){return}if(by){bB=bx.call(this,bA,bz.val())}else{bB=bx}if(bB==null){bB=""}else{if(typeof bB==="number"){bB+=""}else{if(b.isArray(bB)){bB=b.map(bB,function(bC){return bC==null?"":bC+""})}}}e=b.valHooks[this.type]||b.valHooks[this.nodeName.toLowerCase()];if(!e||!("set" in e)||e.set(this,bB,"value")===L){this.value=bB}})}});b.extend({valHooks:{option:{get:function(e){var bv=e.attributes.value;return !bv||bv.specified?e.value:e.text}},select:{get:function(e){var bA,bv,bz,bx,by=e.selectedIndex,bB=[],bC=e.options,bw=e.type==="select-one";if(by<0){return null}bv=bw?by:0;bz=bw?by+1:bC.length;for(;bv=0});if(!e.length){bv.selectedIndex=-1}return e}}},attrFn:{val:true,css:true,html:true,text:true,data:true,width:true,height:true,offset:true},attr:function(bA,bx,bB,bz){var bw,e,by,bv=bA.nodeType;if(!bA||bv===3||bv===8||bv===2){return}if(bz&&bx in b.attrFn){return b(bA)[bx](bB)}if(typeof bA.getAttribute==="undefined"){return b.prop(bA,bx,bB)}by=bv!==1||!b.isXMLDoc(bA);if(by){bx=bx.toLowerCase();e=b.attrHooks[bx]||(ao.test(bx)?aZ:bf)}if(bB!==L){if(bB===null){b.removeAttr(bA,bx);return}else{if(e&&"set" in e&&by&&(bw=e.set(bA,bB,bx))!==L){return bw}else{bA.setAttribute(bx,""+bB);return bB}}}else{if(e&&"get" in e&&by&&(bw=e.get(bA,bx))!==null){return bw}else{bw=bA.getAttribute(bx);return bw===null?L:bw}}},removeAttr:function(by,bA){var bz,bB,bw,e,bv,bx=0;if(bA&&by.nodeType===1){bB=bA.toLowerCase().split(ag);e=bB.length;for(;bx=0)}}})});var be=/^(?:textarea|input|select)$/i,n=/^([^\.]*)?(?:\.(.+))?$/,J=/(?:^|\s)hover(\.\S+)?\b/,aP=/^key/,bg=/^(?:mouse|contextmenu)|click/,T=/^(?:focusinfocus|focusoutblur)$/,U=/^(\w*)(?:#([\w\-]+))?(?:\.([\w\-]+))?$/,Y=function(e){var bv=U.exec(e);if(bv){bv[1]=(bv[1]||"").toLowerCase();bv[3]=bv[3]&&new RegExp("(?:^|\\s)"+bv[3]+"(?:\\s|$)")}return bv},j=function(bw,e){var bv=bw.attributes||{};return((!e[1]||bw.nodeName.toLowerCase()===e[1])&&(!e[2]||(bv.id||{}).value===e[2])&&(!e[3]||e[3].test((bv["class"]||{}).value)))},bt=function(e){return b.event.special.hover?e:e.replace(J,"mouseenter$1 mouseleave$1")};b.event={add:function(bx,bC,bJ,bA,by){var bD,bB,bK,bI,bH,bF,e,bG,bv,bz,bw,bE;if(bx.nodeType===3||bx.nodeType===8||!bC||!bJ||!(bD=b._data(bx))){return}if(bJ.handler){bv=bJ;bJ=bv.handler;by=bv.selector}if(!bJ.guid){bJ.guid=b.guid++}bK=bD.events;if(!bK){bD.events=bK={}}bB=bD.handle;if(!bB){bD.handle=bB=function(bL){return typeof b!=="undefined"&&(!bL||b.event.triggered!==bL.type)?b.event.dispatch.apply(bB.elem,arguments):L};bB.elem=bx}bC=b.trim(bt(bC)).split(" ");for(bI=0;bI=0){bG=bG.slice(0,-1);bw=true}if(bG.indexOf(".")>=0){bx=bG.split(".");bG=bx.shift();bx.sort()}if((!bA||b.event.customEvent[bG])&&!b.event.global[bG]){return}bv=typeof bv==="object"?bv[b.expando]?bv:new b.Event(bG,bv):new b.Event(bG);bv.type=bG;bv.isTrigger=true;bv.exclusive=bw;bv.namespace=bx.join(".");bv.namespace_re=bv.namespace?new RegExp("(^|\\.)"+bx.join("\\.(?:.*\\.)?")+"(\\.|$)"):null;by=bG.indexOf(":")<0?"on"+bG:"";if(!bA){e=b.cache;for(bC in e){if(e[bC].events&&e[bC].events[bG]){b.event.trigger(bv,bD,e[bC].handle.elem,true)}}return}bv.result=L;if(!bv.target){bv.target=bA}bD=bD!=null?b.makeArray(bD):[];bD.unshift(bv);bF=b.event.special[bG]||{};if(bF.trigger&&bF.trigger.apply(bA,bD)===false){return}bB=[[bA,bF.bindType||bG]];if(!bJ&&!bF.noBubble&&!b.isWindow(bA)){bI=bF.delegateType||bG;bH=T.test(bI+bG)?bA:bA.parentNode;bz=null;for(;bH;bH=bH.parentNode){bB.push([bH,bI]);bz=bH}if(bz&&bz===bA.ownerDocument){bB.push([bz.defaultView||bz.parentWindow||bd,bI])}}for(bC=0;bCbC){bv.push({elem:this,matches:bD.slice(bC)})}for(bJ=0;bJ0?this.on(e,null,bx,bw):this.trigger(e)};if(b.attrFn){b.attrFn[e]=true}if(aP.test(e)){b.event.fixHooks[e]=b.event.keyHooks}if(bg.test(e)){b.event.fixHooks[e]=b.event.mouseHooks}}); -/*! - * Sizzle CSS Selector Engine - * Copyright 2011, The Dojo Foundation - * Released under the MIT, BSD, and GPL Licenses. - * More information: http://sizzlejs.com/ - */ -(function(){var bH=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,bC="sizcache"+(Math.random()+"").replace(".",""),bI=0,bL=Object.prototype.toString,bB=false,bA=true,bK=/\\/g,bO=/\r\n/g,bQ=/\W/;[0,0].sort(function(){bA=false;return 0});var by=function(bV,e,bY,bZ){bY=bY||[];e=e||av;var b1=e;if(e.nodeType!==1&&e.nodeType!==9){return[]}if(!bV||typeof bV!=="string"){return bY}var bS,b3,b6,bR,b2,b5,b4,bX,bU=true,bT=by.isXML(e),bW=[],b0=bV;do{bH.exec("");bS=bH.exec(b0);if(bS){b0=bS[3];bW.push(bS[1]);if(bS[2]){bR=bS[3];break}}}while(bS);if(bW.length>1&&bD.exec(bV)){if(bW.length===2&&bE.relative[bW[0]]){b3=bM(bW[0]+bW[1],e,bZ)}else{b3=bE.relative[bW[0]]?[e]:by(bW.shift(),e);while(bW.length){bV=bW.shift();if(bE.relative[bV]){bV+=bW.shift()}b3=bM(bV,b3,bZ)}}}else{if(!bZ&&bW.length>1&&e.nodeType===9&&!bT&&bE.match.ID.test(bW[0])&&!bE.match.ID.test(bW[bW.length-1])){b2=by.find(bW.shift(),e,bT);e=b2.expr?by.filter(b2.expr,b2.set)[0]:b2.set[0]}if(e){b2=bZ?{expr:bW.pop(),set:bF(bZ)}:by.find(bW.pop(),bW.length===1&&(bW[0]==="~"||bW[0]==="+")&&e.parentNode?e.parentNode:e,bT);b3=b2.expr?by.filter(b2.expr,b2.set):b2.set;if(bW.length>0){b6=bF(b3)}else{bU=false}while(bW.length){b5=bW.pop();b4=b5;if(!bE.relative[b5]){b5=""}else{b4=bW.pop()}if(b4==null){b4=e}bE.relative[b5](b6,b4,bT)}}else{b6=bW=[]}}if(!b6){b6=b3}if(!b6){by.error(b5||bV)}if(bL.call(b6)==="[object Array]"){if(!bU){bY.push.apply(bY,b6)}else{if(e&&e.nodeType===1){for(bX=0;b6[bX]!=null;bX++){if(b6[bX]&&(b6[bX]===true||b6[bX].nodeType===1&&by.contains(e,b6[bX]))){bY.push(b3[bX])}}}else{for(bX=0;b6[bX]!=null;bX++){if(b6[bX]&&b6[bX].nodeType===1){bY.push(b3[bX])}}}}}else{bF(b6,bY)}if(bR){by(bR,b1,bY,bZ);by.uniqueSort(bY)}return bY};by.uniqueSort=function(bR){if(bJ){bB=bA;bR.sort(bJ);if(bB){for(var e=1;e0};by.find=function(bX,e,bY){var bW,bS,bU,bT,bV,bR;if(!bX){return[]}for(bS=0,bU=bE.order.length;bS":function(bW,bR){var bV,bU=typeof bR==="string",bS=0,e=bW.length;if(bU&&!bQ.test(bR)){bR=bR.toLowerCase();for(;bS=0)){if(!bS){e.push(bV)}}else{if(bS){bR[bU]=false}}}}return false},ID:function(e){return e[1].replace(bK,"")},TAG:function(bR,e){return bR[1].replace(bK,"").toLowerCase()},CHILD:function(e){if(e[1]==="nth"){if(!e[2]){by.error(e[0])}e[2]=e[2].replace(/^\+|\s*/g,"");var bR=/(-?)(\d*)(?:n([+\-]?\d*))?/.exec(e[2]==="even"&&"2n"||e[2]==="odd"&&"2n+1"||!/\D/.test(e[2])&&"0n+"+e[2]||e[2]);e[2]=(bR[1]+(bR[2]||1))-0;e[3]=bR[3]-0}else{if(e[2]){by.error(e[0])}}e[0]=bI++;return e},ATTR:function(bU,bR,bS,e,bV,bW){var bT=bU[1]=bU[1].replace(bK,"");if(!bW&&bE.attrMap[bT]){bU[1]=bE.attrMap[bT]}bU[4]=(bU[4]||bU[5]||"").replace(bK,"");if(bU[2]==="~="){bU[4]=" "+bU[4]+" "}return bU},PSEUDO:function(bU,bR,bS,e,bV){if(bU[1]==="not"){if((bH.exec(bU[3])||"").length>1||/^\w/.test(bU[3])){bU[3]=by(bU[3],null,null,bR)}else{var bT=by.filter(bU[3],bR,bS,true^bV);if(!bS){e.push.apply(e,bT)}return false}}else{if(bE.match.POS.test(bU[0])||bE.match.CHILD.test(bU[0])){return true}}return bU},POS:function(e){e.unshift(true);return e}},filters:{enabled:function(e){return e.disabled===false&&e.type!=="hidden"},disabled:function(e){return e.disabled===true},checked:function(e){return e.checked===true},selected:function(e){if(e.parentNode){e.parentNode.selectedIndex}return e.selected===true},parent:function(e){return !!e.firstChild},empty:function(e){return !e.firstChild},has:function(bS,bR,e){return !!by(e[3],bS).length},header:function(e){return(/h\d/i).test(e.nodeName)},text:function(bS){var e=bS.getAttribute("type"),bR=bS.type;return bS.nodeName.toLowerCase()==="input"&&"text"===bR&&(e===bR||e===null)},radio:function(e){return e.nodeName.toLowerCase()==="input"&&"radio"===e.type},checkbox:function(e){return e.nodeName.toLowerCase()==="input"&&"checkbox"===e.type},file:function(e){return e.nodeName.toLowerCase()==="input"&&"file"===e.type},password:function(e){return e.nodeName.toLowerCase()==="input"&&"password"===e.type},submit:function(bR){var e=bR.nodeName.toLowerCase();return(e==="input"||e==="button")&&"submit"===bR.type},image:function(e){return e.nodeName.toLowerCase()==="input"&&"image"===e.type},reset:function(bR){var e=bR.nodeName.toLowerCase();return(e==="input"||e==="button")&&"reset"===bR.type},button:function(bR){var e=bR.nodeName.toLowerCase();return e==="input"&&"button"===bR.type||e==="button"},input:function(e){return(/input|select|textarea|button/i).test(e.nodeName)},focus:function(e){return e===e.ownerDocument.activeElement}},setFilters:{first:function(bR,e){return e===0},last:function(bS,bR,e,bT){return bR===bT.length-1},even:function(bR,e){return e%2===0},odd:function(bR,e){return e%2===1},lt:function(bS,bR,e){return bRe[3]-0},nth:function(bS,bR,e){return e[3]-0===bR},eq:function(bS,bR,e){return e[3]-0===bR}},filter:{PSEUDO:function(bS,bX,bW,bY){var e=bX[1],bR=bE.filters[e];if(bR){return bR(bS,bW,bX,bY)}else{if(e==="contains"){return(bS.textContent||bS.innerText||bw([bS])||"").indexOf(bX[3])>=0}else{if(e==="not"){var bT=bX[3];for(var bV=0,bU=bT.length;bV=0)}}},ID:function(bR,e){return bR.nodeType===1&&bR.getAttribute("id")===e},TAG:function(bR,e){return(e==="*"&&bR.nodeType===1)||!!bR.nodeName&&bR.nodeName.toLowerCase()===e},CLASS:function(bR,e){return(" "+(bR.className||bR.getAttribute("class"))+" ").indexOf(e)>-1},ATTR:function(bV,bT){var bS=bT[1],e=by.attr?by.attr(bV,bS):bE.attrHandle[bS]?bE.attrHandle[bS](bV):bV[bS]!=null?bV[bS]:bV.getAttribute(bS),bW=e+"",bU=bT[2],bR=bT[4];return e==null?bU==="!=":!bU&&by.attr?e!=null:bU==="="?bW===bR:bU==="*="?bW.indexOf(bR)>=0:bU==="~="?(" "+bW+" ").indexOf(bR)>=0:!bR?bW&&e!==false:bU==="!="?bW!==bR:bU==="^="?bW.indexOf(bR)===0:bU==="$="?bW.substr(bW.length-bR.length)===bR:bU==="|="?bW===bR||bW.substr(0,bR.length+1)===bR+"-":false},POS:function(bU,bR,bS,bV){var e=bR[2],bT=bE.setFilters[e];if(bT){return bT(bU,bS,bR,bV)}}}};var bD=bE.match.POS,bx=function(bR,e){return"\\"+(e-0+1)};for(var bz in bE.match){bE.match[bz]=new RegExp(bE.match[bz].source+(/(?![^\[]*\])(?![^\(]*\))/.source));bE.leftMatch[bz]=new RegExp(/(^(?:.|\r|\n)*?)/.source+bE.match[bz].source.replace(/\\(\d+)/g,bx))}bE.match.globalPOS=bD;var bF=function(bR,e){bR=Array.prototype.slice.call(bR,0);if(e){e.push.apply(e,bR);return e}return bR};try{Array.prototype.slice.call(av.documentElement.childNodes,0)[0].nodeType}catch(bP){bF=function(bU,bT){var bS=0,bR=bT||[];if(bL.call(bU)==="[object Array]"){Array.prototype.push.apply(bR,bU)}else{if(typeof bU.length==="number"){for(var e=bU.length;bS";e.insertBefore(bR,e.firstChild);if(av.getElementById(bS)){bE.find.ID=function(bU,bV,bW){if(typeof bV.getElementById!=="undefined"&&!bW){var bT=bV.getElementById(bU[1]);return bT?bT.id===bU[1]||typeof bT.getAttributeNode!=="undefined"&&bT.getAttributeNode("id").nodeValue===bU[1]?[bT]:L:[]}};bE.filter.ID=function(bV,bT){var bU=typeof bV.getAttributeNode!=="undefined"&&bV.getAttributeNode("id");return bV.nodeType===1&&bU&&bU.nodeValue===bT}}e.removeChild(bR);e=bR=null})();(function(){var e=av.createElement("div");e.appendChild(av.createComment(""));if(e.getElementsByTagName("*").length>0){bE.find.TAG=function(bR,bV){var bU=bV.getElementsByTagName(bR[1]);if(bR[1]==="*"){var bT=[];for(var bS=0;bU[bS];bS++){if(bU[bS].nodeType===1){bT.push(bU[bS])}}bU=bT}return bU}}e.innerHTML="";if(e.firstChild&&typeof e.firstChild.getAttribute!=="undefined"&&e.firstChild.getAttribute("href")!=="#"){bE.attrHandle.href=function(bR){return bR.getAttribute("href",2)}}e=null})();if(av.querySelectorAll){(function(){var e=by,bT=av.createElement("div"),bS="__sizzle__";bT.innerHTML="

      ";if(bT.querySelectorAll&&bT.querySelectorAll(".TEST").length===0){return}by=function(b4,bV,bZ,b3){bV=bV||av;if(!b3&&!by.isXML(bV)){var b2=/^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec(b4);if(b2&&(bV.nodeType===1||bV.nodeType===9)){if(b2[1]){return bF(bV.getElementsByTagName(b4),bZ)}else{if(b2[2]&&bE.find.CLASS&&bV.getElementsByClassName){return bF(bV.getElementsByClassName(b2[2]),bZ)}}}if(bV.nodeType===9){if(b4==="body"&&bV.body){return bF([bV.body],bZ)}else{if(b2&&b2[3]){var bY=bV.getElementById(b2[3]);if(bY&&bY.parentNode){if(bY.id===b2[3]){return bF([bY],bZ)}}else{return bF([],bZ)}}}try{return bF(bV.querySelectorAll(b4),bZ)}catch(b0){}}else{if(bV.nodeType===1&&bV.nodeName.toLowerCase()!=="object"){var bW=bV,bX=bV.getAttribute("id"),bU=bX||bS,b6=bV.parentNode,b5=/^\s*[+~]/.test(b4);if(!bX){bV.setAttribute("id",bU)}else{bU=bU.replace(/'/g,"\\$&")}if(b5&&b6){bV=bV.parentNode}try{if(!b5||b6){return bF(bV.querySelectorAll("[id='"+bU+"'] "+b4),bZ)}}catch(b1){}finally{if(!bX){bW.removeAttribute("id")}}}}}return e(b4,bV,bZ,b3)};for(var bR in e){by[bR]=e[bR]}bT=null})()}(function(){var e=av.documentElement,bS=e.matchesSelector||e.mozMatchesSelector||e.webkitMatchesSelector||e.msMatchesSelector;if(bS){var bU=!bS.call(av.createElement("div"),"div"),bR=false;try{bS.call(av.documentElement,"[test!='']:sizzle")}catch(bT){bR=true}by.matchesSelector=function(bW,bY){bY=bY.replace(/\=\s*([^'"\]]*)\s*\]/g,"='$1']");if(!by.isXML(bW)){try{if(bR||!bE.match.PSEUDO.test(bY)&&!/!=/.test(bY)){var bV=bS.call(bW,bY);if(bV||!bU||bW.document&&bW.document.nodeType!==11){return bV}}}catch(bX){}}return by(bY,null,null,[bW]).length>0}}})();(function(){var e=av.createElement("div");e.innerHTML="
      ";if(!e.getElementsByClassName||e.getElementsByClassName("e").length===0){return}e.lastChild.className="e";if(e.getElementsByClassName("e").length===1){return}bE.order.splice(1,0,"CLASS");bE.find.CLASS=function(bR,bS,bT){if(typeof bS.getElementsByClassName!=="undefined"&&!bT){return bS.getElementsByClassName(bR[1])}};e=null})();function bv(bR,bW,bV,bZ,bX,bY){for(var bT=0,bS=bZ.length;bT0){bU=e;break}}}e=e[bR]}bZ[bT]=bU}}}if(av.documentElement.contains){by.contains=function(bR,e){return bR!==e&&(bR.contains?bR.contains(e):true)}}else{if(av.documentElement.compareDocumentPosition){by.contains=function(bR,e){return !!(bR.compareDocumentPosition(e)&16)}}else{by.contains=function(){return false}}}by.isXML=function(e){var bR=(e?e.ownerDocument||e:0).documentElement;return bR?bR.nodeName!=="HTML":false};var bM=function(bS,e,bW){var bV,bX=[],bU="",bY=e.nodeType?[e]:e;while((bV=bE.match.PSEUDO.exec(bS))){bU+=bV[0];bS=bS.replace(bE.match.PSEUDO,"")}bS=bE.relative[bS]?bS+"*":bS;for(var bT=0,bR=bY.length;bT0){for(bB=bA;bB=0:b.filter(e,this).length>0:this.filter(e).length>0)},closest:function(by,bx){var bv=[],bw,e,bz=this[0];if(b.isArray(by)){var bB=1;while(bz&&bz.ownerDocument&&bz!==bx){for(bw=0;bw-1:b.find.matchesSelector(bz,by)){bv.push(bz);break}else{bz=bz.parentNode;if(!bz||!bz.ownerDocument||bz===bx||bz.nodeType===11){break}}}}bv=bv.length>1?b.unique(bv):bv;return this.pushStack(bv,"closest",by)},index:function(e){if(!e){return(this[0]&&this[0].parentNode)?this.prevAll().length:-1}if(typeof e==="string"){return b.inArray(this[0],b(e))}return b.inArray(e.jquery?e[0]:e,this)},add:function(e,bv){var bx=typeof e==="string"?b(e,bv):b.makeArray(e&&e.nodeType?[e]:e),bw=b.merge(this.get(),bx);return this.pushStack(B(bx[0])||B(bw[0])?bw:b.unique(bw))},andSelf:function(){return this.add(this.prevObject)}});function B(e){return !e||!e.parentNode||e.parentNode.nodeType===11}b.each({parent:function(bv){var e=bv.parentNode;return e&&e.nodeType!==11?e:null},parents:function(e){return b.dir(e,"parentNode")},parentsUntil:function(bv,e,bw){return b.dir(bv,"parentNode",bw)},next:function(e){return b.nth(e,2,"nextSibling")},prev:function(e){return b.nth(e,2,"previousSibling")},nextAll:function(e){return b.dir(e,"nextSibling")},prevAll:function(e){return b.dir(e,"previousSibling")},nextUntil:function(bv,e,bw){return b.dir(bv,"nextSibling",bw)},prevUntil:function(bv,e,bw){return b.dir(bv,"previousSibling",bw)},siblings:function(e){return b.sibling((e.parentNode||{}).firstChild,e)},children:function(e){return b.sibling(e.firstChild)},contents:function(e){return b.nodeName(e,"iframe")?e.contentDocument||e.contentWindow.document:b.makeArray(e.childNodes)}},function(e,bv){b.fn[e]=function(by,bw){var bx=b.map(this,bv,by);if(!ab.test(e)){bw=by}if(bw&&typeof bw==="string"){bx=b.filter(bw,bx)}bx=this.length>1&&!ay[e]?b.unique(bx):bx;if((this.length>1||bb.test(bw))&&aq.test(e)){bx=bx.reverse()}return this.pushStack(bx,e,P.call(arguments).join(","))}});b.extend({filter:function(bw,e,bv){if(bv){bw=":not("+bw+")"}return e.length===1?b.find.matchesSelector(e[0],bw)?[e[0]]:[]:b.find.matches(bw,e)},dir:function(bw,bv,by){var e=[],bx=bw[bv];while(bx&&bx.nodeType!==9&&(by===L||bx.nodeType!==1||!b(bx).is(by))){if(bx.nodeType===1){e.push(bx)}bx=bx[bv]}return e},nth:function(by,e,bw,bx){e=e||1;var bv=0;for(;by;by=by[bw]){if(by.nodeType===1&&++bv===e){break}}return by},sibling:function(bw,bv){var e=[];for(;bw;bw=bw.nextSibling){if(bw.nodeType===1&&bw!==bv){e.push(bw)}}return e}});function aH(bx,bw,e){bw=bw||0;if(b.isFunction(bw)){return b.grep(bx,function(bz,by){var bA=!!bw.call(bz,by,bz);return bA===e})}else{if(bw.nodeType){return b.grep(bx,function(bz,by){return(bz===bw)===e})}else{if(typeof bw==="string"){var bv=b.grep(bx,function(by){return by.nodeType===1});if(bp.test(bw)){return b.filter(bw,bv,!e)}else{bw=b.filter(bw,bv)}}}}return b.grep(bx,function(bz,by){return(b.inArray(bz,bw)>=0)===e})}function a(e){var bw=aS.split("|"),bv=e.createDocumentFragment();if(bv.createElement){while(bw.length){bv.createElement(bw.pop())}}return bv}var aS="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",ah=/ jQuery\d+="(?:\d+|null)"/g,ar=/^\s+/,R=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,d=/<([\w:]+)/,v=/]","i"),o=/checked\s*(?:[^=]|=\s*.checked.)/i,bn=/\/(java|ecma)script/i,aO=/^\s*",""],legend:[1,"
      ","
      "],thead:[1,"","
      "],tr:[2,"","
      "],td:[3,"","
      "],col:[2,"","
      "],area:[1,"",""],_default:[0,"",""]},ac=a(av);ax.optgroup=ax.option;ax.tbody=ax.tfoot=ax.colgroup=ax.caption=ax.thead;ax.th=ax.td;if(!b.support.htmlSerialize){ax._default=[1,"div
      ","
      "]}b.fn.extend({text:function(e){return b.access(this,function(bv){return bv===L?b.text(this):this.empty().append((this[0]&&this[0].ownerDocument||av).createTextNode(bv))},null,e,arguments.length)},wrapAll:function(e){if(b.isFunction(e)){return this.each(function(bw){b(this).wrapAll(e.call(this,bw))})}if(this[0]){var bv=b(e,this[0].ownerDocument).eq(0).clone(true);if(this[0].parentNode){bv.insertBefore(this[0])}bv.map(function(){var bw=this;while(bw.firstChild&&bw.firstChild.nodeType===1){bw=bw.firstChild}return bw}).append(this)}return this},wrapInner:function(e){if(b.isFunction(e)){return this.each(function(bv){b(this).wrapInner(e.call(this,bv))})}return this.each(function(){var bv=b(this),bw=bv.contents();if(bw.length){bw.wrapAll(e)}else{bv.append(e)}})},wrap:function(e){var bv=b.isFunction(e);return this.each(function(bw){b(this).wrapAll(bv?e.call(this,bw):e)})},unwrap:function(){return this.parent().each(function(){if(!b.nodeName(this,"body")){b(this).replaceWith(this.childNodes)}}).end()},append:function(){return this.domManip(arguments,true,function(e){if(this.nodeType===1){this.appendChild(e)}})},prepend:function(){return this.domManip(arguments,true,function(e){if(this.nodeType===1){this.insertBefore(e,this.firstChild)}})},before:function(){if(this[0]&&this[0].parentNode){return this.domManip(arguments,false,function(bv){this.parentNode.insertBefore(bv,this)})}else{if(arguments.length){var e=b.clean(arguments);e.push.apply(e,this.toArray());return this.pushStack(e,"before",arguments)}}},after:function(){if(this[0]&&this[0].parentNode){return this.domManip(arguments,false,function(bv){this.parentNode.insertBefore(bv,this.nextSibling)})}else{if(arguments.length){var e=this.pushStack(this,"after",arguments);e.push.apply(e,b.clean(arguments));return e}}},remove:function(e,bx){for(var bv=0,bw;(bw=this[bv])!=null;bv++){if(!e||b.filter(e,[bw]).length){if(!bx&&bw.nodeType===1){b.cleanData(bw.getElementsByTagName("*"));b.cleanData([bw])}if(bw.parentNode){bw.parentNode.removeChild(bw)}}}return this},empty:function(){for(var e=0,bv;(bv=this[e])!=null;e++){if(bv.nodeType===1){b.cleanData(bv.getElementsByTagName("*"))}while(bv.firstChild){bv.removeChild(bv.firstChild)}}return this},clone:function(bv,e){bv=bv==null?false:bv;e=e==null?bv:e;return this.map(function(){return b.clone(this,bv,e)})},html:function(e){return b.access(this,function(by){var bx=this[0]||{},bw=0,bv=this.length;if(by===L){return bx.nodeType===1?bx.innerHTML.replace(ah,""):null}if(typeof by==="string"&&!ae.test(by)&&(b.support.leadingWhitespace||!ar.test(by))&&!ax[(d.exec(by)||["",""])[1].toLowerCase()]){by=by.replace(R,"<$1>");try{for(;bw1&&bw0?this.clone(true):this).get();b(bC[bA])[bv](by);bz=bz.concat(by)}return this.pushStack(bz,e,bC.selector)}}});function bh(e){if(typeof e.getElementsByTagName!=="undefined"){return e.getElementsByTagName("*")}else{if(typeof e.querySelectorAll!=="undefined"){return e.querySelectorAll("*")}else{return[]}}}function az(e){if(e.type==="checkbox"||e.type==="radio"){e.defaultChecked=e.checked}}function D(e){var bv=(e.nodeName||"").toLowerCase();if(bv==="input"){az(e)}else{if(bv!=="script"&&typeof e.getElementsByTagName!=="undefined"){b.grep(e.getElementsByTagName("input"),az)}}}function am(e){var bv=av.createElement("div");ac.appendChild(bv);bv.innerHTML=e.outerHTML;return bv.firstChild}b.extend({clone:function(by,bA,bw){var e,bv,bx,bz=b.support.html5Clone||b.isXMLDoc(by)||!ai.test("<"+by.nodeName+">")?by.cloneNode(true):am(by);if((!b.support.noCloneEvent||!b.support.noCloneChecked)&&(by.nodeType===1||by.nodeType===11)&&!b.isXMLDoc(by)){aj(by,bz);e=bh(by);bv=bh(bz);for(bx=0;e[bx];++bx){if(bv[bx]){aj(e[bx],bv[bx])}}}if(bA){s(by,bz);if(bw){e=bh(by);bv=bh(bz);for(bx=0;e[bx];++bx){s(e[bx],bv[bx])}}}e=bv=null;return bz},clean:function(bI,bw,bv,bx){var bA,bH,bD,bJ=[];bw=bw||av;if(typeof bw.createElement==="undefined"){bw=bw.ownerDocument||bw[0]&&bw[0].ownerDocument||av}for(var bE=0,bG;(bG=bI[bE])!=null;bE++){if(typeof bG==="number"){bG+=""}if(!bG){continue}if(typeof bG==="string"){if(!W.test(bG)){bG=bw.createTextNode(bG)}else{bG=bG.replace(R,"<$1>");var bN=(d.exec(bG)||["",""])[1].toLowerCase(),bz=ax[bN]||ax._default,bK=bz[0],bB=bw.createElement("div"),bL=ac.childNodes,bM;if(bw===av){ac.appendChild(bB)}else{a(bw).appendChild(bB)}bB.innerHTML=bz[1]+bG+bz[2];while(bK--){bB=bB.lastChild}if(!b.support.tbody){var by=v.test(bG),e=bN==="table"&&!by?bB.firstChild&&bB.firstChild.childNodes:bz[1]===""&&!by?bB.childNodes:[];for(bD=e.length-1;bD>=0;--bD){if(b.nodeName(e[bD],"tbody")&&!e[bD].childNodes.length){e[bD].parentNode.removeChild(e[bD])}}}if(!b.support.leadingWhitespace&&ar.test(bG)){bB.insertBefore(bw.createTextNode(ar.exec(bG)[0]),bB.firstChild)}bG=bB.childNodes;if(bB){bB.parentNode.removeChild(bB);if(bL.length>0){bM=bL[bL.length-1];if(bM&&bM.parentNode){bM.parentNode.removeChild(bM)}}}}}var bF;if(!b.support.appendChecked){if(bG[0]&&typeof(bF=bG.length)==="number"){for(bD=0;bD1)};b.extend({cssHooks:{opacity:{get:function(bw,bv){if(bv){var e=Z(bw,"opacity");return e===""?"1":e}else{return bw.style.opacity}}}},cssNumber:{fillOpacity:true,fontWeight:true,lineHeight:true,opacity:true,orphans:true,widows:true,zIndex:true,zoom:true},cssProps:{"float":b.support.cssFloat?"cssFloat":"styleFloat"},style:function(bx,bw,bD,by){if(!bx||bx.nodeType===3||bx.nodeType===8||!bx.style){return}var bB,bC,bz=b.camelCase(bw),bv=bx.style,bE=b.cssHooks[bz];bw=b.cssProps[bz]||bz;if(bD!==L){bC=typeof bD;if(bC==="string"&&(bB=I.exec(bD))){bD=(+(bB[1]+1)*+bB[2])+parseFloat(b.css(bx,bw));bC="number"}if(bD==null||bC==="number"&&isNaN(bD)){return}if(bC==="number"&&!b.cssNumber[bz]){bD+="px"}if(!bE||!("set" in bE)||(bD=bE.set(bx,bD))!==L){try{bv[bw]=bD}catch(bA){}}}else{if(bE&&"get" in bE&&(bB=bE.get(bx,false,by))!==L){return bB}return bv[bw]}},css:function(by,bx,bv){var bw,e;bx=b.camelCase(bx);e=b.cssHooks[bx];bx=b.cssProps[bx]||bx;if(bx==="cssFloat"){bx="float"}if(e&&"get" in e&&(bw=e.get(by,true,bv))!==L){return bw}else{if(Z){return Z(by,bx)}}},swap:function(by,bx,bz){var e={},bw,bv;for(bv in bx){e[bv]=by.style[bv];by.style[bv]=bx[bv]}bw=bz.call(by);for(bv in bx){by.style[bv]=e[bv]}return bw}});b.curCSS=b.css;if(av.defaultView&&av.defaultView.getComputedStyle){aJ=function(bA,bw){var bv,bz,e,by,bx=bA.style;bw=bw.replace(y,"-$1").toLowerCase();if((bz=bA.ownerDocument.defaultView)&&(e=bz.getComputedStyle(bA,null))){bv=e.getPropertyValue(bw);if(bv===""&&!b.contains(bA.ownerDocument.documentElement,bA)){bv=b.style(bA,bw)}}if(!b.support.pixelMargin&&e&&aE.test(bw)&&a1.test(bv)){by=bx.width;bx.width=bv;bv=e.width;bx.width=by}return bv}}if(av.documentElement.currentStyle){aY=function(bz,bw){var bA,e,by,bv=bz.currentStyle&&bz.currentStyle[bw],bx=bz.style;if(bv==null&&bx&&(by=bx[bw])){bv=by}if(a1.test(bv)){bA=bx.left;e=bz.runtimeStyle&&bz.runtimeStyle.left;if(e){bz.runtimeStyle.left=bz.currentStyle.left}bx.left=bw==="fontSize"?"1em":bv;bv=bx.pixelLeft+"px";bx.left=bA;if(e){bz.runtimeStyle.left=e}}return bv===""?"auto":bv}}Z=aJ||aY;function af(by,bw,bv){var bz=bw==="width"?by.offsetWidth:by.offsetHeight,bx=bw==="width"?1:0,e=4;if(bz>0){if(bv!=="border"){for(;bx=1&&b.trim(bw.replace(al,""))===""){bx.removeAttribute("filter");if(bv&&!bv.filter){return}}bx.filter=al.test(bw)?bw.replace(al,e):bw+" "+e}}}b(function(){if(!b.support.reliableMarginRight){b.cssHooks.marginRight={get:function(bv,e){return b.swap(bv,{display:"inline-block"},function(){if(e){return Z(bv,"margin-right")}else{return bv.style.marginRight}})}}}});if(b.expr&&b.expr.filters){b.expr.filters.hidden=function(bw){var bv=bw.offsetWidth,e=bw.offsetHeight;return(bv===0&&e===0)||(!b.support.reliableHiddenOffsets&&((bw.style&&bw.style.display)||b.css(bw,"display"))==="none")};b.expr.filters.visible=function(e){return !b.expr.filters.hidden(e)}}b.each({margin:"",padding:"",border:"Width"},function(e,bv){b.cssHooks[e+bv]={expand:function(by){var bx,bz=typeof by==="string"?by.split(" "):[by],bw={};for(bx=0;bx<4;bx++){bw[e+G[bx]+bv]=bz[bx]||bz[bx-2]||bz[0]}return bw}}});var k=/%20/g,ap=/\[\]$/,bs=/\r?\n/g,bq=/#.*$/,aD=/^(.*?):[ \t]*([^\r\n]*)\r?$/mg,a0=/^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,aN=/^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/,aR=/^(?:GET|HEAD)$/,c=/^\/\//,M=/\?/,a7=/)<[^<]*)*<\/script>/gi,p=/^(?:select|textarea)/i,h=/\s+/,br=/([?&])_=[^&]*/,K=/^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+))?)?/,z=b.fn.load,aa={},q={},aF,r,aW=["*/"]+["*"];try{aF=bm.href}catch(aw){aF=av.createElement("a");aF.href="";aF=aF.href}r=K.exec(aF.toLowerCase())||[];function f(e){return function(by,bA){if(typeof by!=="string"){bA=by;by="*"}if(b.isFunction(bA)){var bx=by.toLowerCase().split(h),bw=0,bz=bx.length,bv,bB,bC;for(;bw=0){var e=bw.slice(by,bw.length);bw=bw.slice(0,by)}var bx="GET";if(bz){if(b.isFunction(bz)){bA=bz;bz=L}else{if(typeof bz==="object"){bz=b.param(bz,b.ajaxSettings.traditional);bx="POST"}}}var bv=this;b.ajax({url:bw,type:bx,dataType:"html",data:bz,complete:function(bC,bB,bD){bD=bC.responseText;if(bC.isResolved()){bC.done(function(bE){bD=bE});bv.html(e?b("
      ").append(bD.replace(a7,"")).find(e):bD)}if(bA){bv.each(bA,[bD,bB,bC])}}});return this},serialize:function(){return b.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?b.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||p.test(this.nodeName)||a0.test(this.type))}).map(function(e,bv){var bw=b(this).val();return bw==null?null:b.isArray(bw)?b.map(bw,function(by,bx){return{name:bv.name,value:by.replace(bs,"\r\n")}}):{name:bv.name,value:bw.replace(bs,"\r\n")}}).get()}});b.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(e,bv){b.fn[bv]=function(bw){return this.on(bv,bw)}});b.each(["get","post"],function(e,bv){b[bv]=function(bw,by,bz,bx){if(b.isFunction(by)){bx=bx||bz;bz=by;by=L}return b.ajax({type:bv,url:bw,data:by,success:bz,dataType:bx})}});b.extend({getScript:function(e,bv){return b.get(e,L,bv,"script")},getJSON:function(e,bv,bw){return b.get(e,bv,bw,"json")},ajaxSetup:function(bv,e){if(e){an(bv,b.ajaxSettings)}else{e=bv;bv=b.ajaxSettings}an(bv,e);return bv},ajaxSettings:{url:aF,isLocal:aN.test(r[1]),global:true,type:"GET",contentType:"application/x-www-form-urlencoded; charset=UTF-8",processData:true,async:true,accepts:{xml:"application/xml, text/xml",html:"text/html",text:"text/plain",json:"application/json, text/javascript","*":aW},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText"},converters:{"* text":bd.String,"text html":true,"text json":b.parseJSON,"text xml":b.parseXML},flatOptions:{context:true,url:true}},ajaxPrefilter:f(aa),ajaxTransport:f(q),ajax:function(bz,bx){if(typeof bz==="object"){bx=bz;bz=L}bx=bx||{};var bD=b.ajaxSetup({},bx),bS=bD.context||bD,bG=bS!==bD&&(bS.nodeType||bS instanceof b)?b(bS):b.event,bR=b.Deferred(),bN=b.Callbacks("once memory"),bB=bD.statusCode||{},bC,bH={},bO={},bQ,by,bL,bE,bI,bA=0,bw,bK,bJ={readyState:0,setRequestHeader:function(bT,bU){if(!bA){var e=bT.toLowerCase();bT=bO[e]=bO[e]||bT;bH[bT]=bU}return this},getAllResponseHeaders:function(){return bA===2?bQ:null},getResponseHeader:function(bT){var e;if(bA===2){if(!by){by={};while((e=aD.exec(bQ))){by[e[1].toLowerCase()]=e[2]}}e=by[bT.toLowerCase()]}return e===L?null:e},overrideMimeType:function(e){if(!bA){bD.mimeType=e}return this},abort:function(e){e=e||"abort";if(bL){bL.abort(e)}bF(0,e);return this}};function bF(bZ,bU,b0,bW){if(bA===2){return}bA=2;if(bE){clearTimeout(bE)}bL=L;bQ=bW||"";bJ.readyState=bZ>0?4:0;var bT,b4,b3,bX=bU,bY=b0?bk(bD,bJ,b0):L,bV,b2;if(bZ>=200&&bZ<300||bZ===304){if(bD.ifModified){if((bV=bJ.getResponseHeader("Last-Modified"))){b.lastModified[bC]=bV}if((b2=bJ.getResponseHeader("Etag"))){b.etag[bC]=b2}}if(bZ===304){bX="notmodified";bT=true}else{try{b4=F(bD,bY);bX="success";bT=true}catch(b1){bX="parsererror";b3=b1}}}else{b3=bX;if(!bX||bZ){bX="error";if(bZ<0){bZ=0}}}bJ.status=bZ;bJ.statusText=""+(bU||bX);if(bT){bR.resolveWith(bS,[b4,bX,bJ])}else{bR.rejectWith(bS,[bJ,bX,b3])}bJ.statusCode(bB);bB=L;if(bw){bG.trigger("ajax"+(bT?"Success":"Error"),[bJ,bD,bT?b4:b3])}bN.fireWith(bS,[bJ,bX]);if(bw){bG.trigger("ajaxComplete",[bJ,bD]);if(!(--b.active)){b.event.trigger("ajaxStop")}}}bR.promise(bJ);bJ.success=bJ.done;bJ.error=bJ.fail;bJ.complete=bN.add;bJ.statusCode=function(bT){if(bT){var e;if(bA<2){for(e in bT){bB[e]=[bB[e],bT[e]]}}else{e=bT[bJ.status];bJ.then(e,e)}}return this};bD.url=((bz||bD.url)+"").replace(bq,"").replace(c,r[1]+"//");bD.dataTypes=b.trim(bD.dataType||"*").toLowerCase().split(h);if(bD.crossDomain==null){bI=K.exec(bD.url.toLowerCase());bD.crossDomain=!!(bI&&(bI[1]!=r[1]||bI[2]!=r[2]||(bI[3]||(bI[1]==="http:"?80:443))!=(r[3]||(r[1]==="http:"?80:443))))}if(bD.data&&bD.processData&&typeof bD.data!=="string"){bD.data=b.param(bD.data,bD.traditional)}aX(aa,bD,bx,bJ);if(bA===2){return false}bw=bD.global;bD.type=bD.type.toUpperCase();bD.hasContent=!aR.test(bD.type);if(bw&&b.active++===0){b.event.trigger("ajaxStart")}if(!bD.hasContent){if(bD.data){bD.url+=(M.test(bD.url)?"&":"?")+bD.data;delete bD.data}bC=bD.url;if(bD.cache===false){var bv=b.now(),bP=bD.url.replace(br,"$1_="+bv);bD.url=bP+((bP===bD.url)?(M.test(bD.url)?"&":"?")+"_="+bv:"")}}if(bD.data&&bD.hasContent&&bD.contentType!==false||bx.contentType){bJ.setRequestHeader("Content-Type",bD.contentType)}if(bD.ifModified){bC=bC||bD.url;if(b.lastModified[bC]){bJ.setRequestHeader("If-Modified-Since",b.lastModified[bC])}if(b.etag[bC]){bJ.setRequestHeader("If-None-Match",b.etag[bC])}}bJ.setRequestHeader("Accept",bD.dataTypes[0]&&bD.accepts[bD.dataTypes[0]]?bD.accepts[bD.dataTypes[0]]+(bD.dataTypes[0]!=="*"?", "+aW+"; q=0.01":""):bD.accepts["*"]);for(bK in bD.headers){bJ.setRequestHeader(bK,bD.headers[bK])}if(bD.beforeSend&&(bD.beforeSend.call(bS,bJ,bD)===false||bA===2)){bJ.abort();return false}for(bK in {success:1,error:1,complete:1}){bJ[bK](bD[bK])}bL=aX(q,bD,bx,bJ);if(!bL){bF(-1,"No Transport")}else{bJ.readyState=1;if(bw){bG.trigger("ajaxSend",[bJ,bD])}if(bD.async&&bD.timeout>0){bE=setTimeout(function(){bJ.abort("timeout")},bD.timeout)}try{bA=1;bL.send(bH,bF)}catch(bM){if(bA<2){bF(-1,bM)}else{throw bM}}}return bJ},param:function(e,bw){var bv=[],by=function(bz,bA){bA=b.isFunction(bA)?bA():bA;bv[bv.length]=encodeURIComponent(bz)+"="+encodeURIComponent(bA)};if(bw===L){bw=b.ajaxSettings.traditional}if(b.isArray(e)||(e.jquery&&!b.isPlainObject(e))){b.each(e,function(){by(this.name,this.value)})}else{for(var bx in e){u(bx,e[bx],bw,by)}}return bv.join("&").replace(k,"+")}});function u(bw,by,bv,bx){if(b.isArray(by)){b.each(by,function(bA,bz){if(bv||ap.test(bw)){bx(bw,bz)}else{u(bw+"["+(typeof bz==="object"?bA:"")+"]",bz,bv,bx)}})}else{if(!bv&&b.type(by)==="object"){for(var e in by){u(bw+"["+e+"]",by[e],bv,bx)}}else{bx(bw,by)}}}b.extend({active:0,lastModified:{},etag:{}});function bk(bD,bC,bz){var bv=bD.contents,bB=bD.dataTypes,bw=bD.responseFields,by,bA,bx,e;for(bA in bw){if(bA in bz){bC[bw[bA]]=bz[bA]}}while(bB[0]==="*"){bB.shift();if(by===L){by=bD.mimeType||bC.getResponseHeader("content-type")}}if(by){for(bA in bv){if(bv[bA]&&bv[bA].test(by)){bB.unshift(bA);break}}}if(bB[0] in bz){bx=bB[0]}else{for(bA in bz){if(!bB[0]||bD.converters[bA+" "+bB[0]]){bx=bA;break}if(!e){e=bA}}bx=bx||e}if(bx){if(bx!==bB[0]){bB.unshift(bx)}return bz[bx]}}function F(bH,bz){if(bH.dataFilter){bz=bH.dataFilter(bz,bH.dataType)}var bD=bH.dataTypes,bG={},bA,bE,bw=bD.length,bB,bC=bD[0],bx,by,bF,bv,e;for(bA=1;bA=bw.duration+this.startTime){this.now=this.end;this.pos=this.state=1;this.update();bw.animatedProperties[this.prop]=true;for(bA in bw.animatedProperties){if(bw.animatedProperties[bA]!==true){e=false}}if(e){if(bw.overflow!=null&&!b.support.shrinkWrapBlocks){b.each(["","X","Y"],function(bC,bD){bz.style["overflow"+bD]=bw.overflow[bC]})}if(bw.hide){b(bz).hide()}if(bw.hide||bw.show){for(bA in bw.animatedProperties){b.style(bz,bA,bw.orig[bA]);b.removeData(bz,"fxshow"+bA,true);b.removeData(bz,"toggle"+bA,true)}}bv=bw.complete;if(bv){bw.complete=false;bv.call(bz)}}return false}else{if(bw.duration==Infinity){this.now=bx}else{bB=bx-this.startTime;this.state=bB/bw.duration;this.pos=b.easing[bw.animatedProperties[this.prop]](this.state,bB,0,1,bw.duration);this.now=this.start+((this.end-this.start)*this.pos)}this.update()}return true}};b.extend(b.fx,{tick:function(){var bw,bv=b.timers,e=0;for(;e").appendTo(e),bw=bv.css("display");bv.remove();if(bw==="none"||bw===""){if(!ba){ba=av.createElement("iframe");ba.frameBorder=ba.width=ba.height=0}e.appendChild(ba);if(!m||!ba.createElement){m=(ba.contentWindow||ba.contentDocument).document;m.write((b.support.boxModel?"":"")+"");m.close()}bv=m.createElement(bx);m.body.appendChild(bv);bw=b.css(bv,"display");e.removeChild(ba)}Q[bx]=bw}return Q[bx]}var a8,V=/^t(?:able|d|h)$/i,ad=/^(?:body|html)$/i;if("getBoundingClientRect" in av.documentElement){a8=function(by,bH,bw,bB){try{bB=by.getBoundingClientRect()}catch(bF){}if(!bB||!b.contains(bw,by)){return bB?{top:bB.top,left:bB.left}:{top:0,left:0}}var bC=bH.body,bD=aL(bH),bA=bw.clientTop||bC.clientTop||0,bE=bw.clientLeft||bC.clientLeft||0,bv=bD.pageYOffset||b.support.boxModel&&bw.scrollTop||bC.scrollTop,bz=bD.pageXOffset||b.support.boxModel&&bw.scrollLeft||bC.scrollLeft,bG=bB.top+bv-bA,bx=bB.left+bz-bE;return{top:bG,left:bx}}}else{a8=function(bz,bE,bx){var bC,bw=bz.offsetParent,bv=bz,bA=bE.body,bB=bE.defaultView,e=bB?bB.getComputedStyle(bz,null):bz.currentStyle,bD=bz.offsetTop,by=bz.offsetLeft;while((bz=bz.parentNode)&&bz!==bA&&bz!==bx){if(b.support.fixedPosition&&e.position==="fixed"){break}bC=bB?bB.getComputedStyle(bz,null):bz.currentStyle;bD-=bz.scrollTop;by-=bz.scrollLeft;if(bz===bw){bD+=bz.offsetTop;by+=bz.offsetLeft;if(b.support.doesNotAddBorder&&!(b.support.doesAddBorderForTableAndCells&&V.test(bz.nodeName))){bD+=parseFloat(bC.borderTopWidth)||0;by+=parseFloat(bC.borderLeftWidth)||0}bv=bw;bw=bz.offsetParent}if(b.support.subtractsBorderForOverflowNotVisible&&bC.overflow!=="visible"){bD+=parseFloat(bC.borderTopWidth)||0;by+=parseFloat(bC.borderLeftWidth)||0}e=bC}if(e.position==="relative"||e.position==="static"){bD+=bA.offsetTop;by+=bA.offsetLeft}if(b.support.fixedPosition&&e.position==="fixed"){bD+=Math.max(bx.scrollTop,bA.scrollTop);by+=Math.max(bx.scrollLeft,bA.scrollLeft)}return{top:bD,left:by}}}b.fn.offset=function(e){if(arguments.length){return e===L?this:this.each(function(bx){b.offset.setOffset(this,e,bx)})}var bv=this[0],bw=bv&&bv.ownerDocument;if(!bw){return null}if(bv===bw.body){return b.offset.bodyOffset(bv)}return a8(bv,bw,bw.documentElement)};b.offset={bodyOffset:function(e){var bw=e.offsetTop,bv=e.offsetLeft;if(b.support.doesNotIncludeMarginInBodyOffset){bw+=parseFloat(b.css(e,"marginTop"))||0;bv+=parseFloat(b.css(e,"marginLeft"))||0}return{top:bw,left:bv}},setOffset:function(bx,bG,bA){var bB=b.css(bx,"position");if(bB==="static"){bx.style.position="relative"}var bz=b(bx),bv=bz.offset(),e=b.css(bx,"top"),bE=b.css(bx,"left"),bF=(bB==="absolute"||bB==="fixed")&&b.inArray("auto",[e,bE])>-1,bD={},bC={},bw,by;if(bF){bC=bz.position();bw=bC.top;by=bC.left}else{bw=parseFloat(e)||0;by=parseFloat(bE)||0}if(b.isFunction(bG)){bG=bG.call(bx,bA,bv)}if(bG.top!=null){bD.top=(bG.top-bv.top)+bw}if(bG.left!=null){bD.left=(bG.left-bv.left)+by}if("using" in bG){bG.using.call(bx,bD)}else{bz.css(bD)}}};b.fn.extend({position:function(){if(!this[0]){return null}var bw=this[0],bv=this.offsetParent(),bx=this.offset(),e=ad.test(bv[0].nodeName)?{top:0,left:0}:bv.offset();bx.top-=parseFloat(b.css(bw,"marginTop"))||0;bx.left-=parseFloat(b.css(bw,"marginLeft"))||0;e.top+=parseFloat(b.css(bv[0],"borderTopWidth"))||0;e.left+=parseFloat(b.css(bv[0],"borderLeftWidth"))||0;return{top:bx.top-e.top,left:bx.left-e.left}},offsetParent:function(){return this.map(function(){var e=this.offsetParent||av.body;while(e&&(!ad.test(e.nodeName)&&b.css(e,"position")==="static")){e=e.offsetParent}return e})}});b.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(bw,bv){var e=/Y/.test(bv);b.fn[bw]=function(bx){return b.access(this,function(by,bB,bA){var bz=aL(by);if(bA===L){return bz?(bv in bz)?bz[bv]:b.support.boxModel&&bz.document.documentElement[bB]||bz.document.body[bB]:by[bB]}if(bz){bz.scrollTo(!e?bA:b(bz).scrollLeft(),e?bA:b(bz).scrollTop())}else{by[bB]=bA}},bw,bx,arguments.length,null)}});function aL(e){return b.isWindow(e)?e:e.nodeType===9?e.defaultView||e.parentWindow:false}b.each({Height:"height",Width:"width"},function(bw,bx){var bv="client"+bw,e="scroll"+bw,by="offset"+bw;b.fn["inner"+bw]=function(){var bz=this[0];return bz?bz.style?parseFloat(b.css(bz,bx,"padding")):this[bx]():null};b.fn["outer"+bw]=function(bA){var bz=this[0];return bz?bz.style?parseFloat(b.css(bz,bx,bA?"margin":"border")):this[bx]():null};b.fn[bx]=function(bz){return b.access(this,function(bC,bB,bD){var bF,bE,bG,bA;if(b.isWindow(bC)){bF=bC.document;bE=bF.documentElement[bv];return b.support.boxModel&&bE||bF.body&&bF.body[bv]||bE}if(bC.nodeType===9){bF=bC.documentElement;if(bF[bv]>=bF[e]){return bF[bv]}return Math.max(bC.body[e],bF[e],bC.body[by],bF[by])}if(bD===L){bG=b.css(bC,bB);bA=parseFloat(bG);return b.isNumeric(bA)?bA:bG}b(bC).css(bB,bD)},bx,bz,arguments.length,null)}});bd.jQuery=bd.$=b;if(typeof define==="function"&&define.amd&&define.amd.jQuery){define("jquery",[],function(){return b})}})(window);/*! - * jQuery UI 1.8.18 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI - */ -(function(a,d){a.ui=a.ui||{};if(a.ui.version){return}a.extend(a.ui,{version:"1.8.18",keyCode:{ALT:18,BACKSPACE:8,CAPS_LOCK:20,COMMA:188,COMMAND:91,COMMAND_LEFT:91,COMMAND_RIGHT:93,CONTROL:17,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,MENU:93,NUMPAD_ADD:107,NUMPAD_DECIMAL:110,NUMPAD_DIVIDE:111,NUMPAD_ENTER:108,NUMPAD_MULTIPLY:106,NUMPAD_SUBTRACT:109,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,WINDOWS:91}});a.fn.extend({propAttr:a.fn.prop||a.fn.attr,_focus:a.fn.focus,focus:function(e,f){return typeof e==="number"?this.each(function(){var g=this;setTimeout(function(){a(g).focus();if(f){f.call(g)}},e)}):this._focus.apply(this,arguments)},scrollParent:function(){var e;if((a.browser.msie&&(/(static|relative)/).test(this.css("position")))||(/absolute/).test(this.css("position"))){e=this.parents().filter(function(){return(/(relative|absolute|fixed)/).test(a.curCSS(this,"position",1))&&(/(auto|scroll)/).test(a.curCSS(this,"overflow",1)+a.curCSS(this,"overflow-y",1)+a.curCSS(this,"overflow-x",1))}).eq(0)}else{e=this.parents().filter(function(){return(/(auto|scroll)/).test(a.curCSS(this,"overflow",1)+a.curCSS(this,"overflow-y",1)+a.curCSS(this,"overflow-x",1))}).eq(0)}return(/fixed/).test(this.css("position"))||!e.length?a(document):e},zIndex:function(h){if(h!==d){return this.css("zIndex",h)}if(this.length){var f=a(this[0]),e,g;while(f.length&&f[0]!==document){e=f.css("position");if(e==="absolute"||e==="relative"||e==="fixed"){g=parseInt(f.css("zIndex"),10);if(!isNaN(g)&&g!==0){return g}}f=f.parent()}}return 0},disableSelection:function(){return this.bind((a.support.selectstart?"selectstart":"mousedown")+".ui-disableSelection",function(e){e.preventDefault()})},enableSelection:function(){return this.unbind(".ui-disableSelection")}});a.each(["Width","Height"],function(g,e){var f=e==="Width"?["Left","Right"]:["Top","Bottom"],h=e.toLowerCase(),k={innerWidth:a.fn.innerWidth,innerHeight:a.fn.innerHeight,outerWidth:a.fn.outerWidth,outerHeight:a.fn.outerHeight};function j(m,l,i,n){a.each(f,function(){l-=parseFloat(a.curCSS(m,"padding"+this,true))||0;if(i){l-=parseFloat(a.curCSS(m,"border"+this+"Width",true))||0}if(n){l-=parseFloat(a.curCSS(m,"margin"+this,true))||0}});return l}a.fn["inner"+e]=function(i){if(i===d){return k["inner"+e].call(this)}return this.each(function(){a(this).css(h,j(this,i)+"px")})};a.fn["outer"+e]=function(i,l){if(typeof i!=="number"){return k["outer"+e].call(this,i)}return this.each(function(){a(this).css(h,j(this,i,true,l)+"px")})}});function c(g,e){var j=g.nodeName.toLowerCase();if("area"===j){var i=g.parentNode,h=i.name,f;if(!g.href||!h||i.nodeName.toLowerCase()!=="map"){return false}f=a("img[usemap=#"+h+"]")[0];return !!f&&b(f)}return(/input|select|textarea|button|object/.test(j)?!g.disabled:"a"==j?g.href||e:e)&&b(g)}function b(e){return !a(e).parents().andSelf().filter(function(){return a.curCSS(this,"visibility")==="hidden"||a.expr.filters.hidden(this)}).length}a.extend(a.expr[":"],{data:function(g,f,e){return !!a.data(g,e[3])},focusable:function(e){return c(e,!isNaN(a.attr(e,"tabindex")))},tabbable:function(g){var e=a.attr(g,"tabindex"),f=isNaN(e);return(f||e>=0)&&c(g,!f)}});a(function(){var e=document.body,f=e.appendChild(f=document.createElement("div"));f.offsetHeight;a.extend(f.style,{minHeight:"100px",height:"auto",padding:0,borderWidth:0});a.support.minHeight=f.offsetHeight===100;a.support.selectstart="onselectstart" in f;e.removeChild(f).style.display="none"});a.extend(a.ui,{plugin:{add:function(f,g,j){var h=a.ui[f].prototype;for(var e in j){h.plugins[e]=h.plugins[e]||[];h.plugins[e].push([g,j[e]])}},call:function(e,g,f){var j=e.plugins[g];if(!j||!e.element[0].parentNode){return}for(var h=0;h0){return true}h[e]=1;g=(h[e]>0);h[e]=0;return g},isOverAxis:function(f,e,g){return(f>e)&&(f<(e+g))},isOver:function(j,f,i,h,e,g){return a.ui.isOverAxis(j,i,e)&&a.ui.isOverAxis(f,h,g)}})})(jQuery);/*! - * jQuery UI Widget 1.8.18 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI/Widget - */ -(function(b,d){if(b.cleanData){var c=b.cleanData;b.cleanData=function(f){for(var g=0,h;(h=f[g])!=null;g++){try{b(h).triggerHandler("remove")}catch(j){}}c(f)}}else{var a=b.fn.remove;b.fn.remove=function(e,f){return this.each(function(){if(!f){if(!e||b.filter(e,[this]).length){b("*",this).add([this]).each(function(){try{b(this).triggerHandler("remove")}catch(g){}})}}return a.call(b(this),e,f)})}}b.widget=function(f,h,e){var g=f.split(".")[0],j;f=f.split(".")[1];j=g+"-"+f;if(!e){e=h;h=b.Widget}b.expr[":"][j]=function(k){return !!b.data(k,f)};b[g]=b[g]||{};b[g][f]=function(k,l){if(arguments.length){this._createWidget(k,l)}};var i=new h();i.options=b.extend(true,{},i.options);b[g][f].prototype=b.extend(true,i,{namespace:g,widgetName:f,widgetEventPrefix:b[g][f].prototype.widgetEventPrefix||f,widgetBaseClass:j},e);b.widget.bridge(f,b[g][f])};b.widget.bridge=function(f,e){b.fn[f]=function(i){var g=typeof i==="string",h=Array.prototype.slice.call(arguments,1),j=this;i=!g&&h.length?b.extend.apply(null,[true,i].concat(h)):i;if(g&&i.charAt(0)==="_"){return j}if(g){this.each(function(){var k=b.data(this,f),l=k&&b.isFunction(k[i])?k[i].apply(k,h):k;if(l!==k&&l!==d){j=l;return false}})}else{this.each(function(){var k=b.data(this,f);if(k){k.option(i||{})._init()}else{b.data(this,f,new e(i,this))}})}return j}};b.Widget=function(e,f){if(arguments.length){this._createWidget(e,f)}};b.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",options:{disabled:false},_createWidget:function(f,g){b.data(g,this.widgetName,this);this.element=b(g);this.options=b.extend(true,{},this.options,this._getCreateOptions(),f);var e=this;this.element.bind("remove."+this.widgetName,function(){e.destroy()});this._create();this._trigger("create");this._init()},_getCreateOptions:function(){return b.metadata&&b.metadata.get(this.element[0])[this.widgetName]},_create:function(){},_init:function(){},destroy:function(){this.element.unbind("."+this.widgetName).removeData(this.widgetName);this.widget().unbind("."+this.widgetName).removeAttr("aria-disabled").removeClass(this.widgetBaseClass+"-disabled ui-state-disabled")},widget:function(){return this.element},option:function(f,g){var e=f;if(arguments.length===0){return b.extend({},this.options)}if(typeof f==="string"){if(g===d){return this.options[f]}e={};e[f]=g}this._setOptions(e);return this},_setOptions:function(f){var e=this;b.each(f,function(g,h){e._setOption(g,h)});return this},_setOption:function(e,f){this.options[e]=f;if(e==="disabled"){this.widget()[f?"addClass":"removeClass"](this.widgetBaseClass+"-disabled ui-state-disabled").attr("aria-disabled",f)}return this},enable:function(){return this._setOption("disabled",false)},disable:function(){return this._setOption("disabled",true)},_trigger:function(e,f,g){var j,i,h=this.options[e];g=g||{};f=b.Event(f);f.type=(e===this.widgetEventPrefix?e:this.widgetEventPrefix+e).toLowerCase();f.target=this.element[0];i=f.originalEvent;if(i){for(j in i){if(!(j in f)){f[j]=i[j]}}}this.element.trigger(f,g);return !(b.isFunction(h)&&h.call(this.element[0],f,g)===false||f.isDefaultPrevented())}}})(jQuery);/*! - * jQuery UI Mouse 1.8.18 - * - * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about) - * Dual licensed under the MIT or GPL Version 2 licenses. - * http://jquery.org/license - * - * http://docs.jquery.com/UI/Mouse - * - * Depends: - * jquery.ui.widget.js - */ -(function(b,c){var a=false;b(document).mouseup(function(d){a=false});b.widget("ui.mouse",{options:{cancel:":input,option",distance:1,delay:0},_mouseInit:function(){var d=this;this.element.bind("mousedown."+this.widgetName,function(e){return d._mouseDown(e)}).bind("click."+this.widgetName,function(e){if(true===b.data(e.target,d.widgetName+".preventClickEvent")){b.removeData(e.target,d.widgetName+".preventClickEvent");e.stopImmediatePropagation();return false}});this.started=false},_mouseDestroy:function(){this.element.unbind("."+this.widgetName)},_mouseDown:function(f){if(a){return}(this._mouseStarted&&this._mouseUp(f));this._mouseDownEvent=f;var e=this,g=(f.which==1),d=(typeof this.options.cancel=="string"&&f.target.nodeName?b(f.target).closest(this.options.cancel).length:false);if(!g||d||!this._mouseCapture(f)){return true}this.mouseDelayMet=!this.options.delay;if(!this.mouseDelayMet){this._mouseDelayTimer=setTimeout(function(){e.mouseDelayMet=true},this.options.delay)}if(this._mouseDistanceMet(f)&&this._mouseDelayMet(f)){this._mouseStarted=(this._mouseStart(f)!==false);if(!this._mouseStarted){f.preventDefault();return true}}if(true===b.data(f.target,this.widgetName+".preventClickEvent")){b.removeData(f.target,this.widgetName+".preventClickEvent")}this._mouseMoveDelegate=function(h){return e._mouseMove(h)};this._mouseUpDelegate=function(h){return e._mouseUp(h)};b(document).bind("mousemove."+this.widgetName,this._mouseMoveDelegate).bind("mouseup."+this.widgetName,this._mouseUpDelegate);f.preventDefault();a=true;return true},_mouseMove:function(d){if(b.browser.msie&&!(document.documentMode>=9)&&!d.button){return this._mouseUp(d)}if(this._mouseStarted){this._mouseDrag(d);return d.preventDefault()}if(this._mouseDistanceMet(d)&&this._mouseDelayMet(d)){this._mouseStarted=(this._mouseStart(this._mouseDownEvent,d)!==false);(this._mouseStarted?this._mouseDrag(d):this._mouseUp(d))}return !this._mouseStarted},_mouseUp:function(d){b(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate);if(this._mouseStarted){this._mouseStarted=false;if(d.target==this._mouseDownEvent.target){b.data(d.target,this.widgetName+".preventClickEvent",true)}this._mouseStop(d)}return false},_mouseDistanceMet:function(d){return(Math.max(Math.abs(this._mouseDownEvent.pageX-d.pageX),Math.abs(this._mouseDownEvent.pageY-d.pageY))>=this.options.distance)},_mouseDelayMet:function(d){return this.mouseDelayMet},_mouseStart:function(d){},_mouseDrag:function(d){},_mouseStop:function(d){},_mouseCapture:function(d){return true}})})(jQuery);(function(c,d){c.widget("ui.resizable",c.ui.mouse,{widgetEventPrefix:"resize",options:{alsoResize:false,animate:false,animateDuration:"slow",animateEasing:"swing",aspectRatio:false,autoHide:false,containment:false,ghost:false,grid:false,handles:"e,s,se",helper:false,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:1000},_create:function(){var f=this,k=this.options;this.element.addClass("ui-resizable");c.extend(this,{_aspectRatio:!!(k.aspectRatio),aspectRatio:k.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:k.helper||k.ghost||k.animate?k.helper||"ui-resizable-helper":null});if(this.element[0].nodeName.match(/canvas|textarea|input|select|button|img/i)){this.element.wrap(c('
      ').css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")}));this.element=this.element.parent().data("resizable",this.element.data("resizable"));this.elementIsWrapper=true;this.element.css({marginLeft:this.originalElement.css("marginLeft"),marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom")});this.originalElement.css({marginLeft:0,marginTop:0,marginRight:0,marginBottom:0});this.originalResizeStyle=this.originalElement.css("resize");this.originalElement.css("resize","none");this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"}));this.originalElement.css({margin:this.originalElement.css("margin")});this._proportionallyResize()}this.handles=k.handles||(!c(".ui-resizable-handle",this.element).length?"e,s,se":{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"});if(this.handles.constructor==String){if(this.handles=="all"){this.handles="n,e,s,w,se,sw,ne,nw"}var l=this.handles.split(",");this.handles={};for(var g=0;g
      ');if(/sw|se|ne|nw/.test(j)){h.css({zIndex:++k.zIndex})}if("se"==j){h.addClass("ui-icon ui-icon-gripsmall-diagonal-se")}this.handles[j]=".ui-resizable-"+j;this.element.append(h)}}this._renderAxis=function(q){q=q||this.element;for(var n in this.handles){if(this.handles[n].constructor==String){this.handles[n]=c(this.handles[n],this.element).show()}if(this.elementIsWrapper&&this.originalElement[0].nodeName.match(/textarea|input|select|button/i)){var o=c(this.handles[n],this.element),p=0;p=/sw|ne|nw|se|n|s/.test(n)?o.outerHeight():o.outerWidth();var m=["padding",/ne|nw|n/.test(n)?"Top":/se|sw|s/.test(n)?"Bottom":/^e$/.test(n)?"Right":"Left"].join("");q.css(m,p);this._proportionallyResize()}if(!c(this.handles[n]).length){continue}}};this._renderAxis(this.element);this._handles=c(".ui-resizable-handle",this.element).disableSelection();this._handles.mouseover(function(){if(!f.resizing){if(this.className){var i=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i)}f.axis=i&&i[1]?i[1]:"se"}});if(k.autoHide){this._handles.hide();c(this.element).addClass("ui-resizable-autohide").hover(function(){if(k.disabled){return}c(this).removeClass("ui-resizable-autohide");f._handles.show()},function(){if(k.disabled){return}if(!f.resizing){c(this).addClass("ui-resizable-autohide");f._handles.hide()}})}this._mouseInit()},destroy:function(){this._mouseDestroy();var e=function(g){c(g).removeClass("ui-resizable ui-resizable-disabled ui-resizable-resizing").removeData("resizable").unbind(".resizable").find(".ui-resizable-handle").remove()};if(this.elementIsWrapper){e(this.element);var f=this.element;f.after(this.originalElement.css({position:f.css("position"),width:f.outerWidth(),height:f.outerHeight(),top:f.css("top"),left:f.css("left")})).remove()}this.originalElement.css("resize",this.originalResizeStyle);e(this.originalElement);return this},_mouseCapture:function(f){var g=false;for(var e in this.handles){if(c(this.handles[e])[0]==f.target){g=true}}return !this.options.disabled&&g},_mouseStart:function(g){var j=this.options,f=this.element.position(),e=this.element;this.resizing=true;this.documentScroll={top:c(document).scrollTop(),left:c(document).scrollLeft()};if(e.is(".ui-draggable")||(/absolute/).test(e.css("position"))){e.css({position:"absolute",top:f.top,left:f.left})}this._renderProxy();var k=b(this.helper.css("left")),h=b(this.helper.css("top"));if(j.containment){k+=c(j.containment).scrollLeft()||0;h+=c(j.containment).scrollTop()||0}this.offset=this.helper.offset();this.position={left:k,top:h};this.size=this._helper?{width:e.outerWidth(),height:e.outerHeight()}:{width:e.width(),height:e.height()};this.originalSize=this._helper?{width:e.outerWidth(),height:e.outerHeight()}:{width:e.width(),height:e.height()};this.originalPosition={left:k,top:h};this.sizeDiff={width:e.outerWidth()-e.width(),height:e.outerHeight()-e.height()};this.originalMousePosition={left:g.pageX,top:g.pageY};this.aspectRatio=(typeof j.aspectRatio=="number")?j.aspectRatio:((this.originalSize.width/this.originalSize.height)||1);var i=c(".ui-resizable-"+this.axis).css("cursor");c("body").css("cursor",i=="auto"?this.axis+"-resize":i);e.addClass("ui-resizable-resizing");this._propagate("start",g);return true},_mouseDrag:function(e){var h=this.helper,g=this.options,m={},q=this,j=this.originalMousePosition,n=this.axis;var r=(e.pageX-j.left)||0,p=(e.pageY-j.top)||0;var i=this._change[n];if(!i){return false}var l=i.apply(this,[e,r,p]),k=c.browser.msie&&c.browser.version<7,f=this.sizeDiff;this._updateVirtualBoundaries(e.shiftKey);if(this._aspectRatio||e.shiftKey){l=this._updateRatio(l,e)}l=this._respectSize(l,e);this._propagate("resize",e);h.css({top:this.position.top+"px",left:this.position.left+"px",width:this.size.width+"px",height:this.size.height+"px"});if(!this._helper&&this._proportionallyResizeElements.length){this._proportionallyResize()}this._updateCache(l);this._trigger("resize",e,this.ui());return false},_mouseStop:function(h){this.resizing=false;var i=this.options,m=this;if(this._helper){var g=this._proportionallyResizeElements,e=g.length&&(/textarea/i).test(g[0].nodeName),f=e&&c.ui.hasScroll(g[0],"left")?0:m.sizeDiff.height,k=e?0:m.sizeDiff.width;var n={width:(m.helper.width()-k),height:(m.helper.height()-f)},j=(parseInt(m.element.css("left"),10)+(m.position.left-m.originalPosition.left))||null,l=(parseInt(m.element.css("top"),10)+(m.position.top-m.originalPosition.top))||null;if(!i.animate){this.element.css(c.extend(n,{top:l,left:j}))}m.helper.height(m.size.height);m.helper.width(m.size.width);if(this._helper&&!i.animate){this._proportionallyResize()}}c("body").css("cursor","auto");this.element.removeClass("ui-resizable-resizing");this._propagate("stop",h);if(this._helper){this.helper.remove()}return false},_updateVirtualBoundaries:function(g){var j=this.options,i,h,f,k,e;e={minWidth:a(j.minWidth)?j.minWidth:0,maxWidth:a(j.maxWidth)?j.maxWidth:Infinity,minHeight:a(j.minHeight)?j.minHeight:0,maxHeight:a(j.maxHeight)?j.maxHeight:Infinity};if(this._aspectRatio||g){i=e.minHeight*this.aspectRatio;f=e.minWidth/this.aspectRatio;h=e.maxHeight*this.aspectRatio;k=e.maxWidth/this.aspectRatio;if(i>e.minWidth){e.minWidth=i}if(f>e.minHeight){e.minHeight=f}if(hl.width),s=a(l.height)&&i.minHeight&&(i.minHeight>l.height);if(h){l.width=i.minWidth}if(s){l.height=i.minHeight}if(t){l.width=i.maxWidth}if(m){l.height=i.maxHeight}var f=this.originalPosition.left+this.originalSize.width,p=this.position.top+this.size.height;var k=/sw|nw|w/.test(q),e=/nw|ne|n/.test(q);if(h&&k){l.left=f-i.minWidth}if(t&&k){l.left=f-i.maxWidth}if(s&&e){l.top=p-i.minHeight}if(m&&e){l.top=p-i.maxHeight}var n=!l.width&&!l.height;if(n&&!l.left&&l.top){l.top=null}else{if(n&&!l.top&&l.left){l.left=null}}return l},_proportionallyResize:function(){var k=this.options;if(!this._proportionallyResizeElements.length){return}var g=this.helper||this.element;for(var f=0;f');var e=c.browser.msie&&c.browser.version<7,g=(e?1:0),h=(e?2:-1);this.helper.addClass(this._helper).css({width:this.element.outerWidth()+h,height:this.element.outerHeight()+h,position:"absolute",left:this.elementOffset.left-g+"px",top:this.elementOffset.top-g+"px",zIndex:++i.zIndex});this.helper.appendTo("body").disableSelection()}else{this.helper=this.element}},_change:{e:function(g,f,e){return{width:this.originalSize.width+f}},w:function(h,f,e){var j=this.options,g=this.originalSize,i=this.originalPosition;return{left:i.left+f,width:g.width-f}},n:function(h,f,e){var j=this.options,g=this.originalSize,i=this.originalPosition;return{top:i.top+e,height:g.height-e}},s:function(g,f,e){return{height:this.originalSize.height+e}},se:function(g,f,e){return c.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[g,f,e]))},sw:function(g,f,e){return c.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[g,f,e]))},ne:function(g,f,e){return c.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[g,f,e]))},nw:function(g,f,e){return c.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[g,f,e]))}},_propagate:function(f,e){c.ui.plugin.call(this,f,[e,this.ui()]);(f!="resize"&&this._trigger(f,e,this.ui()))},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}});c.extend(c.ui.resizable,{version:"1.8.18"});c.ui.plugin.add("resizable","alsoResize",{start:function(f,g){var e=c(this).data("resizable"),i=e.options;var h=function(j){c(j).each(function(){var k=c(this);k.data("resizable-alsoresize",{width:parseInt(k.width(),10),height:parseInt(k.height(),10),left:parseInt(k.css("left"),10),top:parseInt(k.css("top"),10)})})};if(typeof(i.alsoResize)=="object"&&!i.alsoResize.parentNode){if(i.alsoResize.length){i.alsoResize=i.alsoResize[0];h(i.alsoResize)}else{c.each(i.alsoResize,function(j){h(j)})}}else{h(i.alsoResize)}},resize:function(g,i){var f=c(this).data("resizable"),j=f.options,h=f.originalSize,l=f.originalPosition;var k={height:(f.size.height-h.height)||0,width:(f.size.width-h.width)||0,top:(f.position.top-l.top)||0,left:(f.position.left-l.left)||0},e=function(m,n){c(m).each(function(){var q=c(this),r=c(this).data("resizable-alsoresize"),p={},o=n&&n.length?n:q.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];c.each(o,function(s,u){var t=(r[u]||0)+(k[u]||0);if(t&&t>=0){p[u]=t||null}});q.css(p)})};if(typeof(j.alsoResize)=="object"&&!j.alsoResize.nodeType){c.each(j.alsoResize,function(m,n){e(m,n)})}else{e(j.alsoResize)}},stop:function(e,f){c(this).removeData("resizable-alsoresize")}});c.ui.plugin.add("resizable","animate",{stop:function(i,n){var p=c(this).data("resizable"),j=p.options;var h=p._proportionallyResizeElements,e=h.length&&(/textarea/i).test(h[0].nodeName),f=e&&c.ui.hasScroll(h[0],"left")?0:p.sizeDiff.height,l=e?0:p.sizeDiff.width;var g={width:(p.size.width-l),height:(p.size.height-f)},k=(parseInt(p.element.css("left"),10)+(p.position.left-p.originalPosition.left))||null,m=(parseInt(p.element.css("top"),10)+(p.position.top-p.originalPosition.top))||null;p.element.animate(c.extend(g,m&&k?{top:m,left:k}:{}),{duration:j.animateDuration,easing:j.animateEasing,step:function(){var o={width:parseInt(p.element.css("width"),10),height:parseInt(p.element.css("height"),10),top:parseInt(p.element.css("top"),10),left:parseInt(p.element.css("left"),10)};if(h&&h.length){c(h[0]).css({width:o.width,height:o.height})}p._updateCache(o);p._propagate("resize",i)}})}});c.ui.plugin.add("resizable","containment",{start:function(f,r){var t=c(this).data("resizable"),j=t.options,l=t.element;var g=j.containment,k=(g instanceof c)?g.get(0):(/parent/.test(g))?l.parent().get(0):g;if(!k){return}t.containerElement=c(k);if(/document/.test(g)||g==document){t.containerOffset={left:0,top:0};t.containerPosition={left:0,top:0};t.parentData={element:c(document),left:0,top:0,width:c(document).width(),height:c(document).height()||document.body.parentNode.scrollHeight}}else{var n=c(k),i=[];c(["Top","Right","Left","Bottom"]).each(function(p,o){i[p]=b(n.css("padding"+o))});t.containerOffset=n.offset();t.containerPosition=n.position();t.containerSize={height:(n.innerHeight()-i[3]),width:(n.innerWidth()-i[1])};var q=t.containerOffset,e=t.containerSize.height,m=t.containerSize.width,h=(c.ui.hasScroll(k,"left")?k.scrollWidth:m),s=(c.ui.hasScroll(k)?k.scrollHeight:e);t.parentData={element:k,left:q.left,top:q.top,width:h,height:s}}},resize:function(g,q){var t=c(this).data("resizable"),i=t.options,f=t.containerSize,p=t.containerOffset,m=t.size,n=t.position,r=t._aspectRatio||g.shiftKey,e={top:0,left:0},h=t.containerElement;if(h[0]!=document&&(/static/).test(h.css("position"))){e=p}if(n.left<(t._helper?p.left:0)){t.size.width=t.size.width+(t._helper?(t.position.left-p.left):(t.position.left-e.left));if(r){t.size.height=t.size.width/i.aspectRatio}t.position.left=i.helper?p.left:0}if(n.top<(t._helper?p.top:0)){t.size.height=t.size.height+(t._helper?(t.position.top-p.top):t.position.top);if(r){t.size.width=t.size.height*i.aspectRatio}t.position.top=t._helper?p.top:0}t.offset.left=t.parentData.left+t.position.left;t.offset.top=t.parentData.top+t.position.top;var l=Math.abs((t._helper?t.offset.left-e.left:(t.offset.left-e.left))+t.sizeDiff.width),s=Math.abs((t._helper?t.offset.top-e.top:(t.offset.top-p.top))+t.sizeDiff.height);var k=t.containerElement.get(0)==t.element.parent().get(0),j=/relative|absolute/.test(t.containerElement.css("position"));if(k&&j){l-=t.parentData.left}if(l+t.size.width>=t.parentData.width){t.size.width=t.parentData.width-l;if(r){t.size.height=t.size.width/t.aspectRatio}}if(s+t.size.height>=t.parentData.height){t.size.height=t.parentData.height-s;if(r){t.size.width=t.size.height*t.aspectRatio}}},stop:function(f,n){var q=c(this).data("resizable"),g=q.options,l=q.position,m=q.containerOffset,e=q.containerPosition,i=q.containerElement;var j=c(q.helper),r=j.offset(),p=j.outerWidth()-q.sizeDiff.width,k=j.outerHeight()-q.sizeDiff.height;if(q._helper&&!g.animate&&(/relative/).test(i.css("position"))){c(this).css({left:r.left-e.left-m.left,width:p,height:k})}if(q._helper&&!g.animate&&(/static/).test(i.css("position"))){c(this).css({left:r.left-e.left-m.left,width:p,height:k})}}});c.ui.plugin.add("resizable","ghost",{start:function(g,h){var e=c(this).data("resizable"),i=e.options,f=e.size;e.ghost=e.originalElement.clone();e.ghost.css({opacity:0.25,display:"block",position:"relative",height:f.height,width:f.width,margin:0,left:0,top:0}).addClass("ui-resizable-ghost").addClass(typeof i.ghost=="string"?i.ghost:"");e.ghost.appendTo(e.helper)},resize:function(f,g){var e=c(this).data("resizable"),h=e.options;if(e.ghost){e.ghost.css({position:"relative",height:e.size.height,width:e.size.width})}},stop:function(f,g){var e=c(this).data("resizable"),h=e.options;if(e.ghost&&e.helper){e.helper.get(0).removeChild(e.ghost.get(0))}}});c.ui.plugin.add("resizable","grid",{resize:function(e,m){var p=c(this).data("resizable"),h=p.options,k=p.size,i=p.originalSize,j=p.originalPosition,n=p.axis,l=h._aspectRatio||e.shiftKey;h.grid=typeof h.grid=="number"?[h.grid,h.grid]:h.grid;var g=Math.round((k.width-i.width)/(h.grid[0]||1))*(h.grid[0]||1),f=Math.round((k.height-i.height)/(h.grid[1]||1))*(h.grid[1]||1);if(/^(se|s|e)$/.test(n)){p.size.width=i.width+g;p.size.height=i.height+f}else{if(/^(ne)$/.test(n)){p.size.width=i.width+g;p.size.height=i.height+f;p.position.top=j.top-f}else{if(/^(sw)$/.test(n)){p.size.width=i.width+g;p.size.height=i.height+f;p.position.left=j.left-g}else{p.size.width=i.width+g;p.size.height=i.height+f;p.position.top=j.top-f;p.position.left=j.left-g}}}}});var b=function(e){return parseInt(e,10)||0};var a=function(e){return !isNaN(parseInt(e,10))}})(jQuery);/*! - * jQuery hashchange event - v1.3 - 7/21/2010 - * http://benalman.com/projects/jquery-hashchange-plugin/ - * - * Copyright (c) 2010 "Cowboy" Ben Alman - * Dual licensed under the MIT and GPL licenses. - * http://benalman.com/about/license/ - */ -(function($,e,b){var c="hashchange",h=document,f,g=$.event.special,i=h.documentMode,d="on"+c in e&&(i===b||i>7);function a(j){j=j||location.href;return"#"+j.replace(/^[^#]*#?(.*)$/,"$1")}$.fn[c]=function(j){return j?this.bind(c,j):this.trigger(c)};$.fn[c].delay=50;g[c]=$.extend(g[c],{setup:function(){if(d){return false}$(f.start)},teardown:function(){if(d){return false}$(f.stop)}});f=(function(){var j={},p,m=a(),k=function(q){return q},l=k,o=k;j.start=function(){p||n()};j.stop=function(){p&&clearTimeout(p);p=b};function n(){var r=a(),q=o(m);if(r!==m){l(m=r,q);$(e).trigger(c)}else{if(q!==m){location.href=location.href.replace(/#.*/,"")+q}}p=setTimeout(n,$.fn[c].delay)}$.browser.msie&&!d&&(function(){var q,r;j.start=function(){if(!q){r=$.fn[c].src;r=r&&r+a();q=$('