From 429025634e7a20983bc5c3b6b946f335b1c5090d Mon Sep 17 00:00:00 2001 From: Haneef Mubarak Date: Tue, 26 May 2020 16:04:28 -0700 Subject: [PATCH 1/3] resolve #201 with a platform-selective REP MOVSB implementation --- include/mimalloc-internal.h | 16 ++++++++++++++++ src/alloc-aligned.c | 2 +- src/alloc-posix.c | 2 +- src/alloc.c | 6 +++--- src/heap.c | 4 ++-- src/init.c | 2 +- src/options.c | 2 +- src/random.c | 2 +- 8 files changed, 26 insertions(+), 10 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 35413315..b191d03d 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -174,6 +174,22 @@ bool _mi_page_is_valid(mi_page_t* page); #define EOVERFLOW (75) #endif +// ------------------------------------------------------ +// Fast `memcpy()` on x86(_64) platforms unavailable, +// use REP MOVSB +// ------------------------------------------------------ + +#if defined(_M_IX86) || defined(_M_X64) +#include +#define _mi_memcpy _mi_memcpy_rep_movsb +static inline void _mi_memcpy_rep_movsb (void *d, const void *s, size_t n) { + __movsb(d, s, n); + return; +} +#else +#define _mi_memcpy memcpy +#endif + /* ----------------------------------------------------------- Inlined definitions diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index ca16d367..10f40355 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -137,7 +137,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne memset((uint8_t*)newp + start, 0, newsize - start); } } - memcpy(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy(newp, p, (newsize > size ? size : newsize)); mi_free(p); // only free if successful } return newp; diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 4395893b..de031928 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -99,7 +99,7 @@ mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noex size_t size = (len+1)*sizeof(unsigned short); unsigned short* p = (unsigned short*)mi_malloc(size); if (p != NULL) { - memcpy(p,s,size); + _mi_memcpy(p,s,size); } return p; } diff --git a/src/alloc.c b/src/alloc.c index 607d15b6..33cd2341 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -571,7 +571,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); memset((uint8_t*)newp + start, 0, newsize - start); } - memcpy(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy(newp, p, (newsize > size ? size : newsize)); mi_free(p); // only free if successful } return newp; @@ -638,7 +638,7 @@ mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_no if (s == NULL) return NULL; size_t n = strlen(s); char* t = (char*)mi_heap_malloc(heap,n+1); - if (t != NULL) memcpy(t, s, n + 1); + if (t != NULL) _mi_memcpy(t, s, n + 1); return t; } @@ -654,7 +654,7 @@ mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_assert_internal(m <= n); char* t = (char*)mi_heap_malloc(heap, m+1); if (t == NULL) return NULL; - memcpy(t, s, m); + _mi_memcpy(t, s, m); t[m] = 0; return t; } diff --git a/src/heap.c b/src/heap.c index 5d0d4b8a..e9518196 100644 --- a/src/heap.c +++ b/src/heap.c @@ -193,7 +193,7 @@ mi_heap_t* mi_heap_new(void) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap==NULL) return NULL; - memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); + _mi_memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); heap->tld = bheap->tld; heap->thread_id = _mi_thread_id(); _mi_random_split(&bheap->random, &heap->random); @@ -219,7 +219,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) { #ifdef MI_MEDIUM_DIRECT memset(&heap->pages_free_medium, 0, sizeof(heap->pages_free_medium)); #endif - memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); + _mi_memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); heap->thread_delayed_free = NULL; heap->page_count = 0; } diff --git a/src/init.c b/src/init.c index 132043e8..35a6d4de 100644 --- a/src/init.c +++ b/src/init.c @@ -188,7 +188,7 @@ static bool _mi_heap_init(void) { // OS allocated so already zero initialized mi_tld_t* tld = &td->tld; mi_heap_t* heap = &td->heap; - memcpy(heap, &_mi_heap_empty, sizeof(*heap)); + _mi_memcpy(heap, &_mi_heap_empty, sizeof(*heap)); heap->thread_id = _mi_thread_id(); _mi_random_init(&heap->random); heap->cookie = _mi_heap_random_next(heap) | 1; diff --git a/src/options.c b/src/options.c index f29b387c..7bbb5a70 100644 --- a/src/options.c +++ b/src/options.c @@ -183,7 +183,7 @@ static void mi_out_buf(const char* msg, void* arg) { if (start+n >= MI_MAX_DELAY_OUTPUT) { n = MI_MAX_DELAY_OUTPUT-start-1; } - memcpy(&out_buf[start], msg, n); + _mi_memcpy(&out_buf[start], msg, n); } static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { diff --git a/src/random.c b/src/random.c index b3dbf4f8..684559c8 100644 --- a/src/random.c +++ b/src/random.c @@ -115,7 +115,7 @@ static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t no static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { memset(ctx_new, 0, sizeof(*ctx_new)); - memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); + _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); ctx_new->input[12] = 0; ctx_new->input[13] = 0; ctx_new->input[14] = (uint32_t)nonce; From 6c92690914094ca6e784ff8d0cd8ee1cfc87d5ed Mon Sep 17 00:00:00 2001 From: Haneef Mubarak Date: Tue, 26 May 2020 16:08:33 -0700 Subject: [PATCH 2/3] fix REP MOVSB doc comment typo --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index b191d03d..43f72a11 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -175,8 +175,8 @@ bool _mi_page_is_valid(mi_page_t* page); #endif // ------------------------------------------------------ -// Fast `memcpy()` on x86(_64) platforms unavailable, -// use REP MOVSB +// Fast `memcpy()` on x86(_64) platforms unavailable +// on Windows, use REP MOVSB if necessary // ------------------------------------------------------ #if defined(_M_IX86) || defined(_M_X64) From 4c45793ec141378f916faef0052356034bc7d678 Mon Sep 17 00:00:00 2001 From: Haneef Mubarak Date: Tue, 26 May 2020 16:16:19 -0700 Subject: [PATCH 3/3] fix __movsb typecast error MSVC --- include/mimalloc-internal.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 43f72a11..6cdf0c03 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -183,7 +183,10 @@ bool _mi_page_is_valid(mi_page_t* page); #include #define _mi_memcpy _mi_memcpy_rep_movsb static inline void _mi_memcpy_rep_movsb (void *d, const void *s, size_t n) { - __movsb(d, s, n); + unsigned char* Destination = (unsigned char*) d; + unsigned const char* Source = (unsigned const char*) s; + size_t Count = n; + __movsb(Destination, Source, Count); return; } #else