diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 79d04d2d..e54c4171 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -139,7 +139,8 @@ typedef int32_t mi_ssize_t; // Derived constants #define MI_SEGMENT_SIZE (MI_ZU(1)<= 655360) #error "mimalloc internal: define more bins" #endif -#if (MI_ALIGNMENT_MAX > MI_SEGMENT_SIZE/2) -#error "mimalloc internal: the max aligned boundary is too large for the segment size" -#endif // Used as a special value to encode block sizes in 32 bits. #define MI_HUGE_BLOCK_SIZE ((uint32_t)MI_HUGE_OBJ_SIZE_MAX) +// Alignments over MI_ALIGNMENT_MAX are allocated in dedicated huge page segments +#define MI_ALIGNMENT_MAX (MI_SEGMENT_SIZE >> 1) + // ------------------------------------------------------ // Mimalloc pages contain allocated blocks diff --git a/include/mimalloc.h b/include/mimalloc.h index 17fd1c60..34644183 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -166,7 +166,6 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s // Note that `alignment` always follows `size` for consistency with unaligned // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. // ------------------------------------------------------------------------------------- -#define MI_ALIGNMENT_MAX (2*1024*1024UL) // maximum supported alignment is 1MiB mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 3ce01f5c..86b0112b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -36,9 +36,9 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_fallback(mi_heap_t* // use OS allocation for very large alignment and allocate inside a huge page (dedicated segment with 1 page) if mi_unlikely(offset != 0) { // todo: cannot support offset alignment for very large alignments yet -#if MI_DEBUG > 0 + #if MI_DEBUG > 0 _mi_error_message(EOVERFLOW, "aligned allocation with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); -#endif + #endif return NULL; } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); diff --git a/src/os.c b/src/os.c index 57b34a2c..c2d53e5b 100644 --- a/src/os.c +++ b/src/os.c @@ -842,29 +842,30 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool* lar /* ----------------------------------------------------------- OS aligned allocation with an offset. This is used - for large alignments > MI_SEGMENT_SIZE so we can align - the first page at an offset from the start of the segment. - As we may need to overallocate, we need to free such pointers - using `mi_free_aligned` to use the actual start of the - memory region. + for large alignments > MI_ALIGNMENT_MAX. We use a large mimalloc + page where the object can be aligned at an offset from the start of the segment. + As we may need to overallocate, we need to free such pointers using `mi_free_aligned` + to use the actual start of the memory region. ----------------------------------------------------------- */ - void* _mi_os_alloc_aligned_offset(size_t size, size_t alignment, size_t offset, bool commit, bool* large, mi_stats_t* tld_stats) { mi_assert(offset <= MI_SEGMENT_SIZE); mi_assert(offset <= size); mi_assert((alignment % _mi_os_page_size()) == 0); if (offset > MI_SEGMENT_SIZE) return NULL; if (offset == 0) { + // regular aligned allocation return _mi_os_alloc_aligned(size, alignment, commit, large, tld_stats); } else { + // overallocate to align at an offset const size_t extra = _mi_align_up(offset, alignment) - offset; const size_t oversize = size + extra; void* start = _mi_os_alloc_aligned(oversize, alignment, commit, large, tld_stats); if (start == NULL) return NULL; void* p = (uint8_t*)start + extra; mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); + // decommit the overallocation at the start if (commit && extra > _mi_os_page_size()) { _mi_os_decommit(start, extra, tld_stats); } diff --git a/src/segment.c b/src/segment.c index 5274aa1d..798aa756 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1317,9 +1317,7 @@ mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t pag mi_assert_internal(_mi_is_power_of_two(page_alignment)); mi_assert_internal(page_alignment >= MI_SEGMENT_SIZE); //mi_assert_internal((MI_SEGMENT_SIZE % page_alignment) == 0); - if (page_alignment < MI_SEGMENT_SIZE) { - page_alignment = MI_SEGMENT_SIZE; - } + if (page_alignment < MI_SEGMENT_SIZE) { page_alignment = MI_SEGMENT_SIZE; } page = mi_segment_huge_page_alloc(block_size, page_alignment, tld, os_tld); } else if (block_size <= MI_SMALL_OBJ_SIZE_MAX) { diff --git a/test/test-api.c b/test/test-api.c index 312b3f1b..a16ef381 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -34,7 +34,7 @@ we therefore test the API over various inputs. Please add more tests :-) #include "mimalloc.h" // #include "mimalloc-internal.h" -#include "mimalloc-types.h" // for MI_DEBUG +#include "mimalloc-types.h" // for MI_DEBUG and MI_ALIGNMENT_MAX #include "testhelper.h"