wip: further progress on removing segments

src/page-map.c

@@ -0,0 +1,90 @@
+/*----------------------------------------------------------------------------
+Copyright (c) 2023-2024, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+#include "mimalloc.h"
+#include "mimalloc/internal.h"
+#include "bitmap.h"
+
+mi_decl_cache_align signed char* _mi_page_map = NULL;
+static bool        mi_page_map_all_committed = false;
+static size_t      mi_blocks_per_commit_bit = 1;
+static mi_memid_t  mi_page_map_memid;
+static mi_bitmap_t mi_page_map_commit;
+
+static bool mi_page_map_init(void) {
+  size_t vbits = _mi_os_virtual_address_bits();
+  if (vbits >= 48) vbits = 47;
+  // 1 byte per block =  2 GiB for 128 TiB address space  (48 bit = 256 TiB address space)
+  //                    64 KiB for 4 GiB address space (on 32-bit)
+  const size_t page_map_size = (MI_ZU(1) << (vbits >> MI_ARENA_BLOCK_SHIFT));
+  
+  const size_t min_commit_size = _mi_divide_up(page_map_size,MI_BITMAP_MAX_BITS);
+  mi_blocks_per_commit_bit = mi_block_count_of_size(min_commit_size);
+
+  mi_page_map_all_committed = _mi_os_has_overcommit(); // commit on-access on Linux systems
+  _mi_page_map = (int8_t*)_mi_os_alloc_aligned(page_map_size, 0, mi_page_map_all_committed, true, &mi_page_map_memid, NULL);
+  if (_mi_page_map==NULL) {
+    _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB);
+    return false;
+  }
+  if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) {
+    _mi_warning_message("the page map was committed on-demand but not zero initialized!\n");
+    _mi_memzero_aligned(_mi_page_map, page_map_size);
+  }
+  return true;
+}
+
+static size_t mi_page_map_get_idx(mi_page_t* page, uint8_t** page_start, size_t* block_count) {
+  size_t page_size;
+  *page_start = mi_page_area(page, &page_size);
+  if (page_size > MI_LARGE_PAGE_SIZE) { page_size = MI_LARGE_PAGE_SIZE; }  // furthest interior pointer
+  *block_count = mi_block_count_of_size(page_size);
+  return ((uintptr_t)*page_start >> MI_ARENA_BLOCK_SHIFT);
+}
+
+
+
+void _mi_page_map_register(mi_page_t* page) {
+  if mi_unlikely(_mi_page_map == NULL) {
+    if (!mi_page_map_init()) return;
+  }
+  uint8_t* page_start;
+  size_t   block_count;
+  const size_t idx = mi_page_map_get_idx(page, &page_start, &block_count);
+  
+  // is the page map area that contains the page address committed?
+  if (!mi_page_map_all_committed) {
+    const size_t commit_bit_count = _mi_divide_up(block_count, mi_blocks_per_commit_bit);
+    const size_t commit_bit_idx = idx / mi_blocks_per_commit_bit;
+    for (size_t i = 0; i < commit_bit_count; i++) {  // per bit to avoid crossing over bitmap chunks
+      if (mi_bitmap_is_xsetN(MI_BIT_CLEAR, &mi_page_map_commit, commit_bit_idx + i, 1)) {
+        // this may race, in which case we do multiple commits (which is ok)
+        _mi_os_commit(page_start + (i*mi_blocks_per_commit_bit*MI_ARENA_BLOCK_SIZE), mi_blocks_per_commit_bit* MI_ARENA_BLOCK_SIZE, NULL, NULL);
+        mi_bitmap_xsetN(MI_BIT_SET, &mi_page_map_commit, commit_bit_idx + i, 1, NULL);
+      }
+    }
+  }
+
+  // set the offsets
+  for (int i = 0; i < block_count; i++) {
+    mi_assert_internal(i < 128);
+    _mi_page_map[idx + i] = (int8_t)(-i-1);
+  }
+}
+
+
+void _mi_page_map_unregister(mi_page_t* page) {
+  mi_assert_internal(_mi_page_map != NULL);
+  
+  // get index and count
+  uint8_t* page_start;
+  size_t   block_count;
+  const size_t idx = mi_page_map_get_idx(page, &page_start, &block_count);
+
+  // unset the offsets
+  _mi_memzero(_mi_page_map + idx, block_count);
+}