remove delayed reset option (for now)

include/mimalloc.h

@@ -273,7 +273,7 @@ typedef enum mi_option_e {
   mi_option_page_reset,
   mi_option_segment_reset,
   mi_option_eager_commit_delay,
-  mi_option_reset_delay,
+  mi_option_reset_decommits,
   mi_option_use_numa_nodes,
   mi_option_os_tag,
   mi_option_max_errors,

src/memory.c

@@ -53,9 +53,6 @@ void    _mi_arena_free(void* p, size_t size, size_t memid, mi_stats_t* stats);
 void*   _mi_arena_alloc(size_t size, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
 void*   _mi_arena_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
 
-// local
-static bool mi_delay_remove(mi_delay_slots_t* delay_slots, void* p, size_t size);
-
 
 // Constants
 #if (MI_INTPTR_SIZE==8)
@@ -354,8 +351,6 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_os_tld_t* tld) {
   if (p==NULL) return;
   if (size==0) return;
 
-  mi_delay_remove(tld->reset_delay, p, size);
-
   size_t arena_memid = 0;
   mi_bitmap_index_t bit_idx;
   mem_region_t* region;
@@ -424,7 +419,6 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
         bool is_eager_committed;
         void* start = mi_region_info_read(mi_atomic_read(&regions[i].info), NULL, &is_eager_committed);
         if (start != NULL) { // && !_mi_os_is_huge_reserved(start)) {
-          mi_delay_remove(tld->reset_delay, start, MI_REGION_SIZE);
           _mi_arena_free(start, MI_REGION_SIZE, region->arena_memid, tld->stats);
         }
         // and release
@@ -434,142 +428,23 @@ void _mi_mem_collect(mi_os_tld_t* tld) {
   }
 }
 
-/* ----------------------------------------------------------------------------
-  Delay slots
------------------------------------------------------------------------------*/
-
-typedef void (mi_delay_resolve_fun)(void* addr, size_t size, void* arg);
-
-static void mi_delay_insert(mi_delay_slots_t* ds,
-  mi_msecs_t delay, uint8_t* addr, size_t size,
-  mi_delay_resolve_fun* resolve, void* arg)
-{
-  if (ds == NULL || delay==0 || addr==NULL || size==0) {
-    resolve(addr, size, arg);
-    return;
-  }
-
-  mi_msecs_t now = _mi_clock_now();
-  mi_delay_slot_t* oldest = &ds->slots[0];
-  // walk through all slots, resolving expired ones.
-  // remember the oldest slot to insert the new entry in.
-  size_t newcount = 0;
-  for (size_t i = 0; i < ds->count; i++) {
-    mi_delay_slot_t* slot = &ds->slots[i];
-    
-    if (slot->expire == 0) {
-      // empty slot
-      oldest = slot;
-    }
-    // TODO: should we handle overlapping areas too?
-    else if (slot->addr <= addr && slot->addr + slot->size >= addr + size) {
-      // earlier slot encompasses new area, increase expiration
-      slot->expire = now + delay;
-      delay = 0; 
-    }
-    else if (addr <= slot->addr && addr + size >= slot->addr + slot->size) {
-      // new one encompasses old slot, overwrite
-      slot->expire = now + delay;
-      slot->addr = addr;
-      slot->size = size;
-      delay = 0;
-    }
-    else if (slot->expire < now) {
-      // expired slot, resolve now
-      slot->expire = 0;
-      resolve(slot->addr, slot->size, arg);
-    }
-    else if (oldest->expire > slot->expire) {  
-      oldest = slot;
-      newcount = i+1;
-    }
-    else {
-      newcount = i+1;
-    }
-  }
-  ds->count = newcount;
-  if (delay>0) {
-    // not yet registered, use the oldest slot (or a new one if there is space)
-    if (ds->count < ds->capacity) {
-      oldest = &ds->slots[ds->count];
-      ds->count++;
-    }
-    else if (oldest->expire > 0) { 
-      resolve(oldest->addr, oldest->size, arg);  // evict if not empty
-    }
-    mi_assert_internal((oldest - ds->slots) < (ptrdiff_t)ds->count);
-    oldest->expire = now + delay;
-    oldest->addr = addr;
-    oldest->size = size;
-  }
-}
-
-static bool mi_delay_remove(mi_delay_slots_t* ds, void* p, size_t size)
-{
-  if (ds == NULL || p==NULL || size==0) return false; 
-  
-  uint8_t* addr = (uint8_t*)p;
-  bool done = false;
-  size_t newcount = 0;
-  
-  // walk through all valid slots
-  for (size_t i = 0; i < ds->count; i++) {
-    mi_delay_slot_t* slot = &ds->slots[i];
-    if (slot->addr <= addr && slot->addr + slot->size >= addr + size) {
-      // earlier slot encompasses the area; remove it
-      slot->expire = 0;
-      done = true;
-    }
-    else if (addr <= slot->addr && addr + size >= slot->addr + slot->size) {
-      // new one encompasses old slot, remove it
-      slot->expire = 0;
-    }
-    else if ((addr <= slot->addr && addr + size > slot->addr) ||
-      (addr < slot->addr + slot->size && addr + size >= slot->addr + slot->size)) {
-      // partial overlap
-      // can happen with a large object spanning onto some partial end block
-      // mi_assert_internal(false);
-      slot->expire = 0;
-    }
-    else {
-      newcount = i + 1;
-    }
-  }
-  ds->count = newcount;
-  return done;
-}
-
-static void mi_resolve_reset(void* p, size_t size, void* vtld) {
-  mi_os_tld_t* tld = (mi_os_tld_t*)vtld;
-  _mi_os_reset(p, size, tld->stats);
-}
-
-bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld) {
-  mi_delay_insert(tld->reset_delay, mi_option_get(mi_option_reset_delay),
-    (uint8_t*)p, size, &mi_resolve_reset, tld);
-  return true;
-}
-
-bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
-  if (!mi_delay_remove(tld->reset_delay, (uint8_t*)p, size)) {
-    return _mi_os_unreset(p, size, is_zero, tld->stats);
-  }
-  return true;
-}
-
-
 
 /* ----------------------------------------------------------------------------
   Other
 -----------------------------------------------------------------------------*/
+bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld) {
+  return _mi_os_reset(p, size, tld->stats);
+}
+
+bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
+  return _mi_os_unreset(p, size, is_zero, tld->stats);
+}
 
 bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld) {
-  mi_delay_remove(tld->reset_delay,p, size);
   return _mi_os_commit(p, size, is_zero, tld->stats);
 }
 
 bool _mi_mem_decommit(void* p, size_t size, mi_os_tld_t* tld) {
-  mi_delay_remove(tld->reset_delay, p, size);
   return _mi_os_decommit(p, size, tld->stats);
 }
 

src/options.c

@@ -65,10 +65,10 @@ static mi_option_desc_t options[_mi_option_last] =
   { 0, UNINIT, MI_OPTION(large_os_pages) },      // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
   { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
   { 0, UNINIT, MI_OPTION(segment_cache) },       // cache N segments per thread
-  { 0, UNINIT, MI_OPTION(page_reset) },          // reset pages on free
+  { 1, UNINIT, MI_OPTION(page_reset) },          // reset pages on free
   { 0, UNINIT, MI_OPTION(segment_reset) },       // reset segment memory on free (needs eager commit)
   { 0, UNINIT, MI_OPTION(eager_commit_delay) },  // the first N segments per thread are not eagerly committed
-  { 500, UNINIT, MI_OPTION(reset_delay) },       // reset delay in milli-seconds
+  { 1, UNINIT, MI_OPTION(reset_decommits) },     // reset uses decommit/commit
   { 0,   UNINIT, MI_OPTION(use_numa_nodes) },    // 0 = use available numa nodes, otherwise use at most N nodes. 
   { 100, UNINIT, MI_OPTION(os_tag) },            // only apple specific for now but might serve more or less related purpose
   { 16,  UNINIT, MI_OPTION(max_errors) }         // maximum errors that are output

src/os.c

@@ -300,7 +300,10 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
 #if !defined(MAP_ANONYMOUS)
 #define MAP_ANONYMOUS  MAP_ANON
 #endif
-  int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+#if !defined(MAP_NORESERVE)
+#define MAP_NORESERVE  0
+#endif
+  int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
   int fd = -1;
 #if defined(MAP_ALIGNED)  // BSD
   if (try_alignment > 0) {
@@ -626,26 +629,40 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
   }
 #elif defined(__wasi__)
   // WebAssembly guests can't control memory protection
+#elif defined(MAP_FIXED)
+  if (!commit) {
+    // use mmap with MAP_FIXED to discard the existing memory (and reduce commit charge)
+    void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
+    if (p != start) { err = errno; }
+  }
+  else {
+    // for commit, just change the protection
+    err = mprotect(start, csize, (PROT_READ | PROT_WRITE));
+    if (err != 0) { err = errno; }
+  }
 #else
   err = mprotect(start, csize, (commit ? (PROT_READ | PROT_WRITE) : PROT_NONE));
   if (err != 0) { err = errno; }
 #endif
   if (err != 0) {
-    _mi_warning_message("commit/decommit error: start: 0x%p, csize: 0x%x, err: %i\n", start, csize, err);
+    _mi_warning_message("%s error: start: 0x%p, csize: 0x%x, err: %i\n", commit ? "commit" : "decommit", start, csize, err);
   }
   mi_assert_internal(err == 0);
   return (err == 0);
 }
 
 bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
-  return mi_os_commitx(addr, size, true, false /* conservative? */, is_zero, stats);
+  return mi_os_commitx(addr, size, true, false /* liberal */, is_zero, stats);
 }
 
 bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats) {
   bool is_zero;
-  return mi_os_commitx(addr, size, false, true /* conservative? */, &is_zero, stats);
+  return mi_os_commitx(addr, size, false, true /* conservative */, &is_zero, stats);
 }
 
+bool _mi_os_commit_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
+  return mi_os_commitx(addr, size, true, true /* conservative */, is_zero, stats);
+}
 
 // Signal to the OS that the address range is no longer in use
 // but may be used later again. This will release physical memory
@@ -704,13 +721,23 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats)
 // pages and reduce swapping while keeping the memory committed.
 // We page align to a conservative area inside the range to reset.
 bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) {
+  if (mi_option_is_enabled(mi_option_reset_decommits)) {
+    return _mi_os_decommit(addr, size, stats);
+  }
+  else {
     return mi_os_resetx(addr, size, true, stats);
   }
+}
 
 bool _mi_os_unreset(void* addr, size_t size, bool* is_zero, mi_stats_t* stats) {
+  if (mi_option_is_enabled(mi_option_reset_decommits)) {
+    return _mi_os_commit_unreset(addr, size, is_zero, stats);  // re-commit it (conservatively!)
+  }
+  else {
     *is_zero = false;
     return mi_os_resetx(addr, size, false, stats);
   }
+}
 
 
 // Protect a region in memory to be not accessible.