From 5e9b37dc4ec6bbc89c8643f65aecbc5a67484a8e Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 11 Oct 2019 17:03:09 -0700 Subject: [PATCH 01/57] add delayed output buffer --- src/options.c | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/options.c b/src/options.c index 09524cb4..5e631b8a 100644 --- a/src/options.c +++ b/src/options.c @@ -140,6 +140,46 @@ static void mi_out_stderr(const char* msg) { #endif } +// Since an output function can be registered earliest in the `main` +// function we also buffer output that happens earlier. When +// an output function is registered it is called immediately with +// the output up to that point. +#define MAX_OUT_BUF (8*1024) +static char out_buf[MAX_OUT_BUF+1]; +static _Atomic(uintptr_t) out_len; + +static void mi_out_buf(const char* msg) { + if (msg==NULL) return; + size_t n = strlen(msg); + if (n==0) return; + // claim + if (mi_atomic_read_relaxed(&out_len)>=MAX_OUT_BUF) return; + uintptr_t start = mi_atomic_addu(&out_len, n); + if (start >= MAX_OUT_BUF) return; + // check bound + if (start+n >= MAX_OUT_BUF) { + n = MAX_OUT_BUF-start-1; + } + memcpy(&out_buf[start], msg, n); +} + +static void mi_out_buf_contents(mi_output_fun* out) { + if (out==NULL) return; + // claim all + size_t count = mi_atomic_addu(&out_len, MAX_OUT_BUF); + // and output it + if (count>MAX_OUT_BUF) count = MAX_OUT_BUF; + out_buf[count] = 0; + out(out_buf); +} + +// The initial default output outputs to stderr and the delayed buffer. +static void mi_out_buf_stderr(const char* msg) { + mi_out_stderr(msg); + mi_out_buf(msg); +} + + // -------------------------------------------------------- // Default output handler // -------------------------------------------------------- @@ -151,11 +191,12 @@ static mi_output_fun* volatile mi_out_default; // = NULL static mi_output_fun* mi_out_get_default(void) { mi_output_fun* out = mi_out_default; - return (out == NULL ? &mi_out_stderr : out); + return (out == NULL ? &mi_out_buf_stderr : out); } void mi_register_output(mi_output_fun* out) mi_attr_noexcept { - mi_out_default = out; + mi_out_default = (out == NULL ? &mi_out_stderr : out); + if (out!=NULL) mi_out_buf_contents(out); } From 480f7683a939da42453518ad63a1d7f3bf40ea89 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 12 Oct 2019 10:42:04 -0700 Subject: [PATCH 02/57] cleanup delayed output --- src/options.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/options.c b/src/options.c index 5e631b8a..3e10926c 100644 --- a/src/options.c +++ b/src/options.c @@ -150,10 +150,10 @@ static _Atomic(uintptr_t) out_len; static void mi_out_buf(const char* msg) { if (msg==NULL) return; + if (mi_atomic_read_relaxed(&out_len)>=MAX_OUT_BUF) return; size_t n = strlen(msg); if (n==0) return; - // claim - if (mi_atomic_read_relaxed(&out_len)>=MAX_OUT_BUF) return; + // claim space uintptr_t start = mi_atomic_addu(&out_len, n); if (start >= MAX_OUT_BUF) return; // check bound @@ -163,17 +163,17 @@ static void mi_out_buf(const char* msg) { memcpy(&out_buf[start], msg, n); } -static void mi_out_buf_contents(mi_output_fun* out) { +static void mi_out_buf_flush(mi_output_fun* out) { if (out==NULL) return; - // claim all + // claim all (no more output will be added after this point) size_t count = mi_atomic_addu(&out_len, MAX_OUT_BUF); - // and output it + // and output the current contents if (count>MAX_OUT_BUF) count = MAX_OUT_BUF; out_buf[count] = 0; out(out_buf); } -// The initial default output outputs to stderr and the delayed buffer. +// The initial default output, outputs to stderr and the delayed output buffer. static void mi_out_buf_stderr(const char* msg) { mi_out_stderr(msg); mi_out_buf(msg); @@ -195,13 +195,13 @@ static mi_output_fun* mi_out_get_default(void) { } void mi_register_output(mi_output_fun* out) mi_attr_noexcept { - mi_out_default = (out == NULL ? &mi_out_stderr : out); - if (out!=NULL) mi_out_buf_contents(out); + mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer + if (out!=NULL) mi_out_buf_flush(out); // output the delayed output now } // -------------------------------------------------------- -// Messages +// Messages, all end up calling `_mi_fputs`. // -------------------------------------------------------- #define MAX_ERROR_COUNT (10) static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT stop emitting errors and warnings From 7441aa42128bc33b06efeb5cb55f002e9a00be3d Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 12 Oct 2019 10:42:34 -0700 Subject: [PATCH 03/57] update redirection module to add additional checks --- bin/mimalloc-redirect.dll | Bin 43520 -> 44544 bytes bin/mimalloc-redirect.lib | Bin 2874 -> 2874 bytes bin/mimalloc-redirect32.dll | Bin 31744 -> 32768 bytes bin/mimalloc-redirect32.lib | Bin 2928 -> 2928 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index 45e0fb4843752d1ecaa1cac4c18a485e4caabf56..98949605fdb7aaeedca3807c97b347638d1a4a98 100644 GIT binary patch delta 5232 zcmbtXeOOc189(Pjga9EIA&BVk1|!Cg(Ud3##WpCkmo}Dw)KQ;WK|#SUqG0_N5OG>$ z1&`XzX&-d0>1;P*(^_iP#?=pWcExI2yQjKnze1^fs9QHTZL{CGxlo^N|Ls1{@4UbF zJ@5OT_k7)()L$0rKM`89X?NSy>2+<f_8KqmI;GV}$h%en03Uz7F;Rgl*d)P8z zgccPjgSE5W&1&%@bs6s?e{38XJ|AZBw-|)w|V|#bTew$!z9gp^fYs7tXbQhCq!=6J#&JP zGvSd$Zk)Oil+-45Vl*Nz_7S;7R}LrriE!HFTN!Tk^+w1onnLd}%@yice+*s{AvKw> zE#Av>#hB#gVv{6?U6h;8%gu&4aM2&pZjBM;W{ney9Q4WCG!KFG?nTr@B}$ihjQJyW z2nXI`wUHZ5sc*q=T$kK@Qr?zpvdGQ3CYPKAR+A<-OeN?g5=+$1jD7j&fMbhpH(RA0 zk^J-?UPiIW>}b}kK(}MuAM%^K9W9zU%%dGar?PJCb9ujgGgSBrp*IUf?EV#x(LD#N z{z$PYa%(~)s@)1TRP9F2S~S%mHy1R%S}e|*z?v5)B_*mYW?Wdm!4o zEcy$IwSHW8A3e^RbP~PfJ*Yb^&^ES2lxQVu5t|16@G39Hn-ur9Kn>n2@$od?_^eV; zYDfRRoLY_19%Q2765l=HyBRDe@qw6qvD~dIunUs;lMSprF(Il_HI~M*6Nzb2HdVF6 zvTqW{#e}Kq9i0*~jM;}IM0G*od3-wd)DUCLK2_bteGaD>m@82qvs5)!a3de9dx%~) zRW(k8%wX4(6Sy@-r&=}ruDq-(F`+TKW0X36qpH7#M7m+DJt-yTDb+YDWK3lKq@gh* zRpYRGJ+I-s^NK%&M0lbS{=913ts1@ULklA^Ucym6lWQ{4#$zUatf{_9i#N=IoX&76 zx9l{vV)b##C)5%WnDuEe*yLl|qldBwjfpz7E1q{5n9(?fH5>JH+}h-;9ePjjjT(5o zegnq<9Z_d6f>5b$Jgfy;)T**5uTva+O+(mx`rmeZAR8CTxeEOy&T=xI0jq&sq zNRKNRH3V7k;!v|p=+82NSyGc{^V%A3Ta}?~12x#VJ8H0nYRKlQQbTrsboN%w+J^rg zIE4><<+`#ci&B$WgK_Bpn*Plda9a(x{^{oq44tbEof{fj9vE8gA6mxrN8)yMA_>1m zrQpX@UJ2f%au@jJ6{JpaS}mX5Y^vwU@W?I2JW8i`pp1R~&}A%@{xy>y$#cMz7wshtp{Hj(>|gs)C+ z-fr?BO&|h(fz4>{O%-uQQ-=VmKcZO;iGU%prHbK=-u0JiXU9{)*oP^ zDI@-?YZxDuSmbRy&A1oQe>i0rbD1Kv-{lixJr`)b#_CN&#(x0qO+KT*QXbQI%KXj+ z<&hh|He*h&^M3x-3e331-Z5D-Eiz7(a!ed84wH?)$Sl_Pt>=P<&yEvlKI~7w<)F1KT>}nP5j9nyh!>P*9 z^fH+}T4yieJ*5M0`%2DixV3ibXhA6``RWhBASQUopp;`qIcBmxo-e)~YsqS(Cos`^ zAG4%Y3WpNdp0sqKK7pN2n=U+&!1SXx(v$2LqhAps<5{+OT;lb?_%J9FMRs#&qFG1T z2J>>EcrfcT|3%P#%r1`kvPkCZqTE@2$nYh?l!Asqo%a?I$xoVvtPIca^%n7YSmZ{Z zq|AW#*l7Ekb_at! znjnj|=CSs0yrDIN>+pS2m)rL0lggw5TDU2j%4;^0l`-7Ksq9EEjyC_W@h>-@9TXfv zoC&_oPShLC$9e5NBxo_{!Ffh0unq=g-)kS`juIRN?%4d^1Hm9aVK#17W+nxCzDG2| zN&=Y!VTS&&PnIA|13Ok&@KEO0=6IdfO*DDUyUII}uly>MU80ojFqs2J_Vt8^Q?&oW zK-gv}7o$l>@#za4QMoxXaQ6Z$oH);La|aC1PrybMLF@!cQbDQL)L+r(+IW%j%lPr;{-5wYZ zduSesd^2&=`J7vH>F@}A*%ewgNuAk5=E|xO26wT~vqnbLR>0%tS*D*nMmT$^?A=EbGW!30@b}q5Ah!%C0KwDW z7XzLEyn?=03cQDB#L|SKnYlA>Wfg9jRrJ~;+uwTK^S2$>8oebmmW9!Y?5uOFPD_b) z6H>%d5_^FC-C0dHd2_bT67qw1c=BXyId+$5R0J^kT|k7rN`xIq0NQ{WzykX;pctqG z)_K35`B(btEMGJg*~zjFTF;ix>!2QHoZmtH>>}EF)==I-o$US!ZoiIJWNiz0Z_dIF zYG>b~b+P@GytlBbgZ44HsDoPAVYDu9+2SB&uIg6Sw^V|2+%oQV5^XD6Ufn@`UZblz z3}dv_@LV~yZf*A{QqQMkVc$X{X(6qY?O*BEdelDATln-`ilGj#;$da0c~Nm|+*<#b zkh{-&at$0T%(#x%esP_fwjJLJjX1VHrN1Fb>*SgeM>p{j4{y4sI9Ar^*0!o1tt@V{ zTjL*yb~YtxrE#HX4_n^k*1A-!+iTq7razsLKO@KCD40IiS(IN;lwX`bJ8!O|ASZWj zK~ZjA5h3?aDK2_EZ^{#SGp7_h<~{xM`B8k<<{hHfzGD|nlB#N?s%2HRRb@-6)>f2D zWeaL6R!F02*x;RMMrlUbk|ovir4<$BRVymy*H%?84VHlCgcjYLq0wXzrt=qX&*+&KLJ0uI2`}T0UH8JA-#YE z-VLcYA`o0k#ZOGY0$vY!0C0hOAU^_DgZm*PMxq~Fgv)CzkpsLHvRX!t3h(m56k&4Zx0pSi^{U*f%44eh zlPUwM48IlfPrZfr?1I5veIb>%*}SW-&6xOqCiPbCyDn%bB^xNdr>WHI+TR-aZwYr1 AN&o-= delta 4875 zcmbtXeNuU+Iq?>Kwv7kp~ZFDEh(9iIA^{bR14q<-2jTcNjO~{+- zS;^Qg9}`(NG5prl`a6XPRW8J#<7vhR9fdv>DC* z3^ezVdVth#qS5r_ppi$Tc@@?7Ka!1R8wTos3W@q}5KBkiek-XPb!e8PqPaW{&3{RK zJq68`2sAS(b59tW7f?WdB-!3bHK6?yr4GQ6cZ9wzT;vXI?(`@AfMshjZg`ck)1hHZ z+B&lel+`31$4E?CDTAMcK?3L zWiU%EC1z0yxgfQim0IHG!6m9&ztdonTC_GOQot+i(&m8;>_=3(5@pOd$_I7rn)V}n zTi9lE%n|ra=#g4ZNV|&6R;i`P?3AX1t;>hPfRY#X`JKv#CNnY_R zl~H206t`%Xp}Tm(yLY$r7Q3|bxL2PdOyoZO@8|4$DOmVA&A@aNv3(!K7r^ z+Zh*zYU`i|tF0pD(mot?b3mijI*Emh<-Os1k}qBl1gKJsiH1g(q?Tf{&7UA|j`&i* zVpkX^3q5?VQ4~ItyN&N@geG2Z5`{Ycys3G_w+E;i*%JMZMo5?av9Us2^ixVjF@GG| zm{WBp<;#W90=H@>YqazE!i3umE+aYh_?sd5U~)Y7B*cZ6sz$q!f0&RFZc$ay$bU(g zVEDlh3>kn#$S7`4j0-=m8a)PHm6&R9scJL%oEQ_&8xx`o530uBk&(voB}N%1s>XE4 z9DXf1j;w~8ICE6ikC4M<^{7T~gi;4r)s~>oDDFubW0b>H5n8jiX5=l2 zzgv&kx31ufpo4-Ah^sKVhI3dtQc6f%A+ExAFvvWb>Mn~$_~sO{i| zDlY|>RPF#TQF*0=VV#sma+Oeo@!et+WxpJ}7m|eZ5>w`0G(2t*`QySAVGuJM%&aCy zay=BEL`iXdB)?9H*WR%QxvIdUy1l>p>P~BKLx&L6txbT$(k9~h6$8p@j4ivBoeso@ zK*tznsp%A*gp)ivzKdJab7HkV)P_;4(k@EzpLXT2kK(QAI(_`TjOCv3doFQzdScd( z_b~PbRqc0OLd@1vmXqg{pS1O=MRLVoq<(tr``29JpQq>MS!gwsoh^1@0Og^O`*H4> zKzwKyW0--6*@~>bL~7Y)u0v*8=6qh9kv0+e=(#aHo|XW{;uaeoMJ1>bcDrHLuT;_T@gCfmHIcg#Zt&zwF z_2YT#xJu2}vHZrkNg8h~A8(nh*%!;JEt>^_e``6akz)A%%n1ppF?h0~q6TaeoS;pLIJTQl6y-C}zqba+LQZ1G&{qI9Z1P7;7+4eyE z?Jvnm6UzkQFY<;--wK++i##PegST22=r*lIjOQZ%(mG8uV#uEKC%pylDz44nWx&s~Ppm>3ffsOhq zgAtc*1_A@)k>qc~NmjuZmpGoTyNFZfqS2vFELwI<-6AA!`$%yO&G8OAj){~i z6QM$)`G51~j`{iLK!El~EW)VbpChruG#$K~!u-0Ac-gdN@vk&7R;%*}{a**S-@Z%x zY?ItKtxdT9^G7kJ?mVW580A$a;?3UU-+L~&c9wy|w0ag+YeI$R$=i=E{5eRc31XI1 z$$8plMRggQZUh4U4*2korWbNk!5EG19JdyXE|)5UD`@xM{6Lw}IIPHG)hPMT!U#i- zV(9UwsV`b_pdQx3;n)+u#?BayeKHuER>?gDaTamdx%pS;vBS=bg3jTU{Az({y}mM7 zmCZjFc4c2{6KDf*1!;aR;}}HxbreJk%-3pDaOkG;)*0(HIcK@HaICHzIsC@6d~#u? z=F>BLY2o}yHTVM&NPkRe>y&nb($*{OW~FUZ+AV1DzK@DV)-XOeFRndgY|hZ1YkJlo zIV&my=7Hch)Q2}Q!(C@b(RXy7d`%N6#4TD}w0OzA@1=AVOkHrOYV!QA{#n@~moBIZ z5pwzai^gjs1s1u5|FWoBI3TZGJYRD+u2F!JhGUZ0L`MIM34Iv#Dvb<#IsW%)fjD3{ zPzBH$(^eG$rNAoLxa0?+J;d}{sW(EvpNZvNf|IW;?-JZRwW3St);)aQt!f5U4oAbtGfh|A4BVq%hm)0q$sN<*Jvj>op5$> zM|EraI#KY*sq3mk(BD=|{j(nH)O*ueBTY)=*Psomnw@t%-l=!18j}~+JuG0PV^2_6 z*#@d9dSj=)H!~RQkxy)dgUC}iQRx>pbqcM>_f(rTjGG#gwD_iJ6!5vixwj!n@D6o3 z8K~;0}jxC*9Hhw6jZ)=j?Mw${QY@-s7ZM&s7 zUe?^HZ&W=RdGz*9ZSQcjeMgesG$9!6=4*F!>K&@)l~Z?i3i9<`j&SbSW8%dJ!er;3 zy+Sy~QGfiqy@{jOS5&N7FV<9xtE_N^N&#~D)2D<6 z5uZ0J01G~idLifHy4A@-~(r)87lzt5nzXW0T98xkO2`k za8nw-2mvd&6S5s}g1aH#2kO9kA%6lI!P!{GvVfi7HprDgE4XuP0|=C7ptvF51>E4h zklzC@f-`&trU4#sE98B^ac~FZZomufhP(jufcqc~>Bs>tLM{aQ$RDx|@R2{{alqdI z#S3x;7zD>@XQS}}dL7&fSq89j3=a7f5C+c1VOszbxCnVC>EL$AdO!qkh3p0_;9kh9 zfEAoR<+Ff%aGZ12@BkVc6dY{U0Z8CD^6V494vr(v44HUZfQyg|0SCAPayL*3?uL95 zaDwChvG0L8a9jwM1vG-=IL?ndmzsOrQp4g8?&*xloK){2m3z= z#g>DLa*-TdoWj_1Ko7VVa$X(+DFE_|Y1lLJ$3OUkKp*))9-oett-y>6utGo*xD#^3 z3``2#3OTb7xxpJDWxxvVeE{UMnOJIax}C9@J5V*a9dfJ<1A#jrm(IeTfx98Q?!-Xg zY&K(W17YBUkg0cJ`@o%$3+7G1_sY742$*Or>_e39|E0YuNt)n_rdx diff --git a/bin/mimalloc-redirect.lib b/bin/mimalloc-redirect.lib index 149dabb71bfbc6b15fecd389ebc5e17af10e40a7..1e22ef12aaec87c9544724a5c766ddbc9bd90ac5 100644 GIT binary patch delta 62 zcmdlbwo7coH5SHMlNs4HCjV!#VVt$uhV=*|Sik^C+OSG*_GOob2&hc%<=}t{Zsh1> F0swn<63qYr delta 62 zcmdlbwo7coH5SH6lNs4HCjV!#VVtzthV=*|Sik^C+OSG*_GOob2&hc%<=}t{Zsh1> F0swVZ5}g15 diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index 0b5f515614df59b6ffb87800d0fba95885910459..b19e42cd2a9106bbf80260c29b32ea9f9cfc8a2e 100644 GIT binary patch delta 4050 zcma(UZBSEJ_P$3#5(wl4MWP@C1r!BKAOT`PP9?Tl6g98*tXEFNW~Vf*u_9ztvIcg#v=9gbudMIL?} zh}E!Gz$(VsY=R7QkO)>Yev(blc;P7&*7L9)`V}HnxT?9X@!1^+?F#`0xTIQpJs+@U zrYA#sNEF%{Dn;l{`+NWg76LdE0iYSI-7zu%1TlaX8Gu*g0eln-KtyN>DgYJa>1ql9 zFR`+dX#6(@KEfI{MD8d-9A4&KvQL~1FweJBN4O_66`Le$*tnDUprmdQ5n0Jkb@J0L zCc>x!lni_gVRh44TpMx9-R>#|IRSbSaltP>)~ zZPGd&5k(MXGdqz=W9l+Wom3-W#_>7RN$pYQ+(|%^3vh;Pr^5tK1M4h7fo(35&z9GHSuk=zSU1018Dyxz}a5^da;U8Gh~| z^oR`iJcQb1xc?z^y9{4?2+fz_yHN|UB!&8U>db5?eN-U7x*b6Jt2Vy*pb6Ng?0mVk|2aSN!VS+ zNvH8Xc)s6pS~+TKuhw0p1S@>K^pkzG!+F zUO0?7Q@4mI1vNEy7}tVPT_ZLf_G>h%D?tZW0uHV?Ei)JR)6<}Sfgcq8mTc$Z=;qC! z+i)vU>E=(HtZfb8Wcbcp4UahNBZwUPXwVaxXW(g+db&sul0P|PIQEOsj&SU=aD|Uq zi@~&uA2A7u0c-Cb`~XHkK7q(KUol{u>G+FH?y^YD!m{;LL?o<(-N}Oueilwca$bA@ zq#4CP1ld7+K~O(}G@n13nj998_ZP6MBl6+wQXcs%Y= z#*3FNPGR5XaLM8oY!8QDUc8NsLqXD*f)F31f$i}dS+xR>#P4U{ z`yQJTUVnQ21%8HR%+El`(o%nP9L;vpVV^{ML&*r+Ez0yxTa)%EEen1MCVVm;Pf(^mux%IL=?BNwC-ywj~;gm00lI#Dhy#NZz^`x03F_J58fd!&-&BI zl%G^i4$X2cBK2@l1X7ez5k%*EB&=kC0cR%jhc=K7lHLHzG~HPzyra0_FmZL1aVVq_8s8fYmD_*u<~#BP(NC&(S;L z%z`K&4Ly`(y$D3>wcs`V<0~4LNeKJ-ojdWo^%h|pxl-ZF0GldK5k(*2kZb>AQ1Q}_ z)B<$$M7|Z#+u%&XAz;@*#8i_5s#Qf;q>fsGn!OJ+Ne0=pQ_`(!B1lHs)_`Cj$7m7{ z%|8S90!MjpfisYF0rYeXZc8gl?z$Q{Wy?(GI>L19k)z%7$sj2enXQM3=Eke|PTEp- z-Blc!UO>9ZpA6r!0HqX?N##%GD4b8BibB%Up7!*3CHRW;yuGH5!K3WL%-|O_yvxx} zSf(;q@U=~TL7p6*;X@Uj1Kv5_GsMO-c=Osc*&YrdJFI$7=h`nBZsQwoKBrs$*<-zZ zj!TnYgna6$U9XK`n($py0lsEZ!MD*}Q#q4Cac8NDG2x@7<&2x+xn>n}5SND-WC{?3`>Uu}ag|(L2uAw|0rvr>2CNORGI2FLc`4mg zEf=AH+hTmO8iW8b5!Z)mNcPH_Uxntb0ka@ejfZN=L`^}}>Zj!5gMc*Q=bkDPd*P`K z(kg~RDaY0wmFdQf#WBp5Y zwd$IuZFSA+#W|*&ik#o%*mC#f9?Kohm2B0Y)t@!wU;I4^ncQStN%&=`RX)7rXkm`#!zH< P%wRHXF?h!Is6+k-%YEi5 delta 3652 zcma(UYfxKN`rJ#>gc3-2l~@r11zJETkQZs8pzST_klheWX>DntB)ci4mO^m5BM3C5 z0+C(lai`tU)v(rLXLY3$r*@~04O$D1I>XlP6jzks;?6}_C+^^yv23&7cVo0V`(y9S z`Of*?kMo^#@9pkpy93Nn2{S!hy0xnz>fPncrk6&;rsM4WA~&pu)9#JSci4_Pfbqg6016l-^lwaVb~&u> zdddkY&$l)>8i2N8Gk|(rz}Qan+0;lktVA##qdNctr(_k3R{A8nHl{TKmR=YraMW{_ zH8UqgtGfOzA!Ol50)S>EfYE3G`$21=lmRd;2GAG@;A%2}U#qPSbd=U}K#3p;Z;!E8fVbKWcmer0`uESK$2@{dE`N*_nTnQc}*0GtlDLfnITsb|5MLf zk$nuKqE%5@DOUn0Bkku_oSiGgk?P_CCYMnL(ko{Em}e*|iDjprqUWN!SA~M{Y>auE zjly-<-CRx?RmGm9Q?V5^DgHHjJua5sicgMdV)*T_)ewP+FO^aze$DDIdY_c0#eI+d zM?xO}6na1q{#ZgA?n19fXwO~f?<92OF0@lZ$H8(~w@%`ji+^3TVH;DFKJ zXAH_OSc1Jb9NLRWIyeda_DnQd2AVSL!;O|Sf+cCM*8!H6P#P^Q!3?1-k@iUQ1ji$W zWIp78hfVG^w-x-c5&H)>447k2AINoS`I z+)i*aVYL}Sjhp%X6tv+4=e-6VCb(-ig=un0nBF5#B5?|J294fodv_Cru}=sC`sb#z z^CrOzr^BueOrxmFGTm4PqNv?~P zKxX6@5^;ulu5p&4AaO*yIGHJ!hzN8F^s4K#x;JHb7Yx$!HS6Ab zPk1b@B_Y@*!cXvYFgb-C<>{+1J;l@alQXhcpix3u1%$qDB26TiD034R)d}{qc{-%}jX^cY%|jT{YABPC!4fuEr+DE!q-a+i zhk8FTkHv?3fgiN^;7;JNq;C2}+JfY-w;>6>+Ve?z4I^2c6Iu-J6D2ie6iNzE`K>wH zmysiB`i{4r`Z8XPvI7tA4gQdip3dCHcKGNgnQ2iiNQR4tzdcJ=XH~JRkM7J$iRwTy zY$bnumiA;7voFkgE@iO{vyon1Z;%xCf}g<0^v?Q2UFs&eJr#MY=yIP znY2q?7HPl5zZeM{0k^eKVn4LebuI{fCpFu<(Nc3r$vy-KUT?f3M1Yp-eaDfDyJvq6 z1-FLipv}0rB`Bh-x)`~98y=)N+m$k;2T+PYVtqrwscuK(xEV+w^vbW8&9-_qteo-2Mu( zjf!gi4|geEL`5%haNC~_E6#l_*oODaVyS&nu&Xfft-!WHl!D7?)bt?>`=}N2X5J7Ilwa`TW?(!*&H>=_{#)o(1|@`?%jlRWiPq>* zTfmDSQ8L}M`jJ}3E8slfgff;|#wXyXElSWl4|q9H%V^~eC0K6QQ43m#2LN}|*jm&- zZz-qeYX1!Q_|6JCwo?I;<2#Xqc>OK~Ge(c>s-?%6?AByf>!RUr2IY^Gla&53q-WMUIyRO-y)|084DwRK>T5(vOy*;0JB5%d_x6w z1Ev711+W^i307kQ-LxN7KmsklydQ)higI**vJtb7ZTwMiV~t=IWK49tu|`xMin)^3 zTV>*IKJcNAFR=Q$t~_i);p|3lF;+1e8!h8IjH15KRHcv^mO zTGXX3v=%lM9w~Gd4i)~hC|D#>%hU<#WOcgwSL(0Sk(x?PwdPUH0nK5}lbX*o|I*yn zFxu5xxi(RI$}ncQXVdDE`I65|J}4zB_>qBNMm3*m?$`EcztDcAUC~D9R_oU4s&sWa zhpt0+LPvF0y+i+u{&oG=`cuVEZ1P(+N0cO&oZoV#j1<717#P@CaJFE5k*CO8G*c9z zR;g>X&Dy7R7jz%#w(EE6>-Alr{#*S;{rmbG`rG<@i$5){GMEiJ4SNk%LzAJ!@VJ2x LkFP@&0p0LlA}&@o diff --git a/bin/mimalloc-redirect32.lib b/bin/mimalloc-redirect32.lib index 2bc4b0bc66f9eefadcb14beee53034bf76e3797d..c99aa32ce499240310b8ff1274d1690c85ec9a96 100644 GIT binary patch delta 62 zcmew$_CajJHx|ZOlQr2jCd;$hG0xf?$9jqpEMNd6<5;CPm$J)31Y{ Date: Thu, 17 Oct 2019 09:20:57 -0700 Subject: [PATCH 04/57] updated redirection module on windows --- bin/mimalloc-redirect.dll | Bin 44544 -> 46592 bytes bin/mimalloc-redirect32.dll | Bin 32768 -> 33792 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index 98949605fdb7aaeedca3807c97b347638d1a4a98..ec718f91c10dfe4c47b83739df2764fd25ad8581 100644 GIT binary patch delta 4901 zcmbtYeN;1hSh9Px7C7D9~lYkpe-45)8x<9!8AYl{;DzbZqc#am9l z1W-gbTr{TXs&NW z^Ij5~*LR_DigceUrywU6yze12i5fJY3TS@28jYX1_$8A~BGTOlnCxxZrTVq{&EyED zrZwu^{ZhAjWC%0!q>ScqWLBjiQv3cZpv0LqgO*5jAtH6BoP$!@dn&FY8vl{{)YI#Mho zb*XE>7W!cItZbF?*SQ@1I`OrH4|E*u(6s6w(?N4e?J-^60nrq12Za4>OQaU!4MYe zd#4GB9Hs?1#jA{r-bb5q%2p*Z`W6l46zg-+rc3oX9ig<2bpFrHXIh|mlxFLi*Z+Oz zpASFIhCid@y0TUGG3ugAy5cq+D)RX$gvTmb!0*Jl4g9xpZU_G+&Rf9MFqgW{>D?^t zo9FZhCO=a55#}Rh3CAv%!#L5UlFO2N7cn`Z1vms7XgTK z9h;u6BjikxJb$MTKolutdfl1h@c0fZV5AfRcrAz1a^gywRY$Ro+`U=Ziarcv0?C#na&h_7JDWn0Fo|<>%@uxqbr*`dYtYaE_Xkb4yfwOQz zWZ~mjENwc3tyiYps@*9^nc^QIFUQ=COe!yl5cy^a#;~f046j&aE?bo_t6xE07+T;1 z9E~O&pqAaK4~VQ3?3#z6NuC{XB?h>=Q}l5mFxuVcL2O=~b;l zDU--^Wey_4(310YsSxPM!Y2z)&tg|`H?a+IVJRNOSQs8eco4~%9Ogm%)>{JWvk54@J6Y6YKPo(xexi!1kKE)DQmY2vZb(eA?TN$hvYuIjNGfHNfax9c+ zucVQMzehCWQ9)eUaqSi@ab>LNbx9hSenQ{dqu+PsqgV`CJa;N*)I^*+mdmo4t+7;eD`H(KT?TUcn?~gH;)k)=47*ttvg7K`WUPg< zn(Ttpqblt7(o?0XDF~WzsC6}`6itI+jGDXD$00(c%*1(uokjVGEa0AHr-S`LXIP77 z`AFeFJgmr?vzw$L=u@lNCm0=hVlAG%MWc+r(^H! zrai@5QTTU@v(SE4T)-F3Qgumg;<@9nO_`-TO7i*7e@34xdAh)fUyQN5lXBZ4x8IlB zGjjW~+`8rV6}0$~lg1*X(dbiadlNPmF8|_ZyYv{|(n`FCK=3S&tpUfMG%}8{@k7C{ z`BZN0GlxqL?>uy}<7C5*gWUz&PF}y5s1H^gRP#wk@dHW45!KOP&7o@?ca)|ceq5cx zk(5`0PakgNgEyZ2T?!3VxqA}?YO2OPQZ3Ucs9=?;*DBIEg_>(b>aJnZ-eV#)9FxZb zpz{i~*hJc6lcm~Pk(z2{8Ul?f)P7u~-s7@Vs1vEVPNs8pVsE|7-1Q<2DAdp(QcHs@ zoo@*ACS=i`Mj1>^GDH>d+U1`41cP3O+zBXPX_k9hrwkzl>@K+{oRncs0rx4nXK0aO zUV)y|a?kXA8KMe!Uy^&~GYooL5v{G}JaXBiKE5j;8v0;b>)5 z9XvRkwFq`vsI=SbYmtlW)7qRv$cxBQ&f~?SmCd-N= za6z|V@?g3MCCq)h=YI%WKvV`q2e^`OvQ}6w$C|CwaT0YYo!BuI0RJ3X|oDdO$`l|jnz!k`e;7$U1hUY)v;pLG&uxkrGw2iubI%c ziw|99Kr)?8|NY-<|jjt|CMzB%~2<1$Ymz ztO&j_OE)k9->QrFlWqb|?8Ezjwqx5*P`M*xE+zqb;MIqlPi#m}$&Bwm3${ zHRD<1q4Dr|=S0thd9q?MYf6}!pNdRHrwG1x{j4xvzc=8W@rJ#2UyIM}>-6>bjQys5 zso&gR(N6|61KNSC0bw9KFgMUMcxli(7#OS=vJBaVT84ySao8|y8jcJ{hZl$2$TiKi zz_pocEq=GZ(|^ft8j(iKBbE`(sCHBsog2M98X3Jb<{b-+S;lSSE#u;ZVZu0JLXDym zixXs0GpR+DLX+XiIn>HM)j4%(iY16$1(3XEZ-v)_ifMgWJ^?km?VI;Sd{NZR+wboW lWcSaYc(wuifP26&XdE;RN+{ssAQ{pOX@|0ga2hTA{{iR7gy;YO delta 4310 zcmbsse^is@`u#pO7$|HiC~4Zq7&^jefIo(SO(XkC%&FvE_he^W9fF`N7>1=XLb{_x zflt>hyy;b(@jr%x>q%y;-}@`)z35`{&&`&yVl< z{XXyWz29Ez_pJ3(wtZ|KBX@2g(O-xtxoK>Yf;@Qtq(FEOzGJd7A42X|KOG^u z>}_S1GL#LMBQhhbMliric5aecitIQ-EW%+ZXYYu|_=FbXfalpeatP+8x*mjI{d6>J zt*N?ZD_S98&p;Ghg6Pcch~A1vv?mYI z&|ury(-X5SOUTio*PC5)T~)Tl|7~yARV!N>%C-yR;VdC0T!az&={iCf#xvPRB1`iw?t=@rTy9Pe!V* zWhe5nl0z?1uf#v2RDPmVC9@^Vuw)xPMoQjKxi)D{#LSFOv}b3aOESwpnlLC$=m0{n zN$f->v?#b}pA%qIu`%|SS7<7~dd8)g9UcPEr!tiY`s@gFk?#>b%=ip|@Nh^q*McEXnOB-oBmh5fU3v9r(7F*wHDcigXsm0>DYx~ ze4;yJME+EiuRwkz%A1kD9OWDM^9NK`ESLWHwo>YpC@;ZqFOP@AG>U<5-m5YYOTzhL zRS6xv2^r<6Jt|5_Jyq!GBPtq^r8ab3dK-3#GHERV%LcwJWe?41FHsqSS~0dX(1(KN zqG2S&GMk#FqiG^j+(nfS(pK z2?dcirRl5-C1+qp$~b5;R!OpY34zlK=0ST#`f@ea^&uWOCu!BeouA{Ho6oAxyd(bj z=I^QgIMvfLA-HZz56>=K6Z7?(1THOH&F0UBIg5_511T`PXq+8LhQsP?)|w0#)%UWG zB!fJ2KYMx>ypTD_%4R{~omq3foQZSaR-)3Dv=yt=c^~9m2&d$~PGJ9?X0~)Dgzo%; zjsFdYCJAQa z?K61$NRYeae>>+IQz(cLOYjtFE@X)Oz=!m?vRI|NrHGk2ZnQL{wz)w~a#FNIsue*Gtg#iO6DeaVw}NnMgnjE*zK|9Lc_vVk!|=kB++F z0c!Qhga$oS^cyXDsKO)Yp~7G)vRCv_-SXHddeGDfV(wO6jRouFt3k0mJ?lNx(pDR% zTCL~XiWv>v3N`gni^CUL!b{6l^Y*+O4%1Ipw0HxlpH7>k4yOGJt}kCBiT^k5$2x6l z?2UKAVX-l5!JI!U{pvvy>BM)K5aa4wrJtY(k7SO)y7k1e1-kO*-<%(uUK}h-HXSO> z)KD3Caj0-@dOD1E;SPzrSudq=$b>La3;T=aYj4?$LD-2;=~Su7?gAQ4eM|kDaOZA7 z^$I0!_utG zP2av-zH4v#?uz*4Mt%QreacsdzM6s#amzXHrgLt@d2ZBs;HESEZ&E=|?tI<)y5XowP`KEq`c^Z-)zSsZA!&PjDM&>{JyB`xl zFxkLRu7=TycxWnb1a7+;blYtZ+^$9&P*hk!sGw>aG*pTrN2L{XJJev@VFOu}8Y9F* zPE{j}R*CA#Dl2qWi)^6U3dWskuhdKZlwu?!)vsIkFtjc7JrV+E^O4Gyyn_{RlM z)Cw?$;HVRz!XkjCUVur2z-|F-PY9rI5FmAr0OJU*Cj~G+C4hFXaB!ahBM9yL1u!%U zz*z+dA_z?aG(0VUu2}%t0Rcu4I$H!VJ|lqoj{<}chHU~^TLs{s#q@SekKlL?)3;&z z=LP@d^F{`2?H8c3;~xKYZ$iu~K1PItG2cVz$QmXAj=kI@t2J$_tTk=5!1klcxCFXw z_}RGa=zXYu;pp61QOVAlZS~bAPMJ}sOn_5Im8kvVXl|yZqOy+LTDFU;HkEDTEH#|D z#$wuKsVuAJOtrN&wRPFty0U7#H-O^U+<7$g)(X?ss>)s4x$>G?&Qeyp-DKf1>Jp&w z*nB3zf6c3iIT*9)pi{!^2C4Hej32t3bC`1fN#}Kj(ZgSR(i<=4;PnVEBBMyZLogs8 zLOPg>;c>Xfk#5Z+WO5tcYLL!eiZbM_NPj{YL7rPih-W#*KwhCmJA@GOqe!JI@C8BM zi1aMN81mzk77(H_kw(H567o0#hsoNJE-WG>6?r|7T zM~G?_Ar;6Qk>=sF^T?Z#?zk7PS;z}W-!3LZhP>iFLOw=NP<{;|tL|4|5PWFApO6|J zgP8nH4=-R>R9vIoC1h`u9*NRZQTkq#2BTE+L&RSB170)BXO4s-j}#YDRj4Cv_Wv2w zU*Z0el`;(3$6$?0#&7n#C~3qSA!#4%91INVhWH`#P-tj!i1Ro+F3+e(?=^U>UgDGa zG(N#M>>J0M2C))?H!WgxR5;olimucyU00xMq$|{Ab~ZR2POe+st?wS~9_uDO)*f4r z(4*?Ow0k(O+N<&EyrbSR@3=SU4SB6To3Gv1>2vrFnIVYW@ThZ<4c6B!j-NW61ZheoThxE#N z6}?=q&^z24=pF4f_Eq#X^eOsN`_=t{{*nH%ezU8=Wpi z)&bi<`#|S_V?Z;g9pnec2ZMv5!O1}~)IQWXR`eHA`4ZLcG8HWP=;p>U)+IETOM*kkN}01G7X^Z)<= diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index b19e42cd2a9106bbf80260c29b32ea9f9cfc8a2e..9dfb934e8b241c583184d215f183872f17709023 100644 GIT binary patch delta 5422 zcma)AeNBYF)Ob^qhv#HtVy_ZBg ze{Bx;$^AWF@AEwG^L_-sU*I1XOfHr@f9AqLV)e5`2r6^QbrfL%#>Z1%>=ty9Xe(Kt zCnJd(tYaYwgK(;HgTzJ>#|6N!5rhU^71Rkq!!}h}y1w4$8u5Xk#sq}ICg@{wXYC$% zyYFKGe!bJ-a{%wszXIukrw*RKM)iV<9B@Vm0Ubm80l}ce8bD8>?+HrQeU=5UFg&N= zpZPloajZy05X-Mifz*`%`E4GMXTW6nAm}Y8vVjD0fYet2d8`bG6!Xl*Kzi}phueWf zv6rW@MG+SKu^fnJEs#g?+s`rYCU)A6$JmOY36oce0-%aoAY-LKUIY)Qpx{L$%PuaM z1D}C8Nw1jw2Z%c`Cq$2ASE8_R4I0cA%7hY9vL+s|OZ zsmQRfF%rw3#Q>27Wiyd%BdOz*{RAU z;08i3uc<{{YfV`oLocs&mSCNoOgzY}gsdxiYw=Lk2|gA-st{>4yf|@sJQX6SaqHbs zA;+;NLQFi10Y3IRei}(ahkb14xyt-kxPkH2?*0Gh}CjIowka`ISbxR;8$dj z^T!ESO95FHAxlnQ@${clDJU5qSxd++X@z8M?CzO&a>hxLK+U<^Houg>I-($x42&iT zUqy&X*g{qbXeV3QLo<`P8wKJ7#0GT~#Ol!NdC~$k6quir^8e)qSe=s`6wfOl-$H_Q z+t=`cquuMYq;IBY-2h2`iWc+tk}n~evXM8?FS{4s3;s;OT zU-S5K8o$QlAExmHkB_BsKZnKp)7Uz*wBetHHEqz~*O__T=7CWOnSbgf*ht($z!Rq~ z-c6-!?&KIju>tgE@oovvDE0%R3H_;9u@N&)z2_@V>ngD5>)Xh(NkQL*iMV^}z5G+} zfU|FT+=I1VMQ1iC$cyORO${4yOYj;K*kg;k9c%1F>q}OLh`n$!Q2VN4;bxdzW4HwGbFxAck|UWxoPeWURz3JOM+! z#)*X>6AZ6~D&s)%FXv&MQ}6nU(qrV|#gr=KhMg0JnEBAB2Y)3Y(23IJhbKQvr9xIw zketR=2j5xXKK9TZJ_dsQ5h%ctOG`5Hz#Y+Qn5>UiLGR}5h0RWg=rwEdd6|Hf{)KF? zA{%{LwpAb_k+@tzu1Dtb<2n9Xg6bvMXV#;+^7CW?a&O)S^!d#@fxfl5188wYA4#E; z72Bk6%1l_}Efe-cYcJ7DqzWbA>g|e$KrEu4C9~1<^sCsruJZf!GLi~1cLJk#h|%oj z*P%G>x$i^SsQ={FZjfOo>`SoAu(niT8JK)t`8k<4^M@_F$sHoXCxWsBv;=GfK8d2aX-fTo6fsK9o57>(Y_OD`OtJ?gqh&U#QG0^1M$4Yv$amn3LJX$-icb$>I z*+alI!-f@X65aC|D!@BkAR@7`fBt>i=XSJI^;)iI9Jh|7h5hCxI$OO53URY~OD=`E z;GYfMMB;7Df*;*P?b|ks%P^Y(FH+>+rX>~V&22A_IYV|TU=S1eCw~B=_c1HZECz~_ zxXvU2!q9ZlhTkq5jZZE629CJ?5+_K_So1boOoZzn&4uT;0m;`e*?!NBpzhq zfzdeOQ}o%vnn{GL@nACrev*(*BGiNn6))#Dao&O35QK94_;)zP;^p>KDuq7Tp+%OO z4Wtvb)$ANQiRBb07r7?^KqNM{m}lf)o`;^cV)yQISn;!uu_F;2Oc2)U5fFma4WV&W z$l~mTyj>w@7shruyuXG>Ob{kvC9Jmm!UFodR*ovRiVERsLG;ZNFhg8LhS!G4<)5ME z+D&6HyM#}Fok`%@#d098z*RB`bTlCxt%MRnx~o+>hpRg$$-f*u_1>@$MtdX{jXiXY zNs-r?6@04!b7m#LLZMdzqbtPmN;r`39eene#+gIb!$L44V$u9uIC@sS3D76CX0&<7 z#<8zKvyXwoTSXIXq%>$x;tA9cT-Y~27_ufqEkeCr!g2l$JBBodH(|9A5itxMmV|Z+ zwg@55e*`h)hwoz(aJcqbpp`*f(8}If#I?moYdKhkM62td++78I$5#CZS9!>@pYy&B ztXkMQ6x^Y%`QYo-nM!i4FT`sXpWGSOUA${9rLeF7S#re=uGL0yuydkuCKVWkB`L>e zFBG5_R~a^pPF~DS@vzk0fdeVydc5xsNmSQs`w zJtN|(^EQ^jc9@5m+?dTUUBJZCWnaNJ82budMUY9rKoY*4&?T+>vVxWOW|GU44+u~C z<1n>iZu*bJei?J{)1U4hSaIObYESeriHt)*jD>C@j_u$bCt$sT<3rzajN1XP*V_Uas?FJK9dj^QOXR?OXs@ck@y z%zX)qtyBxU^FG4}Hk`*2yn#v5hQHyiNt~h49p(H|Huj4TQYjbAKR4tOwq&sgZ;0=M z2qZT4>^)0-bynev;qWeZoEt*a%Km~^%)cA~FNHWBTv&G!{Oo=7 zKlSEaaP*|A;dc^H5l7*ef_I9eQjQjL6b>G0#<9D)0EW+*dDikh5=8}_4e1|hxQ+YZ zvF(FrFK$GFNXsPfj`yl7AiJ`6=Bi*bdA{wh1k_{YzqEe!_#@VfwO`DY{b1HQ<78xc zB`p~w+b1QvGH4u-KZBM4EoG3VO-e>GXtJ#kk#^oA3ld8PS?p4h z&Y-h^x-uvZ$e%$?`=t<9+RHScNCqVUEoG4H(L!{!eeaY^gf2YVJ|&a9(2gl;Z>KiN#2STBN+1Qj3Pz6KB6=X@SO z#yQ;~K_@$!vF=ERBzur1sOhxFC@^SX*wT!egPb@2K?;iWKxfmGEEg>T$as@=QlyQS zNKmjdS4eGzpMRFlhp^^SCugnsE(K*=wVnhpm0s0RL(rFLd9R zQhT@LTUms=VK7jhOh+wpN`%vCS6xm?b_Bq5S~?4iI55JP0Sn?Na=ZRTJml&g_PXkw z9uF!#{D92sa`t##-M*fq1N31>kH_VtckQBwhk6E%&|cS(VUNR0_d7hEqusj?dtI*j z8OPzTvccJpQM*Ti+#VJ5$4@=whld9ohdeIYca$D<_`2P6^$=ZCJy@%!?+K2$eDo9E z9-oUIbb0%GhK68F;-1)h)Z?KKIlBA6gV)ysV?Iec1_wPo5EAFpHB^b-^mvi1-vmFv zysn|6o?|XF(!T}PH~L9`$qrb?{YRa{5XP43p)JI4x7T;bG2{aFsy0rF6QTd=SM9q$ z^`0Tx*)xPwaMp`yy2j^y66fOS8R(;*=<&Jf{vN&xw9Dazki0b1%jb5{-ADWT9Rp6R z=`z=%j)4*=$P)wQWc|!l29Xu>@Sm9*@Fd|WlaoX}JWcR8;dw?*LKDb0jDRftIR|_~ zMov{0{(nqo=DjDfvL7IcQ4+SUE#%DL6E6zKPRdWqr{w45VR>F-MdS9yx<*B#q4B}S ze`+jL)GFMH#}!X0f{N!Af2+t=u2+^SE0sHxyOm1i8``M$A)U@3HI^8&Oax63%@EdS ziY-cy@~X04WmW}M^QwPWeWbdnx}{oCiPfd*O0`kFU)`yGN_|p2r9Pv1MRR>jvt9f5 z+5@@+hTj^yOhcyU@ISU>+<(4^O8KsapEeXVKC8%AmMJyLL8V=NQ1ganL6g*O(><)a zru&_4Nte<|^_%n>y;a|;e_H<&{UO7G;XT8m;rE7HhTDd?;Y$Pl_ZR#Te%(pk8Qm=% zt4rvb^v(JX{YCv)SRc_-2C1RS;4_RE#=-E0;f}#zv>0v1Y2z!#OGcqdWGXhfO&-&6 z(}F2#S~AJZ3iBTG8S`251#`ljG!uh(8VEun>yUNHBCzhfywTsd)Of3LkD^I&L2*$LR?I3SN=hk( ztOu1o<%sgQ(y#nLxv0FMTvFarYE%Z5MYTuOq&laXRuO8UTBN4bPPJR@QTx>M>IHRF zy{N9!$TSL#Mq|)SYR+g@Gzm>oBhuvfreLEBv z0z}{eG8xYGPFaq#(K*XV>TGgh4qZqq5{{uK&N-w_7PoN|PUmi7Vym{&R%>Nz-+d1) z$(7UPn^z?F_H)@B;+-e4fJXmkk4{~jDZKN&3*}M zl2w_nfKT6|Vu_DIvLifin zS#HCQ)OJv}dSYxPbW-5KVERBKKAdj+e30}xhX2KGEc`*twlbKDuhf6Zcwn!1q(obkp`vKRurR`wts|I$1L_txOr>=$t) zyC~(0OJBQnOIm3f34(Tn3b14Aw)8Ct_#hQGY|TmkC)lT=_-7KnoxpEP_|*h{S;T3V z61WdXz}@KtJRym{n!q-!d1PC9X#y5V-nJz01GzN1xLqN*pGo+30>3TcR}=VU313R! zJ`ty#PT&(WjPmd3-458(xA}*ZuTMc`4ZjK%L!f~*J{))$k2?gR4>6%z_+M2A6cS^I zFrLNbJJ{`nG5jO1Dxo_?MEk?Idk3q20Qr9~{G+VlJK*d~9s@+{9$wnPQUM&>QMH|H z0ZC(=cx(_BiN;fy&)pm%2t#a&?h{egzu<{nwm-JS&u#!-;^*k+$YVZVd;~S(V-dCz z8s}#n!ay4sEv{{D#${jMGV-P_@hcD+3d8D?QiXhdE2sy!cL*R{5;z*O-+}iE=JF6G z`j}7;p8fo?BUHQ=KTG!Z^@+cMx7kK0@+z}#*<-^$^eX&Ld1I(d9FPvx!E=NMAN<4N zpf}}q@?}t_vTkLNCuna=m}ii;D4rvFT%_m7@cv(%ct?RSrCq|!`Fr~V{~V9|?ebA- zirAib_m22;`q#xNwxQ6Qpa4-NB>zKBg}(KF%4MN%{x9;u^?V2ij}{ck0tmlez*5=x zvx45#)pEqZu+L{>cj0w->*YnIKo1n{1A4ru3Fur=JC%iZsY{hvup0JA!?-ir*oN9r zJVybpo>D&w;%W6I>aXyD;@61xpBDddWdJdw{JO7i9rY!z-GoF4t|3^%z|YC_py-6g zovToALXLP&KA8MT$uB9zO#jXU)Lt5S<&Ywu89`0ShlTLh(NJ{cz6Ckd9l2lT2<5F3 zFA6ASDjeBW_&ANu%fdYD|K#(zZOLR21c8he{Y=zb;uUcIASiE8yGHBj&wcu{NzwBK zq3*pT>4MKPJh1zX^eJ#aN?qvXPqC@=AaxV>m6oJW6E64b~&P5 zh}i9u>~Md@S8&Ca)NOO*GSj|{V1<0WX)4r{5}?+u;(L2`^h50a1|{cI^w3qoa3?G_ z@@h7-7fyxf;6#?rDY|bPB<}MQtq*)cAmLBKl}mh-+-yJ})?f`X8};=?QSUZL3M5D& zCMaZ2=#zcID@N;-0m_ekd#pIPH|NMSNc}t`=2<=tIUaS!$mk|1*ijI}DCFKWZjUy! z)sp=fBb)ZEpOAcrR}Pqfn@pf5lLQ*_;%D|5bMwXXSUxTuv0$07YxoI%Z{JsOV1G*g zB-9L!my20>jl={y~LJ z+k6QY^xcm|myi$`(yttB4CUM<9C+iS;WtULd$NTKMDtf4v0G9WpElSue{sy{HxJG*(t002*04IsepPL-whuKK;d zNL@d|%a}*M^=HrM%YX3L>rLKjS{6$spAhdGLa>2H!uT6cHn$dZ&amBD%4O z93ofc&dydhR zM?{(@B{U(TtD7hwqUe*G!x6KlV=f$ow(B?p*PWK8LwE#w98BF*ijxU@&q+lxqee_E9z1y{b>5AbJJ43~#gk89Q?nvD zQiK>ak>P92t`}PvIF7Sh4q-R+qyUzorv_dQ(RVu7od%Z*;nQI$G5wySqyO!QG`h4;+4{+pxzgob6VF zsk)=J#jW;ssy&_F)^pyoEgkCCuCC6m?n<@2rK6+s6khGlg)d};P6@52+RvUlqdwi) z_4z1i_z73HFC0HJ%(V$ky8HdP6&9-154^EY9+tF~SMQQ~(E*eGzN zc-#NOaAx`ZU{dk|LVXmx)g{!7=h>I&ey{eAw1Zl#ozgz2%F=Du7Rmvig9xNdme@b?DVm}AT{)*BthW5zQ^uW88iifPf5Vt%XN{1#`l1#LgJ zt=o_Wp~K*0P&2NLYtL8xR@cv7W~bRzw#;ZXHkmG&zGEsibLLv}Nwe2{(L8B>!~9+I zC+6Rq-?8lHsyGu@%^l<(;~d;kj{F58fUg0|kY&R2^X z_Sp zkoF1VRpX>_$`~@P8#j!#rh}%#rb*K^ScJBSX~PsV)te8S9p)oumwC$kih0_6!yGV2 z%`r2w&=$GHVL4)HvP@fUSOS(=%bW#qG$-d6&c!ux&77MPxBxfH&2jTwkds>#R;9Js a>b45j4y(sHXPvhOt&7%W>x{N;g8DD65%kCa From 0e188a18a7f8f2c4cf1e2294a5f4c1641d7e5518 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 09:21:20 -0700 Subject: [PATCH 05/57] update test to match malloc with free --- test/main-override.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index 2cafd2cd..4bc91ae8 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -41,7 +41,7 @@ int main() { p2 = malloc(16); p1 = realloc(p1, 32); free(p1); - mi_free(p2); + free(p2); mi_free(s); Test* t = new Test(42); delete t; From a96c90db5dc29223936cf32e74b2aa78aab93837 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 09:22:23 -0700 Subject: [PATCH 06/57] remove old windows overriding method --- src/alloc-override-win.c | 715 --------------------------------------- 1 file changed, 715 deletions(-) delete mode 100644 src/alloc-override-win.c diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c deleted file mode 100644 index dc4796ab..00000000 --- a/src/alloc-override-win.c +++ /dev/null @@ -1,715 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2018, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ - -#include "mimalloc.h" -#include "mimalloc-internal.h" - -#if !defined(_WIN32) -#error "this file should only be included on Windows" -#endif - -#include -#include - -#include // getenv -#include // _setmaxstdio -#include // strstr - - -/* -To override the C runtime `malloc` on Windows we need to patch the allocation -functions at runtime initialization. Unfortunately we can never patch before the -runtime initializes itself, because as soon as we call `GetProcAddress` on the -runtime module (a DLL or EXE in Windows speak), it will first load and initialize -(by the OS calling `DllMain` on it). - -This means that some things might be already allocated by the C runtime itself -(and possibly other DLL's) before we get to resolve runtime adresses. This is -no problem if everyone unwinds in order: when we unload, we unpatch and restore -the original crt `free` routines and crt malloc'd memory is freed correctly. - -But things go wrong if such early CRT alloc'd memory is freed or re-allocated -_after_ we patch, but _before_ we unload (and unpatch), or if any memory allocated -by us is freed after we unpatched. - -There are two tricky situations to deal with: - -1. The Thread Local Storage (TLS): when the main thread stops it will call registered - callbacks on TLS entries (allocated by `FlsAlloc`). This is done by the OS - before any DLL's are unloaded. Unfortunately, the C runtime registers such - TLS entries with CRT allocated memory which is freed in the callback. - -2. Inside the CRT: - a. Some variables might get initialized by patched allocated - blocks but freed during CRT unloading after we unpatched - (like temporary file buffers). - b. Some blocks are allocated at CRT and freed by the CRT (like the - environment storage). - c. And some blocks are allocated by the CRT and then reallocated - while patched, and finally freed after unpatching! This - happens with the `atexit` functions for example to grow the array - of registered functions. - -In principle situation 2 is hopeless: since we cannot patch before CRT initialization, -we can never be sure how to free or reallocate a pointer during CRT unloading. -However, in practice there is a good solution: when terminating, we just patch -the reallocation and free routines to no-ops -- we are winding down anyway! This leaves -just the reallocation problm of CRT alloc'd memory once we are patched. Here, a study of the -CRT reveals that there seem to be just three such situations: - -1. When registering `atexit` routines (to grow the exit function table), -2. When calling `_setmaxstdio` (to grow the file handle table), -3. and `_popen`/`_wpopen` (to grow handle pairs). These turn out not to be - a problem as these are NULL initialized. - -We fix these by providing wrappers: - -1. We first register a _global_ `atexit` routine ourselves (`mi_patches_at_exit`) before patching, - and then patch the `_crt_atexit` function to implement our own global exit list (and the - same for `_crt_at_quick_exit`). All module local lists are no problem since they are always fully - (un)patched from initialization to end. We can register in the global list by dynamically - getting the global `_crt_atexit` entry from `ucrtbase.dll`. - -2. The `_setmaxstdio` is _detoured_: we patch it by a stub that unpatches first, - calls the original routine and repatches again. - -That leaves us to reliably shutdown and enter "termination mode": - -1. Using our trick to get the global exit list entry point, we register an exit function `mi_patches_atexit` - that first executes all our home brew list of exit functions, and then enters a _termination_ - phase that patches realloc/free variants with no-ops. Patching later again with special no-ops for - `free` also improves efficiency during the program run since no flags need to be checked. - -2. That is not quite good enough yet since after executing exit routines after us on the - global exit list (registered by the CRT), - the OS starts to unwind the TLS callbacks and we would like to run callbacks registered after loading - our DLL to be done in patched mode. So, we also allocate a TLS entry when our DLL is loaded and when its - callback is called, we re-enable the original patches again. Since TLS is destroyed in FIFO order - this runs any callbacks in later DLL's in patched mode. - -3. Finally the DLL's get unloaded by the OS in order (still patched) until our DLL gets unloaded - and then we start a termination phase again, and patch realloc/free with no-ops for good this time. - -*/ - -static int __cdecl mi_setmaxstdio(int newmax); - -// ------------------------------------------------------ -// Microsoft allocation extensions -// ------------------------------------------------------ - - -typedef size_t mi_nothrow_t; - -static void mi_free_nothrow(void* p, mi_nothrow_t tag) { - UNUSED(tag); - mi_free(p); -} - -// Versions of `free`, `realloc`, `recalloc`, `expand` and `msize` -// that are used during termination and are no-ops. -static void mi_free_term(void* p) { - UNUSED(p); -} - -static void mi_free_size_term(void* p, size_t size) { - UNUSED(size); - UNUSED(p); -} - -static void mi_free_nothrow_term(void* p, mi_nothrow_t tag) { - UNUSED(tag); - UNUSED(p); -} - -static void* mi_realloc_term(void* p, size_t newsize) { - UNUSED(p); UNUSED(newsize); - return NULL; -} - -static void* mi__recalloc_term(void* p, size_t newcount, size_t newsize) { - UNUSED(p); UNUSED(newcount); UNUSED(newsize); - return NULL; -} - -static void* mi__expand_term(void* p, size_t newsize) { - UNUSED(p); UNUSED(newsize); - return NULL; -} - -static size_t mi__msize_term(void* p) { - UNUSED(p); - return 0; -} - - -static void* mi__malloc_dbg(size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return _malloc_base(size); -} - -static void* mi__calloc_dbg(size_t count, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return _calloc_base(count, size); -} - -static void* mi__realloc_dbg(void* p, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return _realloc_base(p, size); -} - -static void mi__free_dbg(void* p, int block_type) { - UNUSED(block_type); - _free_base(p); -} - - -// the `recalloc`,`expand`, and `msize` don't have base versions and thus need a separate term version - -static void* mi__recalloc_dbg(void* p, size_t count, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi_recalloc(p, count, size); -} - -static void* mi__expand_dbg(void* p, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi__expand(p, size); -} - -static size_t mi__msize_dbg(void* p, int block_type) { - UNUSED(block_type); - return mi_usable_size(p); -} - -static void* mi__recalloc_dbg_term(void* p, size_t count, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi__recalloc_term(p, count, size); -} - -static void* mi__expand_dbg_term(void* p, size_t size, int block_type, const char* fname, int line) { - UNUSED(block_type); UNUSED(fname); UNUSED(line); - return mi__expand_term(p, size); -} - -static size_t mi__msize_dbg_term(void* p, int block_type) { - UNUSED(block_type); - return mi__msize_term(p); -} - - -// ------------------------------------------------------ -// implement our own global atexit handler -// ------------------------------------------------------ -typedef void (cbfun_t)(void); -typedef int (atexit_fun_t)(cbfun_t* fn); -typedef uintptr_t encoded_t; - -typedef struct exit_list_s { - encoded_t functions; // encoded pointer to array of encoded function pointers - size_t count; - size_t capacity; -} exit_list_t; - -#define MI_EXIT_INC (64) - -static exit_list_t atexit_list = { 0, 0, 0 }; -static exit_list_t at_quick_exit_list = { 0, 0, 0 }; -static CRITICAL_SECTION atexit_lock; - -// encode/decode function pointers with a random canary for security -static encoded_t canary; - -static inline void *decode(encoded_t x) { - return (void*)(x^canary); -} - -static inline encoded_t encode(void* p) { - return ((uintptr_t)p ^ canary); -} - - -static void init_canary() -{ - canary = _mi_random_init(0); - atexit_list.functions = at_quick_exit_list.functions = encode(NULL); -} - - -// initialize the list -static void mi_initialize_atexit(void) { - InitializeCriticalSection(&atexit_lock); - init_canary(); -} - -// register an exit function -static int mi_register_atexit(exit_list_t* list, cbfun_t* fn) { - if (fn == NULL) return EINVAL; - EnterCriticalSection(&atexit_lock); - encoded_t* functions = (encoded_t*)decode(list->functions); - if (list->count >= list->capacity) { // at first `functions == decode(0) == NULL` - encoded_t* newf = (encoded_t*)mi_recalloc(functions, list->capacity + MI_EXIT_INC, sizeof(cbfun_t*)); - if (newf != NULL) { - list->capacity += MI_EXIT_INC; - list->functions = encode(newf); - functions = newf; - } - } - int result; - if (list->count < list->capacity && functions != NULL) { - functions[list->count] = encode(fn); - list->count++; - result = 0; // success - } - else { - result = ENOMEM; - } - LeaveCriticalSection(&atexit_lock); - return result; -} - -// Register a global `atexit` function -static int mi_atexit(cbfun_t* fn) { - return mi_register_atexit(&atexit_list,fn); -} - -static int mi_at_quick_exit(cbfun_t* fn) { - return mi_register_atexit(&at_quick_exit_list,fn); -} - -static int mi_register_onexit(void* table, cbfun_t* fn) { - // TODO: how can we distinguish a quick_exit from atexit? - return mi_atexit(fn); -} - -// Execute exit functions in a list -static void mi_execute_exit_list(exit_list_t* list) { - // copy and zero the list structure - EnterCriticalSection(&atexit_lock); - exit_list_t clist = *list; - memset(list,0,sizeof(*list)); - LeaveCriticalSection(&atexit_lock); - - // now execute the functions outside of the lock - encoded_t* functions = (encoded_t*)decode(clist.functions); - if (functions != NULL) { - for (size_t i = clist.count; i > 0; i--) { // careful with unsigned count down.. - cbfun_t* fn = (cbfun_t*)decode(functions[i-1]); - if (fn==NULL) break; // corrupted! - fn(); - } - mi_free(functions); - } -} - - - -// ------------------------------------------------------ -// Jump assembly instructions for patches -// ------------------------------------------------------ - -#if defined(_M_IX86) || defined(_M_X64) - -#define MI_JUMP_SIZE 14 // at most 2+4+8 for a long jump or 1+5 for a short one - -typedef struct mi_jump_s { - uint8_t opcodes[MI_JUMP_SIZE]; -} mi_jump_t; - -void mi_jump_restore(void* current, const mi_jump_t* saved) { - memcpy(current, &saved->opcodes, MI_JUMP_SIZE); -} - -void mi_jump_write(void* current, void* target, mi_jump_t* save) { - if (save != NULL) { - memcpy(&save->opcodes, current, MI_JUMP_SIZE); - } - uint8_t* opcodes = ((mi_jump_t*)current)->opcodes; - ptrdiff_t diff = (uint8_t*)target - (uint8_t*)current; - uint32_t ofs32 = (uint32_t)diff; - #ifdef _M_X64 - uint64_t ofs64 = (uint64_t)diff; - if (ofs64 != (uint64_t)ofs32) { - // use long jump - opcodes[0] = 0xFF; - opcodes[1] = 0x25; - *((uint32_t*)&opcodes[2]) = 0; - *((uint64_t*)&opcodes[6]) = (uint64_t)target; - } - else - #endif - { - // use short jump - opcodes[0] = 0xE9; - *((uint32_t*)&opcodes[1]) = ofs32 - 5 /* size of the short jump instruction */; - } -} - -#elif defined(_M_ARM64) - -#define MI_JUMP_SIZE 16 - -typedef struct mi_jump_s { - uint8_t opcodes[MI_JUMP_SIZE]; -} mi_jump_t; - -void mi_jump_restore(void* current, const mi_jump_t* saved) { - memcpy(current, &saved->opcodes, MI_JUMP_SIZE); -} - -void mi_jump_write(void* current, void* target, mi_jump_t* save) { - if (save != NULL) { - memcpy(&save->opcodes, current, MI_JUMP_SIZE); - } - uint8_t* opcodes = ((mi_jump_t*)current)->opcodes; - uint64_t diff = (uint8_t*)target - (uint8_t*)current; - - // 0x50 0x00 0x00 0x58 ldr x16, .+8 # load PC relative +8 - // 0x00 0x02 0x3F 0xD6 blr x16 # and jump - //
- //
- static const uint8_t jump_opcodes[8] = { 0x50, 0x00, 0x00, 0x58, 0x00, 0x02, 0x3F, 0xD6 }; - memcpy(&opcodes[0], jump_opcodes, sizeof(jump_opcodes)); - *((uint64_t*)&opcodes[8]) = diff; -} - -#else -#error "define jump instructions for this platform" -#endif - - -// ------------------------------------------------------ -// Patches -// ------------------------------------------------------ -typedef enum patch_apply_e { - PATCH_NONE, - PATCH_TARGET, - PATCH_TARGET_TERM -} patch_apply_t; - -#define MAX_ENTRIES 4 // maximum number of patched entry points (like `malloc` in ucrtbase and msvcrt) - -typedef struct mi_patch_s { - const char* name; // name of the function to patch - void* target; // the address of the new target (never NULL) - void* target_term; // the address of the target during termination (or NULL) - patch_apply_t applied; // what target has been applied? - void* originals[MAX_ENTRIES]; // the resolved addresses of the function (or NULLs) - mi_jump_t saves[MAX_ENTRIES]; // the saved instructions in case it was applied -} mi_patch_t; - -#define MI_PATCH_NAME3(name,target,term) { name, &target, &term, PATCH_NONE, {NULL,NULL,NULL,NULL} } -#define MI_PATCH_NAME2(name,target) { name, &target, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} } -#define MI_PATCH3(name,target,term) MI_PATCH_NAME3(#name, target, term) -#define MI_PATCH2(name,target) MI_PATCH_NAME2(#name, target) -#define MI_PATCH1(name) MI_PATCH2(name,mi_##name) - -static mi_patch_t patches[] = { - // we implement our own global exit handler (as the CRT versions do a realloc internally) - //MI_PATCH2(_crt_atexit, mi_atexit), - //MI_PATCH2(_crt_at_quick_exit, mi_at_quick_exit), - MI_PATCH2(_setmaxstdio, mi_setmaxstdio), - MI_PATCH2(_register_onexit_function, mi_register_onexit), - - // override higher level atexit functions so we can implement at_quick_exit correcty - MI_PATCH2(atexit, mi_atexit), - MI_PATCH2(at_quick_exit, mi_at_quick_exit), - - // regular entries - MI_PATCH2(malloc, mi_malloc), - MI_PATCH2(calloc, mi_calloc), - MI_PATCH3(realloc, mi_realloc,mi_realloc_term), - MI_PATCH3(free, mi_free,mi_free_term), - - // extended api - MI_PATCH2(_strdup, mi_strdup), - MI_PATCH2(_strndup, mi_strndup), - MI_PATCH3(_expand, mi__expand,mi__expand_term), - MI_PATCH3(_recalloc, mi_recalloc,mi__recalloc_term), - MI_PATCH3(_msize, mi_usable_size,mi__msize_term), - - // base versions - MI_PATCH2(_malloc_base, mi_malloc), - MI_PATCH2(_calloc_base, mi_calloc), - MI_PATCH3(_realloc_base, mi_realloc,mi_realloc_term), - MI_PATCH3(_free_base, mi_free,mi_free_term), - - // these base versions are in the crt but without import records - MI_PATCH_NAME3("_recalloc_base", mi_recalloc,mi__recalloc_term), - MI_PATCH_NAME3("_msize_base", mi_usable_size,mi__msize_term), - - // debug - MI_PATCH2(_malloc_dbg, mi__malloc_dbg), - MI_PATCH2(_realloc_dbg, mi__realloc_dbg), - MI_PATCH2(_calloc_dbg, mi__calloc_dbg), - MI_PATCH2(_free_dbg, mi__free_dbg), - - MI_PATCH3(_expand_dbg, mi__expand_dbg, mi__expand_dbg_term), - MI_PATCH3(_recalloc_dbg, mi__recalloc_dbg, mi__recalloc_dbg_term), - MI_PATCH3(_msize_dbg, mi__msize_dbg, mi__msize_dbg_term), - -#if 0 - // override new/delete variants for efficiency (?) -#ifdef _WIN64 - // 64 bit new/delete - MI_PATCH_NAME2("??2@YAPEAX_K@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPEAX_K@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPEAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPEAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??3@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized - MI_PATCH_NAME3("??_V@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized - MI_PATCH_NAME2("??2@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - MI_PATCH_NAME3("??_V@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - - -#else - // 32 bit new/delete - MI_PATCH_NAME2("??2@YAPAXI@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPAXI@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??3@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized - MI_PATCH_NAME3("??_V@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized - - MI_PATCH_NAME2("??2@YAPAXIABUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME2("??_U@YAPAXIABUnothrow_t@std@@@Z", mi_new), - MI_PATCH_NAME3("??3@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), - -#endif -#endif - { NULL, NULL, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} } -}; - - -// Apply a patch -static bool mi_patch_apply(mi_patch_t* patch, patch_apply_t apply) -{ - if (patch->originals[0] == NULL) return true; // unresolved - if (apply == PATCH_TARGET_TERM && patch->target_term == NULL) apply = PATCH_TARGET; // avoid re-applying non-term variants - if (patch->applied == apply) return false; - - for (int i = 0; i < MAX_ENTRIES; i++) { - void* original = patch->originals[i]; - if (original == NULL) break; // no more - - DWORD protect = PAGE_READWRITE; - if (!VirtualProtect(original, MI_JUMP_SIZE, PAGE_EXECUTE_READWRITE, &protect)) return false; - if (apply == PATCH_NONE) { - mi_jump_restore(original, &patch->saves[i]); - } - else { - void* target = (apply == PATCH_TARGET ? patch->target : patch->target_term); - mi_assert_internal(target != NULL); - if (target != NULL) mi_jump_write(original, target, &patch->saves[i]); - } - VirtualProtect(original, MI_JUMP_SIZE, protect, &protect); - } - patch->applied = apply; - return true; -} - -// Apply all patches -static bool _mi_patches_apply(patch_apply_t apply, patch_apply_t* previous) { - static patch_apply_t current = PATCH_NONE; - if (previous != NULL) *previous = current; - if (current == apply) return true; - current = apply; - bool ok = true; - for (size_t i = 0; patches[i].name != NULL; i++) { - if (!mi_patch_apply(&patches[i], apply)) ok = false; - } - return ok; -} - -// Export the following three functions just in case -// a user needs that level of control. - -// Disable all patches -mi_decl_export void mi_patches_disable(void) { - _mi_patches_apply(PATCH_NONE, NULL); -} - -// Enable all patches normally -mi_decl_export bool mi_patches_enable(void) { - return _mi_patches_apply( PATCH_TARGET, NULL ); -} - -// Enable all patches in termination phase where free is a no-op -mi_decl_export bool mi_patches_enable_term(void) { - return _mi_patches_apply(PATCH_TARGET_TERM, NULL); -} - -// ------------------------------------------------------ -// Stub for _setmaxstdio -// ------------------------------------------------------ - -static int __cdecl mi_setmaxstdio(int newmax) { - patch_apply_t previous; - _mi_patches_apply(PATCH_NONE, &previous); // disable patches - int result = _setmaxstdio(newmax); // call original function (that calls original CRT recalloc) - _mi_patches_apply(previous,NULL); // and re-enable patches - return result; -} - - -// ------------------------------------------------------ -// Resolve addresses dynamically -// ------------------------------------------------------ - -// Try to resolve patches for a given module (DLL) -static void mi_module_resolve(const char* fname, HMODULE mod, int priority) { - // see if any patches apply - for (size_t i = 0; patches[i].name != NULL; i++) { - mi_patch_t* patch = &patches[i]; - if (patch->applied == PATCH_NONE) { - // find an available entry - int i = 0; - while (i < MAX_ENTRIES && patch->originals[i] != NULL) i++; - if (i < MAX_ENTRIES) { - void* addr = GetProcAddress(mod, patch->name); - if (addr != NULL) { - // found it! set the address - patch->originals[i] = addr; - _mi_trace_message(" found %s at %s!%p (entry %i)\n", patch->name, fname, addr, i); - } - } - } - } -} - -#define MIMALLOC_NAME "mimalloc-override.dll" -#define UCRTBASE_NAME "ucrtbase.dll" -#define UCRTBASED_NAME "ucrtbased.dll" - -// Resolve addresses of all patches by inspecting the loaded modules -static atexit_fun_t* crt_atexit = NULL; -static atexit_fun_t* crt_at_quick_exit = NULL; - - -static bool mi_patches_resolve(void) { - // get all loaded modules - HANDLE process = GetCurrentProcess(); // always -1, no need to release - DWORD needed = 0; - HMODULE modules[400]; // try to stay under 4k to not trigger the guard page - EnumProcessModules(process, modules, sizeof(modules), &needed); - if (needed == 0) return false; - int count = needed / sizeof(HMODULE); - int ucrtbase_index = 0; - int mimalloc_index = 0; - // iterate through the loaded modules - for (int i = 0; i < count; i++) { - HMODULE mod = modules[i]; - char filename[MAX_PATH] = { 0 }; - DWORD slen = GetModuleFileName(mod, filename, MAX_PATH); - if (slen > 0 && slen < MAX_PATH) { - // filter out potential crt modules only - filename[slen] = 0; - const char* lastsep = strrchr(filename, '\\'); - const char* basename = (lastsep==NULL ? filename : lastsep+1); - _mi_trace_message(" %i: dynamic module %s\n", i, filename); - - // remember indices so we can check load order (in debug mode) - if (_stricmp(basename, MIMALLOC_NAME) == 0) mimalloc_index = i; - if (_stricmp(basename, UCRTBASE_NAME) == 0) ucrtbase_index = i; - if (_stricmp(basename, UCRTBASED_NAME) == 0) ucrtbase_index = i; - - // see if we potentially patch in this module - int priority = 0; - if (i == 0) priority = 2; // main module to allow static crt linking - else if (_strnicmp(basename, "ucrt", 4) == 0) priority = 3; // new ucrtbase.dll in windows 10 - // NOTE: don't override msvcr -- leads to crashes in setlocale (needs more testing) - // else if (_strnicmp(basename, "msvcr", 5) == 0) priority = 1; // older runtimes - - if (priority > 0) { - // probably found a crt module, try to patch it - mi_module_resolve(basename,mod,priority); - - // try to find the atexit functions for the main process (in `ucrtbase.dll`) - if (crt_atexit==NULL) crt_atexit = (atexit_fun_t*)GetProcAddress(mod, "_crt_atexit"); - if (crt_at_quick_exit == NULL) crt_at_quick_exit = (atexit_fun_t*)GetProcAddress(mod, "_crt_at_quick_exit"); - } - } - } - int diff = mimalloc_index - ucrtbase_index; - if (diff > 1) { - _mi_warning_message("warning: the \"mimalloc-override\" DLL seems not to load before or right after the C runtime (\"ucrtbase\").\n" - " Try to fix this by changing the linking order.\n"); - } - return true; -} - - -// ------------------------------------------------------ -// Dll Entry -// ------------------------------------------------------ - -extern BOOL WINAPI _DllMainCRTStartup(HINSTANCE inst, DWORD reason, LPVOID reserved); - -static DWORD mi_fls_unwind_entry; -static void NTAPI mi_fls_unwind(PVOID value) { - if (value != NULL) mi_patches_enable(); // and re-enable normal patches again for DLL's loaded after us - return; -} - -static void mi_patches_atexit(void) { - mi_execute_exit_list(&atexit_list); - mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op -} - -static void mi_patches_at_quick_exit(void) { - mi_execute_exit_list(&at_quick_exit_list); - mi_patches_enable_term(); // enter termination phase and patch realloc/free with a no-op -} - -BOOL WINAPI DllEntry(HINSTANCE inst, DWORD reason, LPVOID reserved) { - if (reason == DLL_PROCESS_ATTACH) { - __security_init_cookie(); - } - else if (reason == DLL_PROCESS_DETACH) { - // enter termination phase for good now - mi_patches_enable_term(); - } - // C runtime main - BOOL ok = _DllMainCRTStartup(inst, reason, reserved); - if (reason == DLL_PROCESS_ATTACH && ok) { - // initialize at exit lists - mi_initialize_atexit(); - - // Now resolve patches - ok = mi_patches_resolve(); - if (ok) { - // check if patching is not disabled - #pragma warning(suppress:4996) - const char* s = getenv("MIMALLOC_DISABLE_OVERRIDE"); - bool enabled = (s == NULL || !(strstr("1;TRUE;YES;ON", s) != NULL)); - if (!enabled) { - _mi_verbose_message("override is disabled\n"); - } - else { - // and register our unwind entry (this must be after resolving due to possible delayed DLL initialization from GetProcAddress) - mi_fls_unwind_entry = FlsAlloc(&mi_fls_unwind); - if (mi_fls_unwind_entry != FLS_OUT_OF_INDEXES) { - FlsSetValue(mi_fls_unwind_entry, (void*)1); - } - - // register our patch disabler in the global exit list - if (crt_atexit != NULL) (*crt_atexit)(&mi_patches_atexit); - if (crt_at_quick_exit != NULL) (*crt_at_quick_exit)(&mi_patches_at_quick_exit); - - // and patch ! this also redirects the `atexit` handling for the global exit list - mi_patches_enable(); - _mi_verbose_message("override is enabled\n"); - - // hide internal allocation - mi_stats_reset(); - } - } - } - return ok; -} From 08d83cc33dcb1338428821f94e50bd436ef5f656 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 09:24:57 -0700 Subject: [PATCH 07/57] disallow regular allocation from the huge reserved area --- src/os.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/os.c b/src/os.c index 2ad0648f..e7313c80 100644 --- a/src/os.c +++ b/src/os.c @@ -456,6 +456,7 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo if (!commit) allow_large = false; void* p = NULL; + /* if (commit && allow_large) { p = _mi_os_try_alloc_from_huge_reserved(size, try_alignment); if (p != NULL) { @@ -463,6 +464,7 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo return p; } } + */ #if defined(_WIN32) int flags = MEM_RESERVE; From 4609537b8ae05f135d51c5d608398df303ae7dc6 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 09:47:12 -0700 Subject: [PATCH 08/57] pick better umul_overflow variant based on intptr size --- include/mimalloc-internal.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index e99e6df6..c4f85ca4 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -156,10 +156,13 @@ bool _mi_page_is_valid(mi_page_t* page); #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 -#if (MI_INTPTR_SIZE == 4) +#include // INT_MAX, LONG_MAX +#if (INTPTR_MAX == INT_MAX) return __builtin_umul_overflow(count, size, total); -#else +#elif (INTPTR_MAX == LONG_MAX) return __builtin_umull_overflow(count, size, total); +#else + return __builtin_umulll_overflow(count, size, total); #endif #else /* __builtin_umul_overflow is unavailable */ *total = count * size; From f3a162f09527922aa9c6fdf333cd3a87aafd5682 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 09:52:10 -0700 Subject: [PATCH 09/57] pick better umul_overflow variant based on size_t size --- include/mimalloc-internal.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index c4f85ca4..1a5b639d 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -156,10 +156,10 @@ bool _mi_page_is_valid(mi_page_t* page); #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 -#include // INT_MAX, LONG_MAX -#if (INTPTR_MAX == INT_MAX) +#include // UINT_MAX, ULONG_MAX +#if (SIZE_MAX == UINT_MAX) return __builtin_umul_overflow(count, size, total); -#elif (INTPTR_MAX == LONG_MAX) +#elif (SIZE_MAX == ULONG_MAX) return __builtin_umull_overflow(count, size, total); #else return __builtin_umulll_overflow(count, size, total); From e747a6f3a6283699ee7b51a5dc1ce9b396a1106c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Oct 2019 17:01:56 +0200 Subject: [PATCH 10/57] Use `unsigned` for bit-field variables It is actually non-standard to use `bool` with a bit-field quantifier, and VS 2019 complains about this. Signed-off-by: Johannes Schindelin --- include/mimalloc-types.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 72fb7e7e..4d0ade1b 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -136,8 +136,8 @@ typedef union mi_page_flags_u { uint16_t value; uint8_t full_aligned; struct { - bool in_full:1; - bool has_aligned:1; + unsigned in_full:1; + unsigned has_aligned:1; bool is_zero; // `true` if the blocks in the free list are zero initialized }; } mi_page_flags_t; @@ -167,10 +167,10 @@ typedef uintptr_t mi_thread_free_t; typedef struct mi_page_s { // "owned" by the segment uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` - bool segment_in_use:1; // `true` if the segment allocated this page - bool is_reset:1; // `true` if the page memory was reset - bool is_committed:1; // `true` if the page virtual memory is committed - bool is_zero_init:1; // `true` if the page was zero initialized + unsigned segment_in_use:1; // `true` if the segment allocated this page + unsigned is_reset:1; // `true` if the page memory was reset + unsigned is_committed:1; // `true` if the page virtual memory is committed + unsigned is_zero_init:1; // `true` if the page was zero initialized // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` From 5bd8ea2e4feaa3e6ce8aa96b9c32994182aa812d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 17 Oct 2019 17:02:51 +0200 Subject: [PATCH 11/57] Repeat `mi_decl_allocator` in functions' definitions Quite a few functions are declared with that attribute, and VS 2019 complains if the definition does not repeat it. Signed-off-by: Johannes Schindelin --- src/alloc-aligned.c | 48 ++++++++++++++++++++++----------------------- src/alloc.c | 44 ++++++++++++++++++++--------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 352f07b2..99347933 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -61,53 +61,53 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t } -void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); } -void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_malloc_aligned_at(heap, size, alignment, 0); } -void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); } -void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_zalloc_aligned_at(heap, size, alignment, 0); } -void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count, size, &total)) return NULL; return mi_heap_zalloc_aligned_at(heap, total, alignment, offset); } -void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_calloc_aligned_at(heap,count,size,alignment,0); } -void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_malloc_aligned_at(mi_get_default_heap(), size, alignment, offset); } -void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_malloc_aligned(mi_get_default_heap(), size, alignment); } -void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_zalloc_aligned_at(mi_get_default_heap(), size, alignment, offset); } -void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_zalloc_aligned(mi_get_default_heap(), size, alignment); } -void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_calloc_aligned_at(mi_get_default_heap(), count, size, alignment, offset); } -void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_calloc_aligned(mi_get_default_heap(), count, size, alignment); } @@ -150,55 +150,55 @@ static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsi return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero); } -void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false); } -void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false); } -void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true); } -void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true); } -void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { size_t total; if (mi_mul_overflow(newcount, size, &total)) return NULL; return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset); } -void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { size_t total; if (mi_mul_overflow(newcount, size, &total)) return NULL; return mi_heap_rezalloc_aligned(heap, p, total, alignment); } -void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_realloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); } -void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment); } -void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_rezalloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); } -void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { return mi_heap_rezalloc_aligned(mi_get_default_heap(), p, newsize, alignment); } -void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { +mi_decl_allocator void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_recalloc_aligned_at(mi_get_default_heap(), p, newcount, size, alignment, offset); } -void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_allocator void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept { return mi_heap_recalloc_aligned(mi_get_default_heap(), p, newcount, size, alignment); } diff --git a/src/alloc.c b/src/alloc.c index 9d50bf9f..3950496a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -47,26 +47,26 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } // allocate a small block -extern inline void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { +extern inline mi_decl_allocator void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { mi_assert(size <= MI_SMALL_SIZE_MAX); mi_page_t* page = _mi_heap_get_free_small_page(heap,size); return _mi_page_malloc(heap, page, size); } -extern inline void* mi_malloc_small(size_t size) mi_attr_noexcept { +extern inline mi_decl_allocator void* mi_malloc_small(size_t size) mi_attr_noexcept { return mi_heap_malloc_small(mi_get_default_heap(), size); } // zero initialized small block -void* mi_zalloc_small(size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_zalloc_small(size_t size) mi_attr_noexcept { void* p = mi_malloc_small(size); if (p != NULL) { memset(p, 0, size); } return p; } // The main allocation function -extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +extern inline mi_decl_allocator void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local void* p; @@ -85,7 +85,7 @@ extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcep return p; } -extern inline void* mi_malloc(size_t size) mi_attr_noexcept { +extern inline mi_decl_allocator void* mi_malloc(size_t size) mi_attr_noexcept { return mi_heap_malloc(mi_get_default_heap(), size); } @@ -115,11 +115,11 @@ void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) { return p; } -extern inline void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +extern inline mi_decl_allocator void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { return _mi_heap_malloc_zero(heap, size, true); } -void* mi_zalloc(size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept { return mi_heap_zalloc(mi_get_default_heap(),size); } @@ -360,29 +360,29 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { mi_free(p); } -extern inline void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { +extern inline mi_decl_allocator void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count,size,&total)) return NULL; return mi_heap_zalloc(heap,total); } -void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { return mi_heap_calloc(mi_get_default_heap(),count,size); } // Uninitialized `calloc` -extern void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { +extern mi_decl_allocator void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count,size,&total)) return NULL; return mi_heap_malloc(heap, total); } -void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { return mi_heap_mallocn(mi_get_default_heap(),count,size); } // Expand in place or fail -void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { if (p == NULL) return NULL; size_t size = mi_usable_size(p); if (newsize > size) return NULL; @@ -408,11 +408,11 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) return newp; } -void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { return _mi_heap_realloc_zero(heap, p, newsize, false); } -void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count, size, &total)) return NULL; return mi_heap_realloc(heap, p, total); @@ -420,41 +420,41 @@ void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_a // Reallocate but free `p` on errors -void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { void* newp = mi_heap_realloc(heap, p, newsize); if (newp==NULL && p!=NULL) mi_free(p); return newp; } -void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { return _mi_heap_realloc_zero(heap, p, newsize, true); } -void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { size_t total; if (mi_mul_overflow(count, size, &total)) return NULL; return mi_heap_rezalloc(heap, p, total); } -void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_realloc(mi_get_default_heap(),p,newsize); } -void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { return mi_heap_reallocn(mi_get_default_heap(),p,count,size); } // Reallocate but free `p` on errors -void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_reallocf(mi_get_default_heap(),p,newsize); } -void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { +mi_decl_allocator void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_rezalloc(mi_get_default_heap(), p, newsize); } -void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { +mi_decl_allocator void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { return mi_heap_recalloc(mi_get_default_heap(), p, count, size); } From 0fd0122c0a478d75d08b434ee1e66f51331d3d69 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 16 Oct 2019 22:43:57 +0200 Subject: [PATCH 12/57] Avoid compiler warning when casting the result of `GetProcAddress()` It is most unfortunate that the return type of `GetProcAddress()` is `FARPROC` (which is essentially `intptr_t(*)(void)): this type cannot be cast by GCC without warnings to anything but the generic function pointer type `void(*)(void)`. Let's work around that. Signed-off-by: Johannes Schindelin --- src/os.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index e7313c80..cc69123a 100644 --- a/src/os.c +++ b/src/os.c @@ -145,13 +145,13 @@ void _mi_os_init(void) { hDll = LoadLibrary(TEXT("kernelbase.dll")); if (hDll != NULL) { // use VirtualAlloc2FromApp if possible as it is available to Windows store apps - pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2FromApp"); - if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)GetProcAddress(hDll, "VirtualAlloc2"); + pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); + if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); FreeLibrary(hDll); } hDll = LoadLibrary(TEXT("ntdll.dll")); if (hDll != NULL) { - pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); + pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); FreeLibrary(hDll); } if (mi_option_is_enabled(mi_option_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { From 559688ec6468c26b3831004301d727f0dce2437b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 16 Oct 2019 23:40:25 +0200 Subject: [PATCH 13/57] Suppress warning about unnamed struct This prevents MSVC complaining with warning C4201: nonstandard extension used: nameless struct/union The struct might seem unnecessary to the occasional reader (it did seem so to this commit's author), but it is not! It is required to align the fields to a boundary, which is verified by the test suite. Removing that "unnecessary" `struct` results in this failure: 1: Test command: mimalloc-test-api [...] 1: test: malloc-zero... mimalloc: assertion failed: at src/page.c:591, mi_page_init 1: assertion: "!mi_page_has_aligned(page)" Signed-off-by: Johannes Schindelin --- include/mimalloc-types.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 4d0ade1b..c538d165 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -135,10 +135,11 @@ typedef enum mi_delayed_e { typedef union mi_page_flags_u { uint16_t value; uint8_t full_aligned; - struct { + struct { // force alignment unsigned in_full:1; unsigned has_aligned:1; bool is_zero; // `true` if the blocks in the free list are zero initialized +#pragma warning(suppress:4201) }; } mi_page_flags_t; From 26c27fbf587627caabc1e72c2b7d5a813a097464 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 12:07:26 -0700 Subject: [PATCH 14/57] use uint8_t bit fields, and improve portability of page_flags type --- include/mimalloc-types.h | 25 +++++++++++++------------ src/init.c | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index c538d165..c2df6340 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -132,15 +132,16 @@ typedef enum mi_delayed_e { // The `in_full` and `has_aligned` page flags are put in a union to efficiently // test if both are false (`value == 0`) in the `mi_free` routine. -typedef union mi_page_flags_u { - uint16_t value; - uint8_t full_aligned; - struct { // force alignment - unsigned in_full:1; - unsigned has_aligned:1; - bool is_zero; // `true` if the blocks in the free list are zero initialized -#pragma warning(suppress:4201) +typedef struct mi_page_flags_s { + #pragma warning(suppress:4201) + union { + uint8_t full_aligned; + struct { + uint8_t in_full : 1; + uint8_t has_aligned : 1; + }; }; + bool is_zero; // `true` if the blocks in the free list are zero initialized } mi_page_flags_t; // Thread free list. @@ -168,10 +169,10 @@ typedef uintptr_t mi_thread_free_t; typedef struct mi_page_s { // "owned" by the segment uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` - unsigned segment_in_use:1; // `true` if the segment allocated this page - unsigned is_reset:1; // `true` if the page memory was reset - unsigned is_committed:1; // `true` if the page virtual memory is committed - unsigned is_zero_init:1; // `true` if the page was zero initialized + uint8_t segment_in_use:1; // `true` if the segment allocated this page + uint8_t is_reset:1; // `true` if the page memory was reset + uint8_t is_committed:1; // `true` if the page virtual memory is committed + uint8_t is_zero_init:1; // `true` if the page was zero initialized // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` diff --git a/src/init.c b/src/init.c index 5ab39c28..d62a2d34 100644 --- a/src/init.c +++ b/src/init.c @@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { 0, false, false, false, false, 0, 0, - { 0 }, + { { 0 }, false }, NULL, // free #if MI_SECURE 0, From 6e94950de329f0817ad8853b658aa15dc09984ab Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 12:13:45 -0700 Subject: [PATCH 15/57] update redirection modules --- bin/mimalloc-redirect.dll | Bin 46592 -> 46592 bytes bin/mimalloc-redirect32.dll | Bin 33792 -> 33792 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index ec718f91c10dfe4c47b83739df2764fd25ad8581..a1daf316b56cb3c885dc48f1c5fd1ef0ffa302b9 100644 GIT binary patch delta 690 zcmZp8!_@GGX~GRA=Dizlu`x0p-TaWznTc`7W>uCCN{pG4#ncxt-kQ8!{UMN?siDuf zVe&~07sk%Xe46!4A4MkjYp!Orm~5@3!zeg8L#vIkW%GM2Zx+UY&DQ#FSs1r$)-*|G z75&M;z|ifY641$-#m&gjT%!`eP?9uxp`{L^!Q^w6T5P>Q`Pb_v^IF9+PMn-=)iU{m zRivqQ9s|P*os<9ndjPe1cy#-yD0p=9oOrSI1W>5cMMdNFG7q3p8VscZ9=*JxAl1Al zC)ZiavIYM9|Nq6Y$qTJ>8O0~_+sM{8WHT^;3p6E8k}gIJ*Ox&|m(0@r&Ds0^yN7_N67L~j&S?>4wz2cX^vh_ZU1#j_Baqv4v9 zAex<^nycWNS&R?7wg&l5fuU3nXpjL^i^Gc;pcejZ2Y~Jn^8p2^M|X${&?;Uzpo$mb zAoGBt3LeczEDoQ6fyr_FPLnsVuuLwnEoO9@{KQs;ZAunUx$I;#0m;n=?Sut^M)FL);%_jCnO}~1?}}JpBr-4oaXJvY zBm!|DNQ8lbdGBVW05wJdlQhxv#v6kE3*XEPeDr{6(IU!noZVU_osSFGcfb>dt i1_l8T8%PU!GB88{*$F_}-iv`DBXx6Qs1Vbl6?_2Ufdkk8 delta 702 zcmZp8!_@GGX~GSr`BOIDVq;{yviTvSGZW*9&8jRPlo%@~i>WVQd^LHy`a>W&Q$wHe zz~qw}E{roL^J&&IF^W#^*Ido$G1*#6hf#5IhE^Nnl+Ew8yjd6%He2hzWnnzBS<@t$ zRg{B~fuY+)C7_cvi<^<5xke>`p`>u~LQ5S+`^o1lwM6GIFfe!kg$=k$*^Lgo>;-aP z&!7CyGM2Gva*S2WC({{Qa*G|9uG+ebygqnqc%i=`)kLY*!u8n2gu zjL={x74YcgWd*7Jdt!2)wJe+C&;S2lY@0mMI+u}s@_%dD`hskrbwC3GUN8YQw;m`} z_vjU^h6;MTICUIoVy|cvSm4BqN8cb8D7>x#il)Hz9s??a>P?30?Sts`g6drc*Xsb( z>j6<#545-oq1hX*IS8WJ2&y>?u9?O7z-w!e?-UqH^?(K`K(#o$@BwP!-*y1#4ly54 z2zzvgr~s|vihTV>69Iw#;&qCA}}XJYpxpu sLqIA6!v`R})184q0K^8;ik=J%5kPhUkoNatV8}?_+!!jvv}gq%09^zIeEXT>FI!%u0W~f3lda6=TxmK;6{M z2Xr$;HtRUuV4P^cBG*4 Date: Thu, 17 Oct 2019 12:14:15 -0700 Subject: [PATCH 16/57] increase delayed output buffer to 32k --- src/options.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/options.c b/src/options.c index 3e10926c..413ea765 100644 --- a/src/options.c +++ b/src/options.c @@ -144,21 +144,23 @@ static void mi_out_stderr(const char* msg) { // function we also buffer output that happens earlier. When // an output function is registered it is called immediately with // the output up to that point. -#define MAX_OUT_BUF (8*1024) -static char out_buf[MAX_OUT_BUF+1]; +#ifndef MI_MAX_DELAY_OUTPUT +#define MI_MAX_DELAY_OUTPUT (32*1024) +#endif +static char out_buf[MI_MAX_DELAY_OUTPUT+1]; static _Atomic(uintptr_t) out_len; static void mi_out_buf(const char* msg) { if (msg==NULL) return; - if (mi_atomic_read_relaxed(&out_len)>=MAX_OUT_BUF) return; + if (mi_atomic_read_relaxed(&out_len)>=MI_MAX_DELAY_OUTPUT) return; size_t n = strlen(msg); if (n==0) return; // claim space uintptr_t start = mi_atomic_addu(&out_len, n); - if (start >= MAX_OUT_BUF) return; + if (start >= MI_MAX_DELAY_OUTPUT) return; // check bound - if (start+n >= MAX_OUT_BUF) { - n = MAX_OUT_BUF-start-1; + if (start+n >= MI_MAX_DELAY_OUTPUT) { + n = MI_MAX_DELAY_OUTPUT-start-1; } memcpy(&out_buf[start], msg, n); } @@ -166,9 +168,9 @@ static void mi_out_buf(const char* msg) { static void mi_out_buf_flush(mi_output_fun* out) { if (out==NULL) return; // claim all (no more output will be added after this point) - size_t count = mi_atomic_addu(&out_len, MAX_OUT_BUF); + size_t count = mi_atomic_addu(&out_len, MI_MAX_DELAY_OUTPUT); // and output the current contents - if (count>MAX_OUT_BUF) count = MAX_OUT_BUF; + if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT; out_buf[count] = 0; out(out_buf); } From 93b4281b82768023563bc2eb4a1441d83f53efa4 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 12:35:35 -0700 Subject: [PATCH 17/57] ensure randomized huge page start address in 1GiB aligned --- src/os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os.c b/src/os.c index cc69123a..c6ab4ab6 100644 --- a/src/os.c +++ b/src/os.c @@ -886,7 +886,7 @@ int mi_reserve_huge_os_pages( size_t pages, double max_secs, size_t* pages_reser uint8_t* start = (uint8_t*)((uintptr_t)32 << 40); // 32TiB virtual start address #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode uintptr_t r = _mi_random_init((uintptr_t)&mi_reserve_huge_os_pages); - start = start + ((uintptr_t)MI_SEGMENT_SIZE * ((r>>17) & 0xFFFF)); // (randomly 0-64k)*4MiB == 0 to 256GiB + start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x3FF)); // (randomly 0-1024)*1GiB == 0 to 1TiB #endif // Allocate one page at the time but try to place them contiguously From 5de851a84d20835de2429ed60a6eeac3b0a8b6eb Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 16:48:16 -0700 Subject: [PATCH 18/57] update page_flags to have more portable definition --- ide/vs2019/mimalloc.vcxproj | 2 +- include/mimalloc-internal.h | 8 ++++---- include/mimalloc-types.h | 21 +++++++++------------ src/alloc-aligned.c | 2 +- src/alloc.c | 6 +++--- src/init.c | 2 +- src/page.c | 8 ++++---- 7 files changed, 23 insertions(+), 26 deletions(-) diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 5658b536..56beeff9 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -111,7 +111,7 @@ - Level3 + Level4 Disabled true true diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 1a5b639d..4c47af94 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -345,19 +345,19 @@ static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) // Page flags //----------------------------------------------------------- static inline bool mi_page_is_in_full(const mi_page_t* page) { - return page->flags.in_full; + return page->flags.x.in_full; } static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { - page->flags.in_full = in_full; + page->flags.x.in_full = in_full; } static inline bool mi_page_has_aligned(const mi_page_t* page) { - return page->flags.has_aligned; + return page->flags.x.has_aligned; } static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { - page->flags.has_aligned = has_aligned; + page->flags.x.has_aligned = has_aligned; } diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index c2df6340..eea76a25 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -131,17 +131,13 @@ typedef enum mi_delayed_e { // The `in_full` and `has_aligned` page flags are put in a union to efficiently -// test if both are false (`value == 0`) in the `mi_free` routine. -typedef struct mi_page_flags_s { - #pragma warning(suppress:4201) - union { - uint8_t full_aligned; - struct { - uint8_t in_full : 1; - uint8_t has_aligned : 1; - }; - }; - bool is_zero; // `true` if the blocks in the free list are zero initialized +// test if both are false (`full_aligned == 0`) in the `mi_free` routine. +typedef union mi_page_flags_s { + uint8_t full_aligned; + struct { + uint8_t in_full : 1; + uint8_t has_aligned : 1; + } x; } mi_page_flags_t; // Thread free list. @@ -177,7 +173,8 @@ typedef struct mi_page_s { // layout like this to optimize access in `mi_malloc` and `mi_free` uint16_t capacity; // number of blocks committed, must be the first field, see `segment.c:page_clear` uint16_t reserved; // number of blocks reserved in memory - mi_page_flags_t flags; // `in_full` and `has_aligned` flags (16 bits) + mi_page_flags_t flags; // `in_full` and `has_aligned` flags (8 bits) + bool is_zero; // `true` if the blocks in the free list are zero initialized mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) #if MI_SECURE diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 99347933..5a59a63a 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -126,7 +126,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne if (newp != NULL) { if (zero && newsize > size) { const mi_page_t* page = _mi_ptr_page(newp); - if (page->flags.is_zero) { + if (page->is_zero) { // already zero initialized mi_assert_expensive(mi_mem_is_zero(newp,newsize)); } diff --git a/src/alloc.c b/src/alloc.c index 3950496a..0c399671 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -33,7 +33,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz page->used++; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); #if (MI_DEBUG) - if (!page->flags.is_zero) { memset(block, MI_DEBUG_UNINIT, size); } + if (!page->is_zero) { memset(block, MI_DEBUG_UNINIT, size); } #elif (MI_SECURE) block->next = 0; #endif @@ -96,7 +96,7 @@ void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size) { mi_assert_internal(p != NULL); mi_assert_internal(size > 0 && page->block_size >= size); mi_assert_internal(_mi_ptr_page(p)==page); - if (page->flags.is_zero) { + if (page->is_zero) { // already zero initialized memory? ((mi_block_t*)p)->next = 0; // clear the free list pointer mi_assert_expensive(mi_mem_is_zero(p,page->block_size)); @@ -147,7 +147,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc mi_block_set_next(page, block, page->free); page->free = block; page->used--; - page->flags.is_zero = false; + page->is_zero = false; _mi_segment_page_free(page,true,&heap->tld->segments); } return; diff --git a/src/init.c b/src/init.c index d62a2d34..75836aca 100644 --- a/src/init.c +++ b/src/init.c @@ -13,7 +13,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array const mi_page_t _mi_page_empty = { 0, false, false, false, false, 0, 0, - { { 0 }, false }, + { 0 }, false, NULL, // free #if MI_SECURE 0, diff --git a/src/page.c b/src/page.c index 25e59977..77d98f11 100644 --- a/src/page.c +++ b/src/page.c @@ -192,7 +192,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // usual case page->free = page->local_free; page->local_free = NULL; - page->flags.is_zero = false; + page->is_zero = false; } else if (force) { // append -- only on shutdown (force) as this is a linear operation @@ -204,7 +204,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { mi_block_set_next(page, tail, page->free); page->free = page->local_free; page->local_free = NULL; - page->flags.is_zero = false; + page->is_zero = false; } } @@ -559,7 +559,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st // extension into zero initialized memory preserves the zero'd free list if (!page->is_zero_init) { - page->flags.is_zero = false; + page->is_zero = false; } mi_assert_expensive(mi_page_is_valid_init(page)); } @@ -579,7 +579,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi #if MI_SECURE page->cookie = _mi_heap_random(heap) | 1; #endif - page->flags.is_zero = page->is_zero_init; + page->is_zero = page->is_zero_init; mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); From fdfa6ed2602f2e6cbd6d5d466b24a7e447cc0e42 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 17 Oct 2019 16:56:57 -0700 Subject: [PATCH 19/57] fix warnings at high warning level in msvc --- ide/vs2019/mimalloc.vcxproj | 2 +- include/mimalloc-atomic.h | 2 +- src/memory.c | 2 +- src/options.c | 2 +- src/os.c | 2 +- src/page-queue.c | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 56beeff9..5658b536 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -111,7 +111,7 @@ - Level4 + Level3 Disabled true true diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index 8b254d3e..dff0f011 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -130,7 +130,7 @@ static inline intptr_t mi_atomic_add(volatile _Atomic(intptr_t)* p, intptr_t add return (intptr_t)RC64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); } static inline bool mi_atomic_cas_strong(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) { - return (expected == RC64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)expected)); + return (expected == (uintptr_t)RC64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)expected)); } static inline bool mi_atomic_cas_weak(volatile _Atomic(uintptr_t)* p, uintptr_t desired, uintptr_t expected) { return mi_atomic_cas_strong(p,desired,expected); diff --git a/src/memory.c b/src/memory.c index 0ad582cd..f9c53782 100644 --- a/src/memory.c +++ b/src/memory.c @@ -71,7 +71,7 @@ bool _mi_os_is_huge_reserved(void* p); typedef uintptr_t mi_region_info_t; static inline mi_region_info_t mi_region_info_create(void* start, bool is_large, bool is_committed) { - return ((uintptr_t)start | ((is_large?1:0) << 1) | (is_committed?1:0)); + return ((uintptr_t)start | ((uintptr_t)(is_large?1:0) << 1) | (is_committed?1:0)); } static inline void* mi_region_info_read(mi_region_info_t info, bool* is_large, bool* is_committed) { diff --git a/src/options.c b/src/options.c index 413ea765..4e2bdeaa 100644 --- a/src/options.c +++ b/src/options.c @@ -346,7 +346,7 @@ static void mi_option_init(mi_option_desc_t* desc) { size_t len = strlen(s); if (len >= sizeof(buf)) len = sizeof(buf) - 1; for (size_t i = 0; i < len; i++) { - buf[i] = toupper(s[i]); + buf[i] = (char)toupper(s[i]); } buf[len] = 0; if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) { diff --git a/src/os.c b/src/os.c index c6ab4ab6..ed938221 100644 --- a/src/os.c +++ b/src/os.c @@ -700,7 +700,7 @@ static bool mi_os_resetx(void* addr, size_t size, bool reset, mi_stats_t* stats) void* p = VirtualAlloc(start, csize, MEM_RESET, PAGE_READWRITE); mi_assert_internal(p == start); #if 1 - if (p == start) { + if (p == start && start != NULL) { VirtualUnlock(start,csize); // VirtualUnlock after MEM_RESET removes the memory from the working set } #endif diff --git a/src/page-queue.c b/src/page-queue.c index d613095f..4af70b50 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -57,7 +57,7 @@ static inline uint8_t mi_bsr32(uint32_t x); static inline uint8_t mi_bsr32(uint32_t x) { uint32_t idx; _BitScanReverse((DWORD*)&idx, x); - return idx; + return (uint8_t)idx; } #elif defined(__GNUC__) || defined(__clang__) static inline uint8_t mi_bsr32(uint32_t x) { From 2affdbbd2e6c26fba92c380375d2e1f7c8578ffe Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 18 Oct 2019 18:11:04 -0700 Subject: [PATCH 20/57] stronger secure mode when defining MI_SECURE=4: checks for double free, corrupted free list, and invalid pointer frees. Performance is impacted but not too much -- more perf testing is needed --- CMakeLists.txt | 2 +- ide/vs2019/mimalloc.vcxproj | 4 +-- include/mimalloc-internal.h | 45 ++++++++++++++++++++++++---------- include/mimalloc-types.h | 7 ++++-- src/alloc.c | 49 +++++++++++++++++++++++++++++++++++-- src/options.c | 8 ++++++ test/main-override-static.c | 20 +++++++++++++++ test/main-override.cpp | 6 +++++ 8 files changed, 121 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 443476f0..81cc339a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,7 +68,7 @@ endif() if(MI_SECURE MATCHES "ON") message(STATUS "Set secure build (MI_SECURE=ON)") - list(APPEND mi_defines MI_SECURE=2) + list(APPEND mi_defines MI_SECURE=3) endif() if(MI_SEE_ASM MATCHES "ON") diff --git a/ide/vs2019/mimalloc.vcxproj b/ide/vs2019/mimalloc.vcxproj index 5658b536..28e96d71 100644 --- a/ide/vs2019/mimalloc.vcxproj +++ b/ide/vs2019/mimalloc.vcxproj @@ -111,12 +111,12 @@ - Level3 + Level2 Disabled true true ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_DEBUG=1;%(PreprocessorDefinitions); CompileAsCpp false stdcpp17 diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 4c47af94..7bffb6ac 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -20,6 +20,18 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_trace_message(...) #endif +#if defined(_MSC_VER) +#define mi_decl_noinline __declspec(noinline) +#define mi_attr_noreturn +#elif defined(__GNUC__) || defined(__clang__) +#define mi_decl_noinline __attribute__((noinline)) +#define mi_attr_noreturn __attribute__((noreturn)) +#else +#define mi_decl_noinline +#define mi_attr_noreturn +#endif + + // "options.c" void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, const char* fmt, ...); @@ -28,6 +40,7 @@ void _mi_warning_message(const char* fmt, ...); void _mi_verbose_message(const char* fmt, ...); void _mi_trace_message(const char* fmt, ...); void _mi_options_init(void); +void _mi_fatal_error(const char* fmt, ...) mi_attr_noreturn; // "init.c" extern mi_stats_t _mi_stats_main; @@ -124,13 +137,6 @@ bool _mi_page_is_valid(mi_page_t* page); #define __has_builtin(x) 0 #endif -#if defined(_MSC_VER) -#define mi_decl_noinline __declspec(noinline) -#elif defined(__GNUC__) || defined(__clang__) -#define mi_decl_noinline __attribute__((noinline)) -#else -#define mi_decl_noinline -#endif /* ----------------------------------------------------------- @@ -365,8 +371,12 @@ static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { // Encoding/Decoding the free list next pointers // ------------------------------------------------------------------- -static inline mi_block_t* mi_block_nextx( uintptr_t cookie, mi_block_t* block ) { - #if MI_SECURE +static inline bool mi_is_in_same_segment(const void* p, const void* q) { + return (_mi_ptr_segment(p) == _mi_ptr_segment(q)); +} + +static inline mi_block_t* mi_block_nextx( uintptr_t cookie, const mi_block_t* block ) { + #if MI_SECURE return (mi_block_t*)(block->next ^ cookie); #else UNUSED(cookie); @@ -374,7 +384,7 @@ static inline mi_block_t* mi_block_nextx( uintptr_t cookie, mi_block_t* block ) #endif } -static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, mi_block_t* next) { +static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, const mi_block_t* next) { #if MI_SECURE block->next = (mi_encoded_t)next ^ cookie; #else @@ -383,16 +393,25 @@ static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, mi_bl #endif } -static inline mi_block_t* mi_block_next(mi_page_t* page, mi_block_t* block) { +static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { #if MI_SECURE - return mi_block_nextx(page->cookie,block); + mi_block_t* next = mi_block_nextx(page->cookie,block); + #if MI_SECURE >= 4 + // check if next is at least in our segment range + // TODO: it is better to check if it is actually inside our page but that is more expensive + // to calculate. Perhaps with a relative free list this becomes feasible? + if (next!=NULL && !mi_is_in_same_segment(block, next)) { + _mi_fatal_error("corrupted free list entry at %p: %zx\n", block, (uintptr_t)next); + } + #endif + return next; #else UNUSED(page); return mi_block_nextx(0, block); #endif } -static inline void mi_block_set_next(mi_page_t* page, mi_block_t* block, mi_block_t* next) { +static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { #if MI_SECURE mi_block_set_nextx(page->cookie,block,next); #else diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index eea76a25..00a83839 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -22,8 +22,11 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). // #define MI_STAT 1 -// Define MI_SECURE as 1 to encode free lists -// #define MI_SECURE 1 +// Define MI_SECURE to enable security mitigations +// #define MI_SECURE 1 // guard page around metadata +// #define MI_SECURE 2 // guard page around each mimalloc page +// #define MI_SECURE 3 // encode free lists +// #define MI_SECURE 4 // all security enabled (checks for double free, corrupted free list and invalid pointer free) #if !defined(MI_SECURE) #define MI_SECURE 0 diff --git a/src/alloc.c b/src/alloc.c index 0c399671..f5208a0a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -124,10 +124,50 @@ mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept { } +// ------------------------------------------------------ +// Check for double free in secure mode +// ------------------------------------------------------ + +#if MI_SECURE>=4 +static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) { + while (list != NULL) { + if (elem==list) return true; + list = mi_block_next(page, list); + } + return false; +} + +static mi_decl_noinline void mi_free_check_blockx(const mi_page_t* page, const mi_block_t* block, const mi_block_t* n) { + size_t psize; + uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); + if ((uint8_t*)n >= pstart && (uint8_t*)n < (pstart + psize)) { + // Suspicious: the decoded value is in the same page. + // Walk the free lists to see if it is already freed + if (mi_list_contains(page, page->free, n) || + mi_list_contains(page, page->local_free, n) || + mi_list_contains(page, (const mi_block_t*)mi_atomic_read_ptr_relaxed(mi_atomic_cast(void*,&page->thread_free)), n)) + { + _mi_fatal_error("double free detected of block %p with size %zu\n", block, page->block_size); + } + } +} + +static inline void mi_free_check_block(const mi_page_t* page, const mi_block_t* block) { + mi_block_t* n = (mi_block_t*)(block->next ^ page->cookie); + if (n!=NULL && mi_is_in_same_segment(block, n)) { // quick check + // Suspicous: decoded value in block is in the same segment, maybe a double free? + mi_free_check_blockx(page, block, n); + } + return; +} +#endif + + // ------------------------------------------------------ // Free // ------------------------------------------------------ + // multi-threaded free static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) { @@ -251,14 +291,16 @@ void mi_free(void* p) mi_attr_noexcept #if (MI_DEBUG>0) if (mi_unlikely(!mi_is_in_heap_region(p))) { - _mi_warning_message("possibly trying to mi_free a pointer that does not point to a valid heap region: 0x%p\n" + _mi_warning_message("possibly trying to free a pointer that does not point to a valid heap region: 0x%p\n" "(this may still be a valid very large allocation (over 64MiB))\n", p); if (mi_likely(_mi_ptr_cookie(segment) == segment->cookie)) { _mi_warning_message("(yes, the previous pointer 0x%p was valid after all)\n", p); } } +#endif +#if (MI_DEBUG>0 || MI_SECURE>=4) if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { - _mi_error_message("trying to mi_free a pointer that does not point to a valid heap space: %p\n", p); + _mi_error_message("trying to free a pointer that does not point to a valid heap space: %p\n", p); return; } #endif @@ -278,6 +320,9 @@ void mi_free(void* p) mi_attr_noexcept if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned mi_block_t* block = (mi_block_t*)p; + #if MI_SECURE>=4 + mi_free_check_block(page,block); + #endif mi_block_set_next(page, block, page->local_free); page->local_free = block; page->used--; diff --git a/src/options.c b/src/options.c index 4e2bdeaa..e74d9eb5 100644 --- a/src/options.c +++ b/src/options.c @@ -285,6 +285,14 @@ void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, co } #endif +mi_attr_noreturn void _mi_fatal_error(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + mi_vfprintf(NULL, "mimalloc: fatal: ", fmt, args); + va_end(args); + exit(99); +} + // -------------------------------------------------------- // Initialize options by checking the environment // -------------------------------------------------------- diff --git a/test/main-override-static.c b/test/main-override-static.c index 6ddf4f37..d8369389 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -2,12 +2,16 @@ #include #include #include +#include #include #include // redefines malloc etc. +static void double_free(); + int main() { mi_version(); + double_free(); void* p1 = malloc(78); void* p2 = malloc(24); free(p1); @@ -29,3 +33,19 @@ int main() { mi_stats_print(NULL); return 0; } + +static void double_free() { + void* p[256]; + uintptr_t buf[256]; + + p[0] = mi_malloc(622616); + p[1] = mi_malloc(655362); + p[2] = mi_malloc(786432); + mi_free(p[2]); + // [VULN] Double free + mi_free(p[2]); + p[3] = mi_malloc(786456); + // [BUG] Found overlap + // p[3]=0x429b2ea2000 (size=917504), p[1]=0x429b2e42000 (size=786432) + fprintf(stderr, "p3: %p-%p, p1: %p-%p, p2: %p\n", p[3], (uint8_t*)(p[3]) + 786456, p[1], (uint8_t*)(p[1]) + 655362, p[2]); +} diff --git a/test/main-override.cpp b/test/main-override.cpp index 4bc91ae8..ea940061 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -2,10 +2,13 @@ #include #include #include +#include #include #include +static void double_free(); + static void* p = malloc(8); void free_p() { @@ -24,6 +27,7 @@ public: int main() { //mi_stats_reset(); // ignore earlier allocations + double_free(); atexit(free_p); void* p1 = malloc(78); void* p2 = mi_malloc_aligned(16,24); @@ -66,3 +70,5 @@ public: }; static Static s = Static(); + + From 25246070aeb3dc5a8f602a6e34222284b18560b4 Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 19 Oct 2019 08:34:18 -0700 Subject: [PATCH 21/57] fix double free check in secure = 4 mode; inline _mi_ptr_cookie --- include/mimalloc-internal.h | 5 ++++- src/alloc.c | 32 ++++++++++++++++++-------------- src/init.c | 4 ---- test/main-override-static.c | 26 +++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 7bffb6ac..cf0252c6 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -46,7 +46,6 @@ void _mi_fatal_error(const char* fmt, ...) mi_attr_noreturn; extern mi_stats_t _mi_stats_main; extern const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); -uintptr_t _mi_ptr_cookie(const void* p); uintptr_t _mi_random_shuffle(uintptr_t x); uintptr_t _mi_random_init(uintptr_t seed /* can be zero */); bool _mi_preloading(); // true while the C runtime is not ready @@ -244,6 +243,10 @@ static inline bool mi_heap_is_initialized(mi_heap_t* heap) { return (heap != &_mi_heap_empty); } +static inline uintptr_t _mi_ptr_cookie(const void* p) { + return ((uintptr_t)p ^ _mi_heap_main.cookie); +} + /* ----------------------------------------------------------- Pages ----------------------------------------------------------- */ diff --git a/src/alloc.c b/src/alloc.c index f5208a0a..916b1f32 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -137,28 +137,32 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons return false; } -static mi_decl_noinline void mi_free_check_blockx(const mi_page_t* page, const mi_block_t* block, const mi_block_t* n) { +static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block, const mi_block_t* n) { size_t psize; uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); - if ((uint8_t*)n >= pstart && (uint8_t*)n < (pstart + psize)) { - // Suspicious: the decoded value is in the same page. + if (n == NULL || ((uint8_t*)n >= pstart && (uint8_t*)n < (pstart + psize))) { + // Suspicious: the decoded value is in the same page (or NULL). // Walk the free lists to see if it is already freed - if (mi_list_contains(page, page->free, n) || - mi_list_contains(page, page->local_free, n) || - mi_list_contains(page, (const mi_block_t*)mi_atomic_read_ptr_relaxed(mi_atomic_cast(void*,&page->thread_free)), n)) + if (mi_list_contains(page, page->free, block) || + mi_list_contains(page, page->local_free, block) || + mi_list_contains(page, (const mi_block_t*)mi_atomic_read_ptr_relaxed(mi_atomic_cast(void*,&page->thread_free)), block)) { _mi_fatal_error("double free detected of block %p with size %zu\n", block, page->block_size); + return true; } } + return false; } -static inline void mi_free_check_block(const mi_page_t* page, const mi_block_t* block) { +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = (mi_block_t*)(block->next ^ page->cookie); - if (n!=NULL && mi_is_in_same_segment(block, n)) { // quick check - // Suspicous: decoded value in block is in the same segment, maybe a double free? - mi_free_check_blockx(page, block, n); - } - return; + if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check + (n==NULL || mi_is_in_same_segment(block, n))) + { + // Suspicous: decoded value in block is in the same segment (or NULL) -- maybe a double free? + return mi_check_double_freex(page, block, n); + } + return false; } #endif @@ -320,8 +324,8 @@ void mi_free(void* p) mi_attr_noexcept if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned mi_block_t* block = (mi_block_t*)p; - #if MI_SECURE>=4 - mi_free_check_block(page,block); + #if MI_SECURE>=4 + if (mi_check_double_free(page,block)) return; #endif mi_block_set_next(page, block, page->local_free); page->local_free = block; diff --git a/src/init.c b/src/init.c index 75836aca..6514ce53 100644 --- a/src/init.c +++ b/src/init.c @@ -184,10 +184,6 @@ uintptr_t _mi_random_init(uintptr_t seed /* can be zero */) { return x; } -uintptr_t _mi_ptr_cookie(const void* p) { - return ((uintptr_t)p ^ _mi_heap_main.cookie); -} - /* ----------------------------------------------------------- Initialization and freeing of the thread local heaps ----------------------------------------------------------- */ diff --git a/test/main-override-static.c b/test/main-override-static.c index d8369389..ed5048e0 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -7,11 +7,13 @@ #include #include // redefines malloc etc. -static void double_free(); +static void double_free1(); +static void double_free2(); int main() { mi_version(); - double_free(); + //double_free1(); + //double_free2(); void* p1 = malloc(78); void* p2 = malloc(24); free(p1); @@ -34,7 +36,7 @@ int main() { return 0; } -static void double_free() { +static void double_free1() { void* p[256]; uintptr_t buf[256]; @@ -49,3 +51,21 @@ static void double_free() { // p[3]=0x429b2ea2000 (size=917504), p[1]=0x429b2e42000 (size=786432) fprintf(stderr, "p3: %p-%p, p1: %p-%p, p2: %p\n", p[3], (uint8_t*)(p[3]) + 786456, p[1], (uint8_t*)(p[1]) + 655362, p[2]); } + +static void double_free2() { + void* p[256]; + uintptr_t buf[256]; + // [INFO] Command buffer: 0x327b2000 + // [INFO] Input size: 182 + p[0] = malloc(712352); + p[1] = malloc(786432); + free(p[0]); + // [VULN] Double free + free(p[0]); + p[2] = malloc(786440); + p[3] = malloc(917504); + p[4] = malloc(786440); + // [BUG] Found overlap + // p[4]=0x433f1402000 (size=917504), p[1]=0x433f14c2000 (size=786432) + fprintf(stderr, "p1: %p-%p, p2: %p-%p\n", p[4], (uint8_t*)(p[4]) + 917504, p[1], (uint8_t*)(p[1]) + 786432); +} From 5dfdc092b50612abddadc97bf609222bef2ab00f Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 28 Oct 2019 12:26:57 -0700 Subject: [PATCH 22/57] improve windows warning message --- src/os.c | 2 +- test/main-override.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/os.c b/src/os.c index ed938221..0cd3a1ab 100644 --- a/src/os.c +++ b/src/os.c @@ -283,7 +283,7 @@ static void* mi_win_virtual_alloc(void* addr, size_t size, size_t try_alignment, p = mi_win_virtual_allocx(addr, size, try_alignment, flags); } if (p == NULL) { - _mi_warning_message("unable to alloc mem error: err: %i size: 0x%x \n", GetLastError(), size); + _mi_warning_message("unable to allocate memory: error code: %i, addr: %p, size: 0x%x, large only: %d, allow_large: %d\n", GetLastError(), addr, size, large_only, allow_large); } return p; } diff --git a/test/main-override.cpp b/test/main-override.cpp index ea940061..92740c6e 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -7,8 +7,6 @@ #include #include -static void double_free(); - static void* p = malloc(8); void free_p() { From ff9f29660b38ad3dfb9b8a55d7b5fb1837a50c7f Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 28 Oct 2019 12:27:32 -0700 Subject: [PATCH 23/57] remove double_free call --- test/main-override.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/main-override.cpp b/test/main-override.cpp index 92740c6e..e006ad27 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -25,7 +25,6 @@ public: int main() { //mi_stats_reset(); // ignore earlier allocations - double_free(); atexit(free_p); void* p1 = malloc(78); void* p2 = mi_malloc_aligned(16,24); From 28d4ec4c5ad39bc9459cd432c34f8d9234b51431 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 28 Oct 2019 13:14:14 -0700 Subject: [PATCH 24/57] fix statistics accounting of huge pages --- src/alloc.c | 10 +++++++++- src/page.c | 3 ++- test/test-stress.c | 7 ++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 916b1f32..f8dab24d 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -192,7 +192,15 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc page->free = block; page->used--; page->is_zero = false; - _mi_segment_page_free(page,true,&heap->tld->segments); + mi_assert(page->used == 0); + mi_tld_t* tld = heap->tld; + if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { + _mi_stat_decrease(&tld->stats.giant, page->block_size); + } + else { + _mi_stat_decrease(&tld->stats.huge, page->block_size); + } + _mi_segment_page_free(page,true,&tld->segments); } return; } diff --git a/src/page.c b/src/page.c index 77d98f11..b71be522 100644 --- a/src/page.c +++ b/src/page.c @@ -370,6 +370,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { mi_page_set_has_aligned(page, false); // account for huge pages here + // (note: no longer necessary as huge pages are always abandoned) if (page->block_size > MI_LARGE_OBJ_SIZE_MAX) { if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_decrease(&page->heap->tld->stats.giant, page->block_size); @@ -378,7 +379,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force) { _mi_stat_decrease(&page->heap->tld->stats.huge, page->block_size); } } - + // remove from the page list // (no need to do _mi_heap_delayed_free first as all blocks are already free) mi_segments_tld_t* segments_tld = &page->heap->tld->segments; diff --git a/test/test-stress.c b/test/test-stress.c index 354e2b07..bb428072 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -158,6 +158,7 @@ int main(int argc, char** argv) { //printf("(reserve huge: %i\n)", res); //bench_start_program(); + mi_stats_reset(); memset((void*)transfer, 0, TRANSFERS*sizeof(void*)); run_os_threads(THREADS); for (int i = 0; i < TRANSFERS; i++) { @@ -165,7 +166,6 @@ int main(int argc, char** argv) { } #ifndef NDEBUG mi_collect(false); - mi_collect(true); #endif mi_stats_print(NULL); //bench_end_program(); @@ -191,6 +191,11 @@ static void run_os_threads(size_t nthreads) { for (size_t i = 0; i < nthreads; i++) { WaitForSingleObject(thandles[i], INFINITE); } + for (size_t i = 0; i < nthreads; i++) { + CloseHandle(thandles[i]); + } + free(tids); + free(thandles); } static void* atomic_exchange_ptr(volatile void** p, void* newval) { From 081e2d1eb6d517f1949b3245d37f758e8a27ac3f Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 28 Oct 2019 13:43:42 -0700 Subject: [PATCH 25/57] fix statistics display --- include/mimalloc-types.h | 6 +++--- src/init.c | 4 ++-- src/os.c | 4 ++-- src/page.c | 2 +- src/stats.c | 36 ++++++++++++++++++++++++------------ 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 00a83839..2e5d481b 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -340,14 +340,14 @@ typedef struct mi_stats_s { mi_stat_count_t page_committed; mi_stat_count_t segments_abandoned; mi_stat_count_t pages_abandoned; - mi_stat_count_t pages_extended; - mi_stat_count_t mmap_calls; - mi_stat_count_t commit_calls; mi_stat_count_t threads; mi_stat_count_t huge; mi_stat_count_t giant; mi_stat_count_t malloc; mi_stat_count_t segments_cache; + mi_stat_counter_t pages_extended; + mi_stat_counter_t mmap_calls; + mi_stat_counter_t commit_calls; mi_stat_counter_t page_no_retire; mi_stat_counter_t searches; mi_stat_counter_t huge_count; diff --git a/src/init.c b/src/init.c index 6514ce53..d361de3a 100644 --- a/src/init.c +++ b/src/init.c @@ -64,8 +64,8 @@ const mi_page_t _mi_page_empty = { MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), \ + { 0, 0 }, { 0, 0 }, { 0, 0 }, \ { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ MI_STAT_COUNT_END_NULL() diff --git a/src/os.c b/src/os.c index 0cd3a1ab..8f5afc5b 100644 --- a/src/os.c +++ b/src/os.c @@ -477,7 +477,7 @@ static void* mi_os_mem_alloc(size_t size, size_t try_alignment, bool commit, boo int protect_flags = (commit ? (PROT_WRITE | PROT_READ) : PROT_NONE); p = mi_unix_mmap(NULL, size, try_alignment, protect_flags, false, allow_large, is_large); #endif - _mi_stat_increase(&stats->mmap_calls, 1); + mi_stat_counter_increase(stats->mmap_calls, 1); if (p != NULL) { _mi_stat_increase(&stats->reserved, size); if (commit) { _mi_stat_increase(&stats->committed, size); } @@ -632,7 +632,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ int err = 0; if (commit) { _mi_stat_increase(&stats->committed, csize); - _mi_stat_increase(&stats->commit_calls, 1); + _mi_stat_counter_increase(&stats->commit_calls, 1); } else { _mi_stat_decrease(&stats->committed, csize); diff --git a/src/page.c b/src/page.c index b71be522..2a48c64b 100644 --- a/src/page.c +++ b/src/page.c @@ -531,7 +531,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st size_t page_size; _mi_page_start(_mi_page_segment(page), page, &page_size); - mi_stat_increase(stats->pages_extended, 1); + mi_stat_counter_increase(stats->pages_extended, 1); // calculate the extend count size_t extend = page->reserved - page->capacity; diff --git a/src/stats.c b/src/stats.c index 37a7bde4..50bd029d 100644 --- a/src/stats.c +++ b/src/stats.c @@ -95,15 +95,17 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); - mi_stat_add(&stats->mmap_calls, &src->mmap_calls, 1); - mi_stat_add(&stats->commit_calls, &src->commit_calls, 1); mi_stat_add(&stats->threads, &src->threads, 1); - mi_stat_add(&stats->pages_extended, &src->pages_extended, 1); mi_stat_add(&stats->malloc, &src->malloc, 1); mi_stat_add(&stats->segments_cache, &src->segments_cache, 1); mi_stat_add(&stats->huge, &src->huge, 1); mi_stat_add(&stats->giant, &src->giant, 1); + + mi_stat_counter_add(&stats->pages_extended, &src->pages_extended, 1); + mi_stat_counter_add(&stats->mmap_calls, &src->mmap_calls, 1); + mi_stat_counter_add(&stats->commit_calls, &src->commit_calls, 1); + mi_stat_counter_add(&stats->page_no_retire, &src->page_no_retire, 1); mi_stat_counter_add(&stats->searches, &src->searches, 1); mi_stat_counter_add(&stats->huge_count, &src->huge_count, 1); @@ -121,6 +123,9 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { Display statistics ----------------------------------------------------------- */ +// unit > 0 : size in binary bytes +// unit == 0: count as decimal +// unit < 0 : count in binary static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, const char* fmt) { char buf[32]; int len = 32; @@ -165,17 +170,24 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t _mi_fprintf(out, " ok\n"); } else if (unit<0) { - mi_print_amount(stat->peak, 1, out); - mi_print_amount(stat->allocated, 1, out); - mi_print_amount(stat->freed, 1, out); - mi_print_amount(-unit, 1, out); - mi_print_count((stat->allocated / -unit), 0, out); + mi_print_amount(stat->peak, -1, out); + mi_print_amount(stat->allocated, -1, out); + mi_print_amount(stat->freed, -1, out); + if (unit==-1) { + _mi_fprintf(out, "%22s", ""); + } + else { + mi_print_amount(-unit, 1, out); + mi_print_count((stat->allocated / -unit), 0, out); + } if (stat->allocated > stat->freed) _mi_fprintf(out, " not all freed!\n"); else _mi_fprintf(out, " ok\n"); } else { + mi_print_amount(stat->peak, 1, out); + mi_print_amount(stat->allocated, 1, out); _mi_fprintf(out, "\n"); } } @@ -247,11 +259,11 @@ static void _mi_stats_print(mi_stats_t* stats, double secs, mi_output_fun* out) mi_stat_print(&stats->segments_cache, "-cached", -1, out); mi_stat_print(&stats->pages, "pages", -1, out); mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out); - mi_stat_print(&stats->pages_extended, "-extended", 0, out); + mi_stat_counter_print(&stats->pages_extended, "-extended", out); mi_stat_counter_print(&stats->page_no_retire, "-noretire", out); - mi_stat_print(&stats->mmap_calls, "mmaps", 0, out); - mi_stat_print(&stats->commit_calls, "commits", 0, out); - mi_stat_print(&stats->threads, "threads", 0, out); + mi_stat_counter_print(&stats->mmap_calls, "mmaps", out); + mi_stat_counter_print(&stats->commit_calls, "commits", out); + mi_stat_print(&stats->threads, "threads", -1, out); mi_stat_counter_print_avg(&stats->searches, "searches", out); if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs); From 87bdfbb9b6b0a869f9ce8e76fd4fa580bb816840 Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 28 Oct 2019 14:12:06 -0700 Subject: [PATCH 26/57] use more conservative retire strategy --- src/page.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/page.c b/src/page.c index 2a48c64b..5a186727 100644 --- a/src/page.c +++ b/src/page.c @@ -407,16 +407,18 @@ void _mi_page_retire(mi_page_t* page) { // (or we end up retiring and re-allocating most of the time) // NOTE: refine this more: we should not retire if this // is the only page left with free blocks. It is not clear - // how to check this efficiently though... for now we just check - // if its neighbours are almost fully used. + // how to check this efficiently though... + // for now, we don't retire if it is the only page left of this size class. + mi_page_queue_t* pq = mi_page_queue_of(page); if (mi_likely(page->block_size <= (MI_SMALL_SIZE_MAX/4))) { - if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { + // if (mi_page_mostly_used(page->prev) && mi_page_mostly_used(page->next)) { + if (pq->last==page && pq->first==page) { mi_stat_counter_increase(_mi_stats_main.page_no_retire,1); return; // dont't retire after all } } - _mi_page_free(page, mi_page_queue_of(page), false); + _mi_page_free(page, pq, false); } From b052d3b73129fee02e145b7c1b8b2153dd39af0d Mon Sep 17 00:00:00 2001 From: daan Date: Mon, 28 Oct 2019 15:54:33 -0700 Subject: [PATCH 27/57] enable double free and heap corruption detection in debug mode --- include/mimalloc-internal.h | 23 +++++++++--------- include/mimalloc-types.h | 27 ++++++++++++++------- src/alloc.c | 47 +++++++++++++++++++++---------------- src/init.c | 4 ++-- src/options.c | 4 +++- src/page.c | 6 ++--- test/main-override-static.c | 42 +++++++++++++++++++++++++++++++-- 7 files changed, 104 insertions(+), 49 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cf0252c6..ccf12a06 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -379,7 +379,7 @@ static inline bool mi_is_in_same_segment(const void* p, const void* q) { } static inline mi_block_t* mi_block_nextx( uintptr_t cookie, const mi_block_t* block ) { - #if MI_SECURE + #ifdef MI_ENCODE_FREELIST return (mi_block_t*)(block->next ^ cookie); #else UNUSED(cookie); @@ -388,7 +388,7 @@ static inline mi_block_t* mi_block_nextx( uintptr_t cookie, const mi_block_t* bl } static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, const mi_block_t* next) { - #if MI_SECURE + #ifdef MI_ENCODE_FREELIST block->next = (mi_encoded_t)next ^ cookie; #else UNUSED(cookie); @@ -397,16 +397,15 @@ static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, const } static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { - #if MI_SECURE + #ifdef MI_ENCODE_FREELIST mi_block_t* next = mi_block_nextx(page->cookie,block); - #if MI_SECURE >= 4 - // check if next is at least in our segment range - // TODO: it is better to check if it is actually inside our page but that is more expensive - // to calculate. Perhaps with a relative free list this becomes feasible? - if (next!=NULL && !mi_is_in_same_segment(block, next)) { - _mi_fatal_error("corrupted free list entry at %p: %zx\n", block, (uintptr_t)next); - } - #endif + // check for free list corruption: is `next` at least in our segment range? + // TODO: it is better to check if it is actually inside our page but that is more expensive + // to calculate. Perhaps with a relative free list this becomes feasible? + if (next!=NULL && !mi_is_in_same_segment(block, next)) { + _mi_fatal_error("corrupted free list entry of size %zub at %p: value 0x%zx\n", page->block_size, block, (uintptr_t)next); + next = NULL; + } return next; #else UNUSED(page); @@ -415,7 +414,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* } static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { - #if MI_SECURE + #ifdef MI_ENCODE_FREELIST mi_block_set_nextx(page->cookie,block,next); #else UNUSED(page); diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 2e5d481b..99b6b22b 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -25,24 +25,30 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_SECURE to enable security mitigations // #define MI_SECURE 1 // guard page around metadata // #define MI_SECURE 2 // guard page around each mimalloc page -// #define MI_SECURE 3 // encode free lists -// #define MI_SECURE 4 // all security enabled (checks for double free, corrupted free list and invalid pointer free) +// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) +// #define MI_SECURE 4 // experimental, may be more expensive: checks for double free. #if !defined(MI_SECURE) #define MI_SECURE 0 #endif -// Define MI_DEBUG as 1 for basic assert checks and statistics -// set it to 2 to do internal asserts, -// and to 3 to do extensive invariant checking. +// Define MI_DEBUG for debug mode +// #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. +// #define MI_DEBUG 2 // + internal assertion checks +// #define MI_DEBUG 3 // + extensive internal invariant checking #if !defined(MI_DEBUG) #if !defined(NDEBUG) || defined(_DEBUG) -#define MI_DEBUG 1 +#define MI_DEBUG 2 #else #define MI_DEBUG 0 #endif #endif +// Encoded free lists allow detection of corrupted free lists +// and can detect buffer overflows and double `free`s. +#if (MI_SECURE>=3 || MI_DEBUG>=1) +#define MI_ENCODE_FREELIST 1 +#endif // ------------------------------------------------------ // Platform specific values @@ -117,6 +123,8 @@ terms of the MIT license. A copy of the license can be found in the file #error "define more bins" #endif +// The free lists use encoded next fields +// (Only actually encodes when MI_ENCODED_FREELIST is defined.) typedef uintptr_t mi_encoded_t; // free lists contain blocks @@ -125,6 +133,7 @@ typedef struct mi_block_s { } mi_block_t; +// The delayed flags are used for efficient multi-threaded free-ing typedef enum mi_delayed_e { MI_NO_DELAYED_FREE = 0, MI_USE_DELAYED_FREE = 1, @@ -180,7 +189,7 @@ typedef struct mi_page_s { bool is_zero; // `true` if the blocks in the free list are zero initialized mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - #if MI_SECURE + #ifdef MI_ENCODE_FREELIST uintptr_t cookie; // random cookie to encode the free lists #endif size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) @@ -197,8 +206,8 @@ typedef struct mi_page_s { // improve page index calculation // without padding: 10 words on 64-bit, 11 on 32-bit. Secure adds one word - #if (MI_INTPTR_SIZE==8 && MI_SECURE>0) || (MI_INTPTR_SIZE==4 && MI_SECURE==0) - void* padding[1]; // 12 words on 64-bit in secure mode, 12 words on 32-bit plain + #if (MI_INTPTR_SIZE==8 && defined(MI_ENCODE_FREELIST)) || (MI_INTPTR_SIZE==4 && !defined(MI_ENCODE_FREELIST)) + void* padding[1]; // 12 words on 64-bit with cookie, 12 words on 32-bit plain #endif } mi_page_t; diff --git a/src/alloc.c b/src/alloc.c index f8dab24d..d2319f82 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -32,10 +32,10 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz page->free = mi_block_next(page,block); page->used++; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); -#if (MI_DEBUG) +#if (MI_DEBUG!=0) if (!page->is_zero) { memset(block, MI_DEBUG_UNINIT, size); } -#elif (MI_SECURE) - block->next = 0; +#elif (MI_SECURE!=0) + block->next = 0; // don't leak internal data #endif #if (MI_STAT>1) if(size <= MI_LARGE_OBJ_SIZE_MAX) { @@ -125,10 +125,12 @@ mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept { // ------------------------------------------------------ -// Check for double free in secure mode +// Check for double free in secure and debug mode +// This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if MI_SECURE>=4 +#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +// linear check if the free list contains a specific element static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) { while (list != NULL) { if (elem==list) return true; @@ -137,15 +139,15 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons return false; } -static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block, const mi_block_t* n) { +static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block, const mi_block_t* n) { size_t psize; uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); if (n == NULL || ((uint8_t*)n >= pstart && (uint8_t*)n < (pstart + psize))) { // Suspicious: the decoded value is in the same page (or NULL). - // Walk the free lists to see if it is already freed + // Walk the free lists to verify positively if it is already freed if (mi_list_contains(page, page->free, block) || - mi_list_contains(page, page->local_free, block) || - mi_list_contains(page, (const mi_block_t*)mi_atomic_read_ptr_relaxed(mi_atomic_cast(void*,&page->thread_free)), block)) + mi_list_contains(page, page->local_free, block) || + mi_list_contains(page, (const mi_block_t*)mi_atomic_read_ptr_relaxed(mi_atomic_cast(void*,&page->thread_free)), block)) { _mi_fatal_error("double free detected of block %p with size %zu\n", block, page->block_size); return true; @@ -154,16 +156,23 @@ static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const return false; } -static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { - mi_block_t* n = (mi_block_t*)(block->next ^ page->cookie); - if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check - (n==NULL || mi_is_in_same_segment(block, n))) +static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + mi_block_t* n = mi_block_nextx(page->cookie, block); // pretend it is freed, and get the decoded first field + if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + (n==NULL || mi_is_in_same_segment(block, n))) // quick check: in same segment or NULL? { // Suspicous: decoded value in block is in the same segment (or NULL) -- maybe a double free? - return mi_check_double_freex(page, block, n); + // (continue in separate function to improve code generation) + return mi_check_is_double_freex(page, block, n); } return false; } +#else +static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { + UNUSED(page); + UNUSED(block); + return false; +} #endif @@ -171,7 +180,6 @@ static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* // Free // ------------------------------------------------------ - // multi-threaded free static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) { @@ -258,6 +266,7 @@ static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block // and push it on the free list if (mi_likely(local)) { // owning thread can free a block directly + if (mi_check_is_double_free(page, block)) return; mi_block_set_next(page, block, page->local_free); page->local_free = block; page->used--; @@ -301,7 +310,7 @@ void mi_free(void* p) mi_attr_noexcept const mi_segment_t* const segment = _mi_ptr_segment(p); if (mi_unlikely(segment == NULL)) return; // checks for (p==NULL) -#if (MI_DEBUG>0) +#if (MI_DEBUG!=0) if (mi_unlikely(!mi_is_in_heap_region(p))) { _mi_warning_message("possibly trying to free a pointer that does not point to a valid heap region: 0x%p\n" "(this may still be a valid very large allocation (over 64MiB))\n", p); @@ -310,7 +319,7 @@ void mi_free(void* p) mi_attr_noexcept } } #endif -#if (MI_DEBUG>0 || MI_SECURE>=4) +#if (MI_DEBUG!=0 || MI_SECURE>=4) if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { _mi_error_message("trying to free a pointer that does not point to a valid heap space: %p\n", p); return; @@ -332,9 +341,7 @@ void mi_free(void* p) mi_attr_noexcept if (mi_likely(tid == segment->thread_id && page->flags.full_aligned == 0)) { // the thread id matches and it is not a full page, nor has aligned blocks // local, and not full or aligned mi_block_t* block = (mi_block_t*)p; - #if MI_SECURE>=4 - if (mi_check_double_free(page,block)) return; - #endif + if (mi_check_is_double_free(page,block)) return; mi_block_set_next(page, block, page->local_free); page->local_free = block; page->used--; diff --git a/src/init.c b/src/init.c index d361de3a..e15d82eb 100644 --- a/src/init.c +++ b/src/init.c @@ -15,14 +15,14 @@ const mi_page_t _mi_page_empty = { 0, false, false, false, false, 0, 0, { 0 }, false, NULL, // free - #if MI_SECURE + #if MI_ENCODE_FREELIST 0, #endif 0, // used NULL, ATOMIC_VAR_INIT(0), ATOMIC_VAR_INIT(0), 0, NULL, NULL, NULL - #if (MI_INTPTR_SIZE==8 && MI_SECURE>0) || (MI_INTPTR_SIZE==4 && MI_SECURE==0) + #if (MI_INTPTR_SIZE==8 && defined(MI_ENCODE_FREELIST)) || (MI_INTPTR_SIZE==4 && !defined(MI_ENCODE_FREELIST)) , { NULL } // padding #endif }; diff --git a/src/options.c b/src/options.c index e74d9eb5..d71e5d1c 100644 --- a/src/options.c +++ b/src/options.c @@ -290,7 +290,9 @@ mi_attr_noreturn void _mi_fatal_error(const char* fmt, ...) { va_start(args, fmt); mi_vfprintf(NULL, "mimalloc: fatal: ", fmt, args); va_end(args); - exit(99); + #if (MI_SECURE>=0) + abort(); + #endif } // -------------------------------------------------------- diff --git a/src/page.c b/src/page.c index 5a186727..c1d29d46 100644 --- a/src/page.c +++ b/src/page.c @@ -512,7 +512,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* page, size_t e ----------------------------------------------------------- */ #define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well. -#if MI_SECURE +#if (MI_SECURE>0) #define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many #else #define MI_MIN_EXTEND (1) @@ -579,7 +579,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi page->block_size = block_size; mi_assert_internal(page_size / block_size < (1L<<16)); page->reserved = (uint16_t)(page_size / block_size); - #if MI_SECURE + #ifdef MI_ENCODE_FREELIST page->cookie = _mi_heap_random(heap) | 1; #endif page->is_zero = page->is_zero_init; @@ -592,7 +592,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi mi_assert_internal(page->next == NULL); mi_assert_internal(page->prev == NULL); mi_assert_internal(!mi_page_has_aligned(page)); - #if MI_SECURE + #if (MI_ENCODE_FREELIST) mi_assert_internal(page->cookie != 0); #endif mi_assert_expensive(mi_page_is_valid_init(page)); diff --git a/test/main-override-static.c b/test/main-override-static.c index ed5048e0..19712411 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -9,11 +9,16 @@ static void double_free1(); static void double_free2(); +static void corrupt_free(); int main() { mi_version(); + + // detect double frees and heap corruption //double_free1(); //double_free2(); + //corrupt_free(); + void* p1 = malloc(78); void* p2 = malloc(24); free(p1); @@ -36,9 +41,13 @@ int main() { return 0; } + +// The double free samples come ArcHeap [1] by Insu Yun (issue #161) +// [1]: https://arxiv.org/pdf/1903.00503.pdf + static void double_free1() { void* p[256]; - uintptr_t buf[256]; + //uintptr_t buf[256]; p[0] = mi_malloc(622616); p[1] = mi_malloc(655362); @@ -54,7 +63,7 @@ static void double_free1() { static void double_free2() { void* p[256]; - uintptr_t buf[256]; + //uintptr_t buf[256]; // [INFO] Command buffer: 0x327b2000 // [INFO] Input size: 182 p[0] = malloc(712352); @@ -69,3 +78,32 @@ static void double_free2() { // p[4]=0x433f1402000 (size=917504), p[1]=0x433f14c2000 (size=786432) fprintf(stderr, "p1: %p-%p, p2: %p-%p\n", p[4], (uint8_t*)(p[4]) + 917504, p[1], (uint8_t*)(p[1]) + 786432); } + + +// Try to corrupt the heap through buffer overflow +#define N 256 +#define SZ 64 + +static void corrupt_free() { + void* p[N]; + // allocate + for (int i = 0; i < N; i++) { + p[i] = malloc(SZ); + } + // free some + for (int i = 0; i < N; i += (N/10)) { + free(p[i]); + p[i] = NULL; + } + // try to corrupt the free list + for (int i = 0; i < N; i++) { + if (p[i] != NULL) { + memset(p[i], 0, SZ+8); + } + } + // allocate more.. trying to trigger an allocation from a corrupted entry + // this may need many allocations to get there (if at all) + for (int i = 0; i < 4096; i++) { + malloc(SZ); + } +} \ No newline at end of file From 6cf16b1201fdfa47e18020b2166d49c5b97d2097 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 30 Oct 2019 14:32:28 -0700 Subject: [PATCH 28/57] fix reset error on windows when disabling eager commit option --- src/memory.c | 10 +++++++--- src/options.c | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/memory.c b/src/memory.c index f9c53782..dd03cf95 100644 --- a/src/memory.c +++ b/src/memory.c @@ -461,10 +461,14 @@ void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) { // reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large if (!is_large) { if (mi_option_is_enabled(mi_option_segment_reset)) { - _mi_os_reset(p, size, stats); // - // _mi_os_decommit(p,size,stats); // if !is_eager_committed (and clear dirty bits) + if (!is_eager_committed && // cannot reset large pages + (mi_option_is_enabled(mi_option_eager_commit) || // cannot reset halfway committed segments, use `option_page_reset` instead + mi_option_is_enabled(mi_option_reset_decommits))) // but we can decommit halfway committed segments + { + _mi_os_reset(p, size, stats); + //_mi_os_decommit(p, size, stats); // todo: and clear dirty bits? + } } - // else { _mi_os_reset(p,size,stats); } } if (!is_eager_committed) { // adjust commit statistics as we commit again when re-using the same slot diff --git a/src/options.c b/src/options.c index d71e5d1c..a49c46ed 100644 --- a/src/options.c +++ b/src/options.c @@ -65,7 +65,7 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(cache_reset) }, { 0, UNINIT, MI_OPTION(reset_decommits) }, // note: cannot enable this if secure is on { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed - { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free + { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit) { 100, UNINIT, MI_OPTION(os_tag) } // only apple specific for now but might serve more or less related purpose }; From 4a4d74927ccd7c07e68d28ca372c6c30408ad92f Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 30 Oct 2019 14:53:21 -0700 Subject: [PATCH 29/57] protect against double-free in multi-threaded free list --- src/page.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/page.c b/src/page.c index c1d29d46..ab271309 100644 --- a/src/page.c +++ b/src/page.c @@ -161,14 +161,21 @@ static void _mi_page_thread_free_collect(mi_page_t* page) // return if the list is empty if (head == NULL) return; - // find the tail + // find the tail -- also to get a proper count (without data races) + uintptr_t max_count = page->capacity; // cannot collect more than capacity uintptr_t count = 1; mi_block_t* tail = head; mi_block_t* next; - while ((next = mi_block_next(page,tail)) != NULL) { + while ((next = mi_block_next(page,tail)) != NULL && count <= max_count) { count++; tail = next; } + // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free) + if (count > max_count) { + _mi_fatal_error("corrupted thread-free list\n"); + return; // the thread-free items cannot be freed + } + // and append the current local free list mi_block_set_next(page,tail, page->local_free); page->local_free = head; From 8725a88fba49b719078240c65561e7c68bb286c9 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 30 Oct 2019 15:22:40 -0700 Subject: [PATCH 30/57] fix assertion in debug secure mode --- src/page.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/page.c b/src/page.c index ab271309..f7fad764 100644 --- a/src/page.c +++ b/src/page.c @@ -441,8 +441,10 @@ void _mi_page_retire(mi_page_t* page) { static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats) { UNUSED(stats); + #if (MI_SECURE<=2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); + #endif mi_assert_internal(page->capacity + extend <= page->reserved); void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL); size_t bsize = page->block_size; @@ -532,10 +534,12 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* page, size_t e // extra test in malloc? or cache effects?) static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* stats) { UNUSED(stats); + mi_assert_expensive(mi_page_is_valid_init(page)); + #if (MI_SECURE<=2) mi_assert(page->free == NULL); mi_assert(page->local_free == NULL); - mi_assert_expensive(mi_page_is_valid_init(page)); if (page->free != NULL) return; + #endif if (page->capacity >= page->reserved) return; size_t page_size; From aece753dce816e6c6967449eef5de5fd30d4a294 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 3 Nov 2019 12:18:20 -0800 Subject: [PATCH 31/57] fix output during preloading enabling stderr only after the crt has loaded --- src/options.c | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/options.c b/src/options.c index a49c46ed..0b3c6c97 100644 --- a/src/options.c +++ b/src/options.c @@ -14,6 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file #include // toupper #include +static void mi_add_stderr_output(); + int mi_version(void) mi_attr_noexcept { return MI_MALLOC_VERSION; } @@ -72,7 +74,9 @@ static mi_option_desc_t options[_mi_option_last] = static void mi_option_init(mi_option_desc_t* desc); void _mi_options_init(void) { - // called on process load + // called on process load; should not be called before the CRT is initialized! + // (e.g. do not call this from process_init as that may run before CRT initialization) + mi_add_stderr_output(); // now it safe to use stderr for output for(int i = 0; i < _mi_option_last; i++ ) { mi_option_t option = (mi_option_t)i; mi_option_get(option); // initialize @@ -134,7 +138,7 @@ static void mi_out_stderr(const char* msg) { #ifdef _WIN32 // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. - _cputs(msg); + if (!_mi_preloading()) { _cputs(msg); } #else fputs(msg, stderr); #endif @@ -165,23 +169,29 @@ static void mi_out_buf(const char* msg) { memcpy(&out_buf[start], msg, n); } -static void mi_out_buf_flush(mi_output_fun* out) { +static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf) { if (out==NULL) return; - // claim all (no more output will be added after this point) - size_t count = mi_atomic_addu(&out_len, MI_MAX_DELAY_OUTPUT); + // claim (if `no_more_buf == true`, no more output will be added after this point) + size_t count = mi_atomic_addu(&out_len, (no_more_buf ? MI_MAX_DELAY_OUTPUT : 1)); // and output the current contents if (count>MI_MAX_DELAY_OUTPUT) count = MI_MAX_DELAY_OUTPUT; out_buf[count] = 0; out(out_buf); + if (!no_more_buf) { + out_buf[count] = '\n'; // if continue with the buffer, insert a newline + } } -// The initial default output, outputs to stderr and the delayed output buffer. + +// Once this module is loaded, switch to this routine +// which outputs to stderr and the delayed output buffer. static void mi_out_buf_stderr(const char* msg) { mi_out_stderr(msg); mi_out_buf(msg); } + // -------------------------------------------------------- // Default output handler // -------------------------------------------------------- @@ -193,14 +203,19 @@ static mi_output_fun* volatile mi_out_default; // = NULL static mi_output_fun* mi_out_get_default(void) { mi_output_fun* out = mi_out_default; - return (out == NULL ? &mi_out_buf_stderr : out); + return (out == NULL ? &mi_out_buf : out); } void mi_register_output(mi_output_fun* out) mi_attr_noexcept { mi_out_default = (out == NULL ? &mi_out_stderr : out); // stop using the delayed output buffer - if (out!=NULL) mi_out_buf_flush(out); // output the delayed output now + if (out!=NULL) mi_out_buf_flush(out,true); // output all the delayed output now } +// add stderr to the delayed output after the module is loaded +static void mi_add_stderr_output() { + mi_out_buf_flush(&mi_out_stderr, false); // flush current contents to stderr + mi_out_default = &mi_out_buf_stderr; // and add stderr to the delayed output +} // -------------------------------------------------------- // Messages, all end up calling `_mi_fputs`. @@ -213,7 +228,7 @@ static volatile _Atomic(uintptr_t) error_count; // = 0; // when MAX_ERROR_COUNT static mi_decl_thread bool recurse = false; void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message) { - if (_mi_preloading() || recurse) return; + if (recurse) return; if (out==NULL || (FILE*)out==stdout || (FILE*)out==stderr) out = mi_out_get_default(); recurse = true; if (prefix != NULL) out(prefix); @@ -227,7 +242,7 @@ void _mi_fputs(mi_output_fun* out, const char* prefix, const char* message) { static void mi_vfprintf( mi_output_fun* out, const char* prefix, const char* fmt, va_list args ) { char buf[512]; if (fmt==NULL) return; - if (_mi_preloading() || recurse) return; + if (recurse) return; recurse = true; vsnprintf(buf,sizeof(buf)-1,fmt,args); recurse = false; From c38af8f7c14f33d22832bb24fdacfd41b20ad69f Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 3 Nov 2019 13:25:41 -0800 Subject: [PATCH 32/57] merge d1d65fbc: make max error messages configurable --- include/mimalloc.h | 1 + src/options.c | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index b63ed79d..bc817f54 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -271,6 +271,7 @@ typedef enum mi_option_e { mi_option_eager_commit_delay, mi_option_segment_reset, mi_option_os_tag, + mi_option_max_errors, _mi_option_last } mi_option_t; diff --git a/src/options.c b/src/options.c index 0b3c6c97..0bee74e0 100644 --- a/src/options.c +++ b/src/options.c @@ -14,6 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file #include // toupper #include +static uintptr_t mi_max_error_count = 16; // stop outputting errors after this + static void mi_add_stderr_output(); int mi_version(void) mi_attr_noexcept { @@ -68,7 +70,8 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(reset_decommits) }, // note: cannot enable this if secure is on { 0, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed { 0, UNINIT, MI_OPTION(segment_reset) }, // reset segment memory on free (needs eager commit) - { 100, UNINIT, MI_OPTION(os_tag) } // only apple specific for now but might serve more or less related purpose + { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose + { 16, UNINIT, MI_OPTION(max_errors) } // maximum errors that are output }; static void mi_option_init(mi_option_desc_t* desc); @@ -85,6 +88,7 @@ void _mi_options_init(void) { _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value); } } + mi_max_error_count = mi_option_get(mi_option_max_errors); } long mi_option_get(mi_option_t option) { @@ -275,7 +279,7 @@ void _mi_verbose_message(const char* fmt, ...) { void _mi_error_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; - if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return; + if (mi_atomic_increment(&error_count) > mi_max_error_count) return; va_list args; va_start(args,fmt); mi_vfprintf(NULL, "mimalloc: error: ", fmt, args); @@ -285,7 +289,7 @@ void _mi_error_message(const char* fmt, ...) { void _mi_warning_message(const char* fmt, ...) { if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; - if (mi_atomic_increment(&error_count) > MAX_ERROR_COUNT) return; + if (mi_atomic_increment(&error_count) > mi_max_error_count) return; va_list args; va_start(args,fmt); mi_vfprintf(NULL, "mimalloc: warning: ", fmt, args); From 31d11f64d581abfd28818be65f3780506977d889 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 7 Nov 2019 10:33:45 -0800 Subject: [PATCH 33/57] fix secure free list extension where a non-empty initial free list was discarded --- include/mimalloc-types.h | 4 ++-- src/page.c | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 99b6b22b..893dcd67 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -26,7 +26,7 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_SECURE 1 // guard page around metadata // #define MI_SECURE 2 // guard page around each mimalloc page // #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) -// #define MI_SECURE 4 // experimental, may be more expensive: checks for double free. +// #define MI_SECURE 4 // experimental, may be more expensive: checks for double free. (cmake -DMI_SECURE_FULL=ON) #if !defined(MI_SECURE) #define MI_SECURE 0 @@ -35,7 +35,7 @@ terms of the MIT license. A copy of the license can be found in the file // Define MI_DEBUG for debug mode // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. // #define MI_DEBUG 2 // + internal assertion checks -// #define MI_DEBUG 3 // + extensive internal invariant checking +// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_CHECK_FULL=ON) #if !defined(MI_DEBUG) #if !defined(NDEBUG) || defined(_DEBUG) #define MI_DEBUG 2 diff --git a/src/page.c b/src/page.c index f7fad764..cb3a4bf8 100644 --- a/src/page.c +++ b/src/page.c @@ -455,8 +455,8 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si while ((extend >> shift) == 0) { shift--; } - size_t slice_count = (size_t)1U << shift; - size_t slice_extend = extend / slice_count; + const size_t slice_count = (size_t)1U << shift; + const size_t slice_extend = extend / slice_count; mi_assert_internal(slice_extend >= 1); mi_block_t* blocks[MI_MAX_SLICES]; // current start of the slice size_t counts[MI_MAX_SLICES]; // available objects in the slice @@ -470,7 +470,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si // set up first element size_t current = _mi_heap_random(heap) % slice_count; counts[current]--; - page->free = blocks[current]; + mi_block_t* const free_start = blocks[current]; // and iterate through the rest uintptr_t rnd = heap->random; for (size_t i = 1; i < extend; i++) { @@ -490,7 +490,9 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next` current = next; } - mi_block_set_next(page, blocks[current], NULL); // end of the list + // prepend to the free list (usually NULL) + mi_block_set_next(page, blocks[current], page->free); // end of the list + page->free = free_start; heap->random = _mi_random_shuffle(rnd); } From 9b6538880768ccbe0dde86cfc0018a7b035e7911 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 7 Nov 2019 10:59:19 -0800 Subject: [PATCH 34/57] fix space leak in secure mode where a non-null free list would be discarded --- src/page.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/page.c b/src/page.c index cb3a4bf8..aaf1cb91 100644 --- a/src/page.c +++ b/src/page.c @@ -439,15 +439,15 @@ void _mi_page_retire(mi_page_t* page) { #define MI_MAX_SLICES (1UL << MI_MAX_SLICE_SHIFT) #define MI_MIN_SLICES (2) -static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, size_t extend, mi_stats_t* stats) { +static void mi_page_free_list_extend_secure(mi_heap_t* const heap, mi_page_t* const page, const size_t extend, mi_stats_t* const stats) { UNUSED(stats); #if (MI_SECURE<=2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif mi_assert_internal(page->capacity + extend <= page->reserved); - void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL); - size_t bsize = page->block_size; + void* const page_area = _mi_page_start(_mi_page_segment(page), page, NULL); + const size_t bsize = page->block_size; // initialize a randomized free list // set up `slice_count` slices to alternate between @@ -475,7 +475,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si uintptr_t rnd = heap->random; for (size_t i = 1; i < extend; i++) { // call random_shuffle only every INTPTR_SIZE rounds - size_t round = i%MI_INTPTR_SIZE; + const size_t round = i%MI_INTPTR_SIZE; if (round == 0) rnd = _mi_random_shuffle(rnd); // select a random next slice index size_t next = ((rnd >> 8*round) & (slice_count-1)); @@ -485,7 +485,7 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si } // and link the current block to it counts[next]--; - mi_block_t* block = blocks[current]; + mi_block_t* const block = blocks[current]; blocks[current] = (mi_block_t*)((uint8_t*)block + bsize); // bump to the following block mi_block_set_next(page, block, blocks[next]); // and set next; note: we may have `current == next` current = next; @@ -496,25 +496,28 @@ static void mi_page_free_list_extend_secure(mi_heap_t* heap, mi_page_t* page, si heap->random = _mi_random_shuffle(rnd); } -static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* page, size_t extend, mi_stats_t* stats) +static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t extend, mi_stats_t* const stats) { UNUSED(stats); + #if (MI_SECURE <= 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); + #endif mi_assert_internal(page->capacity + extend <= page->reserved); - void* page_area = _mi_page_start(_mi_page_segment(page), page, NULL ); - size_t bsize = page->block_size; - mi_block_t* start = mi_page_block_at(page, page_area, page->capacity); + void* const page_area = _mi_page_start(_mi_page_segment(page), page, NULL ); + const size_t bsize = page->block_size; + mi_block_t* const start = mi_page_block_at(page, page_area, page->capacity); // initialize a sequential free list - mi_block_t* last = mi_page_block_at(page, page_area, page->capacity + extend - 1); + mi_block_t* const last = mi_page_block_at(page, page_area, page->capacity + extend - 1); mi_block_t* block = start; while(block <= last) { mi_block_t* next = (mi_block_t*)((uint8_t*)block + bsize); mi_block_set_next(page,block,next); block = next; } - mi_block_set_next(page, last, NULL); + // prepend to free list (usually `NULL`) + mi_block_set_next(page, last, page->free); page->free = start; } From 56887aeb2f75d0ade86120e448e66a2684c920ff Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 7 Nov 2019 10:59:45 -0800 Subject: [PATCH 35/57] add MI_SECURE_FULL=ON as a cmake option to include double free mitigation --- CMakeLists.txt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81cc339a..59d889b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library" OFF) option(MI_SECURE "Use security mitigations (like guard pages and randomization)" OFF) +option(MI_SECURE_FULL "Use full security mitigations, may be more expensive (includes double-free mitigation)" OFF) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_TESTS "Build test executables" ON) @@ -66,9 +67,15 @@ if(MI_OVERRIDE MATCHES "ON") endif() endif() -if(MI_SECURE MATCHES "ON") - message(STATUS "Set secure build (MI_SECURE=ON)") - list(APPEND mi_defines MI_SECURE=3) +if(MI_SECURE_FULL MATCHES "ON") + message(STATUS "Set full secure build (may be more expensive) (MI_SECURE_FULL=ON)") + list(APPEND mi_defines MI_SECURE=4) + set(MI_SECURE "ON") +else() + if(MI_SECURE MATCHES "ON") + message(STATUS "Set secure build (MI_SECURE=ON)") + list(APPEND mi_defines MI_SECURE=3) + endif() endif() if(MI_SEE_ASM MATCHES "ON") From f0e02bab0344e099fe491eb24690a0b9a08cf6e0 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 8 Nov 2019 12:22:03 -0800 Subject: [PATCH 36/57] pr #168 by @zerodefect to update the install location --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59d889b8..7b455881 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ option(MI_SECURE_FULL "Use full security mitigations, may be more expensiv option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_TESTS "Build test executables" ON) -set(mi_install_dir "lib/mimalloc-${mi_version}") +set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") set(mi_sources src/stats.c From 29919a938dbd6f070ed84b146ad4d712946240ee Mon Sep 17 00:00:00 2001 From: Marco Wang Date: Wed, 13 Nov 2019 13:19:21 +0800 Subject: [PATCH 37/57] Avoid the use of variable argument list function --- include/mimalloc-atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h index dff0f011..10368df3 100644 --- a/include/mimalloc-atomic.h +++ b/include/mimalloc-atomic.h @@ -220,7 +220,7 @@ static inline void mi_atomic_write(volatile _Atomic(uintptr_t)* p, uintptr_t x) #endif #elif defined(__wasi__) #include - static inline void mi_atomic_yield() { + static inline void mi_atomic_yield(void) { sched_yield(); } #else From a4ed63d1273befbe2c8835395f3137564d3af7e9 Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 13 Nov 2019 17:22:03 -0800 Subject: [PATCH 38/57] Adresses pr #165 and issue #164 by @colesbury: On Mac OS, the thread-local _mi_default_heap may get reset before _mi_thread_done is called, leaking the default heap on non-main threads. Now the current default heap is also stored in mi_pthread_key (or mi_fls_key on Windows). The _mi_thread_done function is called with this value. --- include/mimalloc-internal.h | 1 + src/heap.c | 6 ++-- src/init.c | 59 +++++++++++++++++++++---------------- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index ccf12a06..73849337 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -101,6 +101,7 @@ uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD i void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); uintptr_t _mi_heap_random(mi_heap_t* heap); +void _mi_heap_set_default_direct(mi_heap_t* heap); // "stats.c" void _mi_stats_done(mi_stats_t* stats); diff --git a/src/heap.c b/src/heap.c index 15c5d02a..daa9b241 100644 --- a/src/heap.c +++ b/src/heap.c @@ -223,7 +223,7 @@ static void mi_heap_free(mi_heap_t* heap) { // reset default if (mi_heap_is_default(heap)) { - _mi_heap_default = heap->tld->heap_backing; + _mi_heap_set_default_direct(heap->tld->heap_backing); } // and free the used memory mi_free(heap); @@ -354,8 +354,8 @@ mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { mi_assert(mi_heap_is_initialized(heap)); if (!mi_heap_is_initialized(heap)) return NULL; mi_assert_expensive(mi_heap_is_valid(heap)); - mi_heap_t* old = _mi_heap_default; - _mi_heap_default = heap; + mi_heap_t* old = mi_get_default_heap(); + _mi_heap_set_default_direct(heap); return old; } diff --git a/src/init.c b/src/init.c index e15d82eb..081e7ce7 100644 --- a/src/init.c +++ b/src/init.c @@ -90,6 +90,7 @@ const mi_heap_t _mi_heap_empty = { false }; +// the thread-local default heap for allocation mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; @@ -198,8 +199,8 @@ static bool _mi_heap_init(void) { if (mi_heap_is_initialized(_mi_heap_default)) return true; if (_mi_is_main_thread()) { // the main heap is statically allocated - _mi_heap_default = &_mi_heap_main; - mi_assert_internal(_mi_heap_default->tld->heap_backing == _mi_heap_default); + _mi_heap_set_default_direct(&_mi_heap_main); + mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_get_default_heap()); } else { // use `_mi_os_alloc` to allocate directly from the OS @@ -219,18 +220,17 @@ static bool _mi_heap_init(void) { tld->heap_backing = heap; tld->segments.stats = &tld->stats; tld->os.stats = &tld->stats; - _mi_heap_default = heap; + _mi_heap_set_default_direct(heap); } return false; } // Free the thread local default heap (called from `mi_thread_done`) -static bool _mi_heap_done(void) { - mi_heap_t* heap = _mi_heap_default; +static bool _mi_heap_done(mi_heap_t* heap) { if (!mi_heap_is_initialized(heap)) return true; // reset default heap - _mi_heap_default = (_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); + _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); // todo: delete all non-backing heaps? @@ -277,6 +277,8 @@ static bool _mi_heap_done(void) { // to set up the thread local keys. // -------------------------------------------------------- +static void _mi_thread_done(mi_heap_t* default_heap); + #ifdef __wasi__ // no pthreads in the WebAssembly Standard Interface #elif !defined(_WIN32) @@ -291,14 +293,14 @@ static bool _mi_heap_done(void) { #include static DWORD mi_fls_key; static void NTAPI mi_fls_done(PVOID value) { - if (value!=NULL) mi_thread_done(); + if (value!=NULL) _mi_thread_done((mi_heap_t*)value); } #elif defined(MI_USE_PTHREADS) // use pthread locol storage keys to detect thread ending #include static pthread_key_t mi_pthread_key; static void mi_pthread_done(void* value) { - if (value!=NULL) mi_thread_done(); + if (value!=NULL) _mi_thread_done((mi_heap_t*)value); } #elif defined(__wasi__) // no pthreads in the WebAssembly Standard Interface @@ -332,6 +334,8 @@ void mi_thread_init(void) mi_attr_noexcept mi_process_init(); // initialize the thread local default heap + // (this will call `_mi_heap_set_default_direct` and thus set the + // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) if (_mi_heap_init()) return; // returns true if already initialized // don't further initialize for the main thread @@ -339,33 +343,38 @@ void mi_thread_init(void) mi_attr_noexcept _mi_stat_increase(&mi_get_default_heap()->tld->stats.threads, 1); - // set hooks so our mi_thread_done() will be called - #if defined(_WIN32) && defined(MI_SHARED_LIB) - // nothing to do as it is done in DllMain - #elif defined(_WIN32) && !defined(MI_SHARED_LIB) - FlsSetValue(mi_fls_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_fls_done` is called - #elif defined(MI_USE_PTHREADS) - pthread_setspecific(mi_pthread_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_pthread_done` is called - #endif - //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); } void mi_thread_done(void) mi_attr_noexcept { + _mi_thread_done(mi_get_default_heap()); +} + +static void _mi_thread_done(mi_heap_t* heap) { // stats - mi_heap_t* heap = mi_get_default_heap(); if (!_mi_is_main_thread() && mi_heap_is_initialized(heap)) { _mi_stat_decrease(&heap->tld->stats.threads, 1); } - // abandon the thread local heap - if (_mi_heap_done()) return; // returns true if already ran - - //if (!_mi_is_main_thread()) { - // _mi_verbose_message("thread done: 0x%zx\n", _mi_thread_id()); - //} + if (_mi_heap_done(heap)) return; // returns true if already ran } +void _mi_heap_set_default_direct(mi_heap_t* heap) { + mi_assert_internal(heap != NULL); + _mi_heap_default = heap; + + // ensure the default heap is passed to `_mi_thread_done` + // setting to a non-NULL value also ensures `mi_thread_done` is called. + #if defined(_WIN32) && defined(MI_SHARED_LIB) + // nothing to do as it is done in DllMain + #elif defined(_WIN32) && !defined(MI_SHARED_LIB) + FlsSetValue(mi_fls_key, heap); + #elif defined(MI_USE_PTHREADS) + pthread_setspecific(mi_pthread_key, heap); + #endif +} + + // -------------------------------------------------------- // Run functions on process init/done, and thread init/done @@ -446,7 +455,7 @@ void mi_process_init(void) mi_attr_noexcept { // access _mi_heap_default before setting _mi_process_is_initialized to ensure // that the TLS slot is allocated without getting into recursion on macOS // when using dynamic linking with interpose. - mi_heap_t* h = _mi_heap_default; + mi_heap_t* h = mi_get_default_heap(); _mi_process_is_initialized = true; _mi_heap_main.thread_id = _mi_thread_id(); From dccffea66286dfb16e642aef3fea7babee7038e3 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 14 Nov 2019 11:01:05 -0800 Subject: [PATCH 39/57] fix pr #173 by @zerodefect to use case-insensitive matching of the build type; also use MI_DEBUG_FULL option (instead of MI_CHECK_FULL) --- CMakeLists.txt | 74 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b455881..aa9c126f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,15 +6,14 @@ set(CMAKE_CXX_STANDARD 17) option(MI_OVERRIDE "Override the standard malloc interface" ON) option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" ON) -option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode" OFF) -option(MI_USE_CXX "Use the C++ compiler to compile the library" OFF) -option(MI_SECURE "Use security mitigations (like guard pages and randomization)" OFF) +option(MI_DEBUG_FULL "Use full internal heap invariant checking in DEBUG mode" OFF) +option(MI_SECURE "Use security mitigations (like guard pages, allocation randomization, and free-list corruption detection)" OFF) option(MI_SECURE_FULL "Use full security mitigations, may be more expensive (includes double-free mitigation)" OFF) +option(MI_USE_CXX "Use the C++ compiler to compile the library" OFF) +option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) option(MI_BUILD_TESTS "Build test executables" ON) - -set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") +option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) set(mi_sources src/stats.c @@ -29,29 +28,33 @@ set(mi_sources src/options.c src/init.c) -# Set default build type +# ----------------------------------------------------------------------------- +# Converience: set default build type depending on the build directory +# ----------------------------------------------------------------------------- + if (NOT CMAKE_BUILD_TYPE) - if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$") - message(STATUS "No build type selected, default to *** Debug ***") + if ("${CMAKE_BINARY_DIR}" MATCHES ".*(D|d)ebug$" OR MI_DEBUG_FULL MATCHES "ON") + message(STATUS "No build type selected, default to: Debug") set(CMAKE_BUILD_TYPE "Debug") else() - message(STATUS "No build type selected, default to *** Release ***") + message(STATUS "No build type selected, default to: Release") set(CMAKE_BUILD_TYPE "Release") endif() -else() - message(STATUS "Build type specified as *** ${CMAKE_BUILD_TYPE} ***") endif() if("${CMAKE_BINARY_DIR}" MATCHES ".*(S|s)ecure$") + message(STATUS "Default to secure build") set(MI_SECURE "ON") endif() +# ----------------------------------------------------------------------------- +# Process options +# ----------------------------------------------------------------------------- + if(CMAKE_C_COMPILER_ID MATCHES "MSVC") set(MI_USE_CXX "ON") endif() - -# Options if(MI_OVERRIDE MATCHES "ON") message(STATUS "Override standard malloc (MI_OVERRIDE=ON)") if(APPLE) @@ -84,7 +87,12 @@ if(MI_SEE_ASM MATCHES "ON") endif() if(MI_CHECK_FULL MATCHES "ON") - message(STATUS "Set debug level to full invariant checking (MI_CHECK_FULL=ON)") + message(STATUS "The MI_CHECK_FULL option is deprecated, use MI_DEBUG_FULL instead") + set(MI_DEBUG_FULL "ON") +endif() + +if(MI_DEBUG_FULL MATCHES "ON") + message(STATUS "Set debug level to full invariant checking (MI_DEBUG_FULL=ON)") list(APPEND mi_defines MI_DEBUG=3) # full invariant checking endif() @@ -109,19 +117,6 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") endif() endif() -if(NOT(CMAKE_BUILD_TYPE MATCHES "Release|release|RelWithDebInfo|relwithdebinfo")) - string(TOLOWER "${CMAKE_BUILD_TYPE}" build_type) - set(mi_basename "mimalloc-${build_type}") -else() - if(MI_SECURE MATCHES "ON") - set(mi_basename "mimalloc-secure") - else() - set(mi_basename "mimalloc") - endif() -endif() -message(STATUS "Output library name : ${mi_basename}") -message(STATUS "Installation directory: ${mi_install_dir}") - # extra needed libraries if(WIN32) list(APPEND mi_libraries psapi shell32 user32) @@ -134,9 +129,28 @@ else() endif() # ----------------------------------------------------------------------------- -# Main targets +# Install and output names # ----------------------------------------------------------------------------- +set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") +if(MI_SECURE MATCHES "ON") + set(mi_basename "mimalloc-secure") +else() + set(mi_basename "mimalloc") +endif() +string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) +if(NOT(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel)$")) + set(mi_basename "${mi_basename}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version +endif() +message(STATUS "") +message(STATUS "Library base name: ${mi_basename}") +message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +message(STATUS "Install directory: ${mi_install_dir}") +message(STATUS "") + +# ----------------------------------------------------------------------------- +# Main targets +# ----------------------------------------------------------------------------- # shared library add_library(mimalloc SHARED ${mi_sources}) @@ -238,7 +252,7 @@ endif() if (MI_OVERRIDE MATCHES "ON") target_compile_definitions(mimalloc PRIVATE MI_MALLOC_OVERRIDE) if(NOT WIN32) - # It is only possible to override malloc on Windows when building as a DLL. (src/alloc-override.c) + # It is only possible to override malloc on Windows when building as a DLL. target_compile_definitions(mimalloc-static PRIVATE MI_MALLOC_OVERRIDE) target_compile_definitions(mimalloc-obj PRIVATE MI_MALLOC_OVERRIDE) endif() From 8637f113d5ed817fa93e584d716d2b5c91ca723f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 15 Nov 2019 14:09:17 -0800 Subject: [PATCH 40/57] improve test-stress to run multiple iterations --- test/test-stress.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index bb428072..4b6ec22d 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -18,7 +18,8 @@ terms of the MIT license. // argument defaults static int THREADS = 32; // more repeatable if THREADS <= #processors -static int N = 20; // scaling factor +static int N = 20; // scaling factor +static int ITER = 10; // N full iterations re-creating all threads // static int THREADS = 8; // more repeatable if THREADS <= #processors // static int N = 100; // scaling factor @@ -159,14 +160,17 @@ int main(int argc, char** argv) { //bench_start_program(); mi_stats_reset(); - memset((void*)transfer, 0, TRANSFERS*sizeof(void*)); - run_os_threads(THREADS); - for (int i = 0; i < TRANSFERS; i++) { - free_items((void*)transfer[i]); + for (int i = 0; i < ITER; i++) { + memset((void*)transfer, 0, TRANSFERS * sizeof(void*)); + run_os_threads(THREADS); + for (int i = 0; i < TRANSFERS; i++) { + free_items((void*)transfer[i]); + } } - #ifndef NDEBUG +#ifndef NDEBUG mi_collect(false); - #endif +#endif + mi_stats_print(NULL); //bench_end_program(); return 0; From fd3ce5dc7d22bf4155588ac2755a98e4a405303f Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 15 Nov 2019 16:28:11 -0800 Subject: [PATCH 41/57] improve stress test --- ide/vs2019/mimalloc-test-stress.vcxproj | 4 +- test/test-stress.c | 86 ++++++++++++++++--------- 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/ide/vs2019/mimalloc-test-stress.vcxproj b/ide/vs2019/mimalloc-test-stress.vcxproj index afbb6666..ef7ab357 100644 --- a/ide/vs2019/mimalloc-test-stress.vcxproj +++ b/ide/vs2019/mimalloc-test-stress.vcxproj @@ -149,8 +149,8 @@ - - {abb5eae7-b3e6-432e-b636-333449892ea7} + + {abb5eae7-b3e6-432e-b636-333449892ea6} diff --git a/test/test-stress.c b/test/test-stress.c index 4b6ec22d..b6ceaa0a 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -6,7 +6,8 @@ terms of the MIT license. /* This is a stress test for the allocator, using multiple threads and transferring objects between threads. This is not a typical workload - but uses a random linear size distribution. Do not use this test as a benchmark! + but uses a random linear size distribution. Timing can also depend on + (random) thread scheduling. Do not use this test as a benchmark! */ #include @@ -18,16 +19,31 @@ terms of the MIT license. // argument defaults static int THREADS = 32; // more repeatable if THREADS <= #processors -static int N = 20; // scaling factor -static int ITER = 10; // N full iterations re-creating all threads +static int SCALE = 12; // scaling factor +static int ITER = 50; // N full iterations re-creating all threads // static int THREADS = 8; // more repeatable if THREADS <= #processors -// static int N = 100; // scaling factor +// static int SCALE = 100; // scaling factor +static bool allow_large_objects = true; // allow very large objects? +static size_t use_one_size = 0; // use single object size of N uintptr_t? + + +#ifdef USE_STD_MALLOC +#define custom_malloc(s) malloc(s) +#define custom_realloc(p,s) realloc(p,s) +#define custom_free(p) free(p) +#else +#define custom_malloc(s) mi_malloc(s) +#define custom_realloc(p,s) mi_realloc(p,s) +#define custom_free(p) mi_free(p) +#endif + +// transfer pointer between threads #define TRANSFERS (1000) - static volatile void* transfer[TRANSFERS]; + #if (UINTPTR_MAX != UINT32_MAX) const uintptr_t cookie = 0xbf58476d1ce4e5b9UL; #else @@ -64,10 +80,17 @@ static bool chance(size_t perc, random_t r) { } static void* alloc_items(size_t items, random_t r) { - if (chance(1, r)) items *= 100; // 1% huge objects; + if (chance(1, r)) { + if (chance(1, r) && allow_large_objects) items *= 1000; // 0.01% giant + else if (chance(10, r) && allow_large_objects) items *= 100; // 0.1% huge + else items *= 10; // 1% large objects; + } if (items==40) items++; // pthreads uses that size for stack increases - uintptr_t* p = (uintptr_t*)mi_malloc(items*sizeof(uintptr_t)); - for (uintptr_t i = 0; i < items; i++) p[i] = (items - i) ^ cookie; + if (use_one_size>0) items = (use_one_size/sizeof(uintptr_t)); + uintptr_t* p = (uintptr_t*)custom_malloc(items*sizeof(uintptr_t)); + if (p != NULL) { + for (uintptr_t i = 0; i < items; i++) p[i] = (items - i) ^ cookie; + } return p; } @@ -82,7 +105,7 @@ static void free_items(void* p) { } } } - mi_free(p); + custom_free(p); } @@ -91,12 +114,12 @@ static void stress(intptr_t tid) { uintptr_t r = tid ^ 42; const size_t max_item = 128; // in words const size_t max_item_retained = 10*max_item; - size_t allocs = 25*N*(tid%8 + 1); // some threads do more + size_t allocs = 25*SCALE*(tid%8 + 1); // some threads do more size_t retain = allocs/2; void** data = NULL; size_t data_size = 0; size_t data_top = 0; - void** retained = (void**)mi_malloc(retain*sizeof(void*)); + void** retained = (void**)custom_malloc(retain*sizeof(void*)); size_t retain_top = 0; while (allocs>0 || retain>0) { @@ -105,7 +128,7 @@ static void stress(intptr_t tid) { allocs--; if (data_top >= data_size) { data_size += 100000; - data = (void**)mi_realloc(data, data_size*sizeof(void*)); + data = (void**)custom_realloc(data, data_size*sizeof(void*)); } data[data_top++] = alloc_items((pick(&r) % max_item) + 1, &r); } @@ -121,7 +144,7 @@ static void stress(intptr_t tid) { data[idx] = NULL; } if (chance(25, &r) && data_top > 0) { - // 25% transfer-swap + // 25% exchange a local pointer with the (shared) transfer buffer. size_t data_idx = pick(&r) % data_top; size_t transfer_idx = pick(&r) % TRANSFERS; void* p = data[data_idx]; @@ -136,8 +159,8 @@ static void stress(intptr_t tid) { for (size_t i = 0; i < data_top; i++) { free_items(data[i]); } - mi_free(retained); - mi_free(data); + custom_free(retained); + custom_free(data); //bench_end_thread(); } @@ -152,25 +175,29 @@ int main(int argc, char** argv) { if (argc>=3) { char* end; long n = (strtol(argv[2], &end, 10)); - if (n > 0) N = n; + if (n > 0) SCALE = n; } - printf("start with %i threads with a %i%% load-per-thread\n", THREADS, N); + printf("start with %i threads with a %i%% load-per-thread\n", THREADS, SCALE); //int res = mi_reserve_huge_os_pages(4,1); //printf("(reserve huge: %i\n)", res); - //bench_start_program(); + //bench_start_program(); + + // Run ITER full iterations where half the objects in the transfer buffer survive to the next round. mi_stats_reset(); - for (int i = 0; i < ITER; i++) { - memset((void*)transfer, 0, TRANSFERS * sizeof(void*)); + uintptr_t r = 43; + for (int n = 0; n < ITER; n++) { run_os_threads(THREADS); for (int i = 0; i < TRANSFERS; i++) { - free_items((void*)transfer[i]); + if (chance(50, &r) || n+1 == ITER) { // free all on last run, otherwise free half of the transfers + void* p = atomic_exchange_ptr(&transfer[i], NULL); + free_items(p); + } } } -#ifndef NDEBUG - mi_collect(false); -#endif + mi_collect(false); + mi_collect(true); mi_stats_print(NULL); //bench_end_program(); return 0; @@ -187,8 +214,8 @@ static DWORD WINAPI thread_entry(LPVOID param) { } static void run_os_threads(size_t nthreads) { - DWORD* tids = (DWORD*)malloc(nthreads * sizeof(DWORD)); - HANDLE* thandles = (HANDLE*)malloc(nthreads * sizeof(HANDLE)); + DWORD* tids = (DWORD*)custom_malloc(nthreads * sizeof(DWORD)); + HANDLE* thandles = (HANDLE*)custom_malloc(nthreads * sizeof(HANDLE)); for (uintptr_t i = 0; i < nthreads; i++) { thandles[i] = CreateThread(0, 4096, &thread_entry, (void*)(i), 0, &tids[i]); } @@ -198,8 +225,8 @@ static void run_os_threads(size_t nthreads) { for (size_t i = 0; i < nthreads; i++) { CloseHandle(thandles[i]); } - free(tids); - free(thandles); + custom_free(tids); + custom_free(thandles); } static void* atomic_exchange_ptr(volatile void** p, void* newval) { @@ -220,7 +247,7 @@ static void* thread_entry(void* param) { } static void run_os_threads(size_t nthreads) { - pthread_t* threads = (pthread_t*)mi_malloc(nthreads*sizeof(pthread_t)); + pthread_t* threads = (pthread_t*)custom_malloc(nthreads*sizeof(pthread_t)); memset(threads, 0, sizeof(pthread_t)*nthreads); //pthread_setconcurrency(nthreads); for (uintptr_t i = 0; i < nthreads; i++) { @@ -229,6 +256,7 @@ static void run_os_threads(size_t nthreads) { for (size_t i = 0; i < nthreads; i++) { pthread_join(threads[i], NULL); } + custom_free(threads); } static void* atomic_exchange_ptr(volatile void** p, void* newval) { From 94bfb4772575d43bb11247b957ee5c3741a97a1a Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 20 Nov 2019 12:59:26 -0800 Subject: [PATCH 42/57] update stress test for more realisitic size distribution --- test/test-stress.c | 87 ++++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/test/test-stress.c b/test/test-stress.c index b6ceaa0a..6b2fb8c4 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -7,7 +7,7 @@ terms of the MIT license. /* This is a stress test for the allocator, using multiple threads and transferring objects between threads. This is not a typical workload but uses a random linear size distribution. Timing can also depend on - (random) thread scheduling. Do not use this test as a benchmark! + (random) thread scheduling. Do not use this test as a benchmark! */ #include @@ -17,10 +17,12 @@ terms of the MIT license. #include #include +// > mimalloc-test-stress [THREADS] [SCALE] [ITER] +// // argument defaults -static int THREADS = 32; // more repeatable if THREADS <= #processors -static int SCALE = 12; // scaling factor -static int ITER = 50; // N full iterations re-creating all threads +static int THREADS = 32; // more repeatable if THREADS <= #processors +static int SCALE = 50; // scaling factor +static int ITER = 10; // N full iterations re-creating all threads // static int THREADS = 8; // more repeatable if THREADS <= #processors // static int SCALE = 100; // scaling factor @@ -56,21 +58,21 @@ typedef uintptr_t* random_t; static uintptr_t pick(random_t r) { uintptr_t x = *r; - #if (UINTPTR_MAX > UINT32_MAX) - // by Sebastiano Vigna, see: +#if (UINTPTR_MAX > UINT32_MAX) + // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; x ^= x >> 27; x *= 0x94d049bb133111ebUL; x ^= x >> 31; - #else - // by Chris Wellons, see: +#else + // by Chris Wellons, see: x ^= x >> 16; x *= 0x7feb352dUL; x ^= x >> 15; x *= 0x846ca68bUL; x ^= x >> 16; - #endif +#endif *r = x; return x; } @@ -81,13 +83,13 @@ static bool chance(size_t perc, random_t r) { static void* alloc_items(size_t items, random_t r) { if (chance(1, r)) { - if (chance(1, r) && allow_large_objects) items *= 1000; // 0.01% giant - else if (chance(10, r) && allow_large_objects) items *= 100; // 0.1% huge - else items *= 10; // 1% large objects; + if (chance(1, r) && allow_large_objects) items *= 10000; // 0.01% giant + else if (chance(10, r) && allow_large_objects) items *= 1000; // 0.1% huge + else items *= 100; // 1% large objects; } - if (items==40) items++; // pthreads uses that size for stack increases - if (use_one_size>0) items = (use_one_size/sizeof(uintptr_t)); - uintptr_t* p = (uintptr_t*)custom_malloc(items*sizeof(uintptr_t)); + if (items == 40) items++; // pthreads uses that size for stack increases + if (use_one_size > 0) items = (use_one_size / sizeof(uintptr_t)); + uintptr_t* p = (uintptr_t*)custom_malloc(items * sizeof(uintptr_t)); if (p != NULL) { for (uintptr_t i = 0; i < items; i++) p[i] = (items - i) ^ cookie; } @@ -99,7 +101,7 @@ static void free_items(void* p) { uintptr_t* q = (uintptr_t*)p; uintptr_t items = (q[0] ^ cookie); for (uintptr_t i = 0; i < items; i++) { - if ((q[i]^cookie) != items - i) { + if ((q[i] ^ cookie) != items - i) { fprintf(stderr, "memory corruption at block %p at %zu\n", p, i); abort(); } @@ -111,30 +113,30 @@ static void free_items(void* p) { static void stress(intptr_t tid) { //bench_start_thread(); - uintptr_t r = tid ^ 42; - const size_t max_item = 128; // in words - const size_t max_item_retained = 10*max_item; - size_t allocs = 25*SCALE*(tid%8 + 1); // some threads do more - size_t retain = allocs/2; + uintptr_t r = tid * 43; + const size_t max_item_shift = 5; // 128 + const size_t max_item_retained_shift = max_item_shift + 2; + size_t allocs = 100 * ((size_t)SCALE) * (tid % 8 + 1); // some threads do more + size_t retain = allocs / 2; void** data = NULL; size_t data_size = 0; size_t data_top = 0; - void** retained = (void**)custom_malloc(retain*sizeof(void*)); + void** retained = (void**)custom_malloc(retain * sizeof(void*)); size_t retain_top = 0; - while (allocs>0 || retain>0) { + while (allocs > 0 || retain > 0) { if (retain == 0 || (chance(50, &r) && allocs > 0)) { // 50%+ alloc allocs--; if (data_top >= data_size) { data_size += 100000; - data = (void**)custom_realloc(data, data_size*sizeof(void*)); + data = (void**)custom_realloc(data, data_size * sizeof(void*)); } - data[data_top++] = alloc_items((pick(&r) % max_item) + 1, &r); + data[data_top++] = alloc_items( 1ULL << (pick(&r) % max_item_shift), &r); } else { // 25% retain - retained[retain_top++] = alloc_items(10*((pick(&r) % max_item_retained) + 1), &r); + retained[retain_top++] = alloc_items( 1ULL << (pick(&r) % max_item_retained_shift), &r); retain--; } if (chance(66, &r) && data_top > 0) { @@ -167,36 +169,45 @@ static void stress(intptr_t tid) { static void run_os_threads(size_t nthreads); int main(int argc, char** argv) { - if (argc>=2) { + // > mimalloc-test-stress [THREADS] [SCALE] [ITER] + if (argc >= 2) { char* end; long n = strtol(argv[1], &end, 10); if (n > 0) THREADS = n; } - if (argc>=3) { + if (argc >= 3) { char* end; long n = (strtol(argv[2], &end, 10)); if (n > 0) SCALE = n; } - printf("start with %i threads with a %i%% load-per-thread\n", THREADS, SCALE); + if (argc >= 4) { + char* end; + long n = (strtol(argv[3], &end, 10)); + if (n > 0) ITER = n; + } + printf("start with %d threads with a %d%% load-per-thread and %d iterations\n", THREADS, SCALE, ITER); //int res = mi_reserve_huge_os_pages(4,1); //printf("(reserve huge: %i\n)", res); - //bench_start_program(); + //bench_start_program(); // Run ITER full iterations where half the objects in the transfer buffer survive to the next round. mi_stats_reset(); - uintptr_t r = 43; + uintptr_t r = 43 * 43; for (int n = 0; n < ITER; n++) { run_os_threads(THREADS); for (int i = 0; i < TRANSFERS; i++) { - if (chance(50, &r) || n+1 == ITER) { // free all on last run, otherwise free half of the transfers + if (chance(50, &r) || n + 1 == ITER) { // free all on last run, otherwise free half of the transfers void* p = atomic_exchange_ptr(&transfer[i], NULL); free_items(p); } } + mi_collect(false); +#ifndef NDEBUG + if ((n + 1) % 10 == 0) { printf("- iterations: %3d\n", n + 1); } +#endif } - mi_collect(false); mi_collect(true); mi_stats_print(NULL); //bench_end_program(); @@ -230,11 +241,11 @@ static void run_os_threads(size_t nthreads) { } static void* atomic_exchange_ptr(volatile void** p, void* newval) { - #if (INTPTR_MAX == UINT32_MAX) +#if (INTPTR_MAX == UINT32_MAX) return (void*)InterlockedExchange((volatile LONG*)p, (LONG)newval); - #else +#else return (void*)InterlockedExchange64((volatile LONG64*)p, (LONG64)newval); - #endif +#endif } #else @@ -247,8 +258,8 @@ static void* thread_entry(void* param) { } static void run_os_threads(size_t nthreads) { - pthread_t* threads = (pthread_t*)custom_malloc(nthreads*sizeof(pthread_t)); - memset(threads, 0, sizeof(pthread_t)*nthreads); + pthread_t* threads = (pthread_t*)custom_malloc(nthreads * sizeof(pthread_t)); + memset(threads, 0, sizeof(pthread_t) * nthreads); //pthread_setconcurrency(nthreads); for (uintptr_t i = 0; i < nthreads; i++) { pthread_create(&threads[i], NULL, &thread_entry, (void*)i); From 4d4a2885f5ef5d0b3db8de149b472380f495e729 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Wed, 20 Nov 2019 13:19:17 -0800 Subject: [PATCH 43/57] use atomic read/write on the page->heap field where concurrent interaction is possible --- src/alloc.c | 2 +- src/page-queue.c | 6 +++--- src/page.c | 21 ++++++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index d2319f82..c4863115 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -235,7 +235,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc } else { // racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`) - mi_heap_t* heap = page->heap; + mi_heap_t* heap = (mi_heap_t*)mi_atomic_read_ptr(mi_atomic_cast(void*, &page->heap)); mi_assert_internal(heap != NULL); if (heap != NULL) { // add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity) diff --git a/src/page-queue.c b/src/page-queue.c index 4af70b50..95443a69 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -260,7 +260,7 @@ static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { page->heap->page_count--; page->next = NULL; page->prev = NULL; - page->heap = NULL; + mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL); mi_page_set_in_full(page,false); } @@ -274,7 +274,7 @@ static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_ (mi_page_is_in_full(page) && mi_page_queue_is_full(queue))); mi_page_set_in_full(page, mi_page_queue_is_full(queue)); - page->heap = heap; + mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), heap); page->next = queue->first; page->prev = NULL; if (queue->first != NULL) { @@ -338,7 +338,7 @@ size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue // set append pages to new heap and count size_t count = 0; for (mi_page_t* page = append->first; page != NULL; page = page->next) { - page->heap = heap; + mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), heap); count++; } diff --git a/src/page.c b/src/page.c index aaf1cb91..a8115d27 100644 --- a/src/page.c +++ b/src/page.c @@ -343,18 +343,24 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { mi_assert_internal(pq == mi_page_queue_of(page)); mi_assert_internal(page->heap != NULL); - _mi_page_use_delayed_free(page,MI_NEVER_DELAYED_FREE); +#if MI_DEBUG > 1 + mi_heap_t* pheap = (mi_heap_t*)mi_atomic_read_ptr(mi_atomic_cast(void*, &page->heap)); +#endif + + // remove from our page list + mi_segments_tld_t* segments_tld = &page->heap->tld->segments; + mi_page_queue_remove(pq, page); + + // page is no longer associated with our heap + mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL); + #if MI_DEBUG>1 // check there are no references left.. - for (mi_block_t* block = (mi_block_t*)page->heap->thread_delayed_free; block != NULL; block = mi_block_nextx(page->heap->cookie,block)) { + for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap->cookie, block)) { mi_assert_internal(_mi_ptr_page(block) != page); } #endif - // and then remove from our page list - mi_segments_tld_t* segments_tld = &page->heap->tld->segments; - mi_page_queue_remove(pq, page); - // and abandon it mi_assert_internal(page->heap == NULL); _mi_segment_page_abandon(page,segments_tld); @@ -755,7 +761,8 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { mi_assert_internal(_mi_page_segment(page)->page_kind==MI_PAGE_HUGE); mi_assert_internal(_mi_page_segment(page)->used==1); mi_assert_internal(_mi_page_segment(page)->thread_id==0); // abandoned, not in the huge queue - page->heap = NULL; + mi_atomic_write_ptr(mi_atomic_cast(void*, &page->heap), NULL); + if (page->block_size > MI_HUGE_OBJ_SIZE_MAX) { _mi_stat_increase(&heap->tld->stats.giant, block_size); _mi_stat_counter_increase(&heap->tld->stats.giant_count, 1); From 1674d551ffe5dfffd978737786fe8f94ec7b258c Mon Sep 17 00:00:00 2001 From: daan Date: Wed, 20 Nov 2019 20:45:31 -0800 Subject: [PATCH 44/57] add verbose message with secure build level --- src/init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/init.c b/src/init.c index 081e7ce7..81413aa9 100644 --- a/src/init.c +++ b/src/init.c @@ -470,6 +470,7 @@ void mi_process_init(void) mi_attr_noexcept { #if (MI_DEBUG) _mi_verbose_message("debug level : %d\n", MI_DEBUG); #endif + _mi_verbose_message("secure level: %d\n", MI_SECURE); mi_thread_init(); mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) } From 74dbfc30bebc2e7e48e88edf3cf66b35c057b16f Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 15:21:23 -0800 Subject: [PATCH 45/57] improved security by encoding NULL values; double free mitigation on by default; more precise free list corruption detection --- CMakeLists.txt | 15 ++++-------- include/mimalloc-internal.h | 48 ++++++++++++++++++++++++++----------- include/mimalloc-types.h | 6 ++--- src/alloc.c | 4 ++-- src/page.c | 6 ++--- test/main-override-static.c | 6 ++--- 6 files changed, 49 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa9c126f..467fad95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,7 @@ set(CMAKE_CXX_STANDARD 17) option(MI_OVERRIDE "Override the standard malloc interface" ON) option(MI_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_DEBUG_FULL "Use full internal heap invariant checking in DEBUG mode" OFF) -option(MI_SECURE "Use security mitigations (like guard pages, allocation randomization, and free-list corruption detection)" OFF) -option(MI_SECURE_FULL "Use full security mitigations, may be more expensive (includes double-free mitigation)" OFF) +option(MI_SECURE "Use full security mitigations (like guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_LOCAL_DYNAMIC_TLS "Use slightly slower, dlopen-compatible TLS mechanism (Unix)" OFF) @@ -70,15 +69,9 @@ if(MI_OVERRIDE MATCHES "ON") endif() endif() -if(MI_SECURE_FULL MATCHES "ON") - message(STATUS "Set full secure build (may be more expensive) (MI_SECURE_FULL=ON)") +if(MI_SECURE MATCHES "ON") + message(STATUS "Set full secure build (MI_SECURE=ON)") list(APPEND mi_defines MI_SECURE=4) - set(MI_SECURE "ON") -else() - if(MI_SECURE MATCHES "ON") - message(STATUS "Set secure build (MI_SECURE=ON)") - list(APPEND mi_defines MI_SECURE=3) - endif() endif() if(MI_SEE_ASM MATCHES "ON") @@ -92,7 +85,7 @@ if(MI_CHECK_FULL MATCHES "ON") endif() if(MI_DEBUG_FULL MATCHES "ON") - message(STATUS "Set debug level to full invariant checking (MI_DEBUG_FULL=ON)") + message(STATUS "Set debug level to full internal invariant checking (MI_DEBUG_FULL=ON)") list(APPEND mi_defines MI_DEBUG=3) # full invariant checking endif() diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 73849337..452f0b68 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -275,14 +275,20 @@ static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { return segment; } -// Get the page containing the pointer -static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { +// used internally +static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) { // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; mi_assert_internal(diff >= 0 && diff < MI_SEGMENT_SIZE); uintptr_t idx = (uintptr_t)diff >> segment->page_shift; mi_assert_internal(idx < segment->capacity); mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0); + return idx; +} + +// Get the page containing the pointer +static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { + uintptr_t idx = _mi_segment_page_idx_of(segment, p); return &((mi_segment_t*)segment)->pages[idx]; } @@ -373,53 +379,67 @@ static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { // ------------------------------------------------------------------- // Encoding/Decoding the free list next pointers +// Note: we pass a `null` value to be used as the `NULL` value for the +// end of a free list. This is to prevent the cookie itself to ever +// be present among user blocks (as `cookie^0==cookie`). // ------------------------------------------------------------------- static inline bool mi_is_in_same_segment(const void* p, const void* q) { return (_mi_ptr_segment(p) == _mi_ptr_segment(q)); } -static inline mi_block_t* mi_block_nextx( uintptr_t cookie, const mi_block_t* block ) { +static inline bool mi_is_in_same_page(const void* p, const void* q) { + mi_segment_t* segmentp = _mi_ptr_segment(p); + mi_segment_t* segmentq = _mi_ptr_segment(q); + if (segmentp != segmentq) return false; + uintptr_t idxp = _mi_segment_page_idx_of(segmentp, p); + uintptr_t idxq = _mi_segment_page_idx_of(segmentq, q); + return (idxp == idxq); +} + +static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, uintptr_t cookie ) { #ifdef MI_ENCODE_FREELIST - return (mi_block_t*)(block->next ^ cookie); + mi_block_t* b = (mi_block_t*)(block->next ^ cookie); + if (mi_unlikely((void*)b==null)) { b = NULL; } + return b; #else - UNUSED(cookie); + UNUSED(cookie); UNUSED(null); return (mi_block_t*)block->next; #endif } -static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, const mi_block_t* next) { +static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, uintptr_t cookie) { #ifdef MI_ENCODE_FREELIST + if (mi_unlikely(next==NULL)) { next = (mi_block_t*)null; } block->next = (mi_encoded_t)next ^ cookie; #else - UNUSED(cookie); + UNUSED(cookie); UNUSED(null); block->next = (mi_encoded_t)next; #endif } static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { #ifdef MI_ENCODE_FREELIST - mi_block_t* next = mi_block_nextx(page->cookie,block); + mi_block_t* next = mi_block_nextx(page,block,page->cookie); // check for free list corruption: is `next` at least in our segment range? - // TODO: it is better to check if it is actually inside our page but that is more expensive - // to calculate. Perhaps with a relative free list this becomes feasible? - if (next!=NULL && !mi_is_in_same_segment(block, next)) { + // TODO: check if `next` is `page->block_size` aligned? + if (next!=NULL && !mi_is_in_same_page(block, next)) { _mi_fatal_error("corrupted free list entry of size %zub at %p: value 0x%zx\n", page->block_size, block, (uintptr_t)next); next = NULL; } return next; #else UNUSED(page); - return mi_block_nextx(0, block); + return mi_block_nextx(page,block,0); #endif } static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { #ifdef MI_ENCODE_FREELIST - mi_block_set_nextx(page->cookie,block,next); + mi_block_set_nextx(page,block,next, page->cookie); #else UNUSED(page); - mi_block_set_nextx(0, block, next); + mi_block_set_nextx(page,block, next,0); #endif } diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 893dcd67..9c5d3c19 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -26,16 +26,16 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_SECURE 1 // guard page around metadata // #define MI_SECURE 2 // guard page around each mimalloc page // #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) -// #define MI_SECURE 4 // experimental, may be more expensive: checks for double free. (cmake -DMI_SECURE_FULL=ON) +// #define MI_SECURE 4 // checks for double free. (may be more expensive) #if !defined(MI_SECURE) -#define MI_SECURE 0 +#define MI_SECURE 4 #endif // Define MI_DEBUG for debug mode // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. // #define MI_DEBUG 2 // + internal assertion checks -// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_CHECK_FULL=ON) +// #define MI_DEBUG 3 // + extensive internal invariant checking (cmake -DMI_DEBUG_FULL=ON) #if !defined(MI_DEBUG) #if !defined(NDEBUG) || defined(_DEBUG) #define MI_DEBUG 2 diff --git a/src/alloc.c b/src/alloc.c index c4863115..e68b48d2 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -157,7 +157,7 @@ static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, con } static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { - mi_block_t* n = mi_block_nextx(page->cookie, block); // pretend it is freed, and get the decoded first field + mi_block_t* n = mi_block_nextx(page, block, page->cookie); // pretend it is freed, and get the decoded first field if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? (n==NULL || mi_is_in_same_segment(block, n))) // quick check: in same segment or NULL? { @@ -242,7 +242,7 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc mi_block_t* dfree; do { dfree = (mi_block_t*)heap->thread_delayed_free; - mi_block_set_nextx(heap->cookie,block,dfree); + mi_block_set_nextx(heap,block,dfree, heap->cookie); } while (!mi_atomic_cas_ptr_weak(mi_atomic_cast(void*,&heap->thread_delayed_free), block, dfree)); } diff --git a/src/page.c b/src/page.c index a8115d27..437cd0a5 100644 --- a/src/page.c +++ b/src/page.c @@ -283,7 +283,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) { // and free them all while(block != NULL) { - mi_block_t* next = mi_block_nextx(heap->cookie,block); + mi_block_t* next = mi_block_nextx(heap,block, heap->cookie); // use internal free instead of regular one to keep stats etc correct if (!_mi_free_delayed_block(block)) { // we might already start delayed freeing while another thread has not yet @@ -291,7 +291,7 @@ void _mi_heap_delayed_free(mi_heap_t* heap) { mi_block_t* dfree; do { dfree = (mi_block_t*)heap->thread_delayed_free; - mi_block_set_nextx(heap->cookie, block, dfree); + mi_block_set_nextx(heap, block, dfree, heap->cookie); } while (!mi_atomic_cas_ptr_weak(mi_atomic_cast(void*,&heap->thread_delayed_free), block, dfree)); } @@ -356,7 +356,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { #if MI_DEBUG>1 // check there are no references left.. - for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap->cookie, block)) { + for (mi_block_t* block = (mi_block_t*)pheap->thread_delayed_free; block != NULL; block = mi_block_nextx(pheap, block, pheap->cookie)) { mi_assert_internal(_mi_ptr_page(block) != page); } #endif diff --git a/test/main-override-static.c b/test/main-override-static.c index 19712411..b04bfeef 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -15,9 +15,9 @@ int main() { mi_version(); // detect double frees and heap corruption - //double_free1(); - //double_free2(); - //corrupt_free(); + // double_free1(); + // double_free2(); + // corrupt_free(); void* p1 = malloc(78); void* p2 = malloc(24); From bc1ff7e7fd5b3822d36b06fcfb532efd422286ef Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 15:28:10 -0800 Subject: [PATCH 46/57] update windows redirect, issue #170 --- bin/mimalloc-redirect.dll | Bin 46592 -> 55808 bytes bin/mimalloc-redirect.lib | Bin 2874 -> 2874 bytes bin/mimalloc-redirect32.dll | Bin 33792 -> 39424 bytes bin/mimalloc-redirect32.lib | Bin 2928 -> 2928 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index a1daf316b56cb3c885dc48f1c5fd1ef0ffa302b9..77e40c1391dc4e3f6fa08c30c1fe461df1b0fa80 100644 GIT binary patch literal 55808 zcmeHw3w)eqneRIxv1w_V01X61_}WhCU}>3Bg!Cqpl9aEQwnG}&h}FzAnKmQ2408zs zr`6Q7Yq}wOBI0kmuCD1ly+`+0iXvyg+6G0eoLx8W>M@7ab0$0GSdSNitLFTl_wvm* z-%Qdp2M;7petEz5_Po#SectE2e3x1GNH@ERF_w!kGQ!vy#PsLl*Rd}bP(ACGug_xt zW%~E#oUv7YZ%%cCKcoeMEn9=$Cauof+}sk@Hv6<-q*?PfYvmg@YE3QmzQqLvGwc%e z65pqP7RkN%tP1b-ym7jf=ihjCJK|q7#ZL!#eDQRc$Jae8;{VO_RsOmLYO6SPS1?xp zm($qug+Kh5D!atyYFFpXDq@eITy@wYiV-`a#DF=T!%!7<* zq@wh^C@G3E_6I=sblaGN8u!@PO2PwbnnJf&~~W?`WHrV-i-Yyvu$Qrd87Ke`Y$n3T5lnwBn8RWL#wve-;CziO0IVZ? z&?(3F!!)?nsU_LmEA%(C<-Lpz{Zhka@C*_rvc+;bByFD<~Oi?Z<>aS6fDhF%~VADNEJ zS#-tFm#@aui(7>qC*FR zPJMT-of4(Qb3Ub9lqwRbk$%d!^!9D`A{Isq0*2s)r>A!S7`c&{!jR9?K$3Wl8apAG zBaMv=l8T~q{z{0d#v)MbHoJpGWVU1_(icrjdHWEQ@d$W8qH(imhb8NX~K zA(yWM=AzVJqMy&j@MA1-wMCImeOazORHBrv=~wmZoi7qIj*k4qYvZGxBqOw#H`GQ( zlbNRIRYm9pwksCCPv5cX_WNOIXcoE~R`tH~WQ{Y?|J5+-tA3z&en;;tE&vD_!%P@V z9z%vJGw-)+Y%K9bwE8)w1laKZL|)kNmFi6yOnCTNd*7g1cp!udQd zYQ3fKg>t(Ga?BvTdEzuhf5jR9Eux{vZ~?bf;7NydmkJcE)k|L^b8zc#hV31SiiB!yq2KK~EnY)K_M8F>ZB9NvaK$Sajz`{l=Crvav3tp=L{e{%E#D$Y+ z2lQn~g>HrJ`%ueueZ!vWm7RHW(XeviOKUqXRCeVR@P@gz zvX1hikqiyrI7^7D?Q=S2+lD@lp$e)?iLz8O(DH*hE*{ zk5b|`zhgGMT3#u#!j2bHsw?jn0j42GR-8!=lU1T4^9%VPT1lEI5^3JSR89^QWz$!@ z7`ch+4Zq2OW%m{(&hR=D{K6MA(N)?p(WhAmH1zoLVdx#Uh2F!3`cbC}S9}*LFkz5K z)u0c?J*tKT6V8Gv?3Cpdb^?Kw*zM@VjFXjO!W5fbT`8Hc9=|b{k<$eam{K5E;R`5* zd%aNcsFlN<{U;z%oLl||@N7Xz{ z1I87;T~GyAUcr^14wm8X;_E~iw5hG=7yal`TnQvJF#>Vo83WIa1|B|%ay)qk9{a+6 zc!KdO*raPpTIJ*Y|6AGroCRV>6RsjXMZI5m`t^+n<*JTu0uo4$VFe%-|)^T@yJV*Ow; zYPo)}54{BIRv>%{G1MnZ+$=I}X>4dOWx37rMMJ6y^3E2k3GP63X$9QM-MUy!xcPiR zW<3Y?eisb~Ej|Oo3dW9J!D<9G@waH0y%0lJI9BwB#f-z-VAY{<3^3PXO3XoGrkJ)0 zKjVcYfZaT}g#v0BG$i?JRPh ziZC~iKKpb3+KO5X9HG_@fWcbo-0y?a$lWs96}M{1i{deKf>S-a+M& zyrB_{e?rfX1eV6yhi!#Be?(AHk>x~yuhp+e0iR9caB>`XN>yEf64xbhEZMeRaU+re-B&E%Z}{Nn8hg^vSYuLSWmk-Ge-dLTa+lt@)836v(%WO$rOLD_>vRnh z*EQtzYvQl{3O32?VN)2!!0M4o3Pk2&0lBLdLn#0X!dpdV zaq}q>=5wc0!bZchRjcp}*q2U+RDA8B6R2K?B4XA}qgpE`TfP%lZdezn!V~{7N$~{$ zOGJJ{tM!p>WAP1TKo{jOv8A;7DQDZSq4XjQxAHd!s48(S;amcrJDt;nK`$ki$}_UG z^y|dD-q{~t3ziam|KW^p{|(r?>G#k>SKbevqxyo^mKl4Z=vXVeQC;+ut{BaKkK(d@ zI0TMIe>XA$mwnBz$KkTMzozLy5ZZBZ03!gV7e>V3pyP;bg}a`}V{A*=nOvLR^^846 z9oto~glMgL!&pL_78TT(uasdU8YMKi;!-PuB0}){iv|L*iOUCdheF^mAqbZm2R?-_ z?93%DV2d}JIQl6LY2#l6LLC(}fNm!|d;r~!1ljfNVgcl&EU$1XQskiL5U*HO6z=+K z)LZ%OCiIm_esLryk=&(_+-2r>o{6Yc-a~@bL6|Ai^u+m;u2AY7dB6W!Ix~GA z1vB;}GnGcmO4YK`%*dLF7BFwIBXoajtW@&MUtY2okMK&n2dqlQfW_v{TDVgo1?D?JcKE(9LC%d_oU5ZHY zRbBvK1v3c#;_Kt!19KGCn%nGAR=AVT(p|+56E@6?(fC{rkypG92N~_F3J|gI^)%Y& z!By~eDM$3n|3v~7AM!Bv90?x33hjV60K5SvCce37aWgfg#7w9nA5`M!(DGc~aag!5 zaT`JBM6O3&CDk289gK|bPq75G_)98%2BlbrKZzzlGz-Q5nTlUSF$da9xfTKRkN~;? zyg!H}meLTNeDg^0YUD9VhxHw22T+imLf_H#z)O}JH)2|)y9}@iC;Bb^k3WZr!6tFb z!G`YtPY;kqj(Up#_fi}%$YR{0T>5f6LTyuk#l4o|-(V^g23rLX>C`l`VD4!&vOF+= z+X&?CGR@RLdzqsnu2)HK7tzC>fW=?`*`Ji2bw9b|BKa&@$Ob5GkMWyKPGj#_*@bgk zIBgJ;ON{nBP3%{+N8b+xh${b5$TM)Uhs=*wF)M{E(VQr07b_Grna88Jw zF`W|*et<}#MP;W88~kc)KvB+jJC2c}!qYJ*d^!g>=PJUU^(^r?v4NO!eubS0PVb2! zzGG{kR4LyCR)SM7SNxMd;?U#e_=8{&bq+}BVB4-E=j4J)LP26y;pHA?oM2-@4$$r#z8v73k&W%3WyloOy)-N zP{%b;XFByC3{s8zj1r<&V>NeEB8!GZOGR2NRr_L$WpbhhIxY1h#H zpWgNH8@$#**?zIXILrwblF)qW$DK0#iTU?)gJOnuksriqBz65P=#F~1O+vZi+awe} zv&4FspyE`kj*4)DOJRo#E!B7P^R|F7M9{5#k4%?37w$VyPm8-aq;={7H=G6x6v{r6 zPpBV_;YbAs0G;KwfCb(`@Q(IU7KY=XRCFc_e}pGHUEm}IxklxhPSCfCB@Hg2ns(pQha@_$y;tGv(A%D zjy5@{ZDRvjF#0k;F&}|np07r%!|<>lFg&bNTDpsX9q}lDh3kf6a|qY%|9D1bO3A;1 z%J}qT@~1m4@+Bvj?B!z1qsTYy#Q9Ep4;50!h|WXGMJboEBY7xKAUaj(%8oFWsVO~& z04=(e9hu;9Tt{9cs$pgNI>A!5(*;U6#z~rIjp4Aw?Fr)^r-U!zj;FBw%Mt}YxN>&p zc3ALuXo|w+DK2Py!%z4Oy;Wktv2+U61}et|#$5Ppu1?;`Awh9pE(J6|jue$}$ep=6 zER|7e6irMUDia6sJI<22$Ja}5pWjN_H@I}gZ}_orJACd}NGioogK+Dlx{=~jA2nNc-q191gxE)5GaLWJzY30ZYQTF~A1VH%r2#$Y zMe62@72aDlWS88|9AqV>g*(5h42}8;h53a$mEoWTH+3iBKn9B%Ztif+Sfb9NQdB{I z`ffd6IBeP@@e#qN&V0&%S`rH~3X8xg;jAbf9Hjz*0|x63_r+eJ z@tfB}SF=2j*|Ff*aQ#_;_+)nVLuJANArijJ1E$?UR&>N&c+P3Ar zu5vUA;ob`V2MXf92O?9mnL@{Cq$i}l+hdx^x1gqhU>c~*LCX|$r9uNjB*h9YkmGLTJ(+hOL{9y`RN{e30!f zh;|p-Mm~lgHHt$=sD?J}^l;eNkAo=xVS3mNq%_?1v#m-1N#stoc;>4ZUeO-Qp@+Tj zIGown4;=vb(3fygEnqGR)exb&T~!EFow>I5SRO>krBnV~4lE|C#HK`zf;Ax-Atd*z z3KKuW^CJ#boWt?7<7_iBTa~>>;0D1&4qpV7ylp+M^v5RvIJXiqF{nLSf~|;3=QG>| zohNe`hER*Mva<}Z7eryFo*ysxAv(N$)Q!5d8Pj@0Zu+qvoGrYEg&?1h1uLyNiH{IW z>8w_*?943aQSLGeZh8PC=hc4#X&6wcsFB{8oTm~eg=uoVoe#pV_Qz|e@R&bqQog2AEa7pTl-v_1>+SmC~h!^}HY)z&1Ff*VC{E~ZjjM=v-^mWpSThe!< z)6h|QL;uRvLyk^PIdCZO7(}AR^dK4aGxsgL!;vZ_GuHU*81~{Reo`{lYw@*UR9PY~ za$x&ulyEbqlKBdTTd?Ci?fjBsiJLHZpgDW@Y+GX4*YgYl@jqlBFa)-=_IMsYwb37a z8Fa7?-<><-vM#~YTbEC!=#$TI*nH8#{$^5|i)I&Y?^^un|3H5sE4A64dBs1<{?Add z^kq7A(Mbvy@9p@K|27WWOW#UHlM@?XbKv2T81d!S{kRNstzxrIF?o@~o&SQ4NUyu9 zUp`St=v;uN^DkHK_PJyQF09ruM(}VDEX?MDg^@D&T;xjpGY2f}+D>@6OzlwzE8O)A zk)(Vj<}d6bI(YsmdmG}oeL5<^rH4$`!@|B)GUv-}r2yl%a81OaEDw6f<|6V+_t2rM zgm;XXO0hR-qo$md<{wpSd3tkNyC+Qh$UB93nn&H=Q9D?);8|tL)5|thyX&yq40) z1U1a`^ss2*&d;f47-AUivn44PS5b}LIh{ssirPpXvj5alY5qxhfPmYCkO!O)QBcN1 z;X-ecNhO>cXPpHTQTBULA>qAC_%g)H{wm{w*$LOUOw zc2wDOw`0A95{Lxl(>CuApu{mOZSblS-%qT^F{x$)1_$nz3Ji{se(Dooz!~Wr!r(|^ zs5da=QA?J1l@3ug98wZSp_}pU`Npz0zLYd8N%)Ix)4-M|ciXf(V?#8sXk2-elY&`b zxCw(IuEj)1dMuDG%0b<|OfTZfB^L>7H7s%qnI?gku*h{hO>n#QA~bysP*9JSiTc-n^op``2v4)Cl zks@mCR$k31HawKqNT%c%kQ*>ML8K#gj;HZWGG-8$oO4#G?=%aiPrWiS^0rv;SiL8o zFU5*KpWg@}`vGHkk{I&Styl<*+2jsoJ2(gg1jd*T^};}}gXW%fL33|?7pnrCW3j|7 z5}Y4I2#cqoV5gw~Nw4H_WAXf}6)tJGf!ICGfd-=iQ7Vu!^PCW||&pm@?~` zlON?`wxx9yO|@YsO|z!LtNJm=LPGRqe>?|nM~Fy9+(j5qa=YLok;pibWE^r*cMwL0 zg-lN3$22M&taXK@I)&h8v0xDd#gOG@BA+yMB)+NoAUS@=-f4~9@1xc7_c1a}3QDb` z3JwWzpin_vd$_Gb$NemfN7jsj&e{F2mLIraEziF!#(@OwoY^0DQ4tNG#351_kkP38 z20?!bRaoP#BlE-jP3!aP=wOy-y9(~X!<_Ml0b_c4Ao4y^ScAM1{dvOjtA31O;}H1r zd)3i&WG+uv6`#6bLG+~dHc%vyG!jT-Vewq;(C=V4m=60^kqES8Cpfs~1t|Q6EY#!o zQsb`TFYp4`BW=(LS7IAV4;XD-$+r8AwpWR^er$?)z!>hV8jHb@$&gYm#%Cc^(t#PIo?ao-6)0=b$g7;QeWf%_Stb z;v`&GK`v<`$RPzl5r6Maq_I}O(KMwy-9-ZFY7)*s%oaF4j zhM1pJSa-82`gDj!9kNkdhDMp-Yb3lo1AO}UkwoUnM(@ke$YdRv=*96qCJ8l&*eaA& zehK|W?vGHF4-VSpFcTd(ytX#;Iqg>jjZ~ep1GxHUrwjr9Eo{T zgUtg8^uquEbVzEpm#N~#})=s5WG zkJ6M2oq^xK9SeTE{N6Q+y^n+68%Jq68ozIiH9j-gpRk>Oz#26}uZ$+2$sbG~OFvw` z3`DK{`RmN&G5(dkTkQesS1XcVjDjR-&ST(9b#gx&1=SVt^V3m~T%I2|X?e(nqdV*e zNm9OQ)v32!@ox*v=_e!^R4w}V-!k`-1wPHcW|G$oH2Lc>-=z(DSNtDwo+2hJK4G}x zpQ8hoTN5}OSoIh5j9<8fTc6?ZM~cf(eAy|5>pPVmCQ95Rcu$`#aeo5aQ|jJ_4eD~a z{c~)t03hi9^p=eDUq&&}&$x~=nOsUTIlj$I@slApPO8t*o&zkoA#z9r zH~ji0zrMw<|AFh35X!=4HIUUnRs&fLWHpf0Kvn};4P-Tt)j(DQSq)@0kkvp|1AmSh zz&Ta+@156x`c^tUe6?+^Ow!kbzm;Z04U z*1AQ}yYJdm=i695)hl{b|7qFob;NirB6qcnH=AODr}E+) zR%~0rY%_D%OxrqUyC#QSW4lh!c?tCj-wiqJ2HWQa9V(~xgzxGccD3z;LXLgNU!Q}Y zI9QadC;16Z;UPI!%k~sg{?jr)SLQd%nCeLnRIl(*OnIe0!b@l;Q+!Jgzq8{ zh8+CDOfE~C`xe(XHlnna&xqb|OR%=SrP-HS;&1kcsYK0}we|jxcXK0$GRl0-qZHMK zeZeN;A^VrrKvn~pG~nc8l%L2$8oBhpO6+|ZL zjICR{uB@_h!+o{oYd4myuB@o7swiJuRdFBxa3Pwc)o*yXqN-|bd4)u`WJPt=LlrAN zQn7Kx27J6A316{36FTNxv8JqYW5tT~8&+&sv&IAuw5vC4RA`fU+*NBgRIROE`$$D? z<%Y8I+6`6Z6;+IlO4nUheV<;tcAa}eRrSU(;MQ!Yy3Zu772o=nR_#tF{+BG{l5URp z8|#;A^FvZ@)gRo6qDYA@~)=xkgqNhRHbRhtv(WjRcHZkxUPW`nD=bmV3t|$9~j-&pjNR#%7NK-%y`JV*p`Ad-ag84@ zG?}O7-GUV*sLjwc2b@=!CdnrVuF>E8sBk9MQUMQok!B_Vvv9+Zos2Nh3?Q)Bs3|Yu4MiWQrZmAo%a%wA)YRPG`vuXBQOcTZBebt$iBf^uJ) zq;f~4yxkL&TdSPd_;`nxYphqw+do0MJ(E=KK_&0ECf08~lT_{!DR1cn@x}`zN8I^IH zMeeD(Zt0eC_Q?2*l3%XV2B+vec0tNHDC1!n_er^nHcU-_S9DIurODVSV~>ePO2PQOpDJ;`lygAFmt^deat}_?e$W4o^ye~m$hc3+>zShc z?UHiV$XMBNY>LXOfKIg7q4`qv@j zslFd`NjYm|9F?(4+PQ0r+W8(S=NTERao;WFR{5zG4@h~3WnA)#u=fj6ZqF3$Z2Zl-p}V*yzA>%ulOpu;*DPu-U$CBN-cj;Sc!mUreb<$*k9Z1 zdz{`yZwR(LuE-D_Q?Y>ahfrPH;PVE+<5s-lYd~|p(yECS!Y}oMx4^eH`|3rbv|N<$3nR7Wh8e46JH=w3aJESvBWp54DmQ2GKC@_)wWB$AV;CYtdxZO!0X>3F% z&CZk|DttuAkWP@Lc1m7eLimjm`8^)<`|GGqvr|zofR!$lY5;F75_Bx3qr%DF7H{8k zyD{`kh}0l9)u-qivWP(}0YFD}fg!H~NzoHmvck&gj-~(P`}`oF^qy=j?9*JXK6RAT zwSZ35e-cA_xxgTGZvb+*ly4A#pJZN(z};Eljj{~z zRlYL%cA^)*C9kLmJpfl)owdDTHL|y%R%r>+a5&e%r|c-DS9yP_ZcS;x^F=t0n~-%MXXm3lFisZm}l0XSY| z5vmcjlc-e@hclH4XRB%@9kLCHSdD1~!Q0Dkd{CJsZ4fyKNDpxmT0eRjA%*> za!f0-0#1x`UdHhmd4p`yzD>F^^rw2pRA=;-MRy#(g16MA;WO1MP7J&zT~a&P2OoKa zpI1raGT;rF2wL`s;^RujiCL#sJeImNI_7%SABu;R@u#fM!e<5;Lq22NWUy}w4U<|$ z|Bqg;)Sm2rgHEc+rS$74FlqY80A~;+;Ot*k16d7ZHIUUnRs&fLWHpf0Kvn};4P-Tt z)j(DQSq)r~2He*$_9X--eOUTh#vVr~Lb?a>#_JfmLwGkuI9Y*y|49Mmq2T#=d|M zB^bo#5j;p=Kz#E;#u|`zA^vNG0Kp)B^mf!E-Gg}EBF6fW_8|TtKD63P@OLn_-^th> zq}dY2Rv-)@?MD1vgbt)HA%1KrW8Fv(B7Wd5{1yVz0mKJC2s%i!yBRxu5A=t$vy`!C z5PFdAL)^Oz`UwLM;`^3ECn)z&jBp8QZ3SaH5lR5lgZS`D;3Ig%i&jAgNV^b+5nM=j zBR+jE`ish4jLj{B-jMbn{?KaZ6X`a@Kf4dO38tK}=MV-#YY_1t@kMke%H8-@y6b-Q zBg)$lpV6UnlozdK>=y`br11?=R)yd}8sF7qpFwCq8sC~^&mshn#`jg(MT98Q_$DD+ zfY63CzRSqm2pve{n|(HkJkN81^3j~iiiZFn58{&NkgGl4OQ1)YlIMR5Q ziOpG$7-@XxfgMH|MjG!akS(dW_eTOvOC1ooSijc;0+^iL$1Zh0u%Z?*x zNaHzYb`imWG@gNFHz7EY#`BTPi%^O*o)KkVL~tRE=Lgv<2s+Yuc9LE5AeteK=SNv3 zLJiV*wvcT@@F0!nk=R!e8j!{_c2j)T8gEZbXVNWCUAl*~V*zXW}k;c1)>`8<^Do6b6!_Ys{*}r$c1{yvq-g@08 z<1fnic^SVdu)kZ-BRVUIrYUcx{ce}Tm5aNSYw?l%vgVL~W0-yd zfxVi8vShxx<)LPOT}!=BRPUwD7m8}##^)d^6`HF(sbf_~LuO-rz>z8)VJ4~_qu{N{ejrr*sPz;C&MJ%qMY zF6lR%FgIl=pPq|333Y+JHnu%ER8yO0bA|?1-WzRd)bJZE^y?9;7A#rp#7_e>W7NYT ztXl9;^_oSc3pDW~B8~VptW^u1@P!uK`=NpvE4`tRuW2)W&;$sYL#q}H1z8rmG}t>{O$@} zS1s^{);71cJn9QB&?5e_x;l84RSULw8$-SY+8qX-m3NFzW#t_yJgvOL5Dv{(-l6n{ zjL{3(L{5w==`q(a{aDR0=ke0xuH)|G zjuXxkr6=?g+DXUBl2h6#5a|IOL`6L%J^6c!_OgBX`-=8y`-b^WxjB%_c%%2f4t`e$$jQT?8KgvJtwq{EPak7XA?oNM>3e(H z_IB*uvp2f0ZD04k!2am|j{OY>0tea-cn*^M(SvOVYYveP0*9i9+=odI4Tl4V^UseMX H*1-P-7U^^R delta 7909 zcmbtZe{@q-p1<$W*t9fl3Qb!ml-H7Alv*<7S1K(8tnO3l7!)<*`XdEeQhtX1km87q z7TtDZHES)!`>3WWAVP?-sE$*qVIOQml{oI%P0>kY7 zwK?Z|Ki~WPeSf@rUlPyjGdveWpQriY%=f?hY+KJ!l}@F1_kOPQyN`YeylY0F_hSWn zdVjCr14rX{RO#PcyV=Ej3IEt{C1lGp$z;Zx!G~1Y6e*Nuis^b%$s~dzHtPX;axhjX z_-UcAVtIHgA(x@^y9HuqCLV#5B>|lRnVP`_{NoddpT%$zVx`}RMM-e*A1uh5RoqbB z)Bwfl^=Mf5O8S+Mzv9gluuf1a$PT*dV2uSG{BRWd>hvqGmKJn5rG!2-NiRV$q#LEaZp-Dz#5z(w-UCcL` zC|HQ*@QrAawP-$((0n};O^}8El*tBB%EX+REXnK4`d?W5xlq#+NiS2oW{K|f!-Pa# z@H&wi73i}Xb7>DI#H%}Q63Ce(Es$Ri0u7Wxvj!zk-Zg0w_FOY zev$lSxl~ABOSwPe{epy1yY6Iw=BDQ7y%=CMY&4oHy_!SNS1x}0cJF9qyJmlYZb;1+ z9}3VtsV`Q%C&lZo6DLbi!%08$()}}}87DUywXIoN6dOc0UhEOZwQHV_oB7bgjM4FV z2I`#oLf+}=Sd5j4oM>2fS@u>MZPA&RhBMOsQxHzmi@GJko79vp37-z%mVR0kPrOWZ znF~bMJ+wIUfS5mb`1Q)RC|ZR4eaV(&^yp#c4Bm_XnNgl6PoLR%!%1 z3lrAS{51t>zvSItFq1vE8I)8zxv-iESt{R}3|%=FE(ey-z`86ZP1A9yMm0rMDPw8z z9HVq#bx~Ru@4W& zoonw(UisvOAQt!C3ZorBTHY|S{VZfWEzTgMX6;ClAh*9@^g)H~{j+^|DDL@VEcSHx z(1JO1cTRRX*I1cm)*mU&Svi-9o(?z6xymt_sb4!!|CCdamx-tK+Q1`C&~G|VZ^^xZ zCeKONvJfrPOB?Caxh3;7|J3@o9=9-$&(6`a@OUF9UHV^=&SEBw=djyQ4V^!yq`AUa zh<*9xUJM2s&=|q+UkV@%*%>EM5NgxO-e-*=h@vITuHDd& zh`(mb5v_m#rB)(nC234KqnX0fBuwTZ?W>3$)kDv8VxyKVA*(Cnq~-${A#M&*MZl5< z9TJQP*eNl?uOYyLXQzXUpTbkJ{m^V4^jbbfwn}85MUm&pUoJT|X$tf3a zCuHe+QQuh_&d*8v483@DJI~ULw1=o|L2_C(6Jlja^zxUAFtorXR+Q6k7u+J&=FpYW ztKt{gbW9o-zcA2K1;t{>K)))uMQk(BC52Cj-(=CD!h_Uqnk!nf=nm83?943OTkVxb zYh^o*iKU+`eJ^Hssgz141AW`H>c${avBmHJ*+=%iU>srC{;aY<(F3!Y@PIKy^A>%Y z%v90ZF?w;)UNK1Mf6S}Ffvnvi%a}Eb;*Z6|!C7gv#|$+q1;P#}Osttau4OA~A&p5T z;mNz%(Q1)f!V=p+R;ICHA(<4NCM}^Cq~)|FwS>CO*(EEKFcOR;vR^ZijFrH;+1ayD z{ds)5$ZQiiSYA11cw)uyh*=h>rD%>g`x0GWv_Q0H&>cmM!dt`Vi}Hn=b9nkDJ!zQ9 zc`IoYCkxfOa+M{l^sdGAF&Yf9+m#CGMl*q_Tyr>5ocl zM{xXM5;tOS?YjMqsQcn4s2g*}&PiyL%LZ#Cx5-EtpPH*nVB5)?$xb2dm}`lJH?vu{$eUT0U2hFLPsQuNvPD1p zODuNfE@mq!dsA@nB!q&pym4W(Ni`K!>ASeq@loa1sd6@chuoH>om_KDqZG=rx&DuA z6DU(^iE5MwMBe@$l5S*`%xX)@#Q5cRK2Sc?n_5n%EOYc(ER8KX@5lIlkmz7pPID!O znJc$#H0tHHN+UMl1qYjS8>?>;rv+H$re#=*Cc?{d%PnQNlMkF>{l8YsP~Q%$>W*i- zI%$6)n@tN|UvLw%g=~3TFgKHiG~}?v!fd)H{}8uyv%J$017ZLf!g5f(TrcY|leNYTj;W3h~rVH|wxS&0&2ZDNK$K+cjN zFMrmg=%veWhL7efN|q^_mT<>|A`Y+AR6m2@^`riMMs7Ya4Y z*n^E}bgI*6m)k1@WwR;+HY?jaIV4P^GE0LZ%Evan1WJ@Ns&ej67mYGpNWn9E)^a}VxD-G`ws2~V^nf~K)?nxpSd(`_qn z(+^!@1LqYWG7YcH)@q~QzaNV!Bdw-3>#f(_zYCEbpyg-FDpuOhxnz{FP19NQ+EHY_vk^S(Pv3&{tPwH~%*(#uoEzXRF4Z`dP5X zKeY_%R@^MHT6270qjQiFne2>I^AK<1^J58T_T6I@n+}OI79)^_yV#-+!X-Z3Z1zZCkZ2*Ihs>6{$@IpZP zq3ho;{Ee6{WZivF#XZYv3|pTZ%e*Ua;iV1d{~YL|jz=6q((s8#Tmmhqs>s}gciAL? zB&oP=_~A|Kh5QtOq^yI=AQFSHoCs+mNfYiQ0^UtC1p|GxD!w+A}?hi&$jPpm3 z9`af}WS}h2K~O!&0{J@7Mvx2ibJy_0HQx${ws~IblQqN9P(-8ixY;iQgmf^(*M5zD zAsGA+@QNdyUTs5YuLJ3urtEN@ectWM!6VH{)kmd3ua!FieQC0SY%Ena= zRTWC8TwLW*ReGaZp^vLVs%kvp72&FJRb@+fO>>pLj#ubTsLV`+iIP=SD4}w4l}A-g zB~%?;HKM94Cbhy*u9{L+?u06>KAtO=A=ctIIwO}@REf#PB{D|`N0g|1Jd5JKU8Wse z)5OtpjioJN>EoIXj=^gzLkY_;*Nk!;zsAy+u>4NdG{ncG=cv{hRxNcB*H{=Tqn5eE zk&w8##FLQtxWunYOd&1_a~$QUX1p>pQLdR*EgIuBs+Lz~M$a`8M=Qrswq`K5TpzEw z%Vg&oC&wm^gpCuH&V;3pYy2ETD)N|m)lwJXnlXlKW=vCDqJ1fH6D%~*I4R{)R-e&6X7_n;tla?!fJNiG}q|&#(jTdGBXZk+BiBms+C(3 zSv*|h=NP`m(ym(0k8;iQH5Pt8gMIV0P4TR`9Mx=kbG)v;`4+CRU1Kqth+*d%=QS2~ zHHOK}H6D&Wj$vkjlx{XZSide)kZbUhg`yed=;4-@gk_X#ra5Z&#XSh0>yTXb^T{FftQyjIA#bb^0`VxD= zByo+Eql2T5XZNYu=QnXp2S+vQcw!_YTrMhbuH%X24wY#=Tb=6xN zcU09$JF9l!cj+ZHb=B3y!zWJW&ZL%IX~Q4vS}PPZxN7U6-zROZYOJr86vHYdSan_9 z?z;M7`croa4Y_N?+Im{B$56_mU!4=~TWag8HX(6wsw9~k>h?(uyQLkqyS7VvYa3kB z&f0i->e?gGXZGAY{9TPxY_{MB>HQ${W1;xB!8?IJfNbC`cqO`M zz|98Uofq8TA$^qUE{2fROn<-%FTtdc}9C&XYX(rR?`11(4 z1R4jg$B*xKfUIzE0#AV?@T0(t0yu(~3JKW>vVg~X2I&O3!Q;h(`~lPi9&ZvPc_H-R z@tQ))K&{~Mu0ZyJeBeC`n~@xy4s`IQKt2Ka!Q=iXq7nZzfY$>{Kq2tB{K;0(2zcD~ zpFvviIO>TRqz8|ygxn3v1&_;uJPVS*<1iyZ(5w}*vB4oVR9< zw&8!Clgm*(cnN<2y#k7W4+HPCVhX{BfVNT;0NxGkxf$yRo~*(OfyTglfK6qXDI^F3 zGs_VVyajj$lnZ_eczHFJ2E46#4c@*%R&+eT=WVDO{5bISwMfJqfFG{I(ty|B0$Bwd z!H)uOzZH7|yc_u3ZI}`8L=uz%+Gr;O>`cw;yGf5Q&Des@mPJSmX<0=``p3d8P? z4<GgY^lSTb`@{W_{?-BCfPK(0 zm^&m5jSoeKriO?Qdzj6QKj;tnNBj}LJJ1yH1X=^WfVI=sDRKI}LVv7M}zuoWfJ27MWKyE<7oQ(y>1JS?~=FQ((80-vnj$ra!UG6SVm!;d< XZR?gXfz#cjN86+C$?d`Y=oJ44aBk?6 diff --git a/bin/mimalloc-redirect.lib b/bin/mimalloc-redirect.lib index 1e22ef12aaec87c9544724a5c766ddbc9bd90ac5..550db8ec56373737c44eaf236727085670796ba1 100644 GIT binary patch delta 104 zcmdlbwo7b6m>A~?1_lu3a)RPLKzh%_oR-N!jH;7gGx1N>VN{#=L3eUF<8=KKoD2+3 pTnr3*fH(lizQMu3z{|kEz{tSFz`?-Iz{0@Fz=mLLwq^Ru0RRTM6hZ(1 delta 112 zcmdlbwo7b6m>Aa{1_m&80@6Utc>;td=Cn+fV`Q5gz^Fd?H5312FJ|G%=8VDh!4D2(PEn_i((&rM#Dquqc+eq(D!H?F;+yZ5m9fPs6$OxqaDoq_PO^$u*_w4_@&p!K{o6yp#ZaJ>r*Ag;&AHEvY485$ttjbSLY-96JUT((ovn}EYKYI?H z2(ah0m!;>YY+hco(MPhJ{Nv#{u2hx8ok;%EIt4etsnaK_ChNHpU_=Q!L64`NTflLZ z457z_35B_Q#hE&2W zNWrj_roWq>iSYn&zd*w)@NY2jb_}r@815$FBTy)qryLPXiH0;krx!Ia6YkN~P7ps% zpf}A~J`OofcrnQ+v?OXwLkm^<-WKs^$Yt0?6D~0Am(ap(evM0h^Tr8dM8qP0{fYJ|6B?wej(YJMM7n8z)67v2kG^$3>HR59pyoMxM*mA9_U>ekLB%4$Kdw+cv{2|A=bcV0i}_d^ zKOAunJKU*_89U*+uvqKLlSm_bT0er$R`*wt6o#o7iSV%m9(YpNx`(KdidaI$$R7Ty zRzxXuXuWn~<=|n1I=YDYv&XG%WfGE_@EKJb)5nN%){v0tf@l2ZnXA7X9YumAugW47 zx^l$n5&!r_SO54&Rq7W;gmn`$=ekCSON(rUo8w$yUW6ae&PE z4oO!>v)kReHg}w8q|2xA3l}DBM;f~Bo*YxD{DM<|aUyxG__Fi%BB6N7<3enT0o25) zg^PC%?Z9fD?O?hYr>+L z))|D^eIbyla33XF)-|DVrek_Abc>>sM7kUk&dt0nHC^HFC*)6KEbQIg0U=>lR;7CY zYmkl6M!PGH+3Hl_biTgbsNaf0n-t=eg)Oyigc>+?QM3qYx**=AIpB2C3(ed}odvy2j0`^x((dIK)g}sR^Rlf@%SF$P}sk&eGi@9}N zaqBqg3^yNVZgnaKI+B5_cIO+Q&_7K-I72Xe>B{DlnZEKF?1bynG2!o5Srx6f^IQ3L=k9u z6Zb-;D!K^F>fm%=E7AHT8n!CA9G&FrC+w>dHu(e-L7^r4VuXK1!h=oUi0P9G+a%Hq z!_&iaWxvsmb>*WeER-Higg+elVBHxV*QGMAQVIUFSqt>h(b0%o+otLu*>%4;&weT* zLsD>@m=+a&kv3~Q3Czn70_?{qxphJUQ?FPcM93O_W6fs9_Q2fhBEJJPlP`G)B z3y!mP4^mY`D)eZ9Lmsh)(a}F+;pUh`-v>HVrH!}^=<01#utHKBZgY!~3NgYDCoFfi z@mC}6tKrkBZSH|rv~-O*eT9qg12{txO7)d(?vY5vNVtFtEJsfcH|e-QeuQwt31#}W zia4Eed0b!?4!Ot_M@1}{2uw7U9e!x~@#vqy9X@SsaU@oHr+5hnk9JVPg%meW$VzKIQSyCd}{)zwSZzuY_Ax= zs)!znOn7Ym8=;*;PAKt-RFci_%N|K9l{h#DBpkyYaiKzCnFuBhCV+w+qeb1q&7on4 zlS4ysyhJL7%6&Srd;AJ1#Ma`e;bO5zu)Mf_VAalGg|N&av5mhNabLuliEt<`Y(Fiw zAGBVjDQUf`$4i92+Q#=IL5tn}s0ZtXO>GtDPtu{dsg3_bdwUOBV4J(I7KJwuK9f=2 z&Y$;)8&auA2X4vfWg}c8wU@I4TrqmfWsf9{6MeYc=Oo9kzekRP7LRDzaXw5piHc!% z13){8JNS#qyURyMM+J6(FUZvfaZT`u`)3HD1vRP@!r+2mTc)C_Sy_%pF@+!0aJV4M z!Bs1=Tu0>Qqjv8zceL%T6K+)Nd~6fn6mrb5SUQ%6_fQjoZ9R&64+ z2_Np(dU3iWewNqax{6dG)w;G4-A!AOG7Viw!rgk0_=|THTbV*fhPG&o6)RcE=Ymsj zT!wO?4@(%GS*ro9Ow2{hm`i?>6)k+8iIPtG&evVO69TZycA{=EAICmo6);SmYseWmb#MTdsiR>Id{T>!G-lmy=`ZmHuV#q``Sc|D+rAfCklth_Wc%5`% zSF-|5-hCIyzD}fu0uf99%q-z%vaT2HqnCegw3ok9(aXP&_p3v_{0De`i1|mlUS7oe zpu3mUDks3N$k)L?@3^bpOD|QDjB2&JTWD>hxy!4J@aziiQ z2bHhEU*&Rdil3Ny9D!ZCdkxROK>z0mZ+KlTHe(!n z1%3K)$p$mwj)zZ=FB3ohyAmB5EDj+dan5Ws$$N$3shSj&nE_=?HH<`jaG6-?V`GNU zovsZ@Jqo2Y%)T^+H@k;-;7ABNC*bH_iPhQ65AR6e(Oj|dmF}i#p1IjQl1jDnO?pwsI!3E+CdQy42!pz5P)`d`p=&GCLf}6@MgidiiEGD)S#2?z3mu}Ga&f{%@{7L9=Cb^0s7%sP zF~0NEGd+x~!?C-S>1Vr5eClsP!{VHbWzt6<$x$cO(%Ft;^xV|G<4xgz7MBUeY}YtV z5bExJQ>f0qcg7(Uii6iO*XgE=Fxw^_S|mr41ll&$i9FfmTOF zI^wt8E|Eue2zM`eD5R4QMtPgi*Sowa!p*~6umC-kZLB?9U@1u0krBhra-V5CoLcb) zr9gWVEk4gM^vxT6naNM_l@RB{5*}BayJ*>Q#VQluL4}5$)a-H2blAc5c0bd{z9Q&@ zD>xgM00Rmt+2gb^u5R0WYl)2RBbvqt`bf}E6zWM+6v3ULo02YH6H|ocAlI|6lD6IR z7lazq0`<=>2#=YZ=*3Cs#Yq!4;n!T)f*x_5Rw$dAGI5nMcS-Zxl=&QKo}|pLk7N8F zvepUtccgi*GXK>$;TrTEQRdG`^hcC=^EkmgEoI^+Wxh_LuTthNX?~kBpCiqal=<}} z$v%D&$NYasn)fR6UrF;L%KVw6t|^9>G$j!p>$m1hN|)T4q$*06P|*d^^swtN_97RU zPP|fU)8ea&kWo!kZSAISD&eP^6kLK5*XX%mQl~0f5?#eqi(iaUjnf_l`hnZMIZQw&i(`Sps27n=By@tcnd)px8_C3c;@;{~-4uPICC?^(9IlwV)Yue`II zU$F+3IFn=5veJ8b$33eZt5>ZOme;Ndx$kr@D=l5UV!dbCniVDM?{s@sm#?{(>JP_y*` zqrWP!(MK;Ef&nAA8okCUFx5P;wZ6eyZOp0;dX0g4VAsj1|A(k)(%f3{-8c zu4Spg+P<-=!Ry5zE}jf+ z4ZtNiqvRMxRkO8*ysvqviULET>#KuxUMzyKzMUZIKTA2rq1(($)99Y?X==zGyJs?%i6op^p5$k@;JSiq^#6;N9lk z8W0vXF1xM4+gM+>4ePeWj|3a%XEvs1HqJNJY$a!_0>(@~NeJI<^p!Vyy;~Y-wgK!) zfLO1pO#yEM*e4i`S*VCWfickVAo0PC&C+TvR4Q*t{Wg?QO|^GvUCoAt!t{Vq$P47H z{g+iHxkusrzsn4ZbWGcNfYToW>`zbCm{O7)fasBtF$H-M zh?tDj7s<|Re#q&K02Zc#v;pCfksbx<17eqq^ef0iK%A11`xN9wAO-+PSwK64mtkoV zoc1KA*8&UxsziS=WAYiYfG`2_WP(W{lmOuYtd$A-6oMa!9e`Gu(4r6y0dW{`TqbnL z1Z@ux{Q%~bh?fy%qC3dx(*TT;g_k^eVpxfnHGoRMEe;2OXaTg!gc}YY1mY;*6rc|Y z;g-oE_5*PhFaqGM$;e76fpz+eK!`H3LqW!Y&>oWXwkXI{ATnfRw~RE{fG7rdhyY(J0c6bJ1ELAA z7cc&;bc0d4_HvE&_%D9CBq;(qi~A>IuNqNJ*?Scz~z_>;-s$G%1FU1JMr{0knWn zED^4D`R_WV36%%x?yY&qTjH(qbL+R%tRFicdzb*f3J2O)z8mwe@s)d*(H){HAaQew z-haSDE=f8WyB4kfR$|w}Rf|+FHLDT`x{8I@uPqwb{e=@3il+|C{{@`-}op z!Se-AI|HtVTpJ6|7OIO%i=HWZy$DK;97mU`EYlOFBc_-s-yAl-YW|~nrsbQK`z^JW z?^&8GAvg*k8M zoXvL>=$#9kPUkGww_O`tn_Y>8lMCk;awh!02pDTHy>G6toVPq~J7UYWziS_|YjcWo z^735yPv#%Xzg&>ySnRml(d&5IG2j?=7@RYmp**MCS?>J4^GD9-of}-}U0enfHQCy2 zAKLzIyJY{|UYb*$b6?JPbE35!Y_Bn@~C9YDJ&(-5P7jj*76&J27 ztSmfL*j;$JFt5m2bYIc&qLY-u764Y*WU`r#nL16!O{Yv*=4`XcoQDNFYCdK@ZZ=vn zELj$l<&dSra@carl4?!E(jBzwZ3f$3+diAtuCwdysrFs=R{LK20XvtY$i z&0-DY53p!s`D_YdiD_`%Ovz3qj0OTmS$7 delta 6377 zcma)Ae_T}8m49y>^nn9BU@!=%gJ9GM$iNIU%r9u_+p>wXeV}7Zqb)>82E>HwU`Zki zWUv87TFmv+X1g{^8X8S&S`w@=E$NPiMRu2LNn;F6G$mbAA40a%E}KrWld#`=XC5*9 zwey*C-#zDh&yRcVz31JTjAk9Kn$zU0hc{KoXEC@L56+$K7rg6ACC<2h+Mndgo`jheBN3z~c_ zM##r=vQi6u^{qYt`7Sincm;Wf&oxqa)W|*QTFB1+3z`~GKIkq|s96^e zqnsApVX_MxoHHAblE2qvCf!&D0Aa8J9mCIQYWVh50>{m~s7I5ZiRSG@G>^mW%wCL} z3CE%F$D=7+hvwdFG3(XH;fxM{w0g>bM*)^P~jKZ#eRM(iG`~rt&WI7{VF)~zkl_>CkqS6Sv07s zfJax2Y0AphPliniPw*P03y`@K2LMJYPdjic>3T(6KckRqMY&g4%;bPSpdg zPjw|V^2(|NwwAJzm_Xh)rmy@68w_#E%7^iczoPQ5NBMIq|3Z}ip33({`NJxIAf9v< zr?1?l^6n^Js`3T#JN#$0r~CA*z5^3%>8@na>s$WN)M$TiriP{v)+e zAC+WuKvFqN7e;q9NZweNMsm|u-$ucszx8FPiY7YjWa<}KxOVUs`uuIa}r`Y5C5POV_ zvPW==Jw_28*C!3h-i}7t>5bayktY3Zlbr7%ZwR9T?#U`8D~ddOdIK-|owVhA`s8z- zPmviN+u%8DgcgSG=S1wc&y?5>pYXR$a4qp;af8s}XEVLpPPlr3!CGLNmK9hj8c6cF zL^|f^zxr0GJ`(8xT1aPZN3{WIJn+HZi#m=RPq`Qew|ieBO2+!!{u>Jm0k_b@4^aKR zuT8KIN)?+{Bd2`-1}Rvb8BZsP(0n(Dy!xIfr&07b$k6ImzKCqjH1nzC8<|JqJ2qn+ zr#zlYu4g{Wr;zTf+tL0!YYW2$^^tnS- zeV%Z2BiG0+r0a@^bFDSaaY>q>vkXW&D2!Gq1{B7%-`2$aCy8J81P*LGncOJsi_kHB zQUv8ds9H$RF5<-=Y0lp^#~oQtYekE;I5_J>G4Nj$dpu!ZDoTgHRm=I}FG=%syUik< z^VEpHZ60~QB!wtLr6CIUN+HzI4Nn?U3O+a87B}-EFz@ISSQ_yUbmdUNls?ZiN{%Aa zFo?_W78zcz7o`8i@lk3M$#2#-@V_Qyx&K_0f+YA`1oTT*&Fk$@EGk-~6- zWGk)(wq<|G4(&yb4oex+iZe9BN*p_d7vATnyVM?S3bL3-LUD2J`E(L3D@uYh4?q)c zWM%NvSPEHrPd<7V0I$$Z50kD+`Scp|gW|q-Ou8NzlRj7&lm6%#lWw4|%#BGic-{m* zs~eN%(ErdqCjFN%CjDt|qYmz?`=sl#k`kTa(e_|VYAnl2!J^-pDY~8_b&32bfkl`1 zk4cwd@-4W$0vB(iKLwY+1OHCTnDnkVCS8T>8nE}^{`crl!~Oe0V-lTOpZju^@_l9v zMG~Gp?SlVeQnfN1QKvjx>&>QqvCkF#qTlUfQV6#G3_pK??f*u99)AA^&yQgLub5tV zZ(}@Nh(1rKS~1@+{n_|{G>=;Z-opFc z^K}7_=yxP=$z9(;Qua&pb;~3is&MRHgHxq#TfN^M(FZ(X|B(be=UWK}Og%FDK7)4z z++nu6bnILze}{_|Hgb;?^0$RJpJ~4b+ETbU;12a{7h#CLtHXgxy7NPHQ$mdz@H%?1 zk^kXcnyZYN-PmyCn`_DtdYY#0UGj^~cO2-V14K4J@b-g2D6*0f6(pYG|Y4r14b0Ny5ujpctC)6+)v(wW{6)r*hXBQeU!edu)$~& zp8drgeEY{NUzm?u=-bKbqX&i_)fGeh%vFE^+)Jvoc8`up{tXfwTdrj_%^wOv3pPA1 zF1ns7EfjtV_>vT+H|0LucN#0`xr-nx%G?wUnO=9+(4`e(^fQln zJqcP>O8&>hlV6&9`QumyBCHx{Y28KY49Z@)3N469` z+Ad&vFoapMGETQ)Y#-6VEw*Sj{ElAES4b!7)&=yP)n02ag37&e%S5PHX<-r+;_&Rt zOh3gd(~&!(a@?=C#&%Yf@+vBff88i8hNzY%jQZ7E%wF=jAks=NT^SVIQ3 zCboA*arFv{`WTt{0I_?Iv7#YXD|OKkP|x$8gwT7MBJ^0Jd|>>3)N$Rj`<0G{?d0AeJfIG1%C`Q0P^5E;sCXR+CUAUi;!Oi{SovDD6x9@Y4seEFqL<>~id-}ox2`|2KEJDk&5u6eS$dAR%Vh<145v7K=l%{83Z1%j`| z&%j#z(7T}7p3x6rU$-rzz!9h10dj*yqG(522?N`fOQEN zR6!X)(-Lr61-k$YE&(}1be=*Gz&Hp?v`4s%&ea&BSI`N8A|M*kbW}ywi~Be+17rYE zU_b?mAh3eA#Q-jc*_HLXA*cg2$AFNk(FMUV(8(Ae#x#U85R8CWSd-{XtIRw)+PWAb zqGO0L`h0X8pi0me0^S3G7t|aB76a~ppdT~{sz6N5YQQ59On|0A%>V{eAPj-Fo)h&T z0{~eKI3EHBhz6{e(WwKqf@mZad9g&CfM5`GCI&15T+91!dTF@H_ybb(HSsJ|N3Ul4+;pl}Rmjsar!gPgbtL`P3X=de?>LQokKitCx} zo#Gw{yfI-&9oY&&S4>zG8>v45!2oCo)C}Qd6p;1f5KMvQKr)0wDxmdZRv-hYAHq-! z5KRzNf@pX}m^Nm)Q(pr?E9eABhOk4OEL6ab=v6$6q2e<&DB-p!)iO}-yxAd@30gd ztgmZ0SijHL@bE*12WlId>gx;z1%{SG4G%qNI9UH+OH=JZ!-3kSrib@!ec)hyeIaUV z|8k;ubi=K?@_BOlo;686Z^I#sdBm`mvQS}QR1a}sjA@{Z@E|9@16$9p@M#eIqA z+W6rMn)Uo}^LI~Z+hrL0va!q9Ydmgz);MHLELm5wu_V95RN^SPv*c$b>88yluj!kn zM@>DZ@0y-A#hFvhS>_z`9poO6j%I>!tIh36?BNj>T!&W%;`0QA?NQc)R6GOPsOWBzHLf2XPd3&X0yk#*ZQ(`!Wt>N-FCNa%=WHr+O}ZR z+gIDIcDKFS{+RtA?e{w-9KUr;Io@+zcieP@93MI8pT+oc9kwpp3EOp>Vhh_U?d{v` zd+cZIL-wFubm$#<4xgjd(GA6`jyZ?JS>cqO1I|I`S*O+|xH4Q`SCi|AYr-|@nsybH zo65J9pD2H-{B(J^JW|dz)5YL8UD2MRnxbIQXi=VVlQG|DGIkq}8T*Y-8Rv{4K3jagy?CT}v^b+AyW~hoN6B=_^^$F-O4Di68PkyIyh&#k&3eqb+3Yj7 znva+}%$Lnm=Bwsu^L4Ye)KOYdx~;Ub^knHkDQD4I1dC{?vv@5{7N2F@GGUptOj+`+ zMOKs5YIRuqttYJW*042V)t2op+f!Ck<}Dj3YY&!B051_lu3DuUv7K>FRpoX*M7jOvq^`6nAQYD|95#5S3qnQQVfE`A`* zK3STPbMjP1fyt8?yY#b%7 delta 121 zcmew$_Cahym>AbP1_m%L0@6Ut`2~b0=5$Ve&%`#FpP6g&F)n@}%|2O}(P6R&v*=_l zMxn`583iU!V(ij?$Hl-<1jJuB85k;n_zwpI11|#u10w?y0|x^;0~-S?0}Fz+*_%m~ F698;87}@{; From 9a5189aa834a62a8ee96867693d622ab0d6d158e Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 15:42:22 -0800 Subject: [PATCH 47/57] add vs2019 project filters --- ide/vs2019/mimalloc-override.vcxproj.filters | 72 +++++++++++++++++++ ide/vs2019/mimalloc.vcxproj.filters | 75 ++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 ide/vs2019/mimalloc-override.vcxproj.filters create mode 100644 ide/vs2019/mimalloc.vcxproj.filters diff --git a/ide/vs2019/mimalloc-override.vcxproj.filters b/ide/vs2019/mimalloc-override.vcxproj.filters new file mode 100644 index 00000000..bc1e4c60 --- /dev/null +++ b/ide/vs2019/mimalloc-override.vcxproj.filters @@ -0,0 +1,72 @@ + + + + + Header Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + {f1fccf27-17b9-42dd-ba51-6070baff85c6} + + + {39cb7e38-69d0-43fb-8406-6a0f7cefc3b4} + + + \ No newline at end of file diff --git a/ide/vs2019/mimalloc.vcxproj.filters b/ide/vs2019/mimalloc.vcxproj.filters new file mode 100644 index 00000000..b2282df3 --- /dev/null +++ b/ide/vs2019/mimalloc.vcxproj.filters @@ -0,0 +1,75 @@ + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + {2b556b10-f559-4b2d-896e-142652adbf0c} + + + {852a14ae-6dde-4e95-8077-ca705e97e5af} + + + \ No newline at end of file From 5c8721f0b80f6f5218f37b102f50c31ddbdda7a7 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 15:59:54 -0800 Subject: [PATCH 48/57] update documentation --- readme.md | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/readme.md b/readme.md index 0d11db16..44f62230 100644 --- a/readme.md +++ b/readme.md @@ -56,6 +56,7 @@ Enjoy! ### Releases +* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode. * 2019-10-07, `v1.1.0`: stable release 1.1. * 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. * 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. @@ -138,6 +139,10 @@ target_link_libraries(myapp PUBLIC mimalloc-static) ``` to link with the static library. See `test\CMakeLists.txt` for an example. +For best performance in C++ programs, it is also recommended to override the +global `new` and `delete` operators. For convience, mimalloc provides +[mimalloc-new-delete.h](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) which does this for you -- just include it in +a single(!) source file in your project. You can pass environment variables to print verbose messages (`MIMALLOC_VERBOSE=1`) and statistics (`MIMALLOC_SHOW_STATS=1`) (in the debug version): @@ -188,18 +193,18 @@ or via environment variables. - `MIMALLOC_SHOW_STATS=1`: show statistics when the program terminates. - `MIMALLOC_VERBOSE=1`: show verbose messages. - `MIMALLOC_SHOW_ERRORS=1`: show error and warning messages. -- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages when available; for some workloads this can significantly +- `MIMALLOC_LARGE_OS_PAGES=1`: use large OS pages when available; for some workloads this can significantly improve performance. Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs to explicitly allow large OS pages (as on [Windows][windows-huge] and [Linux][linux-huge]). However, sometimes the OS is very slow to reserve contiguous physical memory for large OS pages so use with care on systems that can have fragmented memory. - `MIMALLOC_EAGER_REGION_COMMIT=1`: on Windows, commit large (256MiB) regions eagerly. On Windows, these regions - show in the working set even though usually just a small part is committed to physical memory. This is why it - turned off by default on Windows as it looks not good in the task manager. However, in reality it is always better + show in the working set even though usually just a small part is committed to physical memory. This is why it + turned off by default on Windows as it looks not good in the task manager. However, in reality it is always better to turn it on as it improves performance and has no other drawbacks. - `MIMALLOC_RESERVE_HUGE_OS_PAGES=N`: where N is the number of 1GiB huge OS pages. This reserves the huge pages at - startup and can give quite a performance improvement on long running workloads. Usually it is better to not use - `MIMALLOC_LARGE_OS_PAGES` in combination with this setting. Just like large OS pages, use with care as reserving + startup and can give quite a performance improvement on long running workloads. Usually it is better to not use + `MIMALLOC_LARGE_OS_PAGES` in combination with this setting. Just like large OS pages, use with care as reserving contiguous physical memory can take a long time when memory is fragmented. Still experimental. [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 @@ -211,7 +216,7 @@ Overriding the standard `malloc` can be done either _dynamically_ or _statically ## Dynamic override -This is the recommended way to override the standard malloc interface. +This is the recommended way to override the standard malloc interface. ### Linux, BSD @@ -244,29 +249,29 @@ resolved to the _mimalloc_ library. Note that certain security restrictions may apply when doing this from the [shell](https://stackoverflow.com/questions/43941322/dyld-insert-libraries-ignored-when-calling-application-through-bash). -Note: unfortunately, at this time, dynamic overriding on macOS seems broken but it is actively worked on to fix this +Note: unfortunately, at this time, dynamic overriding on macOS seems broken but it is actively worked on to fix this (see issue [`#50`](https://github.com/microsoft/mimalloc/issues/50)). ### Windows On Windows you need to link your program explicitly with the mimalloc -DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). -Moreover, you need to ensure the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) is available -in the same folder as the mimalloc DLL at runtime (as it as referred to by the mimalloc DLL). -The redirection DLL's ensure all calls to the C runtime malloc API get redirected to mimalloc. +DLL and use the C-runtime library as a DLL (using the `/MD` or `/MDd` switch). +Moreover, you need to ensure the `mimalloc-redirect.dll` (or `mimalloc-redirect32.dll`) is available +in the same folder as the main `mimalloc-override.dll` at runtime (as it is a dependency). +The redirection DLL ensures that all calls to the C runtime malloc API get redirected to +mimalloc (in `mimalloc-override.dll`). To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some -call to the mimalloc API in the `main` function, like `mi_version()` +call to the mimalloc API in the `main` function, like `mi_version()` (or use the `/INCLUDE:mi_version` switch on the linker). See the `mimalloc-override-test` project for an example on how to use this. The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic -overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc successfully redirected. +overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. -(Note: in principle, it should be possible to patch existing executables -that are linked with the dynamic C runtime (`ucrtbase.dll`) by just putting the mimalloc DLL into -the import table (and putting `mimalloc-redirect.dll` in the same folder) -Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388)). +(Note: in principle, it is possible to patch existing executables +that are linked with the dynamic C runtime (`ucrtbase.dll`) by just putting the `mimalloc-override.dll` into the import table (and putting `mimalloc-redirect.dll` in the same folder) +Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388)). ## Static override @@ -282,6 +287,12 @@ object file. For example: > gcc -o myprogram mimalloc-override.o myfile1.c ... ``` +Another way to override statically that works on all platforms, is to +link statically to mimalloc (as shown in the introduction) and include a +header file in each source file that re-defines `malloc` etc. to `mi_malloc`. +This is provided by [`mimalloc-override.h`](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-override.h). This only works reliably though if all sources are +under your control or otherwise mixing of pointers from different heaps may occur! + # Performance From b820009df733afdd933cc70d29392593da837466 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:09:34 -0800 Subject: [PATCH 49/57] update documentation --- test/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/readme.md b/test/readme.md index b74364ff..db3524cd 100644 --- a/test/readme.md +++ b/test/readme.md @@ -1,7 +1,7 @@ Testing allocators is difficult as bugs may only surface after particular allocation patterns. The main approach to testing _mimalloc_ is therefore to have extensive internal invariant checking (see `page_is_valid` in `page.c` -for example), which is enabled in debug mode with `-DMI_CHECK_FULL=ON`. +for example), which is enabled in debug mode with `-DMI_DEBUG_FULL=ON`. The main testing strategy is then to run [`mimalloc-bench`][bench] using full invariant checking to catch any potential problems over a wide range of intensive allocation benchmarks and programs. From d55ab50a84250e335337724b6e002fd349e35226 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:10:04 -0800 Subject: [PATCH 50/57] update version to 1.2 --- cmake/mimalloc-config-version.cmake | 2 +- include/mimalloc.h | 2 +- test/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 03316948..9d78b5a0 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,5 +1,5 @@ set(mi_version_major 1) -set(mi_version_minor 1) +set(mi_version_minor 2) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index bc817f54..7f26896c 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 110 // major + 2 digits minor +#define MI_MALLOC_VERSION 120 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8bf36521..a80dde58 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,7 +13,7 @@ if (NOT CMAKE_BUILD_TYPE) endif() # Import mimalloc (if installed) -find_package(mimalloc 1.0 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) +find_package(mimalloc 1.2 REQUIRED NO_SYSTEM_ENVIRONMENT_PATH) message(STATUS "Found mimalloc installed at: ${MIMALLOC_TARGET_DIR}") # overriding with a dynamic library From 7586225fc5c6327e4b16a0abd2b4d75c37e497f6 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:10:21 -0800 Subject: [PATCH 51/57] add secure build to azure pipeline --- azure-pipelines.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 79228c41..41d67f86 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,22 +35,32 @@ jobs: CC: gcc CXX: g++ BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_CHECK_FULL=ON + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON Release: CC: gcc CXX: g++ BuildType: release cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Secure: + CC: gcc + CXX: g++ + BuildType: secure + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON Debug Clang: CC: clang CXX: clang++ BuildType: debug-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_CHECK_FULL=ON + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON Release Clang: CC: clang CXX: clang++ BuildType: release-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Secure Clang: + CC: clang + CXX: clang++ + BuildType: secure-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON steps: - task: CMake@1 From c6c24f9c2efb793a201e531057f25ea914792d3c Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:17:47 -0800 Subject: [PATCH 52/57] update documentation --- readme.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 44f62230..e5a870b8 100644 --- a/readme.md +++ b/readme.md @@ -37,7 +37,7 @@ Notable aspects of the design include: programs. - __secure__: _mimalloc_ can be built in secure mode, adding guard pages, randomized allocation, encrypted free lists, etc. to protect against various - heap vulnerabilities. The performance penalty is only around 3% on average + heap vulnerabilities. The performance penalty is usually around 10% on average over our benchmarks. - __first-class heaps__: efficiently create and use multiple heaps to allocate across different regions. A heap can be destroyed at once instead of deallocating each object separately. @@ -65,7 +65,7 @@ Enjoy! ## Windows -Open `ide/vs2017/mimalloc.sln` in Visual Studio 2017 and build. +Open `ide/vs2019/mimalloc.sln` in Visual Studio 2019 and build (or `ide/vs2017/mimalloc.sln`). The `mimalloc` project builds a static library (in `out/msvc-x64`), while the `mimalloc-override` project builds a DLL for overriding malloc in the entire program. @@ -98,7 +98,7 @@ maintains detailed statistics as: This will name the shared library as `libmimalloc-debug.so`. Finally, you can build a _secure_ version that uses guard pages, encrypted -free lists, etc, as: +free lists, etc., as: ``` > mkdir -p out/secure > cd out/secure @@ -141,8 +141,7 @@ to link with the static library. See `test\CMakeLists.txt` for an example. For best performance in C++ programs, it is also recommended to override the global `new` and `delete` operators. For convience, mimalloc provides -[mimalloc-new-delete.h](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) which does this for you -- just include it in -a single(!) source file in your project. +[mimalloc-new-delete.h](https://github.com/microsoft/mimalloc/blob/master/include/mimalloc-new-delete.h) which does this for you -- just include it in a single(!) source file in your project. You can pass environment variables to print verbose messages (`MIMALLOC_VERBOSE=1`) and statistics (`MIMALLOC_SHOW_STATS=1`) (in the debug version): @@ -264,7 +263,9 @@ mimalloc (in `mimalloc-override.dll`). To ensure the mimalloc DLL is loaded at run-time it is easiest to insert some call to the mimalloc API in the `main` function, like `mi_version()` (or use the `/INCLUDE:mi_version` switch on the linker). See the `mimalloc-override-test` project -for an example on how to use this. +for an example on how to use this. For best performance on Windows with C++, it +is highly recommended to also override the `new`/`delete` operations (as described +in the introduction). The environment variable `MIMALLOC_DISABLE_REDIRECT=1` can be used to disable dynamic overriding at run-time. Use `MIMALLOC_VERBOSE=1` to check if mimalloc was successfully redirected. From f27c87c03cac0b5344c5f715377478375e145b3f Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:19:47 -0800 Subject: [PATCH 53/57] update documentation --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index e5a870b8..f68d79a1 100644 --- a/readme.md +++ b/readme.md @@ -56,7 +56,7 @@ Enjoy! ### Releases -* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode. +* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). * 2019-10-07, `v1.1.0`: stable release 1.1. * 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. * 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. From b7d9ee8830f1e77eba002f26ac65f498e5cce0e6 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:20:35 -0800 Subject: [PATCH 54/57] azure pipeline logo shows dev branch --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index f68d79a1..feee8704 100644 --- a/readme.md +++ b/readme.md @@ -1,7 +1,7 @@ -[](https://dev.azure.com/Daan0324/mimalloc/_build?definitionId=1&_a=summary) +[](https://dev.azure.com/Daan0324/mimalloc/_build?definitionId=1&_a=summary) # mimalloc From 42a0666770688c4c39197320712e9d5c9bcc9dd7 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:24:10 -0800 Subject: [PATCH 55/57] update documentation --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index feee8704..9d3974c9 100644 --- a/readme.md +++ b/readme.md @@ -56,7 +56,7 @@ Enjoy! ### Releases -* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). +* 2019-11-22, `v1.2.0`: stable release 1.2: bug fixes, improved secure mode (free list corruption checks, double free mitigation). Improved dynamic overriding on Windows. * 2019-10-07, `v1.1.0`: stable release 1.1. * 2019-09-01, `v1.0.8`: pre-release 8: more robust windows dynamic overriding, initial huge page support. * 2019-08-10, `v1.0.6`: pre-release 6: various performance improvements. From 41caf6d0f8487ff856f6a10adf4a7fb016df9341 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 21 Nov 2019 16:29:46 -0800 Subject: [PATCH 56/57] set secure default to 0 --- include/mimalloc-types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index 9c5d3c19..96e1860f 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -29,7 +29,7 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_SECURE 4 // checks for double free. (may be more expensive) #if !defined(MI_SECURE) -#define MI_SECURE 4 +#define MI_SECURE 0 #endif // Define MI_DEBUG for debug mode From acb03c54971c4b0a43a6d17ea55a9d5feb88972f Mon Sep 17 00:00:00 2001 From: daan Date: Fri, 22 Nov 2019 08:57:51 -0800 Subject: [PATCH 57/57] update redirection module on Windows for improved trace messages --- bin/mimalloc-redirect.dll | Bin 55808 -> 55808 bytes bin/mimalloc-redirect32.dll | Bin 39424 -> 39424 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/bin/mimalloc-redirect.dll b/bin/mimalloc-redirect.dll index 77e40c1391dc4e3f6fa08c30c1fe461df1b0fa80..b7bf1d099681fc73d15ee8dd1b86a48de0aca1d1 100644 GIT binary patch delta 748 zcmZqJ!rZWhdBP3m&q6mg-ezTFtls>PF-(-PezJwQ^5i+<@{_B?l|*zHfZ#R9aTgUG zMj(rS8;~)1gSefp0f-BhFaRlOu2C^yDCGjGGXZhI5+*PmCX=Nl3`H#95*9ECi^&NR zn$|XOqijG*fJWIsjB=x|X{1i3=7^#FhCxyeT)lto0HDuwCvTM$)D3~_41wwlfjB7wt}6njD`N6z zNfnV8xI_$0A_i<+gp{Ce0^GO+sBsAp<5J+dQee7LCQp-66Ul%}WWXdcCSQ=U)6Ib! zl>;>@2VztKTt@*+N5N!k={1b2CclvOuU7#Dno{e55@}#Cs4$d>c=YlLMlmqFIRE_r ze_*t!KuuF&DCIoHy3B`>f$_B_NGVgP7*MGjNa+%Y(h{&uJV4O~kZ2!Jlz$s*2~Z2r zcs`(TCrJ1VQ1~@S(QFV;ITNDFAEHX!quJIQ$Ydzt?e0-|0c89awb^`Grk!zff}FJ7 z=GvHcMnwh&VA6~QS;hp!=|JpK2*mLq5wP55ifk~K5PLR;FwtyRD z0a5}q$^v4P4P1u}OovT}i;5!1OvTm%{H-S@ACXX=yhuWP@;?b>T^G1HE>Lq^Am(_$ z&GCSl<1yJ!Qcc7MF5v@{@R_^;=*z8=g1P~4;{u?@1wf1of$Iu^=?a1!y84P`Do?d diff --git a/bin/mimalloc-redirect32.dll b/bin/mimalloc-redirect32.dll index 9e5fa86580c786c30acf7e28a84f6ce516fe943a..7ba303af50079c0f0fed334190abf896674d9806 100644 GIT binary patch delta 2091 zcmYLJYfMvT7(QQZ1xHy=xuXb$Rc!Eh-R33_o5KyO++YDAzThYxIim+BEr-O z2ph`EX>@uk4B;3xe@o$O1|c*Vfr&uKr@fzIUSD4y z?9(0<9I7`1$rU&q6bIL1Gz$A;xF3|P^nb0LCbV&(aV!|2Odq(n}fv(Ug+76lR z5LsIX{DleaazeY5&@Lvl$=)`}+XlH|tAmf%Rv%|HyAPwthg2}E%YGbs!nhrynoj%k zx=WQ4YpX|@$`C#gHSg%i%a^s?${yRO6g-N-7gY^&2up^O(5P^R@%7YrRW=?I7Y*bG zD0^07HJzcfrS4j#hWAnal*A7Q^6X}u?#1aOTt?F=M%TG3!)mRK*uhi=y6ozO!v653 z1l-C1@I_1{3iE+<}U$lpwRSh-TFv!wRC*xt8KE?0g zZJC`S^y{Mt;o!j(8(9U$_^ISFtctgj-{Fh+M6w8O#AlLSz$P5=yFNslSd0pX{)8S< z4f_+DU-|7P%0}Hg@&a<=its#2Xh{ktd!Qr9kZce|f4+VTK8bD2;k8PHrn46xrO1^x zmdo!$&FcrRCdZI;n3fzKIz#fTC*G^KXtr`a$h`J|+4I^CEZh5KFT%Ja|VkE6?52(5;r z_X?pUp@P1l=KRG8fXX=9B^n6PU{x(<84-)@&ObHIh8XP&4VHnhe*A}FW zW1aUO)t%2H^I9+evE%MHw=N3BIgv_|F1)iS$PX{DX?c~>V`W%4uUHCvz79XYrser? z2P>Ue6&C2oPN*rUB8B37=?Wcb5vu{LJYwZosUvRKw6Y4jiwg50dll`bRX0{yVwF{> z!-S&3Domrwj3SzYmC#Y-Pry)I3W;l2G^Z7pz%i^^#7bApLdTj?I8e;u1EF>_Vy!}S z?|$n)_g-GD^2}yf7PS0U6`RGdSy+|8DDrOPeY7obUNgfwaL~&sgqn3#xZ~%D2H3v- z8J^>p4JF_yp^<-#P;lDr@Mh=vnEr7z4v(<571R9;K~KA4)%x(V4-S9-78X5oU)udxkyp zv&^o{k(pr@izVN3!!m5~Su~uUOX9M*72Imh#hu{Jaus?jlf^K4464sKnDI0tz;xd< a&Rk$#ZRX7z&6Vc2%{$HS3dRq=Q~nFY_WP6o delta 2068 zcmYLJZA?>V6n@{X9jIl!g#s2Ulv*ty8^zK}N6T=onHh9eHWH^yL2$cL6x5(*5wlB94am-n6ZUzxS){OWzN(=GUmtTrs&Rl>&lwnCLbtX-7q^YZN*(M)ioqj)-i@2wc+GO~aDI zaKvF)bQqGfLlNx|q{kc70}?wRvhD#WjklZnC5QfqL%-2fI8Gky$PsZ@vb}T=TWC=0M8sD4{aJFJbd@;=>%3|Sg;%eHiO`;sk z)9R@X99jeY2i9xX(?h~JZ6>8em`eJd(p&JgE`@Fq#&r%E&4#%tKrcXdN(=RYUH=!Y z2Xm^Hn&72W8|{IH)LuFx`qD&qo`%^rj7Oda*W%b7KFH=NDvpoPLKPamO+mtSVm z=w{cwW;fTYX($eNiH!Um*PY2P1NC=Eotj^<6JNC-bXh4+9<^dR4$HDqXg`!^mD0E2 zY*s4W1feV~eF^@}N`BQmDq1%7*BN=fj&OV}0=u-ZI@0lk8}5pdcU%u9o%}A=c5$BY z-#`67*u(}~S;wth6&{0XbK3l~k-n<8AH8+B-c`*d(Zzfxs=~GKh4~o$14^2&{KH?-CoR_Tzeyv_UEfv?->hX(gTeB9* zO7*lB8cO|m=&WA-x}LU**(hcsV&++=r$N}d&QGVsdc}G+U0${p1~4;=SwWc|9m>o6 z=*Bv^<;({&p{G2WBAH$3g|rF|WA@<9cTh_3&2JecZ(t+r1F- zGRr@`e(-s#VXRuIIQl%{f@1I0%`Lb$rYhRPL1=R|3~f=;GFVkp6EpTPUKQbEwG49;)$s`*a{w+o{MBeaZg1&oE@R;5CZntfR9#5v^J%nIn* zHb)UcnuA|9d%?I}I-jH4|6d+%dpY6sqT)lJ5;uVhZ8OW&o+E^-#%RItpu{XnTI-a! z!qmDN7?JH3ns!7d@118!w=A`|Ev1%?7M~?xIcWLFqOlsSFIrDpO}25{pgr$d&auGZ za3ncjb8c{Yor?UY@(uaKOvnn{_BQjc*(FPcm$c`c$jP)`vrbu~Z3VX6T&Mj5d$)b! z*(dWB=N0B%%p1=O