From 5c81787c306fa720302b0fd8f245d207502e9bec Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 31 Oct 2019 11:49:12 +0800 Subject: [PATCH 01/20] update new pruners --- docs/en_US/Compressor/Overview.md | 6 +- docs/en_US/Compressor/Pruner.md | 54 ++++++++++- docs/img/filter_pruner.PNG | Bin 0 -> 67884 bytes docs/img/slim_pruner.PNG | Bin 0 -> 63122 bytes .../compression/tensorflow/builtin_pruners.py | 19 ++-- .../nni/compression/torch/builtin_pruners.py | 91 +++++++++++++++++- 6 files changed, 153 insertions(+), 17 deletions(-) create mode 100644 docs/img/filter_pruner.PNG create mode 100644 docs/img/slim_pruner.PNG diff --git a/docs/en_US/Compressor/Overview.md b/docs/en_US/Compressor/Overview.md index b19e4f7975..879c3e2337 100644 --- a/docs/en_US/Compressor/Overview.md +++ b/docs/en_US/Compressor/Overview.md @@ -11,6 +11,8 @@ We have provided two naive compression algorithms and three popular ones for use |---|---| | [Level Pruner](./Pruner.md#level-pruner) | Pruning the specified ratio on each weight based on absolute values of weights | | [AGP Pruner](./Pruner.md#agp-pruner) | Automated gradual pruning (To prune, or not to prune: exploring the efficacy of pruning for model compression) [Reference Paper](https://arxiv.org/abs/1710.01878)| +| [Filter Pruner](./Pruner.md#filter-pruner) | Pruning least important filters in convolution layers(PRUNING FILTERS FOR EFFICIENT CONVNETS)[Reference Paper](https://arxiv.org/abs/1608.08710) | +| [Slim Pruner](./Pruner.md#slim-pruner) | Pruning channels in convolution layers by pruning scaling factors in BN layers(Learning Efficient Convolutional Networks through Network Slimming)[Reference Paper](https://arxiv.org/abs/1708.06519) | | [Naive Quantizer](./Quantizer.md#naive-quantizer) | Quantize weights to default 8 bits | | [QAT Quantizer](./Quantizer.md#qat-quantizer) | Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference. [Reference Paper](http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf)| | [DoReFa Quantizer](./Quantizer.md#dorefa-quantizer) | DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients. [Reference Paper](https://arxiv.org/abs/1606.06160)| @@ -115,7 +117,7 @@ class YourPruner(nni.compression.tensorflow.Pruner): def calc_mask(self, weight, config, **kwargs): # weight is the target weight tensor # config is the selected dict object in config_list for this layer - # kwargs contains op, op_types, and op_name + # kwargs contains op, op_type, and op_name # design your mask and return your mask return your_mask @@ -158,7 +160,7 @@ class YourPruner(nni.compression.tensorflow.Quantizer): def quantize_weight(self, weight, config, **kwargs): # weight is the target weight tensor # config is the selected dict object in config_list for this layer - # kwargs contains op, op_types, and op_name + # kwargs contains op, op_type, and op_name # design your quantizer and return new weight return new_weight diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 6e0a521be2..6e6bd1f800 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -65,7 +65,7 @@ config_list = [{ 'start_epoch': 0, 'end_epoch': 10, 'frequency': 1, - 'op_types': 'default' + 'op_types': ['default'] }] pruner = AGP_Pruner(config_list) pruner(model) @@ -92,3 +92,55 @@ You can view example for more information *** +## Filter Pruner + +In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. + +![](../../img/filter_pruner.png) + +> The procedure of pruning m filters from the ith convolutional layer is as follows: +> 1. For each filter $F_{i,j}$ , calculate the sum of its absolute kernel weights $s_j = \sum_{l=1}^{n_i}\sum|K_l|$ +> 2. Sort the filters by $s_j$. +> 3. Prune $m$ filters with the smallest sum values and their corresponding feature maps. The +> kernels in the next convolutional layer corresponding to the pruned feature maps are also +> removed. +> 4. A new kernel matrix is created for both the $i$th and $i+1$th layers, and the remaining kernel +> weights are copied to the new model. + +### Usage + +PyTorch code + +``` +from nni.compression.torch import FilterPruner +config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'] }] +pruner = FilterPruner(config_list) +pruner(model) +``` + +#### User configuration for Filter Pruner + +- **sparsity:** This is to specify the sparsity operations to be compressed to + +## Slim Pruner + +In ['Learning Efficient Convolutional Networks through Network Slimming'](https://arxiv.org/pdf/1708.06519.pdf), authors Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang. + +![](../../img/slim_pruner.png) + +> Slim Pruner prunes channels in the convolution layers by masking corresponding scaling factors in the later BN layers, L1 regularization on the scaling factors should be applied in batch normalization (BN) layers while training, scaling factors of BN layers are **globally ranked** while pruning, so the sparse model can be automatically found given sparsity. + +### Usage + +PyTorch code + +``` +from nni.compression.torch import SlimPruner +config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'] }] +pruner = SlimPruner(config_list) +pruner(model) +``` + +#### User configuration for Filter Pruner + +- **sparsity:** This is to specify the sparsity operations to be compressed to \ No newline at end of file diff --git a/docs/img/filter_pruner.PNG b/docs/img/filter_pruner.PNG new file mode 100644 index 0000000000000000000000000000000000000000..a4d6c498ed4e50ffec4c2e6d665ca6edb4c1d105 GIT binary patch literal 67884 zcmeFZcUY6@_5}(?gGf_R2wg0oqd+3P1nh`X9GOuRfj|^QYUqS6U7CslQb!ON6)A#% z)X)-;UW7;wL3)d{P;=i8%XsF@Ip_D>Kksv&M??^k_j|wn?!DI9Yk$F4wADFw@$X_{ zV&c%ac&BbtP!>sUD&C}Tk#z5~toU;AwFo9i;D8~Ot z6aFZ7&jD7AopJ0$xfqOQ)6UR`4^IVQu#vt@%xAX4#MH6Lr*FgHmQ+6J#2j&wzL)Be zH)qXXcV>R{m=Juon$bL;Vdl|1K2hql+%%ABu*TpOB;o%0lOBPCHT?87Y22ch$^NhZ zC*fB4o{1Wc~jy@qGu=|G&N$FKcH#my73yLZkxH6Y;;SQ`I)9$qdQj z^=8Jp4!x&*qi2o)YVxmM0lR+(>*ro10bWMhCIEg35hrz&jy zSvX|vtr$-IjR`vzES|WNBxtWmc4*hC>MGw{oa=X+^=B+IHV2Y<>V?Z!8f_{UyNpDZ z`kV){iToCR0@j7kvqF=N)`xhFdkb;KqY2KO&1wvHnd^Ot(v3?JBKkG7#!?rSy%@{5k{qwK0xveNWoNGfz2;Q#Ew6|y>@McjlnB*( zLWPM+p2`aXR{P4nJk6@^zokXDyb9BLQE=%bM;XbJzjxr<8Hqo#l`hf`D#Ai( zscAY%!r@92f7*7Nr~ALoDpLyQ4LEjoZq>PbtL&e5bh%(=;m`x}{5U00gi}1h;i|%H$$X{0P3iuT<|GkL z#o>oY*Du4iOP|ct?T^aF#>_N(F6GW`%zCNu-(nGQx21P?mGKM&Y8)mC&E_C52m_U0 zY4P?=!DynP+w?p76>n~B1yRjdnPaN^2J6I=*cq!?41PoWcq%!3C4HvEbv`Oaa^9`( zNOZON9;1zk>r}JRT;rR@OSm2?r}ETpPq4*m8Kq0zl%PZ5LG7R2M%PLgJN2;!X*$g+ zWnQ5tM_`dN{=9}k)2_BYq0B@#sWSSI?Lw=1U=^2|ilB2Zj#hkU)8zmmha@bZtwO7| zWo#_3MII9RnupE!Dxs7d|6rIMiI)H*_8x}tR{k87=O~3zt43vf%ekR z;?28jh2v>)CRxa|VCIE!9WSM+LTY%cUxmt$4UxZi;??U)9~NOml#%C3_GD)9g+m;= zEc2!z68S+4HOBfp*5((=)9Z%8#Lhp z%hbQ%?HpKd95^Q(A}+Hz7pTUrwlS_#m2o52L|Ik2^yA(7E&-*O1gnBWFAon*L{@6Oq_P)HFPnS zaYtS)6yi$R;lgp2;;boz8BdhScT6iI)_Xj6iK>e{Df;vWSF9kUnF z2;v@7_O$&0%g!JX1U?942}X`>Hi0cx9-xBh`%O0)IQ% zx_CMnrLJ?_f(wZcLxXHe%axAM6o?G#5qZ{W8fV3m6t2-HvN5KfBUxI4Ttdd)ZFXlC z>s+185^)P`4R9TC8@#5jiA<|!AXCS7B=Wv7`)~5 z8rx>{a&^&yE!AyM`Hm~H92`IVy6TA zAIZAG%dJ%p#;wa%Mrb@2@uz5BCy5;G$@2S46ANp0bF=ijP{Zc@rQAjNuPoSAwwT;p zXiJXPyBWPw&6L8+i*7|ej*%-cgmSjIOs5RUj!|;pSMFX#sf{aOKpM7*mot5gFwk^c zn)+nyzyDfLY3JB67EPAiU_)F7HgUjXxetk77*nuu=cItz5|$Jv?1a$W);Fvs@DJ2&))OjKjJPVRy-w564rU6@0v=Un6eZmJ()Y_N7nhTP?_) z8vL4fU$7twns96)r4fnviI+9-*Sv^ww)p^D@z`^}Pv={5LvWRaVY8*n{UHGyX}ksE zZO7SlkIPz|!b{`go-ZtH(-dux3_X@skyrTpLqlC~R%T2b-=iaoVoVkctHz9#5SXab zbA0M-DPb-%??>Ty4dV_J4{w3MNh^E>t2@iWm8#H`0>qrbbJPbU=lCJ;zk;t(VR$W6=x!j;}OOe>$vyCiI#>TpCHB(a7BonWr`A-a@JnAHIaO7>6_ z_Uv0+$gh7PH6^T3`(UAow-*p8*Lkt=#=oJEn{qvtxqi%@o#022`1|H?J$)`@nVWS= z^{o_1Yzj{QHGAjmw|#DU>-!Z%?16=%2#AnLkU}DeT&39vJ+u&uCOn@cW8%Hkq4%g? zy?3UT@tNmhBNLV9;%jE2!+2QDA72`KR$W9&U%8w-%qZZz&t+9`>K+_HwM>O07F~`q zrHPJkqp3_O);(4QVOHY?VnNLL1D! z(9>(E)!2dZZ%19BH0$!y18G9Df#f~~HaGMRI;ep&rz<;67!INpsx z(+7QVr(87Cz~PnR*U|Ll->#Eg%|so4!UHyiH0*Slz@)s;a+ZoP>@sw->SR)09ZN|8 z1+rzKH(7GNMg@BXSA0+>wE9GKOC{p=)JawOfI|fVU-%F`A>%>JPN3kMxfP)}K+H`I zD<4wZq*HTLM|jagF>(XwfFtO!+XuY9bD1cPuUK(sX%W&PpKKD6jj6bQw;JPsTN4gw z?l8EZK zGv|R%3_cw&BBx2LD+0JLh3|Y{oK2Y`zuFe6y-Nm|$r{(&+{)@uF*)GbAep$guF$Aq85GCLbk)U| z3K9(p#4A)MEj)f>bC%&L(>??0I@y?5q=J6qL^cl83+WMiCe@&pxNuQO z^K=93g|s^74-&~8#3$6^LFK2yE(_D3JNWV zfy{l(@|1O7E*{_j*>TlGjI67J-tW}s*kub&@de+ha>iPD)xbvIK+STvFj{Go@$tG5 z2V>EQ@or4$ugjq>NP``y-;U4&Jir7ZRwm%?23C2lP4kLr$0Ixec3gPf7KzaEfj4SLml`|&-h$jvG^IOSpy%nHK z+7KwOjPP7YnbTn`=uiOuU{#(iv6so6DVc9#hE0vCFaa(cWbK{6(GOG(g#~PgzgSv^a?8X=OR>BQhD5Y-27qRi4qY7%0A_o=pDngsmhzYK7{6>BXO+)7u-3mk zq-V6PK*b~~!J$oq@~+EJpm4|I9&rc?l5<0Q1$80yi%)6zc2Iv&@7t!bXB)N!kIK~2 z>v*)R+uv1P)s?yNb-7ROJ(lgW>`Ajwe<}7UxlXYW-?%D)eGE{b9>z@1(m6DWYOfvc&QLjAe9 zlEhG4niyIlevUR$?FA5+0wsoC7jIz`0hgN?m7B=?`7?p;uVc+_FoT0S0#FFuGCoDzZfNh zMe&mMtO5CF%_l5$**FtbsM@}y57E{D;iZZg=Ba_%a>k~@1?uM7tccg(+}v0?*}fTF z1*)Qeup9jfDtN3sm=v$-<@`U(1 zyFy$CP!;SITgsAgM$Vu`rRpP!?9S1F_X;nZQw073P>M-7k!D#L37cbG> zyTT}mQk@pWk&)GSi#dvtToLog3EdNV*$13<7+SsiI5fTeWnB&3AW+cmBH6ZzqpJ74 zJ#X>TM{*8XCBu&|C+`tUkSdI&;~oqL^5P^Cksy6R0qKR*Op=D=x? zz`t{A+i=m2l(^aAV0A+Fe59;|o`cmT*}7Ps6Gd{VvA08bVpqeJXLf=uvuMbety;NU zKvk&c8P9n%`HV&-k%UhswB)NjwrsthW`TWqEH=aZbo;xUj(u94d6?FOtFdA?+V7ut z-Zm+VbvE+aSggWZI=~ukq=Bj}UMXKj3(Ic3AmCpwS$x9!#5P#ZY+k!Ij%l4m*yRBK z%}R9MR^Id7bMP`^+BwOgd(#XI2{3WUk;@&=@h}#!gKwcJ`5+JMFmR8DsD8x~FndMcg{WV6m6XV{dD*@$|7~bb*9NBnHA{F{o5r1^K#>AVpUNQ(oq8B^%OURe zx+DQb9ORe1_Zxr|LOcQjp&Y?9LM8~UIiU1a>s!-BO6$mTL9Evy)3?+42b9?Tu~4Q5rgB;Hf|{n5%(7Y z)rr!oE)s@y>jNnOM-_!C9V8DLRSgcwdOCUlb=PE83UkE-@B7F#3|BQ~y52LeDQEo< zEWN3v(rWAr(_ZZ_C<6W+i(aOdRi2Wf!Gk=jxXiF?AoaF^iX(SZs5Rl-s%mTeFxDHz z?i)Y6c8U&;?8S(%b#erArxsVXgGfgQ_>rvMocSMEdo-#`PW`|u) zVbM!}RF?J9BIRS<_|+FyJq}$C%hgiz-epp6n z+8DTiIB?&gY7DLEft*ll>cyV?aqT%_OcUXQFLeT?g0kq$m`j!NIJUGMyW1V_L-VV> zr5`bq(j$~2ZHGZqcucKN&m5+wc67bD!>w(n@Coqs^tGjl1N1YmzNN;m841da?~Z*u zte@A?0R#|@G6)tAW0D>U3uKk8PGoW*MFFgkxLYI_6`Kqav5bMhWv$k&lyofB{4tcF z#j_2u-X61V0|H1wALu~{>m=7{i?z7+p*ws z0ltsPydRNDRg;~s-u^)fB*&b~gR#IaId z676`i)A{+!KG0ee_qQ@Svvg$ddLb$2kw?w%*A4dD_zcTUm=@>Xoym&bDzjg&r#tOe zux3SrU4R~OSb-okeg?l;Y3FVav)q1A8SUhdp5SzIt5O1;hKdSB-Rt;sZl&2vH3RJS zuUSBD|Kud>%U0HPS*lcKy>DP6!8LtsK~3e^Q^RN#zoD^|h;U-(7N*vW*?kStihJ$q z{ZUxrmS8#s_5{_k(0#l3^CH9?HoXZoFLD=*d&KmFx8#4R1;ytYzT9p@A^?mguPinx z&sIk!g~|6Tgjdy`)ecZXYF#essv32ppY1g0#NtD86YEhD=uFvnP+Vk}rjv*xv|pE` zR2nXahYf7ZII|GB8m1%R+}&m52=TKbrAh;R4bd3|c9yp{I>W=XdYJ@#vQ4rdjx*-~ zEyN$vv3uQSOKH5ec+ovCGxwj=dYcvp!re*l-UBglxT;u=NJ6`VY4S-XY{S1~)^v!={JOaZs-NZi>%|*gGYgtCN9J?i4nP zebU-3`o;ibzc2M%a4sB<+3v~NkLWGNYFE2Q5V>!$}cK!^!Y6jZjzqm4`a z<2kh4qzy}YmD+k$Hqfw7v^G_xbeRA-A*{o6cWTX^d`SNMxQ=x$t3$!FyjpA|B6r_~ge$!ldhB>6dkbag=z z;4b6q5SY2U7dclDsIFF19n4Kl;Ahu)EG!@kZoP3fCUv?4;?ElZ`ZkL30G&hpqA~e8PvCN%&~FKp^HD4X z3-;UWiF{^$vhpIl`5Pm>$fbBQ?38Dd^bG?jZ`2ig6=&1|(gwwrv+}^pOrg}+Vb7S^ zkNImi5#^(0w8KW>=C=kvl%42He-4g${G-cw%NL`okft>I@S#32;2@xWGJrUJdl4u= zUs?pFrU8%^XVtTCr5a;FyYe2!sc=9A>hLL$@U!q3X(qO7aK&NO*&HmXh&ey#v6#ip z?6GdZ#kPPgz_UCO!!mF(iX_Nw2w-SoQ!(?%-8o;T0>7*rB0mMt_mIhu_O)y@mI=88 zBcVL|3@t*?m7JtLKCuROS)#DCVci6jue9>-LR57tGh0K&E(Ji%ubdUU%S%ISdQ}x! z!;KW8M?6J|wuX4F`9abJ{A=l%&!D+gc^$Mp72K8I2^+AE(WP z1q$Pj3JY9z-Y5idjHxj3#;tKt52CK^f+I?WY%jScTy%T`MKTDbj=xb+iP+xnOf*&) zG}Hu#*MXD}HmNeH%5aVsi{E!LoT*?Z@_V=zQ|M^^L~Ui64`a?>^OGX=yhFJw*72S_&Kr8?DC`CU2n|+K z8v{|2pv>P-BH})7EzEhFL1;d=VnuD9R42 z&v~TmIVKdwEQTfo+!J}It(>kLyZ~NC0IGNYh<~oNFEI6h$dT6h<_9;D8Sd`gQXrzkAEA(Ecctt^6C3`LB5R54BI-V|4yckNWjTlK#p>;JRvl> z&QrT5KmZf+l`j7KbpfG?r{8_P5oy{GGv~G0%m|Ly9N9Cdvdi69&?UKzZH2zS7W^g_C1wkyOu7;K10bytj=7qEShTa*6I=>L`Esa@xT( zE51_1?@YHMSvge(R8#-QM5Z;!T)=@04+EE>#mQb^%dfC~&TK-dc;4R+`EeepsO#_C4ldZWe=L;sPj zV!tP__P&a-uUO!-(Q0N-tC3qnz^% zS$-yMMpS*+W9bkJ=NHG~^ic3it!kw$P3MB^RkDQc@J zbBqlyh7sL}G3n(Glx2pqOohwM{iQR&Op;U+xwk*dgVce8S=#=u+c5)=5&F8wA4e-> z=1H3jDCn2EIgsi5B@f2J5ju@SkDiwh?~88%eJG>>(*M^m4-qTaitEgALH#8ufa)6R_cZ&9IW2uLP>T&|kr7cXXwi2nv3fwe~F(>a_DtWU289)YjvC)`V z6gn-zx5(O?L-Id?o@q$!=OGjOm~#Jvj&w$&m|5<=@$)@)d(LRfKJ%|<<>s~Ry;*sF zr%t2E9g6HVo`WCXok`PxPPc9s=hnu3Ki*%fHozjkMcxR!;}PYEHkggWuhVI*OxVyg z$a+G%?afd=#1*AH8Ggd+B=s^5=R#m+sdn^p%-D`D5C+yFm(z8+Pmz^+S>1$MR|pDE z7Yqn%C4!a#QA9NAJiWgSqfd3K`tz~KHAs}Sg={{kk3jL3!MM=2|AKd>bp!7Xq6{=a z{ed7<&7k#Rj@sQaiI?$wLh;+)u4G`XNLH)WP$lL5iopC-eTB#Ko+@>hu#da{&Y3hz z3A=5W!YpTPCn%of`Zy_MWqy?8(W@ojw@>R%6N8*6NE){xZz{^A>G*&SX?*J6OYVA1 z7&DPvDkxCX(w0^eaTb9YU~Dc^b8v;8f_AWsgKv>HY)`a6lH6vuWJOKDaEMfyf}sm& z*c}Q#o7;KQ!e7XVESjkRHUHfIH}o*gm7A2(8KQ0zs5qcXj;A zpf6YQo)1M;tAz8SI{1>o`)mz5cDJ*afMriqWo0a_7IGx9 zZJ08NP_z|E4Jq%Uul@Ul=jzStH7q?P&samF|JM)}7m| zD&XdfB5y#-zOT>BiZ-N1-t{y#$vJ_{i&1sF+OA(F-MYO;w*Gg#cUZUkEwU;<2EoEs zNuM`tn&?KU^6xJaySyRyUG0D0v!d+?@ulT%cOAXk){j{59)ygGDyu4_?eyDt z6Dl3#KW}kPyD#ceFMzg05?8Xb=!f)_q@h}?;02kt2TVBvI)UaU5SEa?{5I2M9pl;R zHU&IvtiY#vFFw%l7VjN_gXD09W5)U^fxNsemY3Hni4jNLy+8tYD|wp89egN4M+*xx@rcUfitU0Moxy%c8niM%6+vy45BO>tk&XyxC~M2?7ZY^u*G zi7^dGjL;z<-ibiVSuVeEHSWzz^24a=Pny?yuH}zat(3DMkCD%>Ot}q(@*Q z#-#$$JkM}!RPJ_*1If$m3~mRw;EQ-*}6ClJI*``(BWrU@<4wkIu1p&9N9Se!R{VDCDF!bu16q3#)NVdh!{Q zJFFD1XaoKmo)i>Zbz0XKhMYWZ$~nSWz}yrodwjysr&+tUPG-J=>lehY?#`k0vW^oS zwzZ;x)gk8N6n{!hmbFUjV_X?1P)>}d!9TC&RTUy+kg(%8tTu!CNZZ>zwT2(M?OFMAfKkxE8pn6@L zwD^0YVN2+O67&3V9yFoQ>H)W_WXq=;Il52e?%x&JKETXr8GoXU%?X(QLUcS-sG-tB zZ`@)|zU91u*`S~^^snE-onOE6RDV{lz^=tG^ax%R=S{p~vSGlFRNH&f$WlF~%3Ts( zTMD#Gs>B!272*We8?6#=;Oi=_f_#!T=SQl2dX$WSOVj0e`odgg^+EDAR4#c0*-FO+ z5PA>JtFf+>#~6MIIlKzoa}kTn-4dJmoRT6}F3b~>x1&S=$+-d4sEAT`U_ z&lQmvrj<{YbGP1*3a|<drE1=tZ)%qN#oSFw4=I zq37aG_*-_l3^u!14m8X(<_oZEF-%UN$aQ^6pHgL!JwGt`b&Fa|+yBk<~IOglE z0;Njk&P@GYCz0GBP(ZYQeH7L)e_Rk7)n`{PK>-^Y(zB<)F3ps;V$^g^lS3-4S?hwEE)xmkOM zA@%t5Q1o{Ri93^4JJNfZCYk;L=qY|Q@oqJE%I1s@vpv|7e-`u@_UO%INM*pPF_PO{ z5wL=N!1N;9`Y_dux4r6ca>3pLp_b#n_L>cjJ4*d3y$#46JfdvD9l4I>)GL9Se2q|Y4zMtVdT=#-uBDvQmDc|8&!fT zj)=}5o!6Z6>LH*vsx6UPIpfGIp!G1L6{uuB6&Q7$CX+_50 zYho^k%-G_=_MVOJ?~LP{IkNY_kHdHcZiONrW(!~vwt(v2C}=upO-b@2PQ|m`2GNzu zTzWd7C@_|@@rD(kVi*Za{g3;UQN*`(D83YCJ`KeUS+x^8kk%%A}L#4lpRaShrR_^g>C^@pi9b*G{=MTW6 z8_@khTAV#bj06MI62K=f-hTCpZiXNdUYM$2}y)HnkHEndtr z_sdzHDsNj=3S1BUb!Y#^5Wsb1pXEZ!Bwn{~s$&4J7yMz!4#}&9M*(}3{`p?~bkdf9 zn@4`>64W2OX?vwca46qi^3IWs1PCb* zuDYN|wohhTxB5_Ze1 z4I46XKpm9?Z5V$d;%sl)N|neaQe|x_mjb3ZBIYaWq{Vlx+qQL9g>Y|coI7|mHw1A) z(|24qntN}Vv^;_5@x+gCh86+!N3^P$dKUf&sme4{Evsd6tAlLsQQlOmRczM_jmTc6 z*FpazR1E(X!Yt;jYwI}yT4V@7R=#owHlCjuqT)xefu)=ydwAm)DwBKU(3#mgY?KH8`Y3jwGlh%V|d9V{GC(?9Wi)FC632yH=k5s1NTEOx2uJ~;h(rCgA(qf;G(AlVD6nUjb$#bL}&Y)@<{1mIkq zI+D)kU5O+B4e1LS*7d`frV<&BEYsP{$CMhCx^)hgj_0cLwHoLEVR#bI ze;pDw#dwOd!1*&+E?5@y{&|$peH#Bj=xgX^- zyo^M7$FR&sEVKNcu9!{T*oV? zn7YU|N}d_X;mv0oso=;yGQjfkyAZdf&ulp&fGdiG%djnRf=zM%msRq4?>NYIUZ&y$ zfps_UncmI&%yP~%y*~Z$aewVs(M}5!?oX2kM%>K3;5Q=9*RM)^I}MXj+b56o@Ur~Y zsmnCI0L>F{>glvpO#&+?2Q(WoMOma%BS?->izU-h7dpo~=91wg-&OHZen?NA5CH$@3~Y*a23^d`=;rY7ko9&QHMBc0DsdUDb$1OtIB;X=;aN^r z79oc#B_08W+qVYOAoDg|?V9j?IiP1x+%Raxnv9VTuAHuEx+!Vc{&bGgPT zxeanLx>Q7fp&XP|g$rt6wX?vaCna3z-k{ex?ey8>e!F>D^_T6|q*co!?+IL5I{d3s}RN#R`}KMIAUwuvj*_|@86)6Hd1 zTk&x-7xMxcyMqVj+a@RT8V>dRGT*p9uSi#E9Gw4>V0&K;P{3@UwN#5+d|8nY9r!)# zf|6J4%;MaN;YmH`xBv(9V#xg>h;g$N7CACZu2l`OhvieDxk=3!q-dgB39yyoL^`5!pYA%IvD=NUrSIurGT{|yY(VX+ zgKx5c9-Q{Kj@H-yWG3j3X9Kh61$*yE7pI5g=V4!uJJ^Ov!AW6s5u$z)G6I-#6ezzP z@C3Log;KLrbx<#KdQxkBs_8O+NUanYepck-rb*QL9aD=?`oMcAfQ4vYgY0X=ViPD}mu0bG&)V&LKv$6#^kHr;6b1(%(yrNxD*sAM+aRVOwD9 zmBM|K1=&X9tc#A##F0m9&-+moE{iPfnUX5pJxX^0cC|apQjLS;7-5k$W20WKq~~2& z^G120y~ubZUU9%mSVXsRC7HdV%2jT(HU7B$)ML3uOf5S*GE+iP?!og^A>S2T(8|i@ za=A!ESk)r+p#WyriK9F!yqcl=55Z#|O2&L`(%pQq_>UcSfm^Fvt$#4c8muh8awN`* zDRw@3l35HMleg!b)N42*X&MY^hL0RT51*)@>so=n1vJe@0oqlFsrEc*s~titYy_&U z8>{S2V|0D>GRS>&SCvHE=o(8n0t_db5sB|1BkWXgMc#_a-u1?I!IeF}T{oh(FRp~U zemGPgWxiW8V&(Bgi`1lai9gTksB7=V@5q-$v7t8TvJsJPo~@BRYQ@0|ORZ^9q4E&U&qdn+8!i~SCTD;$KTz;Mc%I%W1j%sg)1#@<>6y-bmbn3vZ4nD@e%m)dUL z9HSbZQWb9p)pv!?naJny}LYq>0lF0VF?gH|lb%G1nI{FpY#TA8|&Em}zQ zcs+2NRza@eS>~v*?tk5eEh{uEI#K_?=WCo!*T#2%d@*%S{SsGpmys9RiFc%n0Q&Pp zTI#*+1M1q+Eeh0})^~~N`#vxNe?f>_q=d;mFj!W8lkO3aKR9nyLL(Ztx9SI2InzzZ zp6Mt9l9Te@;_7kB+>IvV$!Vuu`ZkxR5c#VLEyM4+R?d_-Mug~Chdmygf2$cMIv+cs zZ4#*^#|q4ec2oYHmu=5f>^LfpgzU`^+cy(S?NL#R;u#9MJUD-;rd(iCobQ;Ogj#OBsy2FA_TvuDfJh$1z1J$90-RzHs1 zGBgvwfy~?(UJb)WH=I?H0=_k<;-A47_X24Z_J27{>u=7aj_LX(RcwH zwPRDR^o8u^9A0NWUjNdF1feC>k;=KxHU@|?hXr7OvoYMgv^ugPVip|1P_?^2@AzQh zQL;I`2go_I?DZ#mLDrllA#E%hd2S7LJC|rd!W-n(&fM|IGfr7&BRX5VdGESj4HtHyzqT|}OF8`=MnzBZTtSa@u|kv>!y>L9 zbDzz&OkOk1@RSmI4km@xt7h^Y$?xshvi$)sCs3isff?YtC-5hqWBQzGlfSEylG=x$ z=SN7%rY>CBdv+SBvibN|u(H7sylZEQFy9*X0Da~C#9n^*QLe&H6wHgJE17v4-+jl2 zsRtfG1m<;S(Y+}laq4-FJ1x{uVN8tCf@<${S2ML5F`B{?VQ6}X z?ZofRe)c0}t&Z+)Gn14a`Es77vhk0$oncvixg#fb>RP)fj^vKGuP>7r%JZv9e$6(8 z1`3;g+*B^ad)HS&@#Nf0@caNZqdXo74{|5@!l!FDn$~8^z2epJx7h=F^EvOX6u)V_ z0$StwcYi2HUyX~9#3F|zjO`%4xpwkymSB<{5GAXspcxMv9W-zrQuV^@G>=%^TzVbb zv|~Vu7ef+qzU4CCD4%PU2Pn`+h|&5%LC~hg8GX>r`^-qttm0P}cD+6Ier_tzC6U=ga{7_=Kc|La(#|Uav^?nKGOJVk2)7q00`(dGM)(z-A1yDul`-Dle-EF)sF=#Ef}k;w_pRjUdUYx_!UxX zxrO?e*3SK-t7O;ge%KT+9XbNRZSQ|y(Me$Y689N@Ix%kTeIDH19;~3SJOOWSWVto1 z@Rudms*t%eu8=vZiwATkmFSw*-i{2ud;i{xu5<%t{U8YLoB;9iolF0v5rH8zC2Q}G zKqE^9gS%{yW9*+U8(cB4YW}!lc|s9`{hN^aQ_9e$lmdwpkoO(`{VIXD&<41_e~_5| z-!l%RhY47FFN#@ve}1%gwk7Q>%xO5X=YdTe1C@;KOuED2z> zbC`RkKmw3WM|EYGu&n!F;g<5pQsHj*3@fRt1H;20gcgZjV)NU|G71b1{$vC ziX@}+bqs;*AbRKn6Se;Kb6k$9dE1NZ8XkQ=A?IGy6qwQZH24mfA5=rZY+^=!_dR3a z$!g4>pBsCp2QHb~#G&ge#HDS$e2NAKp-EqehWnc(x{5)rTgsm);k&TJP}VL3X9IVu ziu*fIGCw@z|6mG#kZ&&IyBLezHR@ug@^MfWx?{=g$ZPD99XAl*xh2*Lz-3ScseaCc z1R_vatUAA%!v2dskY9c2_?vwkzc{I8%ui+HC4gQ3cH&pFg*39mNA6EQTMV}1ZMOgp z`e5;uP>Z^<=RKIB*nc1qx6XEPz6l?@Dl;H1I{Ef9fH!(JLvTGqeGrNE!(>vQGUTVx z+Zvskd=t&l@pzMV!RCIUglF0Vn5ctS(~KD2M$bA3=^)%cAi#W9VTqkRx8s z#R@A1gd9kpPYsZSnej7e%3oAzHCJT50}7$9sAOKAN0;D zavsfVF7dewN410LeW7Jo0=)g;e~no%$gq>$uoGt;IIPg&RQm0zo-h;d4SJXN8(5g7H`>Q#!OY*^UN`yupS%p|Vo=#$;+jvas)7%RM@|vao)rR_a zb^b0!fvpplf1`;-@xncla{t!70INQOrQjaDhaSWQ4MSObANxD#0gW`UaB`k``;Z17 z5#+Hxa0&orehuphj~|pK%!0{mL7jWZ;?Iu8M+>CiaQ?Uc=+#EreU{z0iunGvfqXU4 z9si!sO#@wTi&W4aT7?O>ftvdA=Pdfyhz$nUx}4~O5q>+CyJnV_vFrO z1m*<7a8gF$*;_Dl2fXMG|3yjG-v*lkhk^rG<~qG8FSLB!>RgX*FI6Y6*tQ+uMU-f3 z%ku66M(#5{xX0Q?cCSbgA|f2fM3c&OObm@-7$~F)Cxd}qKDGKQzRX1LsNNZAK{%p5 zf(8p{XG-b!%hF-h9XSys_IsnkZ1^&7d;16kH$$17h}nFY_#oT2=ngod=cxED;086m zWd=RB=WOBTx zG~_O4*GV(4A|g;IWT+11G=juFSl1Fk{;N6D((18d{4hsTK?x+DKG1jPw zH0G#D=BSB0@cxOz%X_t8nq?_hz0-iy)M>?)eJdfpX_4vjAuxR?(E%@}9Db&~~=1Li+N=KPW^|fYCcb zA7C)h5o2&Wu1b*LXZp}~mbvrBkHFLi-hzhDQJ$R}`9^<~vVfY1CKPeaGMfgDr^92G zncDvf1p!#7RWMcPtceR3QUF+&+`R9+`w2mRBi(sBW|^-ArnbV^p`UQ%WoyjRaVTG6 z!WujganK8R$qcvDf9sLDb#KgHFl?ZC&cA5@T3|DP7tLJ%E2vkGs zTH%I;QnQzt>L;}idVq<6ZyP7~m&OqQOn@u6BUXpz)W8nyo@HL5-GAiYsZm=O4d$-@ zha%m*7fdR0f)7dPb4G}r3YHJgmEClxAnD$OVZP5Y`G9!@O^7-N_8{`&K-W*hoKQT27&+4oJF)Dm%?aKi4*UkFBhcG>f%V?X{g>bP zYa$}80^AEykTgM0vGDHbw*G%E)~7*^`k=r%c>9#JtsPviC)mA->2i8f)9W$x5*a*N z3~KQac4&5IUf%ftU(0L0C%!v0?nR~wh58a@> z2%nc^K+51Dztdsja7wq6xQXmVExO<27pL#UpLMb3ZJVb4AfEL~Lb zK;YguB}>@uaL*eT|NpS}pHWTbZ5KBTMuJLHQ3xF?I#PnP&}~?TrZS@_2!S9)iWCK; zMrk5VMJW=%3Ko>mK`DU%5osz4NRbjCA|N10FYj@Jqt4u-&UIh!y`J@~=Yz{xGoz%Q z=l|Hp-oGtW2G{F@>{wCc!or!~Sct*z*vDYUzBA)Lgr#t*4nIGH(gN;fqHl(7RRu%P$V%)Kk~A+#^6J>VZ8%SA`arEL!xKuhA}U>Shgz7?Cb3$I*6suGUxQZq_8NBma$TWo0rzj>wQti z9xm*(i}MosU?HnW@o^vNPq3dVna$8XiIqP7?27B9DM-Le9aVl`DBt;i@Kjh!?FlN+ zuO2$OS(vdqOjvb&O4-I3UxXdQg;>+t$Ij=-%b5bufD@s#2q7%RY_6(|d#hiKBPcW1 zM-iYb&z(ULC*^5ZEGU2v^&Lu9(?>@kIP!n^L_$xhc(QW`i}?-aacLktm1}s9mYM|Z z^2~cH(#$KDAzZ3@fP7J5IJs)8w*^5gkvk&{{DC4MGwBwB=+AV$YgxdZaGkV-XAy;S zFs(lxd8|F>Puuzg(1z+D5B=q`{Z*5Y1yx4qzw#-jNL=g{0Se)8w)h)khy3l9k7HUV zI~N6~YU}l*TF*{T-C=Xr?Iwgl}%oi!s(4Y z2FP#KBpa?s=F+Vl^6^MC`r3=sK8e1)qo_Pg&h#I8qi(^z<6XW2sNUYRYGGVl*;3a> zs)gBF2$v-Hysp#|N86?U;qj6ETy?71at~WoOz-0gmQ{59Sjb>GsFUDb(pRY2KESHn zVp}N9LKp=%e|i2}tB(AWwfiDCc%Mv-i6z756PTa+=t2g`2RS~bq$-MJ-0YfACf%rX zQTOmS3OhO!qF9w#JjltdJT1MEm$c<2H!pZ-NoK+?&$Q_WF_#Wr@m$0p-Jx&J0|oSZ zC&x!{att%tVoE#%_qp#~@8JA;Z~N7Cbto<8L(fU-nK#S<8ZnVGZggnQ>feN}KM%)5 zi$c@9tP{Rb#y1l|E5&52awBpi!V9j&fyFmo?%VPj*_7z4oAySI6ARsT&mnxm^6Bg0 z$WPO$|H;nrg)F5*`8O6zB{)|$tFdf7ES7kLR}ZB%BZZ(FcwCpQ?>XJ5C+Y;yC5d{S z`l^o27pJ7)JOj5UuVuI4&s_KPnr9>Y&jZreC5bGacn40ew5#bMlv#0A%6eRF0vtLS z71K-`YBnm(Z516k-lO$3a7RoTRnOY8ws{>iuKC9e0TE7lAT6i_gg%gaO8Wg{$M>T@ z>GSl)xUSWMWr!<$0jY?Uz&pfqvM(#m_5?aBaR14==};~)kI@(QU8kiEg~&Xb#;x>( zb&3D$+Qy9@Jy8U@6a!g-Or@4SuKFnO+ck7l%c30I9PH-nAUN;0^ZUaocfde&5(P70 zgej33yi^HJ1I?VJOx(Z2`ewUW4fz~%X}D-3I~HM7HOo>KpAkB!it%q`8Zr=#dc17EH5vw_=yn_)!SH0l z7Ko_@F0PVEX;n%l5X^ZaFXvOeC$Cd2?Wn$O-a{(CK;M2jwSpHTLEXsOXH4CD%);a~ z80bq4?BY(a>b@{#Uu~Bm@>_jv6I0Dxn)J)HYd$qCCvm%hgFFNG^v`O*3l6d)5{duv z9KN1L;1mJ#dXxbE$aj+Th!_cPU+K22;mhr}AFe`tb`LgMPk@FPgkC1*uafs0a#&s> zxqNpB|7wH|#(L-KtTh56YPN|vfv!K^Y-n8*OVx1~5pY(1mwrE`XM3V#0!`4~ajbmx zAr8=g2;gISwh|V;UAdXvnXNpBBq=$ur|;qooPK!BVEwGmI7jVKpgcnY;$J*kU*0hu z4p;(!TxGYwNCD!CC>qnU!bdZ8b`wjBZp=pOYX*yt@F1B46$f27B+H!>`~8K?GN*v! zo4W^HFM&mNn?3pMOPH1>Our~O4fQ#WJvkGR;ZU3bn10z13rYlsgYhYsZPF` zhonCyaq-irRqX!nZsYcX?`ay|21NSBDp%YRz;`0xrY}$9%bOO2hBX67;R=-0?`bfg zR-IlxcY?OeBi?7p^Ut1k*u{R%o4!Sp-Bi8)BJ+TTZozajx$7WiaeUBPo!o$BwHhD{ z&c*A$yn25r(qMW_^0uGX(m{IxYpUV5XTEQCB5*i#3C5s;r4CABFfkf&i^|TmW$5^X zZ-!6G9G~I>%qQPRR-|tzW-3>ed%&l1AHnBwfO2TuG;Y75vPujbJ83?!>Ed3dv-M@S zrp~9-iB)@@VbMyRUiB10dz^5g|43-M-zH1BtI*b?2k!(MI<7%q>aB5aztzy&rMKiu znUo3jFo_P}-Uv&gf{#b3u^i0IyCc-?b)EO?;6<`05C=Q9DSo~KvZ#dFGExS*S++lt zC`@5T*@UAcjmB0zb>i_}*74b>BYtL?*@v}jLqO%{0>LQ=Tag}g#Y=-7duG|~OksBC zV_l1uIdvj@%*i&x82-Y?(foLy*#6AcsBVMh-*O$${$?|LGlwljIp!||@o7i}~J`?K{n%-`}pk~gxGU)f(i zvzOsVkeF9XNKO%F*G z!&+PJu49KAv8-y@wIQ1Pt6t^^9d(74^80g<_(mpy6{?gjas8yxLm|uQn#1XhEgZKZ zc5sp5E%=usIC!4pHs`<)j{LNx?wryTQv|qs4|WYNxy4^=qr~e_;;*gXuU0AR-vC#5 zk3Dyov>D4$NsYqz95B~Bu7?M=8(=>M*I6+=rwzEsP3cZY*)qe^SuwMk@Xhrx$#y3O zGS_l6FnGTLqQt&s9({uh08kbB}zeH)BBiMr8;#AedwSTMaO$YVfLFtxvj(fK_{9`7lSH_>EZpg#6R8EJGUe&T|edJ-fQ=cxQq zrZhkKmgxaLH`NoBSups+_&qXs;U!1+xy@F<1`B$GxZJ+?Zh=4`1iy|Uzw7gsFbxUg zHE+IN(Y)_Ah(vI6pQl*?ya2Wnw{2liM}919#z1CKu81Fn6MHW}p1!LU!9#0?XQ5_< zO?|GkRB=3Y9^Uao!L-Po_!@r5>*Md-i4wx#f$7$GEb^}8XC$907D707cFSwPh2 z2g*<&qE75)@)#7C<%t_*n!NqrkTd{JHTOe}4CY>K&}|Mqdh@pz=qmxN2HvyBgknB? zlAvL_6LUXWe{|^2zsOBr24T2gh#e;_w%>Z452uJq>f5DeLA+C=%lLB`9IGoh$jE}G zbclK*^KMY>58mQU-PZKpVqL@palRimyPw zQnLd<0X-t0_wR2^gq!~l2GIXo8ReJHH?;Fm#jv*XTw(m8@qL~P&;zVAZ~L?+^0Mb- z&aB!iWMxf?Bc`27gsp%(A4IOY-!(BBX6ViPwek}`iUt#cC{=c z%=YdZlz6O{T|WFY`C_J_#BI*JrXGPK@Q5A(m2O4s?ul9C+qvh|-!9r;Ky_eHe;1va z9=9kuXkksgOZRMrH4H*8h4J%a5E^dcm%pRnf4yRI%&dC)0WKFJQkWx_vA!gaBj8a& zNVbQ-3cCQJ=l?C&e4o}Q3&@Qg%co!t=<;c;Gm%DBuom|7*Ot6u8rnX$IbhDAdT@jMl0NFgb*|;VqZtvq{kk0h^$>vg5fLD)gZjQ&*&J;~I}mNz zxi@Mp*(2z%3mHQ{JKJC-vW%|!Lx`P%bKj#5AcCD@G@8BL%EKlPMW7lS0Aj+ zk!Fcw?Vj4IQApsg2*RwZSYeBS@NpnEuQ=> z_#Onhq&R~p8Y|7&j?{F>#|D*!iv=JpI3};|9!Sx76v6}fGBKFb4jFrXa#bsK(4J{2 zlTF$li$KGAA?F7BJj8XDpLM$Sh*+)3G21!&UJ}Z{CPl*)s%m&23Vp&0fb9IEX#foW zX#GQ9r_C>%8N?RBIq$B5jl^T-guf?S|MDm9{j^t(;u^v#@8Ce0IRh;pgScW1721ZG z>U7t>n?aJxz_Fk!$aEyeNQ%NJG;4Fm6DO={5~9y-zbY3S1XJ}EesLrwo)m6<%MZLc z)lf@Ch6H)Q!_0=?Mo5?EUD*{k5sKg2WzqARb78%W@`r3f5;c3rRI;s`PauK_L|SJa z)nu3mzytr!p7A^2JB3f48R{x|3`*;Tto=_lXo%GkVT`!0q2gOrOwJbuM%QJs#jpIu zR|fZXaMeQuL1ZL~7nDwTjle??r0iG$Bi>tW&TGEt{NRl^AU(RaJ%3=&wx0I{l2vf+ zz7yFVh4Jvh6=ie(bNZjgF#5Iy-kx;phtA%+ZMl9HtR~@{Hh#?Hd zgLmjCozDoSzW~PbX0|o2*S%(8|Z~|>gGk3|QQE4H!8hfp- zY)}L(X(fwWju_`Q!b%tr3B0q)m__IF+7B{FCXn}3}WhfYG$XT;Q&AMAi(ev_Q& zX4&~jBI`&JeDdaE9m*n-H{Xok9=dYEnT&WN3is_+@+`Y&E((e4FT%Egi7?W7<+`YH zsYHS6Y^U+eaY1*VIFnwp8X+~YK;T9l6|7iY6FmO3P$ybQX`|624rLE^GUNab#_oor z^w1g!dkf;oVh)lusR8!D#wAfyVK#k)1T|23CfWyh#TSd z;N-BPIkqhgLdFJ2+wn%+joAs0cLO>qQ}{}4;6Zz4znr2mEH?m_mW&`ZH5od^^N%b1EK2!oR%l|1&xeVrjZVlGk z@n=aROlhQ!A;p}iHdX`4O=FyZvrEyh-hACKCF$<@S^7OOQuc1*5+5Jti#Ex4Q%|=Q3uo=S# z*P-P(SSenYz^}8X4}@BP!0}`+Ll^k@>rC?ZoZ!N~rMgh1&sJ0QhO+T(W8(HVC^!>A zE?uSW+pbCU`q&*pUh8i%meX9iq**7uLlXs+@7*g*iE*+_2v|d_;ZcmN4bE5vI*KQv zUT7I38dmLJR$lmI!;e!2ZK13E&CkLc>3&{4%)8xl^eIW>Nb1mguSg&X#$4>O^Ip!P{x zjy%R35Vt*;!x^H+#dPVeRWt|1(ybh!jBaUF)|M^wpuuf?2{q^%Gqt;A4 z$NNrb_*$Ma(%^Dl?s2x-v3SKEU#M(S4=qr0O0CXKz*=IneDRm1KkRFiuTrr$x!z|3 zQq%T<9&!{i4Y+Xhn4Tbxi;e{?aFCOuvE{H_rXM*$wCZ3dXO86ImZ%L(a40k_r_HvP z(F9M{j0i@)<0~ye>+b4eO-PLQQu5Ljq$#SYfmwL4b`QIlOHCQvAY) z?{|k$!9g67BYbJW#+mnGiaeYa(fYzjQ%LJf3TMFurvuxLy?1hS0Xi1VVN(>R@;kv_ zIFZUt_FS?g4)RZm+j$y7?_|5{Z-%Ru7@o(HrvYrjTYX&m3NlXFBP+l-;|gANwDeuo z$@T24?+=C%B&o~KTQ z7h^;K6bqGCWP}VBbm^e|!McR9YvzV&>XT{eY)~wFT!n~eto$%A#Lo;T(nX)a+gV4M z%|LMPcA468@nLT9M#p2xRqu{b-3-D+UXB$Vq8Euwbvv3o%%}_P(Q^80GqmMl`av9I zL+MI#e~=&M{BRz^C49VMI#P~=tC3w)otL-XZ5_qvL%xIIY9JAzR)pC!K@OXjq)$(cM{ z6fZs+O@=B|jazAo$@lE4tYhrhGF9{IGR5w3@MLaLj7#B|&=XY-Yx25(&s}vld@G8Y z1s5CZP@MXfj@&AT{}#?zGEcr#Xp7;l0iWfl^@Iv1 z_w_X$o+_&^_xqn=1W280+>mp1%5%bC%4*X++Z*zA_XCocGwu9Va50OkaCXeU1>f5UEVRFo zW5=qoqGr6hWa0w}@#JtV6E~>F@Ipk4!8#{58aCcTzAGmD59dr3)gS|rtcDX0TusaT zcEDB!vxkLFVhgyz?o|lRpwsQl7mi0->&awiH~~HHe9y9$eXz!F4K#l>*{;^IjAnSs z8~5p%$l(yZM;=Q4@51v(&GbL98ThRWoigwJbTKBeVY{wO#jfG~g_#@Cn4WBP7DuRf zfVzTjG0O>v`Ufi;3AkYlvqlO+3zpT=X-g-Wj0na)FQ_9@ z@!@sv!edH)nx|-kmY|W3^0J(9_ zX^YoN{dxjBlBh}XLUU8{;z%vWFOll2dBn%m54S{Xr5a&py(wOLxr`5{$?@V69u7Y0I zWp?VW$+$!Lro49j(48FRXWh8-)i!8Fr-_Ev$X$`1HwJ;fe79j2m6_Ky>ORSpYc*m+ zg|$($hMoHBS93?T$vdq2HB8|_S8klEu27V0LFGNc@{g&FsX38n`%f*O+LE06(j9Tl zJc51Cr`=ygV0SL^oM@W7srhH@09zBBJ5eq%KBk`^jp?WJTDW=X7S6;-S5mQ7->z0>yRqi)|BBL;X)#B+mL>|J%P1Wis}@pS-ze+PCbaOAFE*p|Kf z1Q)rm$f7IKbwndLWe0X|er7zw9o&}{Qj8^rmeNbqb?=%5Yme;I+8LUm2#2V@7Dmz( zXecrIq5U<0>m2NgzAJXYlpMZQXAzF?IXj8O9Pz>9DS{9%Je7znj7~jF-1gB(O}BBM ze6quf;-zjMd`Yni1zH1U5`Yu)^kqYJh14CK6UJ*1%{6|}8@Z&Qd&sFvY9Yy>m%682 zbj-((uvNXO0+^K_S`#hvl5AEA>bM zj*ZSyv;=1V6)G5%*3m#vR9Eyk8& zxS~yZE0I>4{N(X<2SFVAFSsX^3|9^8+QZ3Ym`kqdj6CVwTX_bEy1(*xg%c_nU8Z+V zO_|PgZzz?B^{}vgDjRd)J_KQqDA@nbdJ6}!lXhp#$R6k-#yQ{c+fB?6^?8iAjDBq2 zuyJjjYBd~vpd=BNN{VxlwT?xL=^gq^!Cwvf%lC=l zR+?bkvh|u%>F6eiUi>4K4r!N=rSORBFJ}aVbI-H@6b^1N4j(-tZlF;%2;0NwBUb=e zZUmH_d&&IxS1|*hHk&*nvAnqbJE7}Xf501;Tt1!rjyUq{io^h+QJg5)qi8(VPA{6( z*%-~khEZo;H=Ddq>JP}{Qd?`2)nnA=8|_pSeCDKo^uEUL`*pGl(E`4|Y+YrUS`#3+ za;}qG@)yjj4$sHB_IXJ{eBE<}f3)+Zm&x#QrGmOu>r?os+;E%8BhojnRXvmqd8nC3 zP!H>D45$6xIc0qQ%m&cGzf7w?l;03ULQ@3pttV_zf~GOyY*njE039tB9dODBR|`i& zmE-Xg=$~;_@|+lkRk>|hN{ZsNxxGy>6E)n5-*IOdMA$r*IPVBko3Lch556&j*S<^J zVa<<6d>!92BK~vZx~Y%3mINEu$Ujk9U{%dxmS5{etUQ8!ZlKv-^rVGNVP>Gj{^Jpu zq6q=Jm4}WIn(Zrk-ET#g^f2Cjbi>UyOj5^Na^$sQPitfkTreoQa*x+n`bj89qV}v^ zcI!x;x?L98R+eJttq^>48NgR+mRHhh;#*$ z>_*V|q?1%B#Y{c^zeE%HHKS|;Bsq4TAP#P~UysyPy&G}b*1ldS!11oSKKJ`F7N2>$ z%R&x0IF8I2eSTM^NajvX+tYnc`t}=o+19&;YZpBhu;B?U=$OIwRJjzLsZ;0EQJnQ3 z=wnM>nwFfut|EVYPg_x=F-J>IU$lL1RsnB<;mo!u>Dj)L1O|kGjBZb!H46SR_`o4< zhB?g(NSA|{ETnwOhf~RN?>XxbBm$8HkKE*AM;Vw^O(9^p^w#c>3k(BpF`bG`2t4Qm zJC<(0GN;aZG4BglQyA$()nyU%GXkoHWv42#y-3m9Wn2Fi&QlWU;~xU5pZ6TF&WC82 z3;R9h5m>L5#tJn1K+ot9xoFuIU&s&^v-SS`BaTFy;Me0TZp7GG2g6gN%578qhTQTk zrsY`k(ZIIh{$>$#!=Y_aZiOz?ilF3WX?CvbD;`g0V0#{os@ctMuc$eBMnWrov<6EL z9aWDD9R-(qE&sr~SJX`uqR(CO!|HIRfHOA0PP?dg8K7%jo9F z8Btj6QphtAgrYj#$q$!7z1d!@p4-U^TG zw(N^GJG6iPHvW#TPkPeA71=yvqkobvQDmVu6pfVF`3@S-oWRlVi%6K;SRpkVL7 z8<%lC(H7Ji|W0Eb)R{FLI)>235-@y7m*J}*B_=HIjlyq z)mxWPc?JTOjiJh$NF&Yetg0=xDSh?R)OWF?!St9;qkbfzRbgPnP8dvX6J^g8pD1Gbv*QFHq$H;jTR2NfycaHj3%TU>y|!J%s`#fiSdA2mzmQWu2! zqL2Y#)dnEtYctaT{-;TK-&NmKV{rrau)penzB0mOjX0pe{w82b40)BfP3K8x#z3VW zH0aeDDtipQJLyN|oZMaS_9T{i<&DB_CwR^H{*8;x4|IW<*2`3w?qk3UedL_*&TCaA z6*<*h_o3caM@Kh)Xdlu&GG5?&+AV?%Ng-rShy!sOn#W`W)vcihO+K%%*nX6W8+Xy7`VUz9oMqhdktqqQ1XX3;aI!BV#ieS^l{dPVQ{I`w z%>wc*Zi!?%uM1bAk)uVF>L|@*@>prW!>mSWL)T_5x}K=T+dE+_8%rZCL2t_bB_aU) z_*qyYenbM`6`a3#^G59%(rk){eIzF-*#2BqAe-@9q|P}LipE?WrVy+xm&zae!+&xs zAAmI-`0whDOQVvcm&}|QYGDhAbv4p=>XY)lrK6%;kv%?+9U4(6G7%$j%Qu*v!cC2B zZYwe{X6J2cwj;^5O!q_s?Ph5XeP*kGM?9dN{m1Ty!Gdc#EL&v_IBqrm=16 z$y=_(V{V7*y&@ev-_^v7aigC zuiSld#YT~V5c?51^<^^Zc2XPb?qsQPhvLR_?VJ2u{GCotc2i`5k@j@IoyAYA=>Psn z-$6W*Ca?%+2dkQwAeXo*bDL(gqLcYp6GM%c#zeT$sn+`-IiH`^x62g9vUoVAE?U51 z^4UwMAw#K(nYF_?G<2GzNn~*Z2zj1}a;CGHblq&^vuK|7!i_!_D|+NEEGXY#^T58o zOop12Q|(*sHLx2u>M!C)uX?;m@bt@9_NAjD>@_E6Z@WMkj*ahGpVm)4`N;7L(Y7KG zfv*t=L_dmc5uQR2DR}%Gf`95Q5ma7lWDIZ_Q(fu&>rDZjzH!V$EpW?nZLjIm*whcz z$7_E2q$Trv9+LZSi1{9Kj>qUkNsEyb`X8RHJ3nOSEheZw)P=Qs`PPJSjOM>COj(ZP z-TkLYgI^0*qE6l>(x*K4qBB|fD=>SF-y$^y{*KBu@2?|}@0}X|l_S+kNQ1>oT^P*E zu0I2|ruG2dF+AeJlcgaU!8qXajH&C`;BM&Z{xP;_ECy2%SvUzFxu1W&%lqAxJ)EAo z6m6DC=43Q$^(X5Fiw6evd&p=_zU?zf-Wkfp&84Cw=sCe+QhRl%NW&?flTq!xWduVV zzC7|T^*Kl>DBI}S5wBX$_t5^cZ-gHpfT3$(OV&FB_RD4Q)4di{BxBCy*3D{?SR7>p zA7nQiQhC%v%2o$Z*`WL$XTzy!APbd9!97bPnC&!{_xc>B=qD%QN+#sOG$%!xSGlVi z+SiGp-XuD-ajDt4)G4=BSbGWTOuQwSJjX%``@eo;4GO*NQb?l(bL=xZA4BU082OrzzkkaphUeacz5su%?+m6So1P(Zwden(*RU75r;qlYTT>SiaNxPY5Q30a7RlW6 zZHS(nmrxT)dOt{~0Y)3Bat>ZN#kSeJP*^8>kLlERa~&V>1c~iZ@pjR=1aq<%es7BS z8-=chSM2O+aTCfj4=ns;l4titsm*9Z-ZgQ%+xCmr zn&eq3Q)34)h#Dm$%tlC56X3u~=b_tkSx^XvWMcrraSaTgGK4XnR|sQXQL}du%BEnN zOW%yBy3)Mb`!t34;*^5)P#1&h|AJ}q1g_^<0EDND(-n2$**7|{`qkvS_g@D}Q>M@p zZ%p1$T-z1R56|Y3_WjXWxtP*kPcs;mRjGr#P4WX1`-`)ELfPDV$2-?3P{v*$$lqX^ zS?JZuzSKj*jrrN1Gl+;KhYCspZ(nK!9$pNg+NgH zn|=u&A(b457SB-*#7!Qh>h3({s5eGLdOYfZEl&tr?tBn&cIPtDROoET0=M_H1K`_= z!$oF3WM=UUXrw-eEFeI7{z)Q*oRk&YJ%C;Rp$OO|GYgi=ae~0_ux8{So4cDHQJ~-VgdvF zgbJn}t_cwunJ^z4>O8#6z&7vnl=Mag-3%YwK;v@kr2jG?j#F^vpmcU*7T(zQ3-&{B zimAT+o6Kp8)&~^STA7C zI6Hg9jkwP~NgFoPRne2ozb9y)0x*yqrGx)ev>XMSfi2jm8z=ZAbMW|`0l(HmA0Qnu z7_lchmhwIU$sCbZ&#?I8hZvE%`U z_YA`!p0>}Rarney*4WjD^w-$rk9Rr5lMt-c^S89^qcZX1T4NROd;FwqRMl*pcxH7xkFYHPCU*KE}d`>t3!EAe9WyJ&(112NI}J!puNuWL*%*Ydc zUw?dme2tG=qpp&;2cf~XEh^|kGJnhCcL71&C#xQdC8dv;w>>bPbaU;vv?n5}?hge< zhIUp-eF}lPVvpboH831$SRVjvG;EiDF*OQtP*5mf@$XpqH&5+1d@MA0G6Ca7eBt9O zojb-tl1d(5GltK;GCb41^b+aSfUxv4SE0JNpfn_e5jSRUu)#Rjd4=)0GZ)$5kB;nH zc#nmMys(@>DUFHmu)X`{@8N{V+Q3iYc}+(f?$ z(|3T`1{az%ne}MQQ`VYRx%}W^wta1f`@(gUHU|qRNE$^E?zX9=>h+)#;Eyh(`1u`$ zs}U5-3h-OOLIg=r>z*a0;adYkGIj(*3Uq!~0^*Dgf;E!}TsZ9p4^|yshulnGWHgrV zR0Zh4-+eP*iC+BZMwED;!)x}?3<%;zwMEd5IltNPAwAo&i;ce`TvYK$`}*@&eS(}N zTOmg2mH(NN{0&_SK+DSQh*7kXQpSh375k_S8rg?a6WzHN-hvQ)letvF)R8$nv*%9) zxV@uq@&OS0;AyNt*M}sFo8UR*i>FqqPLH@yuW16lHd~H_t_KZ9MEAtIyrns|_M4vn z4uk)yRm8Z#mbkv`^1%uqE-r@eY3Inq`2__-TsNREMtvunV6Yt#l~JY|46=Kwg`U( zdZ}6y9m>~Ha#R#id%B+f{IJo1A)SX5t-_bJg`t~|F&)-;}#<3cgWi=6#>CV zvxjFd{H_B~22H7#)7zwWe=!nyvXf`Fvy+=g6xWi@jQg0>+DiEy88Vu|abK9h8Q0%* zDts6yV3%i5?^SfTD7wX4v~f2ZkkIv#-M89g^yHhwq_7qna4Sdm|6yPn(>%%+(A`J{8Alr)}S{bVqjTpr2E74rcrEvW2d*CYGfCoqLokh@g3shq}S@a!Amh(J7z zJW~%>9S`q-;M4tLN!n9Qa7{d*UKLGRzJ%BKs!BBm3aWTxG#pE z9W}X{QDkxGn3&B>NuJt_G3!w5iJcziuQQ4g)kep0%e_9FKHZ=wL@4Wbf1=9Cxh14Q zRDAYLb%1teJb54^vzue>p<@=|8Gbhl$!Ww?&@kjJY{49d;DrL)r?=TMOirKjmoWeFLRb)<;;paQy z1bOaL5SuQ0g!_javBrc$,hn>yr@ntJ5gLD2{+_YXQC848J>5+JVeA_H*&wc%H8 zt_b&$WgXkY@V!zan!k|J(!ZukafIst2XGG*09x@fD06zd)ULUPs=sc6{<0>sPgAEl zrM&MwV)Z!|nL{j!Qi4&H{u%dbnuPBfAo>?pxflpKRY1gapYx{K(OV$g0o3VPPeb9Di5oI&}`hp5OAYfX8Yz zrkHNUNawK;Q?Xr*Is%?jBwSMkg)Sz);O;}x_4ACYLJei!0pJO#8~?4ZbZ+v1$?{<7 zJ|;2LwX@+elm?xS7?|9eyN7Getj?neyYE8vc19WtXA}1|e)`Rns5)58{eJW?H#-?} z5cXymyh47e?^(I}$!aFO2@3#fU z`7YP9%%y&cw6s@DHBaHD-6!>=Hx=NPG9p|sWAbv4mj+clS}J2Kgw5XtW8zfD0L*!J z(NRPl4s2-Jx;IROBNv#_fUdk&Rp_D&ThLA2BxW>}dn`lO-v|(HI6+*xLA$CufqCQ8 zo{NB?Z9-$zf9C*_mg7&j(zy3`Z}G@b>ftm}lAC2kyhB^N)T$BE;M;#^$6BpB|KSxA zJrh92k9NM)!5p1Dx^RHuv*HKAfFGuZ|BF1OUvatpe<%v*FF;URf1xPO205Quw*Ftm zNySHBC7BQ0MJnbLn!v^5z@Tc#DvLZ+Q`Uzs;EmSxw`?bT)mdJmkz*!F+2sQnaNY$_ zN5%Pod3WAYn~_S32hX+V`J<7>py36z*e)? zSp|wgzoaW`&50WzsSM)GL4fS}_pC37grwzPK{v7;oW$OD$o7~*N|fKKQQj8Lj+sDu zn!j*cabck?4Qhm_pVyqgvImfJqx}EOcz!#NvXj$Uqo%SiXRa7ChElXi%}Oj?KV}48 zcbQSt6|I8$!Eh-Gqjp{n*@AAwGw^_?0Gj=soLf9Lll=?HcvWZCzbbnVWCHTX|% zhUxM=e|f%;-}QOre))w5l=bH^5gx{`z?)&wI_X;YbVtwnmqF7|`S(*^+dm&!Qcj=5 zL(4{y;T^Y!61(R})_+^#4(I(3!xx+(!wz4u+sms3O^(oRBk@mtB=Z`;U-A>?1QJ9f znI~!8<-^bOo)REL9PL-3t+R{S}Xgl z$6x=vE5*hth)9bMt4LSuGGed5(%*GU|=|*ZiY*suVQu zUxbApf!}_OX8+JU5yAHT1848Xg%w&jjQ#l<1fdXj;v170xN7B?|BWN{4g3%MxAWM& zIh_A(-pLI=^VW08|NSxuB5&*3zdj5fGFdTVz3>3CzBpO_gNvX*f<#Sq_ng*>!0Q0) z8JxH+vv0t<1RMei5U?E4&XJc}2Q3bcWzaHc7`*>vShDs~=ndRVa-mILrRQm+?2mVE zYy=>*vcH~=s_LUggqjO8B==9zTL1Pgf*ky^@pvEs|I(CXgHoLD0jOc3-&BtEQcAu# z3QHGDVjh$Rqj<4aPmDC!$$Gk~a)wToa&52)kSImEhj-T4Iux?-`c`{>aB2g9Iu`3@1vejaLZ5hlTg zw0;5X*TR8~;#;3=tnfHZ8CxmDK^_dJqPSG9^WzsOffuVP)C>@%o6(qyP`Ina+R}$d zn8Z$K%qi#u#NfflX88cF(z1$)G3Y4Ku3d2xQK`2z+3Rr3kR*g zZ3{0x>t<=n?GVw&dZ=E&FbBgxSDYA&O&`gaO9n641b~?9$WXrJkI2eMkF^#F=djnO z5!GFpMh{gk?H7j~>vI9?8f^s*px;cBU(tsSoUEzL;xy>bwyt+(q?{%smVA;|?h~$U z#M8;IYYhS%=%vRQ_neVRG9y=qvkE_qL;n{Cyw)dk6l!P?d*5clDR~gL5A##5@>zqa zQTR}f-Fg{q5y$7QVqjXEbmV?j3DSVXwGWuaJCUV7cx|7*?-)iPK+v)FAlqx}+o&ny zA|+K#8)y|WNvo7+L!e9}r)`mf7s4nx^y9kK{sQyurXo}%f{x1t%l!X_uJ1&e@w2V` zF+~x(P6sd?xQ&~5_YN~P!0bhsL!_C4*OU`k&B`|1MKD@-5-i_8S0=TdJ2+1a<6vLM zUJQgN#3igY*~?uZ^Ij`Xq!yWlXv zn+er(KOiL+;1!p9e7F(Z_DJSc#zaQde#-Fwf=T)Fx6Mf~k^85gjL*#lXvu~;08&{J zQ}C`2Yg)~o{h)Y}de9=fOOu@pms9H$Hh;rLq`2T8geS`(8s{M(mu|w9S;nXpMu}kX zY&_!Uhp1&sXI%sno$6~lTubG@32#*bbxEUf<~r|=#)3~19*LiutH`iAc5ijvDAt-Y zvRF5T?mK-iF%EnlqID6i`u@zBWz4(*(iSQ_r8=JS{CHD0j^ZsIilg_*9Ar z!LcysCgyYDR1jQWk=ldf(N0w_2d@NJ;URvtnO52*cbs2PK$E8 znV$Jd8*ex@;iCQ0iZPF=lz_}t&Gi$Pdzfo8(@YTyf)yHb-WfhZ?+Y^%etm-JV8@0k zfG`2207(z?pu{VJoxC1Evh}BW#)j3sa`J(pR0KT3U-uJgxUTn8f;*$`8TQ5wMgub7#X{P~G zK7;@_c>UD$r@8=5$(;3XpHWj^*K#Miz}s5D;<#@Cq0`*>^@FT!k_GMeO(2&z*InZA zUQ{)=;~sG(v?m7Y);5BPvhgwkL6P49vCZegmwzfmsBWu4tqSZh^z z9k{7g#3-Ch0SikX#Akm%tK}@G!(CwM;bW)&%!Yen5u)kWE#)*7w9M|Gi7>ER%B2Qd za99)2ZCMo%^)o$N-t(vRK7<)M1ksphEGY zFejJ+?7N~Z{O{={N;=zIp%dI}a{DqHiy-O0vf8Sqn`^_5+FP7(caK4zje|`Vd?VF` z&u1b9G3A_j%g5rU8qNi#LSIfXN=Pgio^odCJ^<98S>QeWa2#bbWb4AoYvgL8;;#no z-!^DG<$~^3Gxk!Eb%<+p0I|)5yJrG|>iyxts3Q~XIwvWEmk1jedxmdzMF@N8n<}Py zIjlGXEsr#jOW6so;2))X)kpTd^qGbhfQ8(d{zkHtcW?Y&l%x!T%zd=CeiLnvqWo6b zP(gt=<^etRfaqJW9042Pv#f^#h(eyo9>}l>AA+JuSNnF~IAikJ&Qh-x+{Xb@!VX6u z=PB=HE6_}oENZLR5 z9Ulit-DOJki!fUwDmK!!ni#i{}%M(vJYxqd6bWt z6`x+rr$A$$dJ(6>O{Jt;)Q|Q*{DL^wANLR}j+_=#7o5Yo8f#8{j4TK)3{<84>DK5d z-bD(hZnBOOO_6c+5F#gF8 z&Ij;i!PKoZ^tP|8Q<4AuHf?ukPX*0hSfKd_6(C(?amA9Rwvd)5w5U#M zpbHaSFuc*d>tdeA8C&D74_b4wZsUlMD!5W(5d3L}&)emO`u06bSna)}i_9Q@+hFp8 z8Vux0;r!Oq)WcVqx3>3W4zAU8FInP3SJbndjg8{yf*9j+^zh8h#6@(okJJ^6y~CH| zWUUTngRz{>LL!yh8=A$^i!Ns>qJo{pCo@m4u-p^#QmH6M==2fRGuJEOqbB5jTiSTt zc)k0Y zi?;S*@;{sfn(O?n^NX1@MFf&xUl*8#Ppts$0@#R!F3EfM;!Btaj8dJ4wiCr;gY;X> zrBa<0Tw-EVEqqEHlbx-Oz%}(haBFu7n<}T$W>SX}k-95d*~P9&tP9UKWTe=ZvoG@gJlZu4D@tf%U8qUURg4OkCX@w)D;(&&Va8B>za$A!=ZIE2; z!VdsDMF(h-nCwks1-%W~6=1QdV!gxS+TZC&dS!8od1FG|Tu%gn7{aHZ7@By#jgm?8 zaIm3|MWcCn7Y~WX*L8d9yVMuF^f~*s4BBp6tAKmvmYeZs=%3MKnq;UyDc zmZpTl>L6RZrZ3<8RH|;zov$#V1V%Sv;pD*b)n!>(-kCcDSbJ5;m}}S9p@L&CXYMrrdJwYdE@U1gQ#pgmqfiG={GP&&v$V@@^B>%Nv8l8$ zm5)7XXJV}L(WwoOE1Bp-nj1AxiJo8|)HOZYqHcen?H&0coTr{T&(#Z zjaVz<32kTyV$QTGW6}gGZQY=|MopZ^ir;yKeA8?SKbom)Vs$vWRsxcUjrQk}gxo6b zD|} zb9J?_T@LGu99JOmdZc@zdn9JAee>(f0_gEr5;TQEY4|NLbRtB*`3(6yyRoj9xl}IR zxwn0OPZ91Jt!2C;LHjwXx7>KfF&9cuqk2qggYTRjMNXkd{)Qvh&Pyj`cYz^PaZ!Px z6LStgc1sM{-*7Dxm(jTiN=XThELBjuTq91-e{2)IuBe9bKRL29;z|?CVPQgSMWHmWOi0`@!v~hMogt5l=a4+hmXl2)`d1;eh1HT;~W^|Z7`*0`hqS33s)8brTEBni<7r(d<%n=xTp4;VqcHvN~sr^sKjDxeNT-(J@;-iPo zd%NL#a~N@hffti*4L>uTB|WN1%EDOK_AKi^zG+cT$N^o)`HkP39~t2dM# zC-R(9)qp^i?Sbit7exYM5#@_mc_nC!~$27 zr>S=Nk_EC$Ve1Q5PYfP7%9j_YpsZ9|j7s8VEoAWQ9dYX2a_>cu*mzZITa@!)32#7% ze8Z-Z5nE6rshkd5^P$75bUX1*ll>&Y&!#G9A_Y!n*ZNAy@ow(n2 zCX#g=s(jzbI``JsvqcibXa~_Hj2E*>$-@}Jl(r;$w^gM^Vy4R_2f9^>a+cU*E%r)^|GyTr9?qsD^Tr zq6!q!ou#yGr4$nz3AeqP{Zh4?-Htd^(9fL&KcYiD6QJ^3%O%baPqjB)xUnQueP*nl zTfisybkxvBxbF0uDlP88smnU#cIXG6U6=e~@*2U(NtNK_c<7WJLwj_~v$eO&9TR&9OZQ-aC;KM7{9z0z#BgLAqRIE>B z`vvY@5#En&-mNYTPd3w19JwK`&Ur0KYO8(zVl=N<%urWl&}74~oX*Q=ug5^ZuT}K$ zUbw!sxf~tEj_}32sScM!0u!{NS9*`_lbX^0kc0|8KnvSD+~fxMf?G#4?AbfA_nkaZ zu{regRG8I8gflY6#P5Y&i!DPOX4Nq`i<@E@xy}f7k)G1aIO`-=t6usZX$q(JMX8MW z4<%iCtd?r{A6I$$Q<(@V?2o6NQZoS^QbK#XvUhC#(5v-soDSQIk}uo1lX7#IGkYmf zV!^W|QuR?*B%;%b`x5H4YtV%!)f^74lcPY!TfT|OdVxlX_ibA#hYYNJ$Wuzn0avq{ z)RP|f70%9%RIwaQzNs>sS07|+dpPfaLEj0D2lzuQ#l%(jF^ia2E?IhsNIbB7<+i}m z9V?ctr0oB|dlvoMG7XLW%a;?Fe6}&IXQtct; zq?11s%gCyI($z+q?sEwM-?&V*p>w_2$0l-jXh80I`F+FIExCV2$$OeN^#lqwg&k8Y zuJ>d7HffrACFRKIjqY7r4XnpMb~=a#zB%l{SfWJqP)Ew#?cd!9*nR5)Jv!EJyEDS( z$Wk*Ut*BT_T}-uq0$r;FvDaQSrUOb*8i|*k^y*UCxn)A!7uN)Am3vmoJ*90%>~uO9 zVM(pY*j#3H?PZOBqP5=T8reIEAuBmT1ZXpU$t!E}7cLE-KGIU1E|nNZsEVfxc5Y0=49o^?A3d@&xzLlZSQT zb=snGVTkX!`^w+>u5fR%GMZ|(rWz9-Me6~j+rFDOL+{O(92CCl2N6x>DUYK~+ZfuK z?G-b$Y6Uhb!f7MOn%7@4UFb+Xl4kMS1+MsP9nJHtPvfP3lf(lma*o6m2$VExBR#-? zAOTK2D{EZk%|UBH*6l{deB^=Ak>X2m%687h0ACN~Lutnqte=kFNM2R==aY*x2c&5s znP7N051b?9Bo~h$lJ?!>6bH${(d7Kyp>Qr1RTMXlZfW)9K$GH8tIYbCwBjw#6gcuA zc;XW{Q-(Vgn@f4&AG-^pNhuEi<;%4sr4$d zmULxW>qi&%lQ#~SndEMBB;eSQi(;urhdF1zkw8SQ`@g;0S$pf>?mkalczfw3bcnh} zL_V9Mq!(}9+|>9I^PWOII4~x8JGHU(x+lEO>r?}l1-!B zY^gvb!ZEj+trfrT?$X79PLZyGt-Lk-y47IygE=M>`|2shFP{b!Y}N!zM{ZUaI&lsS z6lchw2~mQ#7g2p4oGaF~`ypcvhR?riWmfLa+a1s$S~AJTw0B%NUei5dgw^*nd#P09 zNaAOgD8twUvuCDK!P28N-^+|#SgnQd$IGwv(y-JgmIJZhU+;Z*fz$?44vs6{Q=u2vMkg%07Yl=1Z?u@^wqbkhQ z_0ZHi#Y1u3!}X-8H)be!bB{#=Zq&^|VBIgTE3lleqlN4+R3uI?-wh^UG}2gh0ISj3 z(3f&AoVPp0a%dRyMJqYMa-6Lgkt<{^L3j?=<&*|G3KC7oF4?&2%1FW8EDDUFf=UvX zi?+?5>Z+f`Gp+E>$_Q_k%q^#eUpN|GQ5YB(iA&EsNf+1MgAbWw-nNm4Atnv1$Gegl$^UVH0$G%EZnE&~ zd0_@^hOS5UnHfXr-FG_#NMi0XQ4EzdEZwSTbhSiGLSf}{2`KiJmCJ3QA?gwl0xLGA z=AVwIoRr}mP_$~Ik$_pr+A5Y5g7E}@ey7`mg?m6HKYhG%N?ZpR%jjFDYQVB ziZ&o;;IsNhCLl*f{LG@}HY#blq1ODptfp~qIMA>&7G;!7t}(_IULCsEYfLBaKc#Jd z^_Te&ES7#B`&6)N)}A3LZ};PjT>0HD-j99V9=M4x@p}42bL*+*Y_nqu`e(HFXw+U5 zYX1Nfhi~k9FZ%Ph{&kfspVVuPSqOZ#Ur@&}d<9WC5r}TH_g&HE3RmVCR~?rTX5d_k zeKL`l{Z;X8u}+YXd*o{9tVbTL@>chH`KO4gV>naAK2YN8<&yCA3$W8KrQHiDN`WbA4cs{C@=>Pgb13ZUkDdLwM}M<=M6iI#*80`= z2_GsaMaOF-NjJV6gK5KcUxU@fJdRDJ8s@IAcd5Qp5Eyo~wEGUd^7N;?aypLrA$fc1 z#R={^*iu~I5!2q;H|*n*cd?2WIc|K)$Zdf;w^$xD_82%_(;(Vqyl`P*bDU=@p^m10 zBpEXIKOOVmjs~Y1kw(DzeRQL!l<#X)V{voS=dcenPkwoREWDe?raN?){gG#jB0)9K z&*lGg{Xg~}c>XJ45ws-Bp3b4Dr~S2oqPix#rl$C z=dUvHW(u}C#^T#&JE^GyBBJhR$9(-B87#p1dS>$m#+*E;ydtyU)>b?XnzjPLkUTID zbS;=RHuc3bxHTAr(W02JsaOVz?b9#wxsJZj-M(uxSCZERS^YM^+%?k73n-*QR@kET zy++tn%e%SnQPYAgWGM#*eYozE0u&Z>Av48*9%GQJRKY9u#KhCW9dkfcEilrp8eY9^ zfBgpdk+mGh7}z>#177Ik{H4KS8W361U8dd+JPDjw@ZWCQ^q8f7MVlWMIGLOTB*_*i zqQLBijTs!?)cecqLZAG0OfR|C5$Z-Z^X^1^TR{3INP(G8Uo(4NoXF z35{>z*4OF?W8_!koxTOO|6lw4pS#x_Pb&m&Ssv5!dsBe#m)OQeo`bE8r|#I_RzNgw z1n#V1?Fo*@+D8dAZwUW)oH^KoL;G z!-*J{%e#-9?qh(orFF|(9jtWGayIps{8*EG!EpCqJz=1wciHes8wN;5E=u0zE>J1| zjRJhHv1&QWes@8y>qIUT=DLb?fTvuLkyqL&CL>}9P-e7fS;9eB*sW_^eO&Fg(9_?d za8O1+gI!P|TWK5AvoP^_MNP-%f-jPdTYa@TjYouBBTuv-dxS7=?$q^MlS?t1Q% zvK&0Sr@7?#@w;t%4RS8fW88~IOn&XtSvqKN7wW}4=YsS2$pmJJC`_@4bN<|`KVA>w zO}qRMleTxd9(1+-9+bzpoV}B?mxWXCD2c%aGkCyc`99Lj@34GF0xey2p>A_;k^v4bk?=0j;7E}fa(2M8g%HY$ zzNrcvT#nPQ zUgBl?yn%4q_CLY)KjD{D$oByx7E)5|Y4t-HNPe}3-;-!THTt#{aTu3YDTu&C5~vi< zpC;iiL~M=>r6s{vpkmtCv6_CsnDoe4T$C|(XENU00X~1A*>7w+U=Tt34a6va=T3eE zKk?GPE9k+jfLPyzJ-tW*he}W$%d)t&0Sf!&-^hx;MG^tv31+;u*N0-l#^y2g^;b7% z-NfL)ll+RU0>F{gH+W|&-Uvn#mDfCU^K%naNMnUo9JvG8gQ%r}hDZ+ulPTM5d?xZs zt0vxcG?t(L1>}F6Z7Qj?8$cw4=nkhpf>y|HbkDTI4o3vIpaK40hbYuN-r=pk1Nif- zm~Bq|4mcZtUtsC}KX2hJZw7wX`)&XpD*g!bexAI)k$Q(m0(?_*-wwX(2huO-pg|tCI#Zv$PD*nLCTOV@-Q;E=E?mF7#`&aa>k56qu-A?s7u2d1>j}+cvO|BkH zXoERXfZOI^%YZrgB`2l~r_3T{;7Bj~2{hJQ2h^8wVjCO>oXX%KIX^=X*CWP~_des5 zOQyTF;fP!?8A6I1U|6GpDu1Kqm*?y>zk(Et_yMR^Ko+WIfRqQbiv}N3-Y4c0h)*W= zsVp4rNw}KOoSp#cY)ZqtzCTHLtH>gK_Gc*aC*mDH0`i--p+ICwH!RMMgc_L4Co_-4 z-aBt;Rrtkm000Yw(-((p8}b3D5jGWsV$X!B_vUP%5*X>~vxm`GKU(qTZBn8_Fx^1k zr<7WJd#+IqRdk(uQDB#!?@(6UE@5cRGCRIRHC;7RMeZj*D5qcAu% zsN60}%JOP#v$lGFxH>PY3)b3n)DLM@)Da^=XrSR@9-B}9XGHL41Vas%e#Azcpk@=4 z%g@FymmbYtnj2U5SqsNSNr59*SR?(_Ic^~l1p=G)SL+J!sbb4*&@2%3z09QUvFzTh zeqR<+6XD}C;lROWbv~v(0JDfd5sMe9g`+eRSUsVVuu~w;XCN~+(%h^-W|_@tK%u4t z2=@5EfHph(m`ju3q>IIRZLO+SE^k*ciXjr#dqxK1BSpr{mgwqzz?H*z_8W8{a^K8m zyRmLDZSua$=7LR0y-Qe7haVy2%uBaL339 zydp|#Fvt7ZHxXbIk`$-jq>FJS{u@N*)A2eV$}}jGlLr$ERNZde2EY!whdahkFgQ{o zX-+aiRG;$>NR$+POuMes3$z*sw$^7@USG|6=%KQ-?F!WnmK}y!c@NaLkU{_6mE?|3 zEFG6QeI38N+h#&ykH*@nxKo;G2wz`f&d{9#tX>bShf8U_ll*eEX2hR$lkq>^itZcb zA>Jp!x7HDp$$6%Y&t;RM*YDHvj?0^RVT!YqdH6{z@-mXJOp($FUOz9@3iYfB5=}So z|3eWn1%q@?gtPci2VQ!bv~&2Xf}^ijs${Mn+Ch65y!+Flr%~27I%S<564ur-m6Ry*T|Ku++C1!s8K-kEOW+O-XGX0@`n<3cR{~5twO##D=kO&b?>m$30RviEM>_>D}x|9xP#gWF&>=&O4RC5rxHdjFG; zgyknK18N`_#G|sZeswiY66y$Cd?ho4@CX8yCE-dnU$7OzU* zoUnr%>O$l~d3w>IL?$zd=NTO>iOM;sMisnG(acWX5{bR{t>MACXWrXz$3yUI`JQj( z#IyKfMmR6*;LgPfen>qNZ--x@K5IB7O=;7j(VShyWB_$&2i!@ZS?{M)Ldq1^4iP4D zH)Fe1HpG4n;e7zsfpdV9^(z7tbha*{;%>+lj$u^jjb^PL?SS*J-g7*)G)X!d-H)VS zj_we}Q3=Syu*^RX+dngzw*d$YGU)Hz={X+(GU@*}jErXzCw%`Ea)zU@L|hM$hcU_| zdecf9Rr!H#kLVK>6H+U8-&u&cDtNwqX_zC-@xyvi-yj$S?V*pdOQ=s;6ydEyvUD^GcVh@3icenCrVsZw^F?7xrP%KQGICE|wsyivDcq;{nm_5X z`4Mth-y9@K-ySJW_=t>zQy^Exl-qhiNkKXt66M*)!dqqSs;XXLa82}wuao{_-)ak3 z2T~`0!`@4Q%0ZR)Ng-z5_oeU<@5S3{k*s7xO&$HPU4K$xKgdHs6oLvvh^3@wiDzm= zVNJc60vBA`exz{zGtUJ2J%3VZqD&dk?+xQ$3qEo`Zmz8We1~1#2PisB)rnzGy^xT7 z5pkt-?Q;&ZS%QaS?9|G}Jh>hOJKmrz0i;VJf0f8MdH(=l63OtB_Zxo6{aZ04BXWC@ z*Wk#Urh?;n_y|5g$IS)LH-~vjaywFt20-ui$!-WA4MO|?G zz6SP`3MZg|>JKKZ%1r`%m1RAm$9?w$KSn+-`0>3&%fQH5d$DGR@~#1hIv$l%pPGf( z+Nbeh6!-Z?A}t*^>(xx%eMeXO9RTMqksWd7`xz7I1}J#Fnq$BN^h)D%-B-UurqC9& z1^NR#_d@m#pbL%vLBCt7OL#>#of})d{pVK}^gFo++5k^&I}+y<0Gcz=qKm^RETy9Z zBTCz=!A?G`#+iVwEZF_ML4u60Pq0`JKH$sNK=C5G^(F~EQVBZjES)=kpZ0}Wyo(T| zpEv-J74!OZi46t3->S*Ze4>Hd)E`nU_d=^z)ld9%zO`$5GPU#iq_#LPhEx~eAKiS@ z5O|O)Ru6@3@EOo~B%6%i`zh+iyW4*_gJO;<(8GZ@48@_q2Cf3p_MyO{dcBy}CWfp8w2DO;P^b5z*%fF#w-B6S;a>m)}-TrkuYFGo&9df+Jt^c`hXJ)i`4I z^8m`U*!J@u*ZhUEiw?W5)*CWI0`r$_GkN+_?_IjfY!BWZ;Qfo6u7NKBKvjPvoriim zdQPn{do8{;WCGl#b?gQqneOVwMWx&IG2)~UTDU!@x$NA<c_e+Q7c_yY8}VA!a;I$ zY=Hvy9xbYGoIzK=770+Dne6QgTKs1@qzCRRAC9=#DrEZF`z3>@5>muYdx8>*U<0`DF5S7LlQtir+iOT~TD7r|rW9hx z7W=y#tGk8niFS}aulw_3cixBe$<_0?@4v6MUP{c*41B@r3tN_0mNun@ciQPU6dcA>1-CZ(mV+$Z#1j;)5-o5sR&(X|Q@iDCdew5cgX*Yj)&* zUfsQQG#KBS%&khvV<8JG(m!vOBh_Di6#eU+fQf$8dYobxs~DlEeT}<$%exo0%~=L* z`tPNCHb`V6(EVDKzTxl#Eb3~_n7vWQTmSdZwxYP-z0mqwB(I+eg1x7;r>|#OaylC2 zt4w(8nOY?w?hGPr>rbPBmZJ|B*5tp;U_Nv-YBLq2VKtTFuHp@rw%-VDzUuH}<(21a zVPKIzoi2IIzLhjO={ewR@TXx7{fs`Ru$7Ui zAUhM;`u>=LVOH+FZy;xr0jw9u-n?9Je@BFpH=}Zf*C=*=ygq=v=^2uoMh%!Qu=sip zKPu3B-mHB1Gx)!J3$~}_NcK}7AYSvSez7C9P>Px-gtpUFP9Xwo&vh>nt|Sp-@wGeC z6r?+=v~%s>z{ron>ETIT_7#CYlS#UL6W=8KbX@yOvONt?4YdQC3;50x?L`+N7xH(@ z>5Ez?`K40rE2n-fIPV}Y_HqJRDX2!cyd^cQ;GAh)XJfB=T+9IZ}O+r zPPEL2*=Ii=&|f~Z!L_DvTVEzY@QT<{0DwwyL@ z4;R$ZI+oXnGjV4C-}Dj=AO$PlK&U~^2&hW*05m}i|4FwuIu9X z>}XP0_Wr*0-sLLLOv>fCl%G&Dv<=1v9|eJlgi+&jsjS-XB_I{R&npbp*5Yfe zUW_2lyKyu<7!oF*wb@>>8K~G^uNX3m2Q&M0&7{Ionr@UhW{r!02f_tFRzt*~R5lg_ z-V@c46&u5|n=@WY?UqWLV;6`#PCffXi~4;cx$i-JbP!R|o}z*mH1U3LLLT(7%^Ar$ zP4_@z;(mZ}5tuCKzJvM1w@#^rZZ{v_<0wPm#mLmM4rbYY?-e)WrOD0#7~jenSx~@N z1x*L`1Vm0X&Yx8!==nyg(&pvN!mYKOZ8>g`;WdIHsmp2SnNH7?-ory(5{`)S^)EBb zOg(x)l5LIx<>&y*RUfabSzNtUIgfSA{fK$72vhK?8O54kxT&@KxyQH-abDbM+$%G_T~&r!fJzA z6_|jvI*9T|ask@%5A2l6clOOgUzwD`)e@$}o6`v#+j|olrc25?p~g zSx(dAh0w)lK+VmLU#pN)yqE?mU76))3AwfdRkonpEdaEN7FOclD zH9wI2o!5c7dL^G$SN?+VILT2Q0k0FVIf}zaXIqSfVr^TXn;s8 zI`43M)8LFIHWhjdI~KlBSg|=p^_adain68C4mpgF929bEG;HfwEJ$t*@NEt78GD5| zgR%WMzL4U%N2<&WUdv8i9P18ICfd5J#v*{my>h?Vo;Mkv?gh3mOrk`(%RGq^zB1Lf zHD&Jo1z;h&Blnqi32y7q83T1l3`j`6in4;F9jIe1i1)nn8=Q@!^+*q9_7M3q&6-_2I+W;;h|2lM+ z87be@#;b4Ye;EubRC**VCI4G z*@~NSKk9gBMZR)X^06qeJ;{loF9t&G&Xv{NWsQ*8YMR+N&s}76onDD}g!ol%>TT;F zC^V%(w12YbpZ6OVhA!GcQ?%Dyupt7Z@U5KzXyG;U#q<*iTPw9*8|o;Nmop$nd!=gTLl^HLgw$LK>01dwL&e%Jo3t?zN^g`XP-o^H zfV3##$7r80CE-(^A}z-4!6xY%A{o3u{q)cq=J6A9G)N=9jea_u{?dE0hiF6WDT(KX zkQ+AUdFosIXl)$e5~H3o32TT4uJn-WTmDJrn30ijMIl9Wy@**R?dG}c`SzPOAs(9J zlc48T9I9PdQjpqz^C>8~!P7VN8&`7^gPV4<{Ep9^D(q9dllcK&B@$dZT|658u*Bz>TL z(6$Zg`skrX+Q5j*j+JKU!Km=7->+(ppUIBCFQ_&?W{+*j6APBx1wQY$7X~;jZ$L#z zZ-851wDQbw^;yvciTo-^CaOq5N3+aFlUi0D*#IsqtaI&g)^4P(@w1D}y~Wxdu62Tu zeh>^TzX)n5FsDQApBTTue-~H?0eGlV&Zm^Q?TErs!Hw~pt=q8-q@Z0kECb4mK*OmT z?p~)M6-TQrC3h5m_S3#*wN zF7w6yvK0_J(K7r9#Qc~gf;T@vHM1{6{NMBT=4zXHb2R6wGjc7oMjyj`g1nV)IamH` z-4xY?DGtv8jzNRQ(n){0%-!P}YnBVNV`;;1Ps)Y;`IF$bxy>D&%JpEDDBt>Kvn9kV zI@hPSzh1o4ainau(O`Gz2QmL}K&u~!Gb^MKy zh^bbKL<1bdUaiySJtNo*f%VMXFd-5O^rZf_c>?mZXT>h6V=OHp9j}V@k_xWAZ{+wW z3Dbrkn6SVcNqt-t9tVy~Hiy?6(Y7?CiIo%dW2HfcUm2e41}!roD=uaA1}bi=-utluWUEwY6GGA7f~-0!-Em3 z;*9vRQp(#x`?LCxd1OE*rswP4uxq=d?hlQzaUWIXd+2seoNWtoOn3aqY6N{PepIZ{ zb@+NT8ipT63Ys1m!F`QOIV3W43`6)m>K5t43g7k|9~4E?UqbjEw>s1=rP zZqNW17bsq@qN`RVXBXj$u1X}80pEUu!o}rssXp}dG!m3OU#N-3wi(U1c9aYnU~@dC zEQui68e*#}5ZtieOF0{Lz8Xa7TxN3fgY3PW(Ojc>P}UIK5xh{_CFCVY3r_ayBh{)m zx9VT=162*sCS`+v!a<#F6L|051I10*wR@AAE={LKs-thDtlK?9VhP%22CPH}BiHMj z_3!3ggl@P4D1-Mn%U=v^wi-gM0wT;wg=SEaR#6+pG*bBi_B0xs<321*gaUIY+ELBw zRPHybMPQaTZ-6+wM|&k_Ydq)WKGLxgaDVg4v5FWh@#X=Lvg(k9;YqSuBIpQoin(4Z zEj38)qpuTZ98W(mjv1FwYpVGEET?)cL@xNAvjhntK{$VE`01iF!I=n}6~MXXZR?vT z=cA!vVAVXUe8q55pst@B=hTa8%mFnqH7Q%|M~UXfGbarcNnHG$98cu#f}+4?XJd=Q z4oqS~L$}1*g4>crUVPPg#0}#n=6$^l&wqt-HqK3-T6S-Gt<-7Y{b1kZbrNEV%ceBX zc`OB40olP_Ea%k-osS~N(u>wN@N|b__78rNcs~0kvP>j2JwJN`ygpsCSD6k%QQtrM zhLiI!7*mRdRrN98WoPq9vg;m3l183$cb~;Mb{%J?pyz}8nnzQg%z~0Hs6x=Mqx74U zotW0rTN4ZV^*b2OMk_SuAmTgV3QhL4rs@R8e19Y9cW+;tKK^1polt&_$cl|LimLSj zO*e`(?jld=!XrL(nBJ+Qp7dQ$=+i4kahQIUvgL0w6{nW+(BbKPOcA`|==oHcvg6Z3 z4G}bLQiz3Fl19V^E)ip1{V?u=``~`N*}(DyovT*l2WN{7NRC)XBCsc;wn|Fo2NFu04I(+56aC8{b0$k6_JyN4~5FF$n1456nMQum< z(mUN+bcr)a!ZGCltcFwf-gnEg=(1C@vtf~~Q4h}VI(9O!YEF{ka`Lp+g3BpJ2V~6d z)?>BwF`&o~6wB=&BxkhB9u0Bn)jfCgen3SH1S!kv?rO!HY9vQnt0kg#8%Jy26fgw! z2`WKs>8;K~Xd9>GCZjOZrS~QkVy3n!THKv6k2{STJWb&gVF1m{1M(szByyq>&&2(v z^r%vxplr3iVxwO7=yio^HqF;Ir@S}=0owj#_Pk=qSzCH75-2i+EJd?cdfw75?tgyo zCKElaIN|I;+l|hZM|Arpav=oW7yl0AWzWIzbV*sCO^Z+Ru<+`ZX2&(#-@MioFI>9$ z@Sc(LoGotbNc|x&(tLXMK8yPyYL_bi^U(9aO0uS4uVOyx5Mze9ArX?E{zx^2d;&$a zejLd;3kvOz#DGMzWK?=d?^=5yNtstI8%4MdkeOsEy=1^==F^j`+2w2FvgxWkLD2yy zlD3$^u6v!gBANE6ar(;JYoII4SJEm9ap{a1>M`IZ05iJF^xzXl-Dx0}#yBn*A)o3Y zvW69LN%o51)k04^j3!H0-Y-Pv2MSf-*kDf%rU&ard@u=u#%RU$mqH}8M@2zjE1z~D z7Z&~CdRyBA2(6Y%uSI`N?E6rm6C_H7wzDtt$T#zQ=R%#T;VTmc%f%Gs{t@|w-(mQD zyCVE&UG)vJN}AQUYOWPZ?5!-eyD4tWq7>_;b^>X){Xt!k%`1&+TnEk!@m>xVBOU-T!JG_3YbHfuL7)FqqzZpB# zV0C-EMD#bAN;*?#j2qL?6r_LCPc=xksbk*-l{0Wx z8>>lq=oD~a*`<8dRqUyEUhPzflN@cMIagNd+zgM65tGL*^@Oqt#*BTd=+hz5ZeC!=v1uo|x0v!>CjauC0a9E-#O6MemHmc2iI?pLF zNd(^tX=r(AF#c*e5j2f(Tj>)RjVW)uH)W4UBbC7>z7km4hZ}yxst*pO2MbV}_s_!& zLG`I=ZEyGwvozS-V8G=tRwX`0^d`T93fyAu6dw=O&M#!Bm%a-}YH7G)25{QGhOxW1yiHO2#z?xTJp%IN9q{qXE-80#)SrjdCE9jTq={8V5hm+0#>O@M%tiuFBmh^S*2+}Qa7G=OiCK0Tfwo}ABjyEYL z5zmY$1LHC(6LpHzU}37sh~cE}Gt;u5Rz!6p^B5A%h*!8Oxvm9HV#WRNnl~)nWHxQH6PK=9}wDn=&~3CuQ{sqw7{Gw@3)LB&+V1N{*)c(4#E4xs)SPy?U--(S*W e2!6BMGzS%H9)H-6xl9B8(L8tlY_~i!jTVFQg#{vwpw#Fbof1>&7Er>VJ4a4H88CG8HaZ3j z961J``Fh`<`@Vnp{sZ6R_#DSLU~Idt=k<7==i`jWxkNJ)LuLj(1_}xa=6iSb|Dm9u z$)%v6yhBGz{*TzV13&UFN}qoWbt$Sw1vbb}Xq>FYcSw8oKwd@nz3T1&b(uM9u!v?h6UA3QmF{-u3)I{@824;k@q z{cPpmx_SdKHANc@aHGweHnP9-BQ1MC193Ovtu_aTu2HVGE(cw|2Hmx5Y>M_Z$=kU* zGu_99KW?$~B>wpRs4P8arl|4opO^iKoRTJvtJ9~25dX7_yKFQ%|MSp|pfipVq5JQL z?p?Oxd;jw?jSd~Xm=gVezfAr^R3}E_e=R^GdM#n>+W#yNB@#o!p8LPIh86zkSL53fWIYha)KdlnRs9ufMyRnUJ{^at_yevi-IDN%ewHd+m$Ao)|M`UxZ1# zU4lI|N{gKHz;V*(*peM6MoKTH>^n?G>7M?--W9`2Y~gp+?>oo|Rq|+-0g;{duVX%p zqceV|^Hsg$@4jsJ62>M^vqO~<$qv{PL-sm0yKA0v+JahV+NuI_Oqe!gG3!3hWGn8A zZ1*CMakTeTyn@LAl|R?@rhf-U)4BCB1I23gwqJXSY{7dsBs4z_w-7vN5 zQyLZR^Ovf0zed(RDtg<~@=Ewmpmamlz|@8kd$gAz@!@Etb;sI}K*t^4@;Z&mnC-xy zexN6zO#z-RS`TwM$iQfBSc;Jdq<ZtB$2~I}(mvl#M-}I+zHL8L1J8h>b zz&Zy(`Fej%-=6-EolwiM^bumUe8j^!Z?Z52p%H*D0{X(}l1Y*gbyHt|^E(tm6jhi;-Y(%NZLv~LjF$$5gU17r(v#v)D zWC?769}d=R@f02$CQ}v)iQ*J~4*0mM1HG9;7Gr`G5lb~MgmKi@M@Fe;JXG4Fqum2{ zeVLg?OVvdy;0s)^1L7A%>TH*;Dr}pj3~B@^>tne-D*D&NYNKX{L|_h|W$scSdEPjf zmCMw~S+PRVJhOX+vD|_o7D8Lt`e|%%;Mc(^=uw%g9t3Yk2*#~>M~zHZGq1(&?cv(N zA-*-x*(-N#_73vSL_w@sUPwcr%0k|0AxULY_W#}n3LaPit&@70P9c`k3LW1jA5s5y zU{EY&$f8DFbq)ln33#4~CMwG8Zk^&_IxGG)ST!%{<|JbOPIfL#zUS~tb6z#|)QH@i zq}A{91r{4>^_Ty01tX0DZ@TV>3*fLg zOoy$6^)LNV!6;IDF0Ud)<(T)GSf@Hz{c6)9D1~ReG-&wDHBC%dnoWgq^O-A0E`ZSu zM;vhR_HbSmg5HW%K+NiDYHV$^!!D=f*yt+ZO4chA;SKmE{x(M7-^;&=KYqy(ggfRp z-zJJ!~O{ z3iUAO9jt^Ncarhg*%P zM{PL>M1CQKqu7Bfu8IGe{$YGAAGdn9A4c=-)1NM z6(3~e@RgLW@q39!{V*7ujBNB$59DA6z9rmlc)^&byANH9B!ylf5FpbIPk&$6cP2F?HnoJU)`D4wp#Hvh+B4Lm$@vcO7yp}7TJ{B*cYaSCJ z8$n7+I6D3Z^FSoKhc%Ayj?ER1iPy2DWASRy!1(m9U5S4{CwY7}byYSfc@0Sc^->k`bZdk z{5I8lK}pdJt>}G33GcbgePR0EG=Yb5IRrW;ebuFaevr=4gQMyEu#vu%BRo&18KNZBHy(POTyXUd^5VP`5>6vu|?t>7d zWR$z?u-L>cdb&Do#G!*jH;K-eodR^$MqXSlRHug7h5~j!+$Rf3qu>xhvf_fb4xr`B zgxLkdiM%$toSPVbSbo**siu}~LsH^t7h1kj0}I zA@*EL-gAX1X5rva9Xy|nx3wiaPS)xJwdU~bR%A|B3$S}5o(`Tjnj|a9goCFxMJ9 zhb(P*@0)`=Hi%B~2u9@W@}&RdtBY^grd%5N$!N$SS^!DX)+`I_KdBK$4X-U8MTsH09+k@LjD@WO<2}eD+HLhH8 zAhV+o_C1I78bxbGLypDp5eOTui0-MlNOGAW+vRu$Wp8S~v}OmOgfBfvXq&Q_D3A&r z3RKVMOxFt{bsW0t1~sq~!Avs@$Su36J$cA9WLvT4pcKNS?(8DlEa*N?%*KYq*7+a9 zAEM*C9x;cX?(osB#r|Rs5LIM57IM% z9#lkX?u|L0WjH?SFWT_?ZX-pbvA=9Up|T2 z!iHhOmyJfhAX(+rJr0F!cN9CJ5LMqYNXtt%A2o?wOi16$)6f_OrW?mrGg?Pc7}oM# z)vGM1`t7mtt+ZX6**LhTN>n29h@wc|T=3X!Z))q%XRzxHNP;>oVg&7D{_W(}Um{ zjyYL3j~#cp58-Ub6n+-=>V%v8PMQP3k9+f zn_FXi!A@cId(wo{s*p$t$2M*Ro{RR#tAQhG=NAy`ibU2tS$KB1k>9%{MB-A3U}v>4 zO8CA}5w8xI`r>e=dyS$<(yN7RQ*)6BCJ>;^Yd-Z~P|DQ;$yrzqsoeok6wp9vduC`s zJc^0*bf`|!Ix4H3*&y~tT`A(mw-(I2f)gNpjU^`L-N#$b^PWo}q-5+I;p*X5x9XYd zSaYLz3u13JDa5`_wzzlWoFPi~IG~@<`LcKpeZ*a)+Kk@Q29=*4Nj}53<6ibg%QqsP z3h#lQ0lU%e?CiO{G)deV>q~l2){gmSY<7}57ZDCxP1tAZ9jxIbJMYi=;6F}FO`F>a zjYaa?eXeKV_0$hT*Qw^7w(UWSD{nDv_OVJ2mzr>W$B|%bzrtxA{{3*=NU>1otv0*v zoeJ;p83b?2aT~n2OdKKJ&(+oCLk>y!Uxpp566o$G=0P<`@?by|Tx~uAc?3gws_rVL zp#6;;>Nw}PkZb(eprwwVsDR!sD58`oJ!e8Xq7ExuuyF$}OmYdPW5eLPv|SfcNk?jB ze{y5wWa8ThO*MP+r=FLaX188Fz(ek6#pGpbWU)QRUjPr$?4en;^GcM}(TMzWA~uPP z%UK>L&N=tuDC%OB%cJOg7PqP*^zu}YE6OmYOwNw`A^y@5YpsU{xW4aGIP8Q+rnHYZ z7wUDwxVG%O(Q;Df_58p;X9fY>c8I6d5{np6CBgMQp&p^P%9=2SDdFI0d?5l3-vnz) zGwrk1gVev1)(4L*TS;X;cNlTke2NwuN#y)7ju^icQ(SFa^gQAe3~iB4T?$h!7y;pn ztw~l>o)>Ojno>zkUU|5+@mb8Iwdg4Z0VVl}l5-WC?gt&LFL$+yw_!!M&y*F8JMhkW zDxoI%U4YQ!jrxA#+}UA?wK++m5_%~P4av@rn(5cizIe)FoV69U{lI01H%F*@TFi&M->Uszvc7b(0AiW1f;Qg_j zCU?Aj2NEiK;5FzgL-L7HJMby~x>1M$rbl;2*U1Sf8u_ievz2h4N0eFjv za6}$8$P}2BcWSiFKobh`>NtOa2S9CF{2SQ)KFHQE7ln{bFAk-<&Q^S#sm44~ID~c< zzu<8;&;vl9Q^LdCeE9C-*1%NJ%)_}|R{nFL(=FeE+l{uJ^AEkW_E@ik(QTWsW4b0% z0Cz&3x0sx@d|+De5&qMda~c``9nZb8$46buf=XF>m?i6gpV`PCr?P3DKUNzz4TOfo zZ!L#V^>t44IKL}!4zXaeD)ZTZ#3SBx6+jIOq{S#n)?Pt8qjs;UyPKPzCYbbNgQLh% zRu0q5Rz!K0sc?+L^L*vUpsm6Xc+Z-@_!E49Jw=cZk^})A&RK>4dj2^mklWS^se!_| zItGud0O+SXH|NJp%%9gl5%F8A-muCdO6b7x=%3y9odT$o?h53P@A1EP0E4LTm@8j* zXqS!LyR=ItA1JQdw#b^RI&#BmZtd93E9kKCcMKeWir; zd~``A8hhc4n~jduvhhcfJk^hrd!k~s$|HNxDm)o; zKRaM;sqT3Zz%>V>PTgRPQL8CJ^tpSTVli!maDOu5q1$~>z$W8z(QU!1yxGyz%nd-s z1-4h79DI3*Q^k#_$U_J99|Wk-VX5o^aR8n@%4>{KBM|AR=bj!CntN3@yq(WnX#N3}RW-O!e|9vS%mNe+y4NQ7<8KL}&p{fqjq!h}vPD z$yzU|4wXm>)foPx+!2peOe7{AV#D^G4Xdf~&f;$o-`H|6iW`J^4`u?wM-K1VN>be7 z67W1aD1d=%h_RTz#%zo6@&@N)bo2ul0&NwTcRCQ;Sy=!MV%8O#8x4=OR8a0k8PpnP zb`(hLVP4nAuEV`nBh)S}D3zFi&Z9Njp-z|Yp`tn5Dv4!5*=+_0;Sa*}1iQ$Xg3+6# zsIWWT8z$EsWD!Mz=4d6hA6^RY+764}XwJt%fWMh>U#2c<*A6$kk2DJNx@60|1h)H>k)CnY5Q;VXZiLe6 zanx2Ew()!cbjMEkC=$6#Hxuc`*b590<2qLZ zMW18Pe)-hy{UPVpQ(HQDLF!#0Q}J6p*%9u*<^>5Mhno4+YDh%5t8HY0PsCQ7S4iO3 z{*$qnT_#*N1B>YMi?)_RYyp1Wyz@^D3Z`BPtDOA#aOC-%3YD^jA~~wJ1RS$`@H{yu zGlEaSPEIPBA1+Le(zK93t?)v;YHgoiADC&==g&M>3FDbbb|^YCPV1St;Q(YU?-gZa~9S%Y3K|p_;x6(6v)gDQs8Ph1Ef)M6+)I=M6(w zgNi)nvaAY6TD;tFYx^kZA*m%iaF;-WIqP3^^?)y^PDUBtLCHN3HlRja?x2#*x`S-^ zF0Ok;twZ)L`o*~nKFHPt%^bQr#?mRxwB67QNAWDV3imA~=It5BlX1(#qZT|Sq5ex@ znA<;MWRATc5=JCC#RFzyUgTW)v%i2fUxbwwVI*gX@3a`L3ZaG;;n|}r)fWdO7fx$+ z$P=vd>}*n9&oNrpvA0OH9P!{mB%-|sr1A)w?`94I>a z&Cxn*dTrL)(bO(0v6ie*m_&1(+MuC6n9h0mhUjpUF^~B@ZzM=m4TI2eMVphG6irlN zN!y&Nr`dLJ6vzf6aBKoJ=G#~Ynbx>_AxCYiFTrzC4_3aY7BIpwDt##xl)B?8wJfK! zkk3TUbym_5FVQTOXfBJ`olmAL^TFP3vMZuLmg^AS_V$qp^)w@kQyrvyU0-T+({0Hz zylVd%JsME#ew8e71olAdn3_gw^h(t8HGG^oW^gOh77EN^Q*rwp2wxfhO@ccUS{-Nw zTmDKZkOYy4Pccwzy{Nqe;x9v3k@3s_+tq5IXWEJKN(4M3`zXhu4^>5&DcRyU(#lqtnkrsvFFApv(kF;#Fys9F!iVo-$f7BEKxbI zzY&!3ukfv5CLR}g5nCx{3#wCL|$|otLx%9i&7*a0&S@KOL zt7Japrnd_c|KZ0=4tW|@iM@D1Zr~$upe^|AvxYv9n?qpJ6_YDjDymMQGPtFXs6XBX zp&%eCn4bsNr<}(5okc9QZ%^tAV@lb&^uKLx{lJcBJEpYtp)vb?$w2Y0?Kzf!xXB@Fn(p~@&dMo2Q8iu5y#fnNeHatbSo*ueXDYAnpg>deE(Ns`DDvnN^$ zSz|OtewfZMA#$hC?63AkS2b=RAqZ3h58~_#%!AwvgR*7eJO??OF!|69PLs8l1m!6~ z&A*3cqrr?6*=qIfi~t3fmSs0S$Vhh6z)$v{pPhgrw(!4b$xE(^{bxX;K|wn7&64cj z2D|QaiZE8A1mpiqO2j;&P0g$pXZO|p{` z|7#-STk^ksMY0KA(RXWY!* z6f_CSKWiL7g*B7D!H6ZI@PX0Xt z z{}<=KM_)gJLA5P-h?mJ{=lZ*UW20ZXg?EmGC+L1DiVKN<40tuQDOM}h@bEW@zqvNK z?Aej`CKvSQ%o82wY9%v=rRD{789H6LWA7Uj0yq72Lsj)CMq@f<{97<%Y*V{r?w`b&Vm}~LN4F@;izx*&%h_ik&X3D-5E7DFK!O!Y$izb`RR(_ zx*B3mzsZ#9Y(Xb*g!*)A?~FG$WyV_-xhrb-)OQP@CLCo(#TTbG=Sg*kyG`c1qJ#ap z*ftPxIjzh!cKw<@7r7fzt%c$F;MO820~4KLQa1o+@_-gRDiD)T8t7S|of^Yn+NIX z{t+C_m(4@>y*As9m1!}*_rNIy8Lyi;~lr0Cf%(IYXz7}*sHK4 z)sxBbwnCFc1HwjBG$GOHcqDFHF8##TgA(GB;i;uww4iqxCB3z4WU7=U8~yB(7vM1S zX+s_VOzEl5OuajEZ1GM&Cl!*LE&EBvjmTFEqdw`uc!up;YC6sc>k`kB&`JTZ>~jnWmAW>j zG?`T6=^>fg2NH0Ohw23h#FHl!`p6u$>8l`Uvv!%Rq(1hr>aIGYFaiyfqzEV*orLH* za(oGcR^IxLBQ;Wl6Sr(Oes6~~_?t_78`0F;dv0O|{UmY*lE`e9an*@MTA zefJ5MZm0%7A;mJk@UN*9=;xx-YDuk1N_vH9W8mvNxMqA;-Iy~VzY&*TM?Z_W!S6ZU zGHI?PF;Mrd+=Y;tru2G?p0Bp`{o?yn{F zM{;{H3C`_zIT_L6Km<5oK~i!=)=I8k z`5>ZvU}b&Z#^U#{o60?)=a0TJ^&;Vn9lB}1l=YQk@4NJN>e5^(sR7x9-laG4;=keI zJ#*c%evRtebX~lRO{G*urns9?>YSe-?Q}kQS?6EVPh-;N!>14^gD*;f!Jr#1qqNsb=21a1beyQF-{yP8 z3NsBsl0Szu9#B8cLK^;IsGLv6(-~Em^JXTW$kEAmjUw#d3$^ygxhHsAb}d!B+1!w8 z)}~i4l9ef|&dhpPedW`nbH>Y?BbB>1OqKrW{$*Hk!Iyn7GecXv`eh|P!)LXHeyjc|l+HnB4s8sgxFzOIP~|O+b~;7Xya;CsJ1OYt!_0zP?WJ)(cIw0Qx;~ZN{sk<8(xnCJKG%_T z$;kq>Qp$6Ba}VYj81Ju-ZnrO}3klkmq+AV|c>{9+|8R_LHffJ2&E3 zvo@AoWv>pV@cZufU!N&_arc3@%-Dz9D>o$B``^5MBb<5t;ibj#Wa|3~ItqKKiI+A* zjg`Ca8VLGjKV2V+cg!%?O(_O(y?>iF)F7-3S+YBItTH@{-BoQ zaYtY=9BjiL@IVE~+g5+cO!n~75}49I1}8duj%o8F&*^1Xv74|XcBTu&;*Lr+4_zn8 za?y`T6|z)X{GoR5xmv)X#wRE4@w$2LSH9k#g8y;tz7juBGIm-m{Yl4lEKeQ0Bcenxc7(zRewtV)c;HOl5)MP5ej$n46(yNV5*kkl>% z6Dh(6JnDXU*em;XVJJIdVxOwmUv|_9IZ|f#;9=(M; zkcHn!gq3=~vOX|z{c;%Q8H^HU6HXpWv&fc}yfy_%&Qp-cPxWcLW3$%N*sI$6^#eb- zu#Cbab)TIehE3JHus6O<+k7jDZoeh-FwF#;oy^JD-D7G%P@Uit8R(H4dCJ$>`aaXC zsI*Q|rS!GJD4OG>iJ@P@NSooE-q#b&G6ynqGLzk(5?tub@^l=xopHIjD!)xy=85S) z=DP3fZ2c9|;rlhLYnLlJ3?Wao3NE)_%y>HmGVfjWFu%lHPwRUPb0@Vlv9kQJ*v{T> zv{xo`^C7ckNP3&4pXe#wL)|U`Rec2=m;D3Z)@LgIxo^Q8-F~v~gc*WL-ZNH>Rn7o> z|NMU8*X{7CGM|M*SMA#ftG=T>ais;Kvm!h5);qb2uQqyNr*o)%K;=rtfs)6n^=TIL z>DZ@S+TACt4}J?gnca@t|M9~h)_8Vrh8wJVP$6N3t|KrKWJ~?|i4>3~(Y1e!-gxLP zN9CyI#Bz#VO3vf1S|~r^EqAF{=_Wk0Dk=HemtpeZQ;H;D)*|e8h$5>Q(IUt+FW{s* zS3`eVs{5rCwS@YfNH%4rshVT7Wcg|w^)KHuZ|WD_UXq)k;>la@v(_!-_7doomp@NL z?kd`C?|lf>(4hm6^L8OIrQ6g$eh6wlH8oecJW-R9xzzAA9h9G>KM9x=!6^azreW`; zd4$$37I0LJ=}ZrP27dz@=_w1Dn>t1Z#*{Z~aCIMibSji=x?XR+MDH<0+viGW6s_{8 zq78ox$%m(`ub8?l_G5~bR@{D0Rp%QM9;koOS}1T*Es`UoIKAf2R~oT!aCPp4%MaiC zCD{DR6t(+%Olxx^=)s%H>VTZN^;;R~{uQYvlXK-skAD$AWqGe&GS)1+t0$~&-(gA&I>~oefv`5pL?-qyqkt0qH>}r39+?! zNx|}4v5k)&`awqvs>|Y8h3^zp$IxF(xPA_cVeh{}{K$VMU?erLIpqQMhx_?wTYsn` zZ6^ZO-G1sdzF@QD+Y1rnXU~noSdy4$o`%M~^!Q$WZ`Q<;ddCjBw5&JQBh5|gtidH5 zl!5_Q816r3p-VLsbJxGgY#nZ7a6N^+XNC@#oQge9`>l3{?@`V>*chl`orYP zZG*{&iqft`SY3eg6)C{YXKM^_ZQD~Sc1P>->tTQBEeoF;h)Y&aRHiwvV@o|t)S~FA z(sr0s=}o}ba3}FRovF4zioQFEy-kpSy?fAB5rgu7cfEVPjw4M(w+C+twYSI86Z5r~ zv#xhobLYJk3B`S8fLlFb+tK&s(4~L5hjHn7TS{SvjT*Cj^rWzO8(=ba_Vv%CYfui~ z8dlPv9v0Ko=NVZs3E%I&;|{uqR2W6h&1boU?Kq8m?uuV$;rXXm#AlSN+*mKmE7AU_L8((1?WEr5LKd}Fl%`YWU-YF?5LyNwfj?*VQ z)cpKyF{y`dGrBr1z8i*)kyX3V`krgI?XGyaXGI9);yk({Kbe{>=Pk_~eKPVHiznZR zpWfx8g^8&j&M_-K!Hn2_GZ)15hr^iq&NEZ4>_InGotuY` z9{P`~&a^it2x$;@ZyG3k|G9iKm%jJjhgZ}X01fpSkG|d310n=_^Cv}9Usf%DDrL6O zl0vrC(zc0q0#fqoQzz?>!qCy%*4;)Q>2JjzYradNGnP?(WTj&PKp+?M8Ep#~H4!_RFbFRAl4l`{bq&aMt6;70mVl~T)Z+S2+vL_j{L2Uye zj$Dk{#>gs|Df6d1P-wOWL9eeU z`gw>xZK_|}%$sA!?8*BC#wfNFfg6<{jGy~ITwK-VOAF0!lezIEO&80SEqC}O_?x^S z`_iysUEYeK9ol?ki`lOyaG&X%HI&yVzt*NSRPYPSu`AmPIV{S_D{=VK5NGsjZ?LLR z$3L$-gEC}#Zc_cu;3d(CKa zv5#D3vTPtF1pB&jk9b8LNB7Jt0S()GrlY_#&PmLvTrZCZ-cr<*3_*E#ZpCn^@~p&b zeWZU((pKhrL}QRK)B5^l?mxbiO}t6+B|gE7wARnUGOx9`9nPh#QH4s1P$T1kA550;i<#7GFWV{xM z?(5|!sZ$jeZ@TJ4(G>82u$&Pk9J%hRZ*}$kcSk`^J5xhJyJ-<}i@Pntu@0PQ=#hL$ zBH>kAwUPszH)FlMqb|4WRdT94Jt#GA5Bci4hRdj<*dX3E;sUE&LZ2Nfeyrw?LDn2Q zzL^^YwF8|1VXSAuf+77VqWPZ;MCs8brdbiSOE+NjQEZ(FV~&`G`F7uyWx7uvw@Ypl zKWh}DGyhOIi$A#}(caS&)fX%0ZO#Gq|9-65FQK`3{U4@OzWfQ6Ok~ouA<+2h`_!A| zulB0`q(d;k7nR+5XpsLXoO63U_5D3t`}!%r6}oLh)^*>k}JMYx~7hiGwJ(NVYyt6Mdd#r@XF7tFDDa> zDA;FtwB%WceCo!F^qg)=oX_RnX%w^N8i@oalK8?VstS|dlSCnTf|3teK*=&P1J$3w zx8F&O&qlG$CX9Wggm3f?JV-BWEPAqgKxYJgR5SUxRne!`k*ye$zHzS)TQQv$-CWjF z8@=?Vyg9T9&vsCsZ$f93HeoV;mdx4Ey-YRlo-kVe{SrfrxD6HO(gPQTpC*@HM{=+f z-?LE$0IlZG%IY4m5+BG+l$_r$LCYd9;0Ux2zNp1o-`}Aox9q8p?{OoZKGYd!AC~&^ zbm>M6eS+@moZRS&k|dd7=^};)Gd225VjP(z2QOY!ao8+Xwz9poWlPoYH_~LgG*oN& zDgTiv)}ZTZQ*CAWFMomP{%QQY+&S^j{Ik1thVENshN`g>I7o@1!bWw$k4MfQ$~NnE zsKv(KGB%($6%H=9-g|N}I_ve`o3)_)`4QiSZ1{Qo5hsRn>2m0syUGw;EICsl_xef4 zwoGeGmyd;uxpIw0Rxzv_B8Hs9$CjYA5z`o#kHk0mxs+WDp;?eu40d$Xv==PWdV9FZKai-)@0STMJnHcSipTtigVBk7Dq%xV)O zm*On6B**EG47`QOoqG?t>whpk)SopVIZZ-eKe6A-*c@`kr{T?5B(Zu1!x;K{#l|x| zpZ$y$!d{f#jR~H`iS=8OuR=dd+BJEwr4N=Bew4pn?vnmE&V6~6vk;YMIsx8lK{2gC|&NnsUyDEOhUV?(@8dl*)SLzu~Lj zbVGqHzM&#bSpead1DaVZl8+4V=8k) za$W@e;WaJ(PVxB4*fUrdBbO~;?!WFa1;r66Nkfhaj!|Qi|C}?Hbg{i!v0zva+LtSoa#kVm9%>%=+J*Jv^TiOH zt86-4`VE?=04h$2rPHc^yYdypn$RQh8|vX%9t&N&sHYsoLj?wJ{w2Eva#-n0^Xs{z zB!Av7G5AwIH^C%|ZN9;xc*8TCbaOuFSn%5r=niL-Kq=_v{mMQKPY+59i8Umft_SN? zPvduq1!V~7awe9I2tPw9k3RX_f}qsx3azUcL?qao_Ka5`Ra@{9 zP`%A7o#TA5bDloB^i!6KW*Iltd!#pTk~}jvZfrtxRjHVjV9{YKU`8ADei=j-DCe}n z-&K9%xSR@#TMi7@9Xv;`N@~$B$?Hobh)BWNjqd8ePO^Zg_Z*Kzv~S%An)&K(teeE~ zw9up}Me0VFW!{^^+QD$~HX0GR+7r%PTvw{50Y$3M%Eh%n4oGwT48yyG()NYC6dEjo zuO#1=4Pj<&plAWVA$*0Z=+{N0we0Xx{(alJ&`*(>)k^*Q8FIW&!HW_F)or>*yXdFq zwOS&R>m!q5XhfS_(sa$; z-EVD61MiJ^%TB!YTiKMa=lNk^`$oo`Ds9EF_u+QqqTE6A&dOK{+-2sl{*nJV%M~u6 zvggenNw8&(aF&>PFR&|E#6HC~MuO}V7>(P2NdUkEnmNH-!uhh$c(jo$Q*r+Kbvr!> z1N`|c?LnWiz$aNQ!G9i1WZ2sCK(_~@(&p2(?Rh6Sjuo!tEZ=f%F5moQ z>@yu7MZ`o8vbo|zMRFAi4HlO(;1uv%xh5xXqRTn>l$gy|aY=D7-ts@#l#hyT4q1<; zJ};)9x{94=Fc9ecCOkqre(x_r!7k>y=1wcT_kV)!)<4}XsU3nBdgn*+b_`e|*uqi` zxvz!_VUM0k#4l&uJGt$PU%nlTuX&jH{ZaxyfGRcm)!@+3*-v|en>`H1B9r!<)P|OY zR+3~1ALx&}>|THNQFY57dHhY1tZ#9gf=zIz_rCw6DW*3aZQPBHh=B+U#Pnm&9MwLu zY7a-;y~~eryl>ms)dG`JCX~ zelOR^Q|$zaD2k6-f+*qh2`23r1%n{HN4>Q$V=*vW!WfNjEq#}R2o<~O$CIQ5uA)0^ zVp+R_0D;N;g7Pq1-HA?bd8K81Usb2`X?kUc;f*&^gOZDEpZe`NOVVuvnc43l z3=>gPsVlz6jFb#cgf_7Owky9?_dX1(&xEHqD|nEFgc!cCCATSz{j=#UuF! z(b^cPtIC@96Jtf%Z9itkPYcJ{TPnlJU-gy1#hW>Cl;X(z8M#X4&d@7=qvd+3F`aR_ z>5C6vHN_iVFPT`@+(_ZbDxb&~zi$7~wm!rCmvrpvt=T(_Puk|myZ(S|!)DknYp zmS_A0^)qO&2J*?N&|9&?T`B+SpxgB8oi7%frznqnLkrdC-@@N|Akh~x4@&fO=6`La z=N3K~YKM*|6BH+%gDUT9r{rt9i%;;g-(!0xqFk$^-CL*DDbTbrvMi#!q5H^{e}|h! zV!3*=uBGLkZEm4|?h{;9$)qNHV||a&*G9jr951dEX%hSnz{l`We@UDVk){xndAkC%0nOApPg zHVO_sX;lj?&+GY)po04>YvZevR~o8Z_R!#G&BN{X28bbxQ^xPHba%z)4L0gOr_)wzmC%%3nqJF*`K~VB&Pn!x6^+sK^j?2I zIMl0|+06BGj}vPHzEFDVW*;c_(}v*xJOc6|`Nz-nceObac}_#33Z@Vjgn68@WoD~S z61=WyVzhv^zR5w$hVw^g;grHoY1w(Lw5)8r$6U*1=j?bio$zR`q<{R$M5aiD|C*KWu$fRFvQMw-N#(Foe?GNHcVo z+Z5ES37$&1m80)qJGt6NG$gbRj=58Y+I|nk#tFWKG?o|PqyLn^+Z{0&6oGy zqlXc%YVUp{92%$Zei@7on;De)H#V5jEzNO-9OhC`l>`5#cFt(PU5TMI+C;CayYKj+(2@6fkM*T^gMUgb~ zcISxQ^t9cp&<9hmW0II8bvp&eujGl5rq5K!^G+E||GaFtvo4+z$1mT{&YlR{I;Q+! zgQJqW)R$1aw#Ig>M;Nr(`S_|-z&~`5eNkQMj}IG%X6#%a{@-s=^K_;#FK5BIlz}zT z`uzpn@kFB3Dg#jk75Y1Ur_|6gVX1WumC{6{G%t(dyFEQ?TUY;x)KvpZBqE-yUZo|@ z4{G!oAl&EThZ%A&r@`xF(a^!!9qe1{LgcEs>sl`4%#AK#>Tku{HMo zoL-fW8NEW03HV$C@8ITMz8N}#kLC4PYJ7|yRKrX7a_3h#MImS(#%mt9>dFjR*=B5Y zI*D|R4>KA(iiH?D&%)GJpcj9lx1&%q8rLz?5$8(jKbytrk4RM_%l8d*I0O0K{>eL_ z-QPW%Iw|u-2c$Va@m2z^MVAm>!7)dW{QEQvSHCqvE&f|oSFau=SFR&jgun))|8JIH zXNb97?LpTn;ZE20?VZIAN;HSyoFdF=!P3yjsM5ZNcnjeu zezfZsko<_P81ap*VDwk!e-|J*!;f9qylHju_;?3XRbl^k!kEE<)1VBZcUhU{w$wth z#z(<&IF#d4KiukysD6<^x3#hRQxty(YIoAaaYz&MP537e;lXla;4a@Goaox=sZ*+R z`;#2I(H3^JQ0~|laD#;1uVcobgoEBks7)w{-^zlYzGiq%81{)EN^|7nM?ZRAk*L<9 z&-HYv8;$w~s=~F`|DZLnYd6=P040*9enyo3>q$^k0Y`$gDLxcFt%!oH4pNPmM!`p6 zV2vY9`>HUZp{im1go-#{t(JLg>WLoh5fBZfTxU#n3DFP;myudy4ArzXregZlC6mrU zLfU?MC-KGVe60y=;vr7*^O2UM_yNri?@%W=AousaO;vUK7?}a`y3n44A9BL72JTXR zJ#}7lzV*ruIpvr@w&B83hWXKMB~8Qj0qLx1?X@37)BQTnDNWB(W-p9Obl{by|2H>3 z;CsW_p8;zbPo^+m1Zq7UhFF*f!R(gECs7#zVmqmaNWDz0{7Oq|!N?M|O zX8A@2QztpzAsls6j@jB=7c-+dYKM(08cR{`u68PNi0s+$qr#}%W0mdk29HYF;fZ4z zOJ~*M5QZ1kavq_&CZ7n5Bo*ulUfdq`bfYBwhQ^K~-CfYK#+L563W&jKB=Im5ti+8H!t@Z)_IH zYFqKK79W!3n9fv|QA|K0>F>9yERoKYORF0%5oa%{pppV4b+?B_$-nnQA7P4ND;wFR z=C39aT?(>C^oxJ`Q#!~KXacqOMz$ITL?UPOb=n0`7kX)%Rphs1Cl2w zh-_u6YeACvP><(IqZc)npSq+=r$X_{y7ARjxN9&hR3!U!L70DYPsVh{eh94;;YMTb zO9iOR{`*ISPkR0cjbWu!`1$_6v_FLeU0K>d3s*dju3y5`*vM)l<5}@nYmRNeroq#X z`Iwojb>jg?RYwQ*=v&l}v5$o_`j@kf2I{=YAQi9ZJ-p>y@1MR^_)GfYscoX!k|{BS z6RivUlcKSFi$Wk!8Wl@*6^gcH5br1QPYYC1kCz$Edv0a8?70{+@v}YOu{t-zR_A)Q zoX;lJf%i~8QnPP-aZwU1^b;RT@eExVA)QC+z>jru=LAT730}p8%N8m0n)K--CU}!Di`C!y1hDH8 z?wz+8@#PnNMZXjZc-v49&6GC>b8fa$L`K=4apS%A(#%l%;_qVveO8Op_+XxO)E97`&)ArfeXr4W6Yan{~Z<;F@o8 zpk-4aR}J?G;~ic$VnEMBi&R87ZYX^kopZNe5I4E*`Nl`MlGN;yuRqE2 z9GP6up0QIk3HC5-82sl%B$7SPFZ4*8URC>JQ^d=K{Xrk@(~<< zl{qU@SadyFqA3SRKXX~{F9*4G}F=fA)x%fJc7u#Z6to=!_fZYoKTPpbqCc48?H_;vmleb%XK5YqY&CS+` z3a<}OTe7@6(znJSW0>LuVtLTx7Bp+x^Gc@McAI@Bf*!k>4|gLW=g`D89|kf9X%f449Vuu)JL zV2YeAp|^P7sDzxL2^lVOq-*ceUyxC{yV5{xk;uw767LQ7cDxAqjjbl`C3@gLK68x4 zl>MyppLwI(5TW`fahs-mkGn=- zr;@VFh+y8fStkVp-h|7#Ms+L7u~hP$VrX(|b?^>Y@LmbPF?Z9g1i$X%s1#)mUM7A{ zSFso&|2UEaX!vdZ_qDQt1Lc8I56~)wtBAyfLQmKv=i;Ok2ecJ18)81H+LAUMN+->L z*4gh_2OY)?&S!0NEQ56xmhL!*d6jE`Z`L2A@yhY_OLP_ZS)46}cppzEAJgq_i^ho( zH9NI6#=yvF;wn_s&*~_!GqISaDQvv!!DxB~7aR~kHmp9jh|J_9(RFcuz3 z&vl3#K1yIbLjFECkKCrj_0khjjcCv{4%wL7Q*cZ#-r!0YHRZHvW-$S2NLl$kpNJ=o zf)86yUhA@1fp*xjy#m>>AH#LA(WGV4`ut0?&pzWnM|h_%PeB`Y=F6^&{n_)9#p<5i zfsRfrmo!o=pA=@TdxNp0KM{LmK|eYbM2GM`Lgy8WdMGq%DZ?6*ZIht{c2$nEb8lkX zyW#G`pEUxe&(pD1LX`UV>*gbPEc=)EAD@0F#$3!e9C==QI@VGyynofcxAZq78I=Aa zlf>N_^p~93XwiL?-RCH{2|2X*NoD_%-H*IM-tL^9&uCZ9?hti89uef8T$cOmUQWG* zfzzmpU+^WTdyVpa=9^7*#=Q?RF?~9gs!j`r$3DNkV-I1|ovg_jBIq!UAWgR>p`}Lg z^(Moyf9re+2yj|=NeF*>!RVN zFXCh8VswO!{>qvr$RIMnw|dI{8n%zoYtZdnGJfW(IM)`zcaU93aVXAmNt@r%V@{B6 z8+ypb#=W-2NcJb79w91pY}zJiY8q|9a8yT(?53exWfS2&cjvv z9x}HXtv$F~D8s0#)gWisC9^OeJi&C|<1OTAN;ox{l`9vG)8Zk{2Uy)uR>Rg7PrJ2J zt3yWYvsCy$Ulz{qI;j7=R&b7?QBUZWS&!t3=j{|q_3q&8=iNcz9X~8s(Ky@?ADe=D07tT@AK3GhBY2PI z$19CealNv<)ECJKdJ5^qZz^B*ylt_FEBraM${WeY$=ErNf*gpbVvR`|L`8}TQQPEG zxS5(np?}S*&jdfN$kD8<9>z%Qc9Ud9N;1(ea&bZYnCUq5pW>H9hb(TVe8#fdJ6Bs0 zKFAOdEg5LKwY_^O)L^8jc3G#D7P&jbG!TB6$`##TOCKuZx!`*&txuoi z7bf^e>*IeVHZsQioe*NC4)(*IxcQOc*UlmBMK7ge!k64FEi#t#3R6?nulC8924izE zGuTBlohBvE2@iXw>|Owo`#j2To#!06xnl2=6ROkpw|^4H@**bI()O+>nPV;*eslz> zy?RqHC7Zxh==<`JSuuD1&gJ5!dBZh!9>W*D-B>N*wQEp{4E`Yrp^pM9BzPgi*h zr0d1re-y2YD+>BnM97zm89RGz{zIAbbWluON;f-r)u})j(6uRXa!bW`iyZd>lzhR& z@Gkp6UH%PC92EuE_4}Hs+Y`c@g+C~yV+GfOPyuxI+%^yVK2Mq;!==#jPe>}9z$qcF zQ}yT>-FkA0v|8>E&;=V(9nLq(ib9YTaLyxZn2vvcO3%9IjhsRx{tkRm3Kk(c@eC4N3e%jB2|y|3vD6DbosgEcu~6 zV!qeeL%DcOq3V>gWqCRLgN}*d8G9FF@D6lc*{!AtH-2?NppXE@peco)Ov{oxf`14t zblcIqAU;~#DC1yjcL%X?5Gj#Hm}?q&F)XzOL_gvEYVr-1gut)@&n^L(8 z#)G(a4iU`d=LCnq3siFgtc6MOLd36CpPP0?ljl!ufPBJfdJh(E+nvZy0j;HYfir*3 z>F(Ov%wH@Cr7Ev1mB7WHPg<{!#33$E#16aMh@@*@N!i>T)eskFDk}M0>rvard!MKihq_GI)syz_{~OJBU>fTE> zbq1qVqM;8HHBWvY!4qv1m3&D{RhDd7B0j0^IkOVA?l@|Kew-4`S-hCNiNR)5l@v|w z<)YDAu?u0Em(|yC_tW=vF?;F{(&dlA^`7&{VO?4|fUIyM5uGd#FOv(KQqHr9LNVsL z->T!7c5c=x1pl?gTPbpy)U89x5AUxad5?YLJ5Tw^0~SD#B z*5h0`P-Uy!=$7w9oKsS4TEPC_YKX}g1K*7-0zLF@ZK2~j5XVH^qm1-ES!66!>*e`X z?^tiwoa@EIOWv{RPr#cY@$KUF)1{90*G%OZ!T?D(O@sU)6aU<^7q^i_uLso_F{$(Y zC4VM7?ck=Gx5&+yx5<6BsBidv_X7paUxJM^(fIjefy(#8KsA)yfeS4`x|e-o?@~aQ zJ$d+;YMd?8v5~I9{jT&+Gg=v|e&dTRDy}BCDU($eFa2g5hka)9N%h`vVkk!?Ab0lt zZ8VgC#_swufLft0MuMcln=Z5Fx?+7p%Ln&VL`Ghj?TG)VAcg#Omou`9OXR4*ghp)!0)}NLDBwq+XFHaLYHca7xqVZTf-k=0!9*>sT{B(gH zG9JH7Fb_%cQs;n-wrKKO=bB^aqJLeDAvbxmhQxI~RX`TNI0ARM6-F+GM98anIMRQQO^ zE0P$_hntS{{d5_E_>!%1nb9})GbBg-C%-ZJZM-tghgwLuaz8!Pbedy)Gmc|ybF5N( z`?1ih()%gKE3H)S%R;q7&!3i|4Smj;?JlZ!)w9rk1|t3E?)GPoi+RlF*8fT7f?tIA zwUAe4(d5w#jLPxm>tCTO*NT0=Ya-vBE)}~dwq#ps^UXY3M3m6X7{5e^w4pHcZb*C= zq0utTOkvEd8bxsmKcxpd)!}a&7*i2R3Y;|<6bVYFjrX$&)%vplYswYhVa`u10bE8&+)qDAPU$aX%e$EHPP>A|64Q&ASu-E@E zIVB9pi@)(lnlgx4lvT`KyJ@T^-ERH{rb_OlX+nn4x`RqBcEu|f<#jukXjz6#WU(a0 zudQ!!#JoP=8gOn9cDc)Fc3$r}L?46FlrL|&`K1!F`Ce=uBsg7xk>aeg!|VAcWk_i{ zR?+AWjvzlG7$&Ockq}xg=_E%>Z^iqipbROQ!%BsBftG6d(W zJ%{!?scKm^pL4oOcfNy$->$wdLgnev6+s!ZR1`)!S^C&!0%zeaLXw73bMh(-G54r} z74CbaEF^D`CZY?McEUEz+5Q+gE|O%28vFj;#Vn@lovbb8PaPb;nOtITe!+&B1o z%2ElMT6n8|?$j89Sr;&dqDLV>J4GbJx7Gqmmm;1*p^>z8P9Sr+E0qSY(A{GA&qqYb zqv)q!CD1WP``OhEPqM6rNsTf`ApkDv7!-1Sy?bY!~BN=e~EP-1d}-WE>iMFG~jw@E^agx2gm|ZbELOYfQU%U z^JG;lYoR^qbxI68ebaAu?*M0K5V}&zL?iNU#dOcgL7EEebw-oc6JyBqAm%8d&J%MkhuFMd8j$(v? zovI)bq**BZi3w9qnPXyp3paIHt?5UvZp_={f#G_x%}cZ*&8Im#Egcsa2`Odi<_n-E z9RpIrkF2Z!AT4qh>JHKMXelXYh+J9LlvVOy3d;rpE8EM?=!{BqFN+!B$>mtrtY0Eg;|6jhq2nq2)DHN|as~b1 z8?RE5Czn#H81FhgHM*Hb5c_%yN0pW8vau3({P(Ic4ak$V}i`bnqgv~%wH}@ z?T??QG%?NK9b>BkGJiXSRzuj6?1!E~$>>RvEI#vg&nqEc?iBW~Lb5oNvR7|vfj5KW z-z{VVNZ2%fG1PTKq#XG~<1_Lb(HNy&y_Fz47h(OT(L=-Gtk#h39^$<86HWrDaB1lN zHsdibK~b??=S3t2fO=W55=aOH{Z}wEY)+W+qNfh%j|acLt*kg7{VWdu^4vt-yp#`i$1S|LfxO2!SqsLRSj^zm?p3>T-HM!OcR`d&hD1b@x=!pn(K@41AGFA75&lH?i&p z=&D)44*5k`5_I^H%AC!W-BS4N4tBw+uf<;@;U^D758Y(Kw=VKFZ`ZASemM^c)}^uh zA^U`sunIU^SdW%Ui^f~iTp3&hmz2Vb5NXHi!#4(iRBoi(v<8zCU76K;Hsl)-FH9+u zh%6|<9?m$8Y6>yNl@U&GQYj@Axu!m{3O)(D4`n=_g7`;Q414EPa2!Z^@L?4x32$rK z{d#HbOQrgfbL~&yf1E+Ew+(stgOaWi1ga)eM#rB#wAhHh$As*TK|nb`x`^Ai;P4?@ z?g5I&4F~aKlWs4dz8vj2sS$kmuv#yz0a%@dkv8?~^X^wk5v*EB^LtGTig0>{IA6x145LJtRT4(1wyMZBM++hq8p zZf48%J1M64XC+Ei2Qs-!Ip^ZjT&6#5C)*OM11_vRjkxkvTG(dPrGWlOR;Uvl64lo7 zM(MS*WR-D3dzHhxl!FnnY!>&s3#a+!F8ofEA<74uUqt~@fNq5)F8@3a3in>lYFecA z)ee0?>8@|Bd=rG8?HwffXd|-Qtu zI79bRePB}duBl!DqYn|XtzbM&d}H&x)A!lTph*`Ndo%&4_U^OTyU~ZoZp+79l=)QA zSav;8+1PfV&D#Ynjx&TXY^*n_f0hnt^3DVZ;KI8e6)m1M4_B{AB`w~AB8oth{t0+q zia-!XXip!r)@H4ZCYCS6WRGYOT+jH3pEZOZ?IMcgQewx~dmUOl^iM*>(U5?9fR^s( zE7Bc&nIElnOeCtzyN9(!Ri7X6;qbu1a*m?1t*spmlS+ zln$JgX9+jApISBFaSnx3xfkO{0bW`=z}&^Ci@8!8~V+Ex937kAlku60!h-ayJ zTgIRy#GSyG#Gri-A3u}Ex18&ADm@tQs;(T9%(z z6NnrfIm-8mwf8=50bqFBf6NMvq$ScCUWWG9v!oQfC(U2G`>vaVUP6B5GO~{Xw4GHurYd(SQRyOhx)d7S?+h6MI0 z-l*RG_rY71R#oddj77+b?t$e-Ls+c-0Kd{ppB9Hc8!jw(|DIQn-{Tmh>urcs_>Mg{ zwD-#yL6i9?9LgZW^>`abx%0Qg}Wp@iErQF?CY0>O6k3yROV-yV$VwF5g9o$VDaTh7~GaF&l5{m*L zP=eKr z4TS30g{EBz&cVtl$BXT4Yfs9~0ZiKX9W9tPe7=NdM2a^Y^)Y7q zn9dzuA{SS?@`$!MsmQ>o&?f^&Q~u-fjE2Frqm0J^g9EfM`~|KG8Sc9`h}nw-E}!D{ zx*+&9;7@j@W#3)(adV#UD{VCsHTbS1cUFS9JJU=~@+jRU%PJ!GG;i0oEQiB|n0Th# z&bP3fV%-qY3g=(aotiLt4_>z$^d7UDP50N|R|3S1xeM=o#-h!Z zcIO{PSc){4E5-0_9H5{j6-ys@OT+DIV?Nj7aF6EqsF6Hem9ynTjj5FN(jek%L_t`Pw2#UQkDoE z135+|%$?DS!^E8GUq3gQ7xs8<7xw0|+tSY5=^7y2Khdmii8&o|%;6{wkn3&}xpS9Y z>q}W+eOG{s3-l&#OG1Vl@Lv(3tB}$6HdQ^m4D0`_@^Us-Zd=3s^y4vZV6I%nZKPIk z|F7+f?UR4w`lM|Bch`-nUNG{Z-jCvErKht9X|?1yJC8HJw;?q%%dqP>$bwvs=$a)B ztMI0NmiXr#1$2b6GoJ(huyRz$xT<6w!3^aN zw-}J?1O(WD>;Yq$u*;d%=KLYpV(nErZ-D5qGC|~r4AB7O;V*S@cqw#azC+1nvr<;O z_d6p?_ZJQAvPbC#|9)I07vqFmh&3m82PGG6Or~>#po5Pcd3!L&B*X(ASNJxnTYaacaNnTx2w#r&M?s!B;=&mVP%`7maoOL-ViX9-<)i zu8gHHa6e}Jq+lH^cnB{4Ehd{Wel<}peErXY^aDsH0s<`l2mR-Z&vDrIj2C4x@L}Iy z$x<2P%g~E4I?Fam<_t(b0-GHK!ulukl0V144M!mQ^B0a84~po8M?xT9jW6n6!U}+4 zZ~r9MGJ1y}g5GmEZ*{hdpG%$h1b}W@aaPqqbw$nr@;367=CyV2OK%5vF~1tX z^9aOFIg!SULlYAc8_&rK-+mfvy^n425Us)J4eVIOiVG|+|NR&G#qw_^5=?_c(DWVs z8?th;)T8cCV*qh7-j!zautiW6tY!~{(HP@1-fW?4T<*GjJacFmSaM^;$pL(jz+%7p z-(sUfPV`ovcf_LY9aKcVpJ|-=1mHUjqaLm?xjbO?aNt9=fu!`iN8HF)1UWAaK6($}h&!yiB46kht!m&~ z1EoLEmq9f@itMODfJEJv5jIDES4sIl)c^I<#~3=lXXoNh0K7JwBAzGzm9l@$i=ge> zaa2gtPW)0b9HVpn6TbEOaq~~W)$(aSFm2|#E_oCH_9|NvPsBZI@B=O%*De$;VW-;H zeyG%HfClsBzpy2h6~AjyQSTcVcjU-;W>oEDz%cG!Dp4A)^8Y`gY}E5jlq?-R$z0h| z)Z@ib6!Jx9rtdxiGfCil4)8j@-T{(3F2XM1u!*b)kpM zxd&-ra!h1}egNF+%KwA#G5=jHn)-^b`t~-vsgZWAkS_*=B|#zpVU6^)SyweJ?1$C- z7VTsJgxO7nwcy%(OV)G)TZZ9XGTcr?ncSShaz6W`qoQ4d9G@h}nt%Pp&#>Xa&qMw5 zHdkUjg}1VbYJ1j#2?lK9%B{&MEc;fiPsUCdZZ%i!{LK;=0%KI1xl72?6uH!-?=2fQ z)$GKzjaJZcbdS z;tOYCc-b=GNZGK?&ptC}xN$i($9vo)b<6Ac5FlC+a9|dtBQjX5+2SdKkp8z$=#Wm4 ziyjO;+pg+dmE-TKc%cY;Nf;`Su|X>roBchpwa;n~ffVs2AhAu`6}FtS=~N7mM|4Dy z3h;hBB{%4wjnhe? zk1ebq$h(rIAyfHX+8K~vW|1}AaBysAXU&ySy@Htjmt<$KA8wqU=p8YxR53O%Z11OM3gW9*d72n?6}8^hs*F%m&>b@)moBqn8yI!A*lQ=cnB=#8qq zkT}2^QSo>Mt!CG=gZ-YDuIuhciPKZQ%ID9ds2`irPmY~Yb1F=gnlx*xy`lt-DZ>D9 zWkqJFw%Ax>VnW@|;^aY&#r(bbecZX?B z6wpo2M@^EopH%lU$nXO&D#7vp1x8|!o>^**a{*Vm3>&eJa2LK`?KfzqGW@Y>Hv<@q z)|6dLFJe`G9z?TgyA!(Un8U$x`hg>fMY81Bdgx@PM*qEH64Ml5D)Tn}(wuh7lFP@D zqAB$i3WQJ??qzBh$nb1T0MJs}u7#!Cz(*l1r?io5LDdy-=(W|mVboxjSB^#K#JwrV zxsk6swZCKJS+KJd1l)RtaJO9j0s@-uJV&SDX9m%;l33U6=SP>Erc%j&9s@IR8gbKC zL7kmWcsoJ3DFd?!Ao z03L#mnp;n=(_78?qZgbrz1iM>v_h@|<~il31P?$kC{bsO`8OxDXTpmq3s7%5&a#9& z8Fywmjjor=T41qT=G}hMC<$V_d^EcPg%oA^e|Z{qdP;z#hpQ#npP_>FPnKOo118xC zMqgt$(Tt%QmO(53iB07PxndT25?WVdJb@#PZ%Fr%@~Kb&{S0-%ouVlHv4BK)2G02G zaFU{75K(Yij{LjS*;Oq`%bV(I)ERfIlr-Lgw8W_U~a4 zIcC|0x|pOy{nv!W&`Qn7yA`9C3!Xa;qso*b}AAA zK#rDj>jy!YEGU@>UC!ZpxnM77jTg$yFna!yF?S%R5hV=8j~H7{E7q~ghMcvb8&CLP z;unL!mjl+$QFnfMBuDD;f#GFY{JvOCB-O=^K$g5>xfy%wX6$7S<;)@(D)!)+*;lnD zD={<0$bC1$IP>|ykTh&ya8QgaIfLY&{;Qdg^~3FOf(nCn1+>@inVkHvA;*lgQvH%;Z9#vwPSkl1DQ$8 zi)T55(HMusg3-3oW{mh@LWg)%Hyd7A2vhhMhhW16mX}FL>Iro>7yXQR2 zbP_IgZt3PR6vpq%dsmzix;S%3EHGT#>6g_|V z-z?k-1MRb3tYwXC=UFsqFCr+}vUy({d8L8hZ4w-DO5v6|lFyOVZO{^M{&OZhCd7zz zZxmpa!mdpypJjC>Nfo~SXqOariLgA30WxG>A>sF0^75f^u{S_Fw@*_tYHW<#&b&A= zP;oJyUO{bJ8By9*g7yH)c9?fw(b9kBY_pYzMuR1+|9 z`a+ad0NTKJv+xZ)LOkjUWxahTp&_160_008rTNYGe227D#3?3T!(H9li7k53HOiZn zEk-dge3%_vLM%*_T&**0RnOnDV56vAQ}H%n;ma#IyuWyW+KaO@956Que9;y{L!-Ah zLB*2IhC}*9lU*}I*TU@v7%exp6iI3?3;Hxvc;|}C8MT^#G!ka6*O%02lZ45gI>DM~ z7P0pfLnoup_7-UB*hs`QRoaFT3u=-H6DnirqJ&l+`lwS=EAZp?=VhPkobNaSW0pjV z#K5uGos}5ILZJ6-5bF7xXK+vvRFC%bIrrcjo?`sfsM?+Y9OekQhL@O%UF__0k{fu* z6M=UZf{NNgIX6jw*pzJ8lo3>|;4x{+pbFD+%~c@u&@Psg0j)a!dA{jWB>Kz5lo&i$ z?a*iQ&bLuTM$m{9g{fD;GU!y$2&UV`2UGeQ6GDIQ_WdgzIs`<#SF!w^oVcmZ&Y)0N z{|Gk~o26ai7R27SKF!(C#LR^oVrirNK~gQ~`Q^({EPo8u;?B>v-gZq9p`ph-9d`>J zm$+|-p=kL`aRYm#Fy)5}qxxwT^8Z}A`hqF1ce{cWSxFm@&>>2~5G5D(@#&?$;=D8C z9&tUwr!>;ju9VS9GD;k&4^$oeYvJ6=?z{ex7&k1i(GOl%R;~wDs0a7RqLhyB!PSi$ z1Q?-hR?JQVF-P6s{eVvZVK|*x%p_@v2}GDv;ZTYdL`2{xCi%cLu&2hWu=*x{zkOxO z`+n-RiezzA2;3j@_2hjK8Trs@AW!J1^CpT;a;gTLjBa-JCq39svjoq3Rm7+sr`E@@ zh0_O3PBzvUh}_M*qYfHy?(u}@$NJ9Rm>U8(}I37q$sb-(#%#`7rc*mv`bS?qE48-AdT9|n(_ ze6Nc{iK7Bn%R}YqUyb0^&(Wr2cGL(8sJ^&NC1bTE<#%xy{dX1w6yf_JAv|vuDe3h@ zhjt8ORgSLh-Ci9|F4*J6{W}8hv}eciBVM48>GiKUUzs_(ONRg3e7#2FgUw6;=K3n6 z{W|QBm8$y0)PXWmr<@rKSmzM%n3|d^wWF~gVERk&rH&Wisn&Q3G&+c;bIHi4$=_rw zR^T1#1D>xnpPk+kB(cj6;I)^`4~7IpP$p%VANUWcpJ=lLWV92_Ymn2c=wP6LBIC@7 zbp;dO%H3uE`i6Tl@Vuia=-aDyJ77dIc7-;K zGcBT}0ux&(BqhsmloK!tCu2me)OlMZ10THMpK=$SANW_7i#7;D5y;FuUQ%=>gziM> zstAd7tr2ztuB$`)p}9ZZQVDuYH4buGi6wq2eGMp}RBN04oz+g~d5;H%XazA9?P=^> zRAUS%&crNf_!a~I=0dXQ`vXsgIgz%0f7QDetUmx{uoXE?4wcwiAT!8LtoAqlEtH8V zRx(N>Xfh9vcMi4NM^xi=Qx@nWG+g*b`b0499wTm@6Q&QkT%|=_I6lCqW)>U?hLuqR zPyoshV|x7~!6B=N+)w_aiS^@b719?gH1QBZd_$F>1ddz8M$sW|c* z#)F9$SAJAvFB2S5;xHD}_(spU+{JN`h7kuWa+2)v|H=aK^$)gP7IMIjOBn)CMo{01 zztW8TQqjSg4VdQpgq?>70po}DyT@K%`MT#0 zU}v%jq%s6zFLbSWO3Fx4Fc58}BkU+d-1Y9jY*0w!_-(3U()S=IVUkDcqQ%6) zl`7wK_hb$v++77gGAeZ@6aSLUxN`h4g1<n!Jp$Z#le(DIVYw$1ew^ zsca$)bpy0+Wq8~(^B(2)cSeGbcN_xlGp_MZP(*8J=YO*c2x34^vXuMKjO}Ry+;Cw= z&oV1pm@pY2s>ql0UCs&r?D?)g&BKcc-fjm8$_CfH|0Dv{3L+s-M@e)q2%#MSK8W*= zY_a@&Dof8vz5OB+U4U$CN3DrV;0q{m)w5dCv?eSmKZ@cOy3WZ1gQjQ7^TfZo#;lD> zrG|E(SiBa@+d33cWwbt;xZH1;WVHh`*4eTH5mkD==)D(T)!IhMse9{&QLan`GsRb< zBps1Dv;X9b76iAQ^DYpUilAn`S)^m29vraB0n{d1{#z;umZ9wcaw8+8f`rE**c}gE z)f%CrlC1mds}NN}Rt2nep91jhZLc4wrqps&+h80m1;{iHT3b3R(qHQ6e4reTT+U3i zRfWQRFDcrPB9*IRTtKqX16-;EKNX3;LyJU68jd*W=~6U>S>0-v1O7u<1KhVWzrsGU zE1*N*VSh+>Q)U7>H09lo3wYaB5Of9C2nLw~UHq+waHi_sdM&F9EDG4G5SSngzub1ZOIo z$X*oF8LWC{Q<;p(Uz-R-U|a&%(~{~WV~%SiV1cZTgjhCXJl0ZKA)|DP(trikK7NGk zqFPv<&5MxKA)>@S3ZmsLl%P`;fiO4tk9NXEO=V!6%b`|ux?5-3B~(% zr;*qTIssvw|AUXtdaRN>h6d8Cm2U9T|C@;C40s3Q3`_$Hhqq*5Q<#+Qw=YqNIg0c7v!f{Fp z;YKg-3arPm1OXjf&B$j(MsML`o8x86c7=l83pN(mJ){U?nXeA5Cw+r?hXlQ z7`hn{knTphJEXgjZtfYspWpr8`-T_8+2@?S*WT;wz1Fjyr)%N8PPBVyn9@=%$F|wd zWyu1gZU8IAfK~ee5(tDQ7Au6<{p%!$HQFA@$VT9fITZ<>D4PnfxAAOG zSH1#`CA!SBgwtg^bk@k|auonBbNpdhrj<-IgjxbpVVk>#Y`N%|lwZ0eZMz>yAH4VQ z(NfyL?yF4=1>@gwzNKMYMYp+WcKjrOD_^XcIT&krvNu;w?vHC^#Xy<*@0*m#*=yaD zNP~)XWk|*1?pZeUdiImMmTRrh_eIrUaR$^e1a$ro5wSIE350T<^?g$tjBTk7 zuMok)XC12Y(qKfZft0f5{CU4tNbv5p%DBxj%jT=}+<14E(Lz9wBZV`aicaMhE;t(E zyO9sh!|<^Vi9>vrK&C+jq>ir1XxE$M%wUToOx5o+f>IOM0FqDq*dKSYc|;=KbS}er z*(FW+wNn}8%E-R3nJ~P6Q6LK*DUR`x(WS6`4X+KbZdvQ9mO&UTGzEhMDM0A)`B&Pp z4rJQ?gAVL_oNJ~N`FljszM};In!j%re!1-Oi^zkY{ zKy?H-xq+;;m-D|Y$%nMP+w2M%3kR~r(+L|9UYyk!*bGvDDaX;gR~^3alW;n<=nUAj zYGBb0PXXV}tew@u-&2$b$ST>c;?D1nCa2{pPoJsa69t z5)!f`GRv+?hb9TE$k&OL#QgI2WRb5&4(4q6GC)JGp$;xvQ-|RzKy|x|K0x;Uhb4Go z)kCRwp=(SfRpQUEb+waYk{*sqS3E@(KGT6nT)RG3(~KldBSP1TU^;36`S%JZ#JoTx#d~3mjrr5@SG*?o8;k)A8_B8qyrnsPxNg_d`pNSQsaKQiEU5ML@$|FEU+PW6` zq|!kcqlG|7xCzrR`C8Zy=Z&QvgptoBZko*SBfF z`Hd`%>AmVQHvk-eRsGv08OlQ%SeWyD0g6gr)5uK)IREIuv!0#FT}Ip_kCkp;gg4=< z^d|-u>;R|K$(qVbMZQO4WapC4vU!5uAh>+dKDejbSgcCT`MV>+Q>D%~*F6XJYMt*9#b! zzX~&d&Jd4PQ&Pb;XNs75k+Fl)`dkRJi4C{HC@9 zu!pgFkf41vw!j>!YjNwb1uAFKIaNXvqaLFT%ztr3T$uSQj>~E|lLm-A_bT>G61ffD zhO})gQl z&wLRHzjWcm&z=Y+Jg24oECyN!zpX3lDOQwUvYW6Iqq-K+UgNuv1V~cNC10 zcKrx%JhVA?xmrV7bo2l&qxUKA;@hgYDpT#S<0kxH04Xay1YoDIVr@9ohxD43;#A}S z(F)H#CtUYi2bZouhr1q^Nz@M3xd(-cuqxf?&f#d(b=Ho^uM@L1Zqf#@O zr-dQ!u0H1iDGcBgd|BC{-DY~l;Rh6avI|5; ziY?_c|LsgkWx}$b31qcx3Xq}Igd>K&z(^8?D78SP-xJTj@aq0DfK8TvEuepAj$JbM zQ52-Eq*7)SJQeG>0Bzgors6e1_$g~3$D|?K8=Yu3Xv`GNFF&C69d_OO{2%x4B7ws_ zn&M^oNJPCZdx1aXEvhh%$FD)mHSA6?9P4!tIj2rr5#@J2G`6%JfFiZZP3U8G?I->W zP#0~p;(kSOlRKbR6#(ECIJvy!w?}kyQ$1;_-j_+C;i1xWvlm}-oI=x=5Vsj|I^SF5 zKMa+N>|<#0cYL{Yg3cd0FxP3ZNo08X()E?FsNbN_=BKGCcQG)vN;ngLbNvjI9&V&3 z;6Ytvi7^aujFC-#GJ%ItJ9yE5hf~;zkUs9w?WOiL^9$3^&yiXQe#xhhzzae4qyMS{ z#kUk;5~s2sz5{BF$W}KJfYj3__TjRYOkW@;m}N8IG?>jar(4%Qgc3g)SWhyx2%B81AXm5n}Zw#UD30J5c2=bZ16pa66X{ z!Q}hq9V6;$Ho)Wdf%Bq@^NH$r8Eby~0D9r_n!v5uqu!p-jQLsBjPHDtAQ{IY`t&H- zhce5?Sk8Ftm85C5DWwIj>c#1!{Qa>S*OK7slB-MwK=P5p)GUJWcp`ycGEMr-m1~z1UWZ{jMz@JbYIvpJxQ`3%Jy4(KkS4BIP+&iYvyoA3zsau8w zdd-@ec)GhH4u9=8;H7VEA`t|brweEH^M!AI@t)x5#Akh) z_`;TygzU}hR(cl05)FfjwL8Wd%fYJ3XzHn zP0Z$>S|3mrbQPcj@oVcE6EiXK$fpW%P^Ek0myA1^#>=DW0}9N5UbnbKIoT}$iFER} zsABuRs7K$mtj5-cT)}TI-bBdpDK%@6XkriV#JQ+!7IeiF(*<2wG0(35(Wb7Z(b$@l zs$CEWx|^PA@b=V3jsp1CIn?G z%72A$Q;^$9;UaWVtq%!SsmujUd4H?CSJ~!MctS1*2}SBFTdhDr=8GK86o>L82jw?_ z5J@!vYy0A2Z8?(0J<7;Gsx6)SrqrlxYWe{^c;fMQ4&*6j%)wyp4PO(>H3|aoUXX73$)^VJ))d#5#2%n-n)Wf$ ztYp75?U8m$(fj)z-aMRvtfci=<#3XFs^uNXTN$7=`WpWfU!E?*_F4XLat3B~8LT1v zq~1_|Vh;d==>&U~GOeTX$Ewk#;|t$}!dD-#X~D5@F(t|l?g(uq6y+*N=0lYasS*2h zk>?4pGU_&K$_LUkLZ-F|s~R9L73XefB4qa!-m1?}gp?frZd&D&&i9b{s>VNFN18Df zaLx)*GuDDuRT3*!Jw`5!1$J9J#Q{`^nYaw(+D6OV(;zgiaWnn5IqkqILZ&D4JfD>E3=qv#4XNeolj|_pjTH8I7-ptpcY605sbK&JH)=wf27h z8r@(24UT?6}}VguKeuxwPFho6QQ{B3k?)>+e>bd2Be|hIl-8qTr$|3YC2S$G&N~ zWqVNFu_|sBNO~$V3RI=Uu;Y(+uYCm56u-b+xJ1O1aZ9_kaEWj${jR%TeWX0;yl%dy z^TOIoTK8(_eE3zyBVokIwO_ZfL)JAZ`9xt=6lAC7%5*fLc6UFQ8|FPuJZZhY-)|#u zi#F}vT%a^juhVg>y-g0OwoaPTtn$=RV%3S^4y#F^RhwSL)NngTSoF4`Gma$@hd>2f zD}do1^@C3snOD#^wPOqU9!j_Gt*El#7IFsM>C4YT&S9mbeR=N56ELgkJ4U$|RKEj7 z-!rmd-fE>hB>iIsiSh^qc=YCp`_A!o)8>>u>UJz>a9?0Hu&+AHc@&3w1Er^}5{c5< zTz~GgnjZ;uiEgutie*#HS06iFS#>Py1a6IrQqJib%oaz8<=scqa#uBCg}$2&T&b)# zr!4W4&mr%Lh|r5Vpl=_Ifj)9`x2oh$kYt4>_O}y-sW*N4gw3NX9JM^L0iKI6K~@}2 z_8KWa$5tXEe^K7Q%w%Zq(-xOu%^Uba9Wt=$3Z~hYr>XKtI}gHR(ottKMf0Z**e8j* z$H!cpPnRli%(expgT{xCzN7^zOSr>^_b5F zcf3j-a)IU!cPcg&!O^EwM^c=={BPcp-!G%GBYtx*d?9nSfrGut+wshbgA^1R zRAX#xRerQNO*_Ckt2kr*`C{pC2dK-(N8mzOlZ*gjgC<0Tn;i_DZOSSflJ z$!3TN$Kx@D_F-MpG=+FJ#^ z)AWJ<8^6lm3l8gqqE3Z8sywB^Z!2n!UDiBy4MCA;fhPgWeKMdW`pQecbUzf;53Ptd&Sv_>0FWD z)V2!JJdQ*LHKMG5^5bG{12N=@)IXwC*)~*0q&mtU9OKVVQ~_1M9-)R&YGD6S^{@P1 zCxDhTa*2z%q_ZZmTPPXAp;*X^m?1@wf`khblNImO<#XJ7W3)j%Rs*nNa+}~N4(-XV zPY9JCtk2dSPCHsX#nI+Ef0#-8@$j1FU{4W^mnbCPyerHcNq{A@-w&+{=0a;k`cXNtOlo-|O%jk<)Mj)%J|ZpRb51zgVlGH2 zzAYkANDOM0AwYRS`3K{aiQj*8^&^)tcNyw4TjQPYWbMY+Bk_gn-?UqTXxuG$1@&{c zAMc(Ng#kXj9rDMA)*SD9yxwDgaK`rIC8W)PGQv{mi0`H4k9V>6Qk}cWF zLL4LC=BD!oe`x(6g{Baz4{)^#-={ z6pg6L)S&HW^nr=TfFE7CoR;f3BY5@J@jXhkLiDeu(HRA@5macxbo)7>XbTJe7*}MN zIpYiI(z<&*&x9Xksm%tf%j4+VBWm-jMpZV;GDPv?s%WoqWEI4MSp5+K)=4aZGyr|Z zBk}Ty!~=C}9+eM{XxCOKD*J*+u`Z4){?VQ+|4TX*(cd0&)g(c#$@)YKAld0jZcXw+ z8Yj7*Zi{>7i%Tca$N8`H1Gs911KO?C-n|J|(1hNh3XLGF>^2lJ^b{I&=fv zuIdqsm$ux~;SaB_A^u#gina?&T4M`4&!RjfI@N`L-G@6f>mQWR#hNdz+cc%1l_#({ zGHWi_kpA7xaG_ni#LB+83m!`lN8goje9tN1!4)qF2Bd3~V0<4%u`gNk_6OY6bUQwo z^(E?T84yZTTj8(H zn*W26AHLQfE7=s>nXWi;FgJDXC%?jdn0R{d9uvO5dy43;dg6`86zQ5eV}$O;d*77F zS-j+{U3dq%~NGar!EAO1w`ba${gI4>aI4jg_ze^~EPP2=wZ=-+UCL86VJ zU7|N(zO5QPD~?R?^20TBj0arV9jX}C?f&I(ifwzplinFV_tjyO{xy$wyDbHpL*8VC z(E3h@hK@ARj}2RV1^*h>eM?^a2d)+=|Jda$+yr%Y_G*74(q?Z#F1H?O#tvgOO>AhT zH3c(ESb@=tp79Ft>1FZjeb?s6`Ib5en3?g7EIPW>D8Q+efSGsWDsFnz((Z6M&UIVc z4LU!3=yce>CA-~Pq6DCou`tVM0~@Cx3)2ES!HAIqF;HbAQ1BrLvEy1uG&5b8Oic$H zIkQ4a6Jx3$jZ3kluEfD5%SpFkMUII&QK)U6 zcJi8tVNI3?)9d%jMy=^a4Yokbd?_olcQ!Y`Bq5E~c=u6APC1^B!t1UFEO6mT6m!ag zIe&_UIX`$=!yVCFbyYBN}hQspvN~!%Sz{w*fT+Z@sdRe-;BgHMjMeX_|tV zbCo+^fMfIL1_|%S)w_@fN|7qGgkQFkp`b?+)D2xj+;&SVN(mm@4Jab-0S*cuWUf zO`X9jW~Mimb22(Vgk$ixqB3t>I|Y#@d_}>6wXA7H^Vt%4f<_#ska7MZOmVJeaaU=) zsmqAFAmNGbY4AB_!aSz(-H$a6p*(;UFz$@Ybw*201R%Hcal8Grh8SXsGbIqTZ<5j~ ztQ2a$qb#U}kHO*8i~HQywBm!Yn-VLumOGL?sw974MoHDu67zvZ0b4%ADGQAj_*w@1 zBRqQtXb7nzE9f}?fuFf?QqDu`Y(f>6JMZqVVak`% z5;DxRCpoj_nE4LmwW&+H{p;00p5RA-5O_C`ZcCRcSO&Ito%jJf9FNwT1<*iW%}U%q z_*Vdj2aSPCio!`HcXc#%t%81~t-N&1Eax!jD1W#oJ%|TA#9%uTnx~s~CPo0zO zy!*GsQje)cJuTU#&BkkGCHqe$4a2wfX>o;4)}1&F83L(f7KDn}H4#}aU|&mG|DIF) zuGsh@=OKn6AM7%IZ-dlh(wE*5++7>2eaID8F)@_4^KB`B}Mm!%@kPMl@%v)?qH9>$9N8$SYDuR?Nj z{MOB}@Lql=C_82`F17NIr^HIQeI$o(DVzUx5F*c_>=+^pp zqCq2~_-9U0@IBTrX~LxI?ev*M)Ae~&JO`(zma^RJq8809OMQ|ef%&_VR zL=fb85*p_nKled0cw~t5XL_motr=mOpmPkFP!stD5(t33p^8usqzx1lx`-p`>T&y( zGGA+i3if*E+C3+fh*2Tj^P&++jtv;#~JGi4+z56O}0a z*uh68f3l?7Gfn4|?fyw>jQcn7PXCyv!#dQ&=@~8~&T9jb5~TOfk3VXd9D?KbXjPnzqmVfSZQg~(>w4emQwDLG#dUC&3| zIa_}^_$IjyJNljB2G_`6`ivh7QCx|GdD&;X8fq=UW@ap-&dY+AOAjZwfQS3{ zHD-o?jqw9$LxioyQLe+5y8i-aW~cJd5&w#IyPp*S4@qNJ@)`?M`{3G zY?eT`QTU;Y3o7m8T1`vwqZ=8mVRP+XcTyCMiJ2jNE$h(iL=@ho%j2oJh4QcNr``H= znlp7L&pZtIeadtLL(>P}=(tZ@l?=j0)Yk$n7+yG3;cRn3pOtfJoKF>4+%P@;3a%sP z{-Q<5K6N!nnWs3WDIb7{;xEV4agA>uM4vjPJE+M#s8Hc`+c}eO<#$vzSuS2}c6SG> zHY*GVpehcnlzBgLESB=hyK5@sFDw*{Rd)@O`J!b%C#$yxNp~2A#GIOIBVDG_G4?~! z?OYm20`g@B+rNQ}hvaNJIBoE3XkIn1;7)1aohNEk&49kz9u_Ee;6-XVek?0*rX_Pt z*V8i8c(@;SU^T)J{L>nzL@tv@JZ18Mg;pvG-;ff83TtVdH6)Imb|NPhfLC9hA-_zR z7d`2)W7iis@%@!6{T=|mMX4E5V2KJC;$#KXtg)Wd>IX|QObC%Sjs`$z@E@)POS$nR z^9{&1_wmzl|I8P}?hR}#kxvAO#-11 z)g0jj46M9L!q|2;m${69*Ph}aGRPn0Q#D(Q*0z8(DYmd_UHytdi%eig6d6V)t!cx?fn#yYeaIwjA@}=Pmc$yVvc=z6`|_p;%-~0KJ>GB)|sxCh;^zsNDirkuZsm$(OzB-Uh#S9V)tuxGGI74MImUn)QrajNnO?5;y=+ zEP2m18i6x5fBAD<@^j3R_0TkAB7Rrb(qCP3SixfZK{b<}`+?06lOD(}vX$w9(U;MR{4xw$*#2s#ijF*-{JhryM z?pn$Xo?lz>lDH1ZOVyrdVul#=ISR>!Dn{eYb%DO(tOVsYNxz-oqt8M!#nI@B22_7v ztU+W~-{^QjkV+VnCCsse%crK@1*mENwyC^M>uM?18HP-K@8gqb#9l!0-Q}Rsp|os+ z`p24TZSwKiwgdn?Q@~c`fJ#i=XHu^;yTX$nW-~$3&}>?#1N|V_e@9(U|2k<_8zOf* znVNy7oaTO%@mT4G`94g)l2Vo@U9Q0TJF0qi3|)~t_a6>a1sUpLB=ra2*ys*yq#6W$ zi?EE_r;4O5^?rOqOQvcPLLqvIm{+_tlB*=A%SL_~D8WJGU*&aP$T;ZzBLd<4VEQp5VEd zIDK)fMEEd2@fa@;%Wnva)YwwD+C-7-J<1`IE>2}I_X+y?%?=~G9gsjf@Mu?wBxN1E zTy1UQdJ`etJgpHp)~8r2qnnh&T|6hYW6-)T!dr{RaK`tlN$=Y>dSnM!qFG}sq7 z-hIF^gk!lthhnj0@sodmDo!9F&ZjO!7kksM99mGKsg+T1NJwn@sSodUJJ^XI&$AU& z1>}5PFoKsazTMoX^b?3vpS4j7BoEy--dNJez(NX3w2iOGep^;zc-L0th-&gj+uWT8 zQs3rh&AzFBB|Em{u_*IBGrOjF2BlL?Ou|F4cc+N#$S;Ih0NL1dRa-}MaE{ z9!tW-UmjMsZSN<)Y;ZX~lJY)}t8wQnVTMY9O4>=VFi<=(9%&1F5}oFTU>=2-+wC`T z;>PR}H!jS(ZETwgl(;;nuXlq8OAjU2r!1DwNi&MRMh}=;^xws)v+D*Xb*gqQhd?jR zbe1pbPz1_S>ASZeaa?`=UxRfq>O zzc_C=8cbUvaz1X_c&rVR^*kC za#x-*J=6sgRs{j3GKyqb{>HJNi_e%cAWS8!P{}}Pr}4wPnT|_`MH$HeT5^hjzej1G z9B$Smi~4kBYq`5s7Y8pjIBt#nc)q?0*Zxct4go|AIPt9BQr7mP2iz-;2)bJoVpzU0 zOk3vDn);Q&AbQYWs1N=)1eqNm>x`YJy-zTk0`2M*Y(Hf@Wh3XYK3-E<9$rb$&tGAm z|0eemdpc=-vOd54{*;|Q=;S)c`g7(<0Z32jPuaD5dAmh=-3UJxvrpJH;B!tr?c5O* z_M3g0>P-|dzCgqnlh^ubE`@heTq&HpA4)y}@GZdH9@rY(EeJTJdWEG*d|6+t9ROlZ z1}n75wP>h{Jot{+9n@f6NgBi@6M`5D{9Rl{r8~!8gOSha?u7UoMqe=p@FCU-vlFnY z)hBy#8}_IVPw#`A#`dxqz(VWxoD^Seu4vf42ulzRR#D>C_Wa0PsATC-Kp4o7NXcAEJX zPf0cS;aPpt`XS0^m2O1=aX2YI1STd+6WE=)CLKRr$qqrNq+@|;CXj~J$BK(3Bv^nA z&)5yoob4P;y#>PIXBodha)z=HylCLK1k~Ogg=%K?St6Zm845=&mM&UpY@R%A4yADK z%=U@Lc{`=xCE?)yq)O`8nF*L!sT6^9bSf#_y`76|-L7uYYwzt_A;@a7BGcc+9Kb%1 z0dX|KUSv=ok;@UkG?^GfcXc>cJwRqf4gs%j*ILUORYiG^@2H~X~sBP;xswA!N zvmkmR^Cy7SRMqi*KqKei_3?5d%BJ)b8G6DGB$J?cKLB=F zwZu^+n#Puv6Hr6_&lih&Q^5dviTrc*#&O~@7pj!m$JOPhnO zfr4|4)F^hf^zL8o@|2Jv<`y_J;ji-&-s>=9q<;#T;S4WgTL2Pb&?STpG)4;!G^^<#>axpdY5xP7)QpH+O50FOM=hIy_LN4+RXt#YIkZMr0|< z#PYQ2tFZX<>o0_hjcLmE&R1F2hiME27F<(PM#XH&+ZSQ})~a&hyw36U=H#u`@329dZ=mH*i`ai3fZnT zw?NU;?S0hlRyK=+c^`p;yQ|hf1NU4B%_2SPqj=cU&J4JCz`71JrFXCk0Rn0kyx5GmTczn>!G&A&f!zt}ioqp*pRGe;v9l}e zysl@R5FsXOp7^u{nPovH^+AQ`5|rzTAb6z3Lj`;9Om+b& zyw~;9)3{l(>(lY*8bFvQP3M$TSO1(>H~EAZV7pX*P4hL(Pz;M=;|@mn@KTrXint8t zOq!ddW!^;bI+anB~v=K$P?m^GkhzdgoK2tbrdwAdSPSP49|=f$n1_^Ks!Br5)Pv zv+Q^ZHcm7}i+dj+VM#f?7S7x?ck;VTvV05s;}xWtk$M<}6`>DgbUi%_`&nj*T>RnDp@DHOx>InHa(QZsa2*ihA; z>v!5*PYF&fc8JgBL|T240uX+Vr?m8eF*94%IVqFLDYww(;#}*{NPElEpa)0sy{{}j z4K+S|gie#DD$$jaW{;nWH8}3`Jyb_*emM;U#LXkFtW}L}?f4qq0{Um{jn<9}-8M!v zB*I6G>=@_H(H(4els|NV@D#K#d-R*OJrqXxXT}u7n+Z?^3+7$>`1CJ&6APs_4ctBw zN>FlD4>blH!)T0@ldBhjINTG(58du+jI+<^`q;zIZP9QQ+#Co3@+F#u7DBM28#qV7N+xG}_CRJwoijoe4P%eBE&itf1+|u3F2zTNE^s4}E zG~=u8ZRx64#xNA_O+zyiu8YanNmsCUT%D&vB-pkX7k?ewG`wIDWfK4|;N--|{5(%F zb*9d*-fxYhlqH=$D&YqRUr^VlEyn)}vN^f8H`Q?SVAZX2;WJBzB1Lbl)!XA*%i}Y{ z_aQoO;8Vv^Oq~bh!|%IkPg`zkHH=IYJN@y7>%?T`ZU<1)5%-*$SY~?aIVDH2bT#Zx z{B@rrlu%|U%%#yCD}SO+LGk6|Q^+LNMkw3WY|>%62GTlvgG=J~3)O*bXR_qtfc%Qe zebc4KcJNi30kqlL=aiIvz_`e>vELZh*A#z;G7%=|PoHP-vtPL{dhaAOU+3`$%#!)b z$@6&fqz&BBIM(2E%<1J-7`gW2DBYf^Ly}Baz}nigoSnAa5{LYj@S~+h8e9dG`KD2a ze2VW@N4lciT9^4irr~5m6{tolcg->OfbQ1eq~N7M59*Df%|iPmLFSA{hwqz7^DUXp z0131JQG*7<#hzlc<02zJ=UC}cf6wo`e|$?cob`}t_3HpTCwGm>;>1vlYGB;w0SD?( zNn%}#mlEH?P_iWRi7*fPA%`OsLVf=S9STO#&mH^f+p&}8Tud43cFd*_R%2F5wg^omb?!}S7GAa> zr;I-`H#Gdac8UwGOxE&WqnhAm>-ZOKr_GX89>iQN$QwsFpf9vM^_`s`niX+D2#IYA zI86E>Mks;v))N7lOyZg7f$ykqArVt$Cr=4E`AdL4zwTrOTjIypdo8h;&1WJCVlRFB zKI*$ENOE1N;r?-H5*|kx93hrPKb0OIN_i2Nso`bymWC9nu#X*CAWo+>y8a^1VNEQi zUoo`z=<7Du=wpuk)D3N}qsol1XQN3+Nwu8-4EV%78OC6WE&F^^aB_Q_G%lpZ%lpwl zM-xXmxkpp=ixEm%qlzpx*<$Q#cC0GdtlgzWevRbTSi70wtcHuXJA0{E4m3;3+1BN~ zUUr?+g`=sTlP>v^Il~HU$nASZ2(?Y zl1Iwy#Qxqt_y-_CzIj=h3bU=+?{b@uDNi-q8y>xaOV6B%@c^L?;DelHgae>F$Ed;8 zQu96$b3G};u=~twri2=5ibJ z*U&Jko>Q(|rlcQZfwTbSZywrC$;08gEa8eLYVSewGt*(AH`$3OWBq<(4mErqLKQ7s zURv+MUhlb4WK!-Ct18I4zMbUE!e;v$G>KS3I8}1nVfLN9SU24x?GO@v8Y-Z=`0i!5 zy3TL*|Is7Ijr&VuF7dPSjO7R9?`!SvXp3TP(y-<>GOWX_oL&?e`=as2p$O99(*&V1 z(Xd6h!5FaBAMTYL%s=QiL~{j38vV=+=4P4Df4nz2UL?534S(hfq1pJt$NAUu7(e^= zyk2o&&oCZ>1MNsgxYv33z{oT&o}!*Ow$&L5b~I6TbM8K+8;<9WK|>0ftZ8ByePpeY;X?Azki2Z{GxZ$gxD zd~l5X%jdm3=z;j&L##loLP)BZojFrt&v+ex0d#+w*DLV;d8)l;>Z-gzx2L@^t#ryD@>uwKI@%AKv3bM51$cY24bCx3l?30GL&Gp%qFJ4ZHSv$Har(qRzUE#Cr< z%+lfpE=fQ+^Xy375T)xi7b-D{Xy=%q#VTH+gZtKf7JKQ8IS+OUpyEIUqDs~=uTQmt za0jqSL<1@d$OImLE`sqO`h8ptZfDuKn5ce0y@M{Vri_+-;@FMj+nw|AvfcBT$7y|S z;j)X1S-kyftILOc_M+69*4-qPB+YG(?a85@o5P;DD9*#$6g?poj_JHwE%VCe)W`H% z*rCT$OFqW{O5vSbC4|G%{VCxp61^R+97pvLQZcd-$d2f;xr;Zio1kEBhX(=F-l|-- zwQs*^s!iPJk;E#-#+r=#g)9Nb1b{%zAwKo`g>z3PNmWZrkoQ&Usp zOPq=9zZI#Cp^1SA+ zI3gnJKJ^W#Xo@ll3##8gR6w7Yy1Y=obH_a04eAdIkn$&7fcGx|0%vrVXvlbTguDWX zU~H3nW%|>&)35I^?tz`fW3kWgcQE?ocRPNh;6*FAlx*h*V9uQnb8L2+h8Tt|+?Glk z%umG{Z(Qa8s%{fY>YsL&-lFNDKBoes`DMbrTr?xNdLzd}@*5uTv}{uNreW91uny!y zknn9{x+4(GZ;tB8X8vKKUW0m~zf_r(9OT{~MB56e@3ePY-i>PK=P3y4I?KC{#BMlI zj=7S@^JT7h5b4%CDGBaK*{KJi;6jl_RsKWGo+k@v^N?>Ae^HuF-A zv3CDEQ)eY2%%tYZT>$_>t7y($psG2|RSqdXFe$0}`J;5Xj*&|J4NjIU4l=M6J_-eA zF#v4|fiDVRNNu=Ul|)g-EFM~$Z!QK-K->S{g@4f6;5NuAQS~lKcXK$D0_cp+tZ;Sw z{v7BQE&r{h3DL-6uv!r@ik6_h4Ly+rTyZKcuwg)w4xkmqc?D0B!~_7kCI|$;j{!Bk z9bK;HIphyAgc5XFMn1g(Lf(%(!k1QPtkLw~)wAvlK@Y6AfimZyPgIsZ@TdgI|CaFWKO8nbx@~7Qa*IHEO_NUfne)O1}hBLv(jcvkK z_oKmyh$M=w)t14ou85>iH0f~51bz%IV}PH;m+%=E*BgxRh@m7&!{`7J3$eygzW^?3 z5xf|kPv*Ot@+H$P@(164h~T66m#=&8J2yP8?a;q_SKQFhP;_?}%9B;v|8wvs6V-5r ze!2kcQ@6v?(?zj9b1O%k4kzUHRNsp?HP~1&Oi!YCGPFn*#U9Br_MGH)U)XGs2rtRA>;2sJgIUem_}i^Zw_waR6hn_Cxk!)Ijhz)*H0{Yi9JX z*(-KacRd4KcEmTkb&ggLoBhwiR%wkt%wCtN6A2=^5V60962E!lM~i?cs%|Sz2*&J2 zUI|b@@QE&r_s`dyc8jgBQDA`Wh2{^??px+7>D2q z>gm*l+1z&12Y=e3 ziA?*KO%rW7*B^r z0IHuq(BAL?efTyVzWXITwe>OO>4Vn`|B`>*$@jj$^G4myinm&@kQ$u~L2E-xMnEVG z|F3ltmk-cd2R;Isup3n9ZA4cl?EfACA133eP}>K!<5=5?AF}PQfzka8TpRvr8yfX# z`!=HL(>975P1FxG1fM-%c+92J3fbt>5t`C%=rNP`FpCi|FhoHhMMd%EeZ@MI6F5vYF_$m#I+bj(fR4N)P}6a(J_9R(9Qof zI8W=;EDhdU1fMmyEe`nJrz3p+qFQq9T9!2l+#(BZ6Eae6uEE%xZf1v}v40a_Vp-Qg zmFLCF6rJA&&lR7!#+8-c-X!>OF5~jfOa5wFFydi1iukgZ`L|R zg^w{dur&QY&|!W1=NBB|HiwUq!{{shzG5Tzj|d2R(xV!G45mUaln|79k*m1=ThS31 z(7pe<>Prb>v|`(nm}1czm}G}fmC@V(d<7>vqzG`X@cW=d0=8-|1nnoo4GO{jZ4fLoxBv>^WZ*_sw~hD-GXM9A5aItGhC&3MVn?i-j*u2&_{M0riH(4;@fLo6 z$852jx_np=@M6mU@7mXC2%mjp!r1f^;TG)%x(Qrq$UK3POQ{|`ri`HT3>Z`&pj*Dk zijrj;fG_AVJ9g?|yjT*Yz>dfN|h%g3&6|F?%n`W!|T$!FV%) z8|W>U^nblw%7pd)b6f~efB4O(L;s2^o?7{SH>cqQ+_&I=W-6wGFxvfZW^chQK;f7z zWz$owNCP;clm8pJ^i0@%jt8)y!l~($Qh!B8Dy2N#v!`%-!vwjHpX(5FKLp(}H#6&< zR+Q+s{$6tB_paE^k^(F)ve#AD0t3U{4_niJzPXqaQp5!xcc19BPHf>S(go-FKfg)2 zR)0Tpx(g@0tTbJ8>YPY9ye+a+pR7p0b}Zh$9>3V1>sG0H{Mux@hJUxaVb%CBM|xRY z&wt20axqu2Zo7A;oPH-c zIz4o;-0fCbI`#b2yW31nyPT|Ou5r8~zI=pnZ0ffHOW?JZa$2EV{`|ksz=_0vHc7~2 zV96YR=enEY{!VxInUB-uTtUXXFSk&YbOI@=Z{vmPR^1*2^|A8?1 zaTU4pVNHt3`Rou!?LzoekHt(V_QB*X_CTx0K78V(^Oev4`*NH}{`VlrXq+?LsBJdR zmj^vHPhGN&S98ox+Xs4$#vS=MDxU@1FcHf=!1Lsjy{FWUH}>@NKi_2b-~Xq!Z;xlPjsNHAtf$ly=^=D_NJ0lBricrbnp`4AGS!J^vwuR&rV`C083pvb;9LHw!yL-OB-#@>?R;e)2s>%mh+B+!rQCpv) zE?L-RSpe$SS{O<7dPvLu1)i6k^5DR52gae&bm$g9Gmoa`@49rU+xgb(d<^|RPvu!v zSEq@7$SAlD)vg)dK?oGB9-nZ~*E)Pg;|7=rB`KSRwhF^lKiINXpP~AYlAq7M)BYsH zzrqa1{(N(R?YKtl!Cvtz_V`nt?s+!wJpvo5jXz8sy~TiSa{wc`Ta?8|UAEAwQvP}A zo!t%X*YwV&%d2hV){ng+)1$r^GDC;{<-FJ0T;yYb$w!HF#$ZsiTJr#dvJ$)Ai@!qN zFfRfNz4cOe?=d?{1l_fA(BIz1{cftU9_O2h9_^dy`eEE2)^U3psYOle!>*Fm$QqOL zh!@HRp^cJXZL00mNZxOScCGvEf7bocs6YEzV2+{hzb3{oGW%>*3q>^jZ6ET_Z(0v; z=M=dP{v*4bDs(vXMxR|M*tF);cl1A989QRB_0*^&yz_HPNCO>lUY|wHWcHcLnQD2~ zY1V@Vj^}9m3071+Tz7_D*2HJlyf@6L###;GbAkSvkS}hvA<^cDR+@VU8#75m&yra~ zrrr%}-`8 z>ks~s=HNAKno%vONi=NqQ`vST`}j0-kR0AJ{9X*V^TO~69(nBj!aMBAk{$!al_C$S z@l*QAf({G(uA*ngHtmWv#RA=SD-xrmKxz^SR9N)A6=AS z7|g*zIAcF#2T zD^_)BV{R7)B{Ll!g$`bU@yi#K8lAq&9TE34Ji7EcuNOx>4?LZqLd~XjSZJB}9q6ar zud1%UM+>9Em#q0Um83=<_kGZS6U=VlO2^&-ovGFM^`{an!FHSarn{kq?K?!gS|{=W zP8>V-ZQlu~9lB!NDarEUo_;jvXz$t73fZvl_SRG2+w3h#s{?5|ssz29)qxlNoZV@4 zq}|CUn6cJ<&bzb6cHwwX^OVbhl1}=*!x~cHjj`N)QY*_82T1hM_-p<$@0R~YxHi0A zgKgIDuuiofISjdu6QvysyAw;%{lo&`)XdR7bE#(+;?QbFVL}3YZuXO#VZ`9q^aMUy zgFSkdnrSlJ(=D1wfbG8WwduivL^D@r@>6!k`m4nXzHT8@R9h?TJPYIRun&|m|An7i z9V*)v6EKuygo90;H3}<7s4RE{br&BokB=u6n!6Zv?9Qg|`U3dB9n$&A;<@kcJXTGM zYsE+aEhxG#Fz2I*@nf5ituE1TZZUCQriNB|A_!(X{VOvCnvOtJ zb^ja!Niz9#HxoDfp?}0Q>YJ%(nX!nH=3gm#(K|KMXb16h)bcBj&`l#9Lr^SKHKrYY zjLYqfdhpiq^V{u~LXY`%&JfxutJvYXOhz$NT=QG}Nd*a3zqR7k?%u76ZG|4_)p=Hs zXAFe2(6O4Z-=?-4J<3unH_m!|jf@+#_j=*+x&7a2$KYu07Hm`N(LFq-(@p0eGcsit z$jmcQ@^8fUpWT&BbBL+YIY>>oThp&AUv!BbfjNUd)Z9A8ecUiCW-k?13Nmn#tgtL` zm($EjB2@}Mb9bF4{`ohsyIGu?yghQSa{z95aWd-Yfu^t0gd$**Sj(6xenfx==V%Ub zZXimU(I4`Mn#o1}rv$^sZT|+Pcq$B~mzdr{5KK46HQ1}X^*Vlmg6y+L+}G(X0UPZ_ zCDhb+9P4?LkHV+6Bfn~G*=rUk)~{!_8bm9e7pbMc-d~m)Ixy!>pdruJ^;hN<-+*4< znbB*{i^~lpG3u`bca{h(E;})%yc37|J33+S_etJvu4Js#JSKc#JKC0TET$%fRv!Fn z--rQEz)35*9zH1KtIz(#w1=G0`20fW4Dn7<5XTWKXW#-X7I-gXEa9@iwYriBAmw=RCZ( zkiD9w;lT%~@TuA8JL*`eLo@o$_FlWwOP)-aY~!&0iIe+=uenA3M|y(K zgJN$_Ut= z@`1=*aiCaSY@v+B_tw^m`IZOj%yyH$*v?hw+3|LzUDv!T>|BZwvf{|AFe05(JHbzJ zN32%Sqh(%(-nJWc9VO{&5n0F->m6x%PeMlcU?NDHi-BHMkV=dFLHzk`s8l@7elCD&+y3@j4+N z`iehDrsaB}z5t8R?GUsmYNzh{Pj5NdWWQ6@+2ZB&ka#x>kz0K51iEDdvT-TFbJ5#! z=M12syo}4KH#&yg$hIwojo>1*?gpgk*ZeyMj8oP`X%CjdR_=0kD3C|-z6jAxNqKIl9FI(pt+8(o>n*89$fppO6{quP1p2*RMlYG5qsyQ}bA5d=ZJ3b9);mrGn-{kzFa@{bwSe)x3Mh$zV4V7uWCgfZg211 zvkzrLwn$5yIW_B}qe8f+TnG1^*A^AW=bdm%ItGCF=6kOT#`!1qWRGb^ZBAusc8TTy zA7-et;EE;O!7`5xx|hy2zK8WRp)j2^G`$R^c{6G zb}YAi!&J3^f6)B40+P3LlJQe2(1t$9%hk8te*;t5)2y$xT6m?qD1TmF?StgygvoH!AcJ9Q1*v@`MDUU z_JZVKuj#ny_tZggBl5X3^yzC*m>sDB8u|uY^6DC|xnpK9&$;|S7)30axpM+woR0eW ze;_JV558T8%~GH+euks3M73nhiu9n*=`^O*(V&D@YHL0`Q8H$G^o`j*_P3IPNrCo= zZ;{sksxle;3G}BWwTdw7%ZQmxun(%ns)s()Oe$#Aln>ALxg5`i)0P!(ozF;~Eyb5{ z1&)tn!!TzUxM<09&e`j$gS$N-B%D#oz`{5_1GU0L@1;0Cmd75{-Lu>d(R!RjQ_V+H zu?{^ydD#5K-6j61QW;=D=5|}?We-}!_D2Ow2E&)8#qyK2bH7b}$A)oYY>t04I+Zx&KHo{LKo) z=JM-)O|WHHN+Y^LXi=1YXJyu2GLg60X2A}zU++^G12%?|tSL+-0R-FN&TPd?O9}z3 zk2Ez}bdO3l8%pY&{s>cHT;((;R)yf4??T^jJ5*=#`hTD?VnHWIKcWcgL9Gn_k$1}x!|SpWJq+YL{#G$ zzO)R#9eCVC+rSu6;{3=jQdlAzI`Da_*6gF6V;-dt-FVt~EMhnsK!3KYX>IJI`ASrR z^MgQxKuu;KNxUP4z&?&7GiaKh0ZqDqyS+jvEXodbv2A|Scr5;^kToUW`mleC=9S=| zZoFkf?P|HxQLiW5X60;ZzykH)Zn-?pXMjiWQ&5K>7ze3@hHu|*%Butfv(s{}z|Y7v z*jMFPoY1)HC?2=H{3Z~a)X#{i9Y71j2Mx(g(L(eU_HqpLYL<3rc^8k$M%A(Fyh$dk z;O|U0C{Pu-R=21Fip2y~hCjAF6|ZiGh;mi+BFJCd-ltP;J68Hd!vOe-rMW*9CXdol zsVSatU$Mc7Lfi{(-BLm>Ny4TiYIpy*)U7M!OGDnR6CAGgMe7hsy+(YgJMC`|$~{B6 z)!NP_VWYil9^$;OEN%KZUYFl?!JZSiS>=sSj#~ea>=kv|c!We%mbLeEc%U05h5=ru zOj-ut6s|KbI$}KJwyDiCQZ_5~B~kuX*=;-%FJ9$@h#HS0_0CN_^A+ZOE9@~mb;9M< z$)Eo5rT9Nx7=3Qyz0ry(DhpWF&%pOJO)8$@`OaC�gj}e|q6U)7QDP^80H2neAFI zg01ThXTWz03q)4{`t#^E6J)xb@y_zlbX&KTv9XiDP^grcXE6I7!tuFctHZw;!1~`q zv7^OWJoIk811|ieQn$Iz4rix%H2V3jBs7_LTj%(zZnLZ%$VeGxp4(=v$S;1?7F5dE zSsK~PxS=nGlxwcs1$8zpkGF|K&Lo3Bw4m!B4jjbI5cx){gFE1B&chAGwNo?L%2U*K z*C4UeUEMS%c;N@ja@+5GV$zIiUXBL26%8+5NgY{wfXNWo=peY)SD86M8D8%}@Gs^O z+8_ZLAkubTc2c&>GlhG48o|4kG~r#s@$jYIKoMm9EY)2)4Z=etKX80iz+#4$aAy{j znw}TV+fSoTCFN$LJ%;DJ>NzUW3$ILW`TIaliR)%xnS7qjPezVDV!MIE=L-?xn{N=K zCMJK~OkM7y%4RTC$;%(a)r1YVr3jYR`RW7~#rZ)jo*=vJx?AQdw#`^a2P6bh!0tST zn_?ef^Fd0x)?ufb_$FkbVof*HOE8aOp>Dk`^vgK@RhaE(AjWH)3yg)|%2K zqn=yFL1b}d=f8Dd4~D4_>!w!X_uGgMAn<#|fv%{FZ*m&dx7n6IF@8W7)rJM;+ZQVN zIVl%CvQoIX;_Q=rb90aos6>8{YO=6++2g~V9RA4W%=1Id&Z#aPnJh2Fgyw|Xpj=rm zcndcUqUD@+a?L?pp!m)5PSTFc3o+#`u?RgeA5d)axm9?!R7FEnz#3J1l z&H-a&5QkP&zWA`089qlCxh|oka~LW}D@! znlSExazAN3HKS&+VG-ZCRYvy(gZx_0%4IOAlYK74^3aRoGuEu>JP+A^&OPMv5dh~bVqLdD`{LAeQpT|UGOv~)(f7Kbo?89sCOY+t zmF?W>i>lHt{|!KWw@~CozN1Ns4K9(D0(Z6v?1nNbA*|0+@H!vYphM<3MYH2MyRuGQ z=raP?hPE%`9uvzE%qFMop8AxYyS&=xW!a95JAUsre{XtRo)T&327PY++1xWSl&TN_Jd3FSRb7YIYk3><4?3cJA&?e8g@g!sUR=UQy zV=|Yp!XHm$XMc?zU)?B7U!6|cF*^x0IqFTnI=sUC^Ed{2A}w_$#S-DBC~Z1m8}?ES zDo}3|hYso2h1lUu&QSXgRt);iBYuf)8?_0%yARuP{oY&m_G4-GmIhqnh|W26r#gqu zB?Z)i!W3IS6ZqyxO-dsUeHbNK%4^heqHazfkGaiOQL4Hdbm&b*k5!a(8u(Db`C8*0 zj3+RPUrFPoqY2Lpyw0LQv2*k)4+?si_bhLoU>6=zrcXt6v!u??hHP`th?_E2O= zn4W@DeZ`2lT$+v@Q>|c_vf#0r(k<;c5|4hCQa7eG>^EFzGf~Y<^0jB0WeJN<&xSf*!j zw$DC9587)ta$wq`utU3MEiT$JY(idu+e2i#6fprC1W*TWZ?~kd^@Aq%)}g=&Y_xit z?KGi7Wp?M4^eVHLqXPiyjTRLM)1GJX92$7qWX#QeS=1SVsLo$<;h8K_ z_@D=^T^nif+9g{fUHmY{(giaRS2WB(tNPb#jcDh36;Y?YM>jNW1E` z>KrS5341YSwF1sHxu13rPCUC?G)n=_1-}p?noYS#4xg%G@l*a~@mK#gA?QPe_zG&Q~r}gd35n7(ZboKb5*D@|ng}BO- zvvzs^JnEPDE|PESG4!#?!q!vSa2ci+u$T0KYfZnAe8#^Sz0dmbo4`z1U&{v4XZe(f z`8n)%rRE+TDs-m$*Vc?FBl`g%+%mw80}-(YQ=joqqN7h5uV@s9-t zzue(calR#~imR_LejlNf0}AW1JhRSl-yl|xrlE|;Q>=|-HXdqKpGgUN%!pCZ$8fnJrbi=B;Kw3OsF_}wi$ zfrZgUUEY}$3Nn^?$(025VnVxD}iQx{Q z>Z{vrv(E%G(8|>ScNflSWRLcIh?d!XJS&mt8vQ+JXO_y`2W(5w2UXJ&^$*Lx`P0F# z@(BU$A@~7F6Mj%VZ=+zNqexNy5b?W7agS}P&ai@+dXQp+?%^$JQ|5n5Nga}e-nJ^r zVm!M4SvPsev243kvTk9Vnu&QJ109${@~*Lsoycb!hk5KgxqKoOp9~IbaN_uB4O@`! zWSaJFZq4B3jvNkrT^ACLH~HX~+IcJNr14nhE(&a_C@k@%zv7EfWMYaC2zvvGB7Rr5 z+A21LPIvG3sJ`>W783SV5)+R16!hM&dpK)YcW2m8#2g?J^kHtvoPEGM?}1a<7qJ(H zg=^Dep4H*3kNeIQAZu$&EK}_k&nfSKj|?^G-@lA!pD%4e8r88R zrDk<8uBhu?Ejao)6m=vckmkXx9Z9Sxa)q0CLEvPtM=hIh4`HDdL8kLK8AwZUhM;Tk zzEj3LX-+&pBr2u7MFs!)Hcl;2)+n!QhrPG-?=Hk1KjtT#W!Uq-C-~0xiq!c(V+f;6 z)3qN2@n{-oAa8S{bWtbev18WQ?E?+gsMcbS$*dzlqJ@;yVsdNS7is6xM^>anjR%UD zw5$HwY6z4C5Mq}4FS;-F5Za}dJn1l9Tw3{Zs|-^&(ds}EPqgLtpA(Pz{c$7RV(H|A zbu2#ipQk!9@8!NulSk>gZ6r#jnI6KP+=6@f`2f5Q!7b+jiTw?gB!m z@t@z%%2WYk@n2Dl%ir}sS9^fC@_$#4o*nV~JxV}ht^ncL|E}_n0@3*YO7}h>=>1>u z+X+MO!M+%ggCI@H6I2u&c{N6I@A;~cW5 zL{oc{^uUH|-s;Lo)cfbr>Xq)TC*)q0?d+zm(ECoon|8me|L+aWg_k&1EkIORN-Dd= zMYK|p^a}SJFlfgCA%>NgME>p9Q$+iTsB|#DjN@q7pMQb|)1dVfiArnG7O6{`iZiCT z3^9xiP9@QW(bWCo=%SPsJxxEw)`)|Epp!2F)rQ(r6CbaQAj!;EzF{LXk>s1*)adu& zGhnw4-A`7X{Q_E9O@+8^@t)2Q6}f^{h}DOohzGYJZH7uM9HAhT_`)V{*p%)Q%oX(e zN*3lfN8<|HsANn+K}U?E6>ntqI!Z3Wo8(OnE^wNtSd={u%bzBKOE?ZJ+e{!6 z>CchUa*&HtUZ?;+e@e$BPpGHWA3f54uWyfg*rj?z-;qKTB8FnaDW`w%EE{@Wm>VDe za00ibUME@PigSXx4|_#}6z82-$*`BpH^-5;!~q-b981NyvLJiVcvuqN3pZK*6;$-$ zpAkxsB`cL$^))PMz9rQX)xzTuRqOCrAD8YTL1?8T1zB>>zfkNK1HLdqp{`1`HB-s> zar{46bg$_~B%Sk@6u_@bzaTtW`NO(%_m)yBc|4gT*b8}I^Of~{w`|)`UeI}RqrAd> z3fzk1kgV{xKR4Y_=Ux4yV92YoFVCr0V)u^uEFz|FeXZYT*OdSH2S!dV>c@EeE%2;g zp8J%x_Gbbd-XeIt!dko7Jw@m2mU?vm7sV6dFl_JJr!o@66@ON7P@NP94y(3=T^h#o zIG;f&B6qDN$%VPn6$&0NZc*?()u$C~yFW>w+OHeUAsv|a*bs3;{!&eqXgBh<@{d=l zhK?ZI>%BlIFh~MH)*F5iHB^{_up`e!Jr!cn>KGuJfD+^_R=05B zA;-cMUM%L_S`$L4kf)~Dm8zMipS2$dwWDb?R6C92uQDb2HPpUdJthzvi+soUk;-LD z#hd!eB(SSVCee7Fl(}|+xxVC6w(Kc=WcTF*udXGP`ER>kODS%?I->u{G|xQGso|A; zfIR!z(y`;(*>|`LqE(mSX#@J??HjRuT0yFiN8GA0UiovY%%}!#vlmm4$_)Xl1RWQf zevCxusI8c6R7pwcdJvI@2NQOs3o8l0StQ7jPoYL{`sTmYTa*!u(P0SH1qoS|CiBU$ zFGW2m2!yq}MzgF(DB~feolW@$?=S>eyvbNdd6>zy@8C(%>I5>toi3gfQ^uyeI1M1B zc~z-Iz<~TU(hyx*9(Nhvw;XZG(~E9r5~Spwr{GDZ$JEj%SVj#AgQDqj=;Bq_GPW6}4EMmn++MoFewuGPO@% z9lcx-#CdYD55hf%!rgN$mR#5Ao&Xxpdg)OTCY}EmU>IJ95B14pgd(hxUev?yB1WDl z1P$dT%ILJxSE{OCmx5ovR-sZ|9~DuTvO8FohSpX*Kh{{sMqSyj<$Hx;<31g4D$@l2 zu9148AurFD?~*#>yeRB;ejP>*S_xY;O(UJgj{HIr=#nb`Bl9qF6rzBSF36e=B4_rc zmVR1wvmIGwEREvDs4fz`S`0z}!Y zM@U^t1k7Xc`*331KY*<791Bw7PLZGA=q|W(+!LgA=mhW3qO;~X`Z#UKS1(HRYpLKK za>7nJdkv$CPMkR5Uc`pQBaMfV%pPPmps)8HjuobbeUEmQa=gbT$LLqjd>VtsRH#B8g z7r{(kE+7X*r)AY0gYz__5TU=De|*leYIVM4hP8w=hKa@ysQO_YDVOpPLS=^L&bFC; z>;lcVZ;=jcjbq!XGfJ0(G%L@dIcxsKN^$G?)8+r%2#sz%`Kugf(!oIVb+!E zzGL=vYlRP?8;}FdQx2R4Qt9MW&-zDSam1Syj`_&EW;)Ub zR9zwd-;+x@;MS#*=BIbVTHUuLQA<+{wQYbQlFmo}aT)LOHZm)%ugCX)`o~;|xm||# z>4E9#yrW-IGYuMofMdVKr%{}~<3}(hd1)0sJrRPC19@I;LQ`$iuC?=H94+1<0D;&n z#HWAete9%;mF-2Xi)MEBsr!2(?B+3CVh481V#d8brC?AwdVJAm1$ohgxfX+=^_#jZ zO^ZM4q(=D_h7nMRE{;=@Xl?D*^gID&zb)pG`)M=u^qhP1AX<>agIL9G6CMb%kyxYgM zcRVBC9ufEB^gF0W`f3_dwM~Q=KSreoOG%x%>1-6~y^mBuvBQshx8(@GR7i$1Oa4zk zDd%urbW_MO`IGPg+>NyaFDAxkl(PE6>GN)w$#Nj@TNY{>`Yzbur>)5L$*~?GMU_an z&!}0t=g#q%_Z6!-a;a$*!qf8td5&9j>==hc_m%zvq?rEg{O8wYLxfHSv4Sp?Np#-# z^%M?!=aLrr;oC6d^lAqy;>uD6Z%sV#=?`>Pbs7m$BAyoex12P*fIJQ8Qx~|GdCC^4 zM-7hGsn`ihAUNpj3|~OM7tL>YrlT2M5rv2xIlJad7oJRa32|}kh)99D_l)mXwl7&B z;H*XMM%&$%scHVKc8}bMmr_6e2U+x*7hF*U(1ykDEM%aB0YFEp_DYE|L4a4PnaV8T>`T3LH%`6d`= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: + self.now_epoch - start_epoch) % freq == 0: target_sparsity = self.compute_target_sparsity(config) threshold = tf.contrib.distributions.percentile(weight, target_sparsity * 100) # stop gradient in case gradient change the mask @@ -92,5 +89,5 @@ def compute_target_sparsity(self, config): def update_epoch(self, epoch, sess): sess.run(self.assign_handler) sess.run(tf.assign(self.now_epoch, int(epoch))) - for k in self.if_init_list: + for k in self.if_init_list.keys(): self.if_init_list[k] = True diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 1ac951c8f9..c1f803b543 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -2,7 +2,7 @@ import torch from .compressor import Pruner -__all__ = ['LevelPruner', 'AGP_Pruner'] +__all__ = ['LevelPruner', 'AGP_Pruner', 'FilterPruner', 'SlimPruner'] logger = logging.getLogger('torch pruner') @@ -63,7 +63,7 @@ def calc_mask(self, weight, config, op_name, **kwargs): start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: + self.now_epoch - start_epoch) % freq == 0: mask = self.mask_list.get(op_name, torch.ones(weight.shape).type_as(weight)) target_sparsity = self.compute_target_sparsity(config) k = int(weight.numel() * target_sparsity) @@ -102,5 +102,90 @@ def compute_target_sparsity(self, config): def update_epoch(self, epoch): if epoch > 0: self.now_epoch = epoch - for k in self.if_init_list: + for k in self.if_init_list.keys(): self.if_init_list[k] = True + + +class FilterPruner(Pruner): + """A structured pruning algorithm that prunes the filters of smallest magnitude + weights sum in the convolution layers to achieve a preset level of network sparsity. + + Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, + "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR + https://arxiv.org/abs/1608.08710 + """ + + def __init__(self, config_list): + """ + config_list: supported keys: + - sparsity + """ + super().__init__(config_list) + self.mask_list = {} + self.if_init_list = {} + + def calc_mask(self, weight, config, op_name, op_type, **kwargs): + assert op_type is 'Conv2d', 'FilterPruner only supports 2d convolution layer pruning' + if self.if_init_list.get(op_name, True): + kernels = weight.shape[0] + w_abs = weight.abs() + k = int(kernels * config['sparsity']) + if k == 0: + return torch.ones(weight.shape).type_as(weight) + w_abs_structured = w_abs.view(kernels, -1).sum(dim=1) + threshold = torch.topk(w_abs_structured.view(-1), k, largest=False).values.max() + mask = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight) + self.mask_list.update({op_name: mask}) + self.if_init_list.update({op_name: False}) + else: + mask = self.mask_list[op_name] + return mask + + +class SlimPruner(Pruner): + """A structured pruning algorithm that prunes channels by pruning the weights of BN layers + + Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang + "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV + https://arxiv.org/pdf/1708.06519.pdf + """ + + def __init__(self, config_list): + """ + config_list: supported keys: + - sparsity + """ + super().__init__(config_list) + self.mask_list = {} + self.if_init_list = {} + + def bind_model(self, model): + weight_list = [] + config = self._config_list[0] + op_types = config.get('op_types') + op_names = config.get('op_names') + if op_types is not None: + assert op_types is 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + for name, m in model.named_modules(): + if type(m).__name__ is 'BatchNorm2d': + weight_list.append(m.weight.data.clone()) + else: + for name, m in model.named_modules(): + if name in op_names: + assert type( + m).__name__ is 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + weight_list.append(m.weight.data.clone()) + all_bn_weights = torch.cat(weight_list) + k = int(all_bn_weights.shape[0] * config['sparsity']) + self.global_threshold = torch.topk(all_bn_weights.view(-1), k, largest=False).values.max() + + def calc_mask(self, weight, config, op_name, op_type, **kwargs): + assert op_type is 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + if self.if_init_list.get(op_name, True): + w_abs = weight.abs() + mask = torch.gt(w_abs, self.global_threshold).type_as(weight) + self.mask_list.update({op_name: mask}) + self.if_init_list.update({op_name: False}) + else: + mask = self.mask_list[op_name] + return mask From fd9561ef691119a38e2f9dccd64cf80b9dcc441b Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 31 Oct 2019 12:03:28 +0800 Subject: [PATCH 02/20] fix --- .../compression/tensorflow/builtin_pruners.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py b/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py index d8e895b3ef..20e57d8260 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py @@ -8,27 +8,29 @@ class LevelPruner(Pruner): - """Prune to an exact pruning level specification - """ - def __init__(self, config_list): """ config_list: supported keys: - sparsity """ super().__init__(config_list) - self.now_epoch = tf.Variable(0) + self.mask_list = {} + self.if_init_list = {} def calc_mask(self, weight, config, op_name, **kwargs): - threshold = tf.contrib.distributions.percentile(tf.abs(weight), config['sparsity'] * 100) - mask = tf.cast(tf.math.greater(tf.abs(weight), threshold), weight.dtype) + if self.if_init_list.get(op_name, True): + threshold = tf.contrib.distributions.percentile(tf.abs(weight), config['sparsity'] * 100) + mask = tf.cast(tf.math.greater(tf.abs(weight), threshold), weight.dtype) + self.mask_list.update({op_name: mask}) + self.if_init_list.update({op_name: False}) + else: + mask = self.mask_list[op_name] return mask class AGP_Pruner(Pruner): """An automated gradual pruning algorithm that prunes the smallest magnitude weights to achieve a preset level of network sparsity. - Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the efficacy of pruning for model compression", 2017 NIPS Workshop on Machine Learning of Phones and other Consumer Devices, @@ -54,7 +56,7 @@ def calc_mask(self, weight, config, op_name, **kwargs): start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: + self.now_epoch - start_epoch) % freq == 0: target_sparsity = self.compute_target_sparsity(config) threshold = tf.contrib.distributions.percentile(weight, target_sparsity * 100) # stop gradient in case gradient change the mask @@ -89,5 +91,5 @@ def compute_target_sparsity(self, config): def update_epoch(self, epoch, sess): sess.run(self.assign_handler) sess.run(tf.assign(self.now_epoch, int(epoch))) - for k in self.if_init_list.keys(): + for k in self.if_init_list: self.if_init_list[k] = True From 42090ed00fdfd710dc889d9bc3b07671253d1bba Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 31 Oct 2019 12:22:33 +0800 Subject: [PATCH 03/20] fix --- src/sdk/pynni/nni/compression/torch/builtin_pruners.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index c1f803b543..f4a125499f 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -63,7 +63,7 @@ def calc_mask(self, weight, config, op_name, **kwargs): start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: + self.now_epoch - start_epoch) % freq == 0: mask = self.mask_list.get(op_name, torch.ones(weight.shape).type_as(weight)) target_sparsity = self.compute_target_sparsity(config) k = int(weight.numel() * target_sparsity) @@ -125,7 +125,7 @@ def __init__(self, config_list): self.if_init_list = {} def calc_mask(self, weight, config, op_name, op_type, **kwargs): - assert op_type is 'Conv2d', 'FilterPruner only supports 2d convolution layer pruning' + assert op_type == 'Conv2d', 'FilterPruner only supports 2d convolution layer pruning' if self.if_init_list.get(op_name, True): kernels = weight.shape[0] w_abs = weight.abs() @@ -165,7 +165,7 @@ def bind_model(self, model): op_types = config.get('op_types') op_names = config.get('op_names') if op_types is not None: - assert op_types is 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + assert op_types == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' for name, m in model.named_modules(): if type(m).__name__ is 'BatchNorm2d': weight_list.append(m.weight.data.clone()) @@ -180,7 +180,7 @@ def bind_model(self, model): self.global_threshold = torch.topk(all_bn_weights.view(-1), k, largest=False).values.max() def calc_mask(self, weight, config, op_name, op_type, **kwargs): - assert op_type is 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + assert op_type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' if self.if_init_list.get(op_name, True): w_abs = weight.abs() mask = torch.gt(w_abs, self.global_threshold).type_as(weight) From e11ab72b892474eee95a9f71361d48e160476729 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 31 Oct 2019 12:54:34 +0800 Subject: [PATCH 04/20] fix string compare --- src/sdk/pynni/nni/compression/torch/builtin_pruners.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index f4a125499f..261d50c417 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -167,13 +167,13 @@ def bind_model(self, model): if op_types is not None: assert op_types == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' for name, m in model.named_modules(): - if type(m).__name__ is 'BatchNorm2d': + if type(m).__name__ == 'BatchNorm2d': weight_list.append(m.weight.data.clone()) else: for name, m in model.named_modules(): if name in op_names: assert type( - m).__name__ is 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + m).__name__ == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' weight_list.append(m.weight.data.clone()) all_bn_weights = torch.cat(weight_list) k = int(all_bn_weights.shape[0] * config['sparsity']) From 4663bbf45aeadf2f250f8e3c2beb5e4cbf424963 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 31 Oct 2019 14:33:39 +0800 Subject: [PATCH 05/20] add docstring --- .../nni/compression/torch/builtin_pruners.py | 204 ++++++++++++++++-- 1 file changed, 184 insertions(+), 20 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 261d50c417..b25dfb6740 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -9,19 +9,52 @@ class LevelPruner(Pruner): """Prune to an exact pruning level specification + + Attributes + ---------- + mask_list : dict + Dictionary for saving masks. + """ def __init__(self, config_list): - """ + """Initiate `LevelPruner` from `config_list` config_list: supported keys: - sparsity + + Parameters + ---------- + config_list : list + List on pruning configs + """ + super().__init__(config_list) self.mask_list = {} - self.if_init_list = {} + self._if_init_list = {} def calc_mask(self, weight, config, op_name, **kwargs): - if self.if_init_list.get(op_name, True): + """Calculate the mask of given layer + + Parameters + ---------- + weight : torch.nn.Parameter + weight of layer to prune + config : dict + layer's pruning config + op_name : str + layer name to be pruned + **kwargs + Arbitrary keyword arguments. + + Returns + ------- + torch.Tensor + mask of the layer's weight + + """ + + if self._if_init_list.get(op_name, True): w_abs = weight.abs() k = int(weight.numel() * config['sparsity']) if k == 0: @@ -29,40 +62,73 @@ def calc_mask(self, weight, config, op_name, **kwargs): threshold = torch.topk(w_abs.view(-1), k, largest=False).values.max() mask = torch.gt(w_abs, threshold).type_as(weight) self.mask_list.update({op_name: mask}) - self.if_init_list.update({op_name: False}) + self._if_init_list.update({op_name: False}) else: mask = self.mask_list[op_name] return mask class AGP_Pruner(Pruner): - """An automated gradual pruning algorithm that prunes the smallest magnitude + """Prune to an exact pruning level specification + An automated gradual pruning algorithm that prunes the smallest magnitude weights to achieve a preset level of network sparsity. Michael Zhu and Suyog Gupta, "To prune, or not to prune: exploring the efficacy of pruning for model compression", 2017 NIPS Workshop on Machine Learning of Phones and other Consumer Devices, https://arxiv.org/pdf/1710.01878.pdf + + Attributes + ---------- + mask_list : dict + Dictionary for saving masks. + """ def __init__(self, config_list): - """ + """Initiate `AGP_Pruner` from `config_list` config_list: supported keys: - initial_sparsity - final_sparsity: you should make sure initial_sparsity <= final_sparsity - start_epoch: start epoch number begin update mask - end_epoch: end epoch number stop update mask, you should make sure start_epoch <= end_epoch - frequency: if you want update every 2 epoch, you can set it 2 + + Parameters + ---------- + config_list : list + List on pruning configs + """ + super().__init__(config_list) self.mask_list = {} self.now_epoch = 0 - self.if_init_list = {} + self._if_init_list = {} def calc_mask(self, weight, config, op_name, **kwargs): + """Calculate the mask of given layer + + Parameters + ---------- + weight : torch.nn.Parameter + Weight of layer to prune + config : dict + Layer's pruning config + op_name : str + Layer name to be pruned + **kwargs + Arbitrary keyword arguments. + + Returns + ------- + torch.Tensor + Mask of the layer's weight + + """ start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) - if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( + if self.now_epoch >= start_epoch and self._if_init_list.get(op_name, True) and ( self.now_epoch - start_epoch) % freq == 0: mask = self.mask_list.get(op_name, torch.ones(weight.shape).type_as(weight)) target_sparsity = self.compute_target_sparsity(config) @@ -74,12 +140,25 @@ def calc_mask(self, weight, config, op_name, **kwargs): threshold = torch.topk(w_abs.view(-1), k, largest=False).values.max() new_mask = torch.gt(w_abs, threshold).type_as(weight) self.mask_list.update({op_name: new_mask}) - self.if_init_list.update({op_name: False}) + self._if_init_list.update({op_name: False}) else: new_mask = self.mask_list.get(op_name, torch.ones(weight.shape).type_as(weight)) return new_mask def compute_target_sparsity(self, config): + """Calculate the sparsity for pruning + + Parameters + ---------- + config : dict + Layer's pruning config + + Returns + ------- + float + Target sparsity to be pruned + + """ end_epoch = config.get('end_epoch', 1) start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) @@ -100,10 +179,18 @@ def compute_target_sparsity(self, config): return target_sparsity def update_epoch(self, epoch): + """Update epoch + + Parameters + ---------- + epoch : int + current training epoch + + """ if epoch > 0: self.now_epoch = epoch - for k in self.if_init_list.keys(): - self.if_init_list[k] = True + for k in self._if_init_list.keys(): + self._if_init_list[k] = True class FilterPruner(Pruner): @@ -113,20 +200,55 @@ class FilterPruner(Pruner): Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR https://arxiv.org/abs/1608.08710 + + Attributes + ---------- + mask_list : dict + Dictionary for saving masks. + """ def __init__(self, config_list): - """ + """Initiate `FilterPruner` from `config_list` config_list: supported keys: - sparsity + + Parameters + ---------- + config_list : list + List of pruning configs + """ + super().__init__(config_list) self.mask_list = {} - self.if_init_list = {} + self._if_init_list = {} def calc_mask(self, weight, config, op_name, op_type, **kwargs): + """Calculate the mask of given layer + + Parameters + ---------- + weight : torch.nn.Parameter + Weight of layer to prune + config : dict + Layer's pruning config + op_name : str + Layer name to be pruned + op_type : str + Layer type to be pruned + **kwargs + Arbitrary keyword arguments. + + Returns + ------- + torch.Tensor + Mask of the layer's weight + + """ + assert op_type == 'Conv2d', 'FilterPruner only supports 2d convolution layer pruning' - if self.if_init_list.get(op_name, True): + if self._if_init_list.get(op_name, True): kernels = weight.shape[0] w_abs = weight.abs() k = int(kernels * config['sparsity']) @@ -136,30 +258,50 @@ def calc_mask(self, weight, config, op_name, op_type, **kwargs): threshold = torch.topk(w_abs_structured.view(-1), k, largest=False).values.max() mask = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight) self.mask_list.update({op_name: mask}) - self.if_init_list.update({op_name: False}) + self._if_init_list.update({op_name: False}) else: mask = self.mask_list[op_name] return mask class SlimPruner(Pruner): - """A structured pruning algorithm that prunes channels by pruning the weights of BN layers + """A structured pruning algorithm that prunes channels by pruning the weights of BN layers. Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV https://arxiv.org/pdf/1708.06519.pdf + + Attributes + ---------- + mask_list : dict + Dictionary for saving masks. + """ def __init__(self, config_list): - """ + """Initiate `FilterPruner` from `config_list` config_list: supported keys: - sparsity + + Parameters + ---------- + config_list : list + List of pruning configs + """ super().__init__(config_list) self.mask_list = {} - self.if_init_list = {} + self._if_init_list = {} def bind_model(self, model): + """Calculate the global threshold for pruning + + Parameters + ---------- + model : torch.nn.Module + Model to be pruned + + """ weight_list = [] config = self._config_list[0] op_types = config.get('op_types') @@ -180,12 +322,34 @@ def bind_model(self, model): self.global_threshold = torch.topk(all_bn_weights.view(-1), k, largest=False).values.max() def calc_mask(self, weight, config, op_name, op_type, **kwargs): + """Calculate the mask of given layer + + Parameters + ---------- + weight : torch.nn.Parameter + Weight of layer to prune + config : dict + Layer's pruning config + op_name : str + Layer name to be pruned + op_type : str + Layer type to be pruned + **kwargs + Arbitrary keyword arguments. + + Returns + ------- + torch.Tensor + Mask of the layer's weight + + """ + assert op_type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' - if self.if_init_list.get(op_name, True): + if self._if_init_list.get(op_name, True): w_abs = weight.abs() mask = torch.gt(w_abs, self.global_threshold).type_as(weight) self.mask_list.update({op_name: mask}) - self.if_init_list.update({op_name: False}) + self._if_init_list.update({op_name: False}) else: mask = self.mask_list[op_name] return mask From f5120a207de745ce25d3124a8f3ad680396386a4 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Mon, 4 Nov 2019 16:53:59 +0800 Subject: [PATCH 06/20] Reproduce the paper --- examples/model_compress/main_filter_pruner.py | 147 ++++++++++++++++++ .../nni/compression/torch/builtin_pruners.py | 147 ++++++++---------- .../pynni/nni/compression/torch/compressor.py | 1 + 3 files changed, 216 insertions(+), 79 deletions(-) create mode 100644 examples/model_compress/main_filter_pruner.py diff --git a/examples/model_compress/main_filter_pruner.py b/examples/model_compress/main_filter_pruner.py new file mode 100644 index 0000000000..723af45739 --- /dev/null +++ b/examples/model_compress/main_filter_pruner.py @@ -0,0 +1,147 @@ +from nni.compression.torch import AGP_Pruner +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import datasets, transforms + + +class vgg(nn.Module): + def __init__(self, init_weights=True): + super(vgg, self).__init__() + cfg = [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512] + self.cfg = cfg + self.feature = self.make_layers(cfg, True) + num_classes = 10 + self.classifier = nn.Sequential( + nn.Linear(cfg[-1], 512), + nn.BatchNorm1d(512), + nn.ReLU(inplace=True), + nn.Linear(512, num_classes) + ) + if init_weights: + self._initialize_weights() + + def make_layers(self, cfg, batch_norm=False): + layers = [] + in_channels = 3 + for v in cfg: + if v == 'M': + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, bias=False) + if batch_norm: + layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] + else: + layers += [conv2d, nn.ReLU(inplace=True)] + in_channels = v + return nn.Sequential(*layers) + + def forward(self, x): + x = self.feature(x) + x = nn.AvgPool2d(2)(x) + x = x.view(x.size(0), -1) + y = self.classifier(x) + return y + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(0.5) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.weight.data.normal_(0, 0.01) + m.bias.data.zero_() + + +def train(model, device, train_loader, optimizer): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = F.cross_entropy(output, target) + loss.backward() + optimizer.step() + if batch_idx % 100 == 0: + print('{:2.0f}% Loss {}'.format(100 * batch_idx / len(train_loader), loss.item())) + + +def test(model, device, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction='sum').item() + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + test_loss /= len(test_loader.dataset) + + print('Loss: {} Accuracy: {}%)\n'.format( + test_loss, 100 * correct / len(test_loader.dataset))) + + +def main(): + torch.manual_seed(0) + device = torch.device('cuda') + train_loader = torch.utils.data.DataLoader( + datasets.CIFAR10('./data.cifar10', train=True, download=True, + transform=transforms.Compose([ + transforms.Pad(4), + transforms.RandomCrop(32), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) + ])), + batch_size=256, shuffle=True) + test_loader = torch.utils.data.DataLoader( + datasets.CIFAR10('./data.cifar10', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) + ])), + batch_size=200, shuffle=False) + + model = vgg() + model.to(device) + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) + lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 150, 0) + for epoch in range(300): + train(model, device, train_loader, optimizer) + test(model, device, test_loader) + lr_scheduler.step(epoch) + torch.save(model.state_dict(), 'vgg16.pth') + '''you can change this to LevelPruner to implement it + pruner = LevelPruner(configure_list) + ''' + # configure_list = [{ + # 'initial_sparsity': 0, + # 'final_sparsity': 0.8, + # 'start_epoch': 0, + # 'end_epoch': 10, + # 'frequency': 1, + # 'op_types': ['default'] + # }] + # + # pruner = AGP_Pruner(model, configure_list) + # pruner.compress() + # # you can also use compress(model) method + # # like that pruner.compress(model) + # + # optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.5) + # for epoch in range(10): + # pruner.update_epoch(epoch) + # print('# Epoch {} #'.format(epoch)) + # train(model, device, train_loader, optimizer) + # test(model, device, test_loader) + + +if __name__ == '__main__': + main() diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 88098bebe2..49c004b3ab 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -8,7 +8,8 @@ class LevelPruner(Pruner): - """Prune to an exact pruning level specification + """ + Prune to an exact pruning level specification Attributes ---------- @@ -24,27 +25,27 @@ def __init__(self, model, config_list): Parameters ---------- + model : torch.nn.module + Model to be pruned config_list : list List on pruning configs """ + super().__init__(model, config_list) self.mask_list = {} self._if_init_list = {} def calc_mask(self, layer, config): - """Calculate the mask of given layer + """ + Calculate the mask of given layer Parameters ---------- - weight : torch.nn.Parameter - weight of layer to prune + layer : LayerInfo + the layer to instrument the compression operation config : dict layer's pruning config - op_name : str - layer name to be pruned - **kwargs - Arbitrary keyword arguments. Returns ------- @@ -70,7 +71,7 @@ def calc_mask(self, layer, config): class AGP_Pruner(Pruner): - """Prune to an exact pruning level specification + """ An automated gradual pruning algorithm that prunes the smallest magnitude weights to achieve a preset level of network sparsity. @@ -97,35 +98,36 @@ def __init__(self, model, config_list): Parameters ---------- + model : torch.nn.module + Model to be pruned config_list : list List on pruning configs """ + super().__init__(model, config_list) self.mask_list = {} self.now_epoch = 0 self._if_init_list = {} def calc_mask(self, layer, config): - """Calculate the mask of given layer + """ + Calculate the mask of given layer Parameters ---------- - weight : torch.nn.Parameter - Weight of layer to prune + layer : LayerInfo + the layer to instrument the compression operation config : dict - Layer's pruning config - op_name : str - Layer name to be pruned - **kwargs - Arbitrary keyword arguments. + layer's pruning config Returns ------- torch.Tensor - Mask of the layer's weight + mask of the layer's weight """ + weight = layer.module.weight.data op_name = layer.name start_epoch = config.get('start_epoch', 0) @@ -148,7 +150,8 @@ def calc_mask(self, layer, config): return new_mask def compute_target_sparsity(self, config): - """Calculate the sparsity for pruning + """ + Calculate the sparsity for pruning Parameters ---------- @@ -196,7 +199,8 @@ def update_epoch(self, epoch): class FilterPruner(Pruner): - """A structured pruning algorithm that prunes the filters of smallest magnitude + """ + A structured pruning algorithm that prunes the filters of smallest magnitude weights sum in the convolution layers to achieve a preset level of network sparsity. Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf, @@ -210,44 +214,42 @@ class FilterPruner(Pruner): """ - def __init__(self, config_list): + def __init__(self, model, config_list): """Initiate `FilterPruner` from `config_list` config_list: supported keys: - sparsity Parameters ---------- + model : torch.nn.module + Model to be pruned config_list : list - List of pruning configs + List on pruning configs """ - super().__init__(config_list) + super().__init__(model, config_list) self.mask_list = {} self._if_init_list = {} def calc_mask(self, layer, config): - """Calculate the mask of given layer - - Parameters - ---------- - weight : torch.nn.Parameter - Weight of layer to prune - config : dict - Layer's pruning config - op_name : str - Layer name to be pruned - op_type : str - Layer type to be pruned - **kwargs - Arbitrary keyword arguments. - - Returns - ------- - torch.Tensor - Mask of the layer's weight - - """ + """ + Calculate the mask of given layer + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the compression operation + config : dict + layer's pruning config + + Returns + ------- + torch.Tensor + mask of the layer's weight + + """ + weight = layer.module.weight.data op_name = layer.name op_type = layer.type @@ -269,8 +271,8 @@ def calc_mask(self, layer, config): class SlimPruner(Pruner): - """A structured pruning algorithm that prunes channels by pruning the weights of BN layers. - + """ + A structured pruning algorithm that prunes channels by pruning the weights of BN layers. Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV https://arxiv.org/pdf/1708.06519.pdf @@ -282,7 +284,7 @@ class SlimPruner(Pruner): """ - def __init__(self, config_list): + def __init__(self, model, config_list): """Initiate `FilterPruner` from `config_list` config_list: supported keys: - sparsity @@ -293,19 +295,10 @@ def __init__(self, config_list): List of pruning configs """ - super().__init__(config_list) + + super().__init__(model, config_list) self.mask_list = {} self._if_init_list = {} - - def bind_model(self, model): - """Calculate the global threshold for pruning - - Parameters - ---------- - model : torch.nn.Module - Model to be pruned - - """ weight_list = [] config = self.config_list[0] op_types = config.get('op_types') @@ -326,27 +319,23 @@ def bind_model(self, model): self.global_threshold = torch.topk(all_bn_weights.view(-1), k, largest=False).values.max() def calc_mask(self, layer, config): - """Calculate the mask of given layer - - Parameters - ---------- - weight : torch.nn.Parameter - Weight of layer to prune - config : dict - Layer's pruning config - op_name : str - Layer name to be pruned - op_type : str - Layer type to be pruned - **kwargs - Arbitrary keyword arguments. - - Returns - ------- - torch.Tensor - Mask of the layer's weight - - """ + """ + Calculate the mask of given layer + + Parameters + ---------- + layer : LayerInfo + the layer to instrument the compression operation + config : dict + layer's pruning config + + Returns + ------- + torch.Tensor + mask of the layer's weight + + """ + weight = layer.module.weight.data op_name = layer.name op_type = layer.type diff --git a/src/sdk/pynni/nni/compression/torch/compressor.py b/src/sdk/pynni/nni/compression/torch/compressor.py index 580b1c1fac..b73fd652d2 100644 --- a/src/sdk/pynni/nni/compression/torch/compressor.py +++ b/src/sdk/pynni/nni/compression/torch/compressor.py @@ -128,6 +128,7 @@ def _expand_config_op_types(self, config): expanded_op_types.append(op_type) return expanded_op_types + class Pruner(Compressor): """ Abstract base PyTorch pruner From 952fc63c52bab3c9aa5c714a26e649e76e66e567 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Mon, 4 Nov 2019 17:24:56 +0800 Subject: [PATCH 07/20] update --- examples/model_compress/main_filter_pruner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/model_compress/main_filter_pruner.py b/examples/model_compress/main_filter_pruner.py index 723af45739..6022e0cae9 100644 --- a/examples/model_compress/main_filter_pruner.py +++ b/examples/model_compress/main_filter_pruner.py @@ -22,7 +22,7 @@ def __init__(self, init_weights=True): if init_weights: self._initialize_weights() - def make_layers(self, cfg, batch_norm=False): + def make_layers(self, cfg, batch_norm=True): layers = [] in_channels = 3 for v in cfg: From 9152178e8761d9f805016a86828cdbb79d00893a Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 7 Nov 2019 15:35:51 +0800 Subject: [PATCH 08/20] update --- src/sdk/pynni/nni/compression/torch/builtin_pruners.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 82b5bd9274..012bda9d94 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -182,7 +182,8 @@ def compute_target_sparsity(self, config): return target_sparsity def update_epoch(self, epoch): - """Update epoch + """ + Update epoch Parameters ---------- @@ -213,7 +214,8 @@ class FilterPruner(Pruner): """ def __init__(self, model, config_list): - """Initiate `FilterPruner` from `config_list` + """ + Initiate `FilterPruner` from `config_list` config_list: supported keys: - sparsity @@ -282,7 +284,8 @@ class SlimPruner(Pruner): """ def __init__(self, model, config_list): - """Initiate `FilterPruner` from `config_list` + """ + Initiate `FilterPruner` from `config_list` config_list: supported keys: - sparsity From 82a9b6272c29d97fd99ee17d478df3821d21d770 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 7 Nov 2019 16:59:24 +0800 Subject: [PATCH 09/20] implement network slimming --- examples/model_compress/main_filter_pruner.py | 10 +- examples/model_compress/main_slim_pruner.py | 174 ++++++++++++++++++ .../nni/compression/torch/builtin_pruners.py | 2 +- 3 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 examples/model_compress/main_slim_pruner.py diff --git a/examples/model_compress/main_filter_pruner.py b/examples/model_compress/main_filter_pruner.py index 7bc8af7f9c..b45dde0a17 100644 --- a/examples/model_compress/main_filter_pruner.py +++ b/examples/model_compress/main_filter_pruner.py @@ -121,12 +121,12 @@ def main(): # train(model, device, train_loader, optimizer) # test(model, device, test_loader) # lr_scheduler.step(epoch) - # torch.save(model.state_dict(), 'vgg16.pth') + # torch.save(model.state_dict(), 'vgg16_cifar10.pth') - # Pretrained model 'vgg16.pth' can also be downloaded from + # Pretrained model 'vgg16_cifar10.pth' can also be downloaded from # https://drive.google.com/open?id=1eaIBg_hu4T0JTqIMpg_51dIWbAvvm5ya - model.load_state_dict(torch.load('vgg16.pth')) + model.load_state_dict(torch.load('vgg16_cifar10.pth')) # Test base model accuracy # top1 = 93.47% @@ -158,12 +158,12 @@ def main(): best_top1 = top1 # Export the best model, 'model_path' stores state_dict of the pruned model, # mask_path stores mask_dict of the pruned model - pruner.export_model(model_path='pruned_vgg16.pth', mask_path='mask_vgg16.pth') + pruner.export_model(model_path='pruned_vgg16_cifar10.pth', mask_path='mask_vgg16_cifar10.pth') # Test the exported model new_model = vgg() new_model.to(device) - new_model.load_state_dict(torch.load('pruned_vgg16.pth')) + new_model.load_state_dict(torch.load('pruned_vgg16_cifar10.pth')) test(new_model, device, test_loader) diff --git a/examples/model_compress/main_slim_pruner.py b/examples/model_compress/main_slim_pruner.py new file mode 100644 index 0000000000..f90db0e4f7 --- /dev/null +++ b/examples/model_compress/main_slim_pruner.py @@ -0,0 +1,174 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import datasets, transforms +from nni.compression.torch import SlimPruner + + +class vgg(nn.Module): + def __init__(self, init_weights=True): + super(vgg, self).__init__() + cfg = [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512] + self.feature = self.make_layers(cfg, True) + num_classes = 10 + self.classifier = nn.Linear(cfg[-1], num_classes) + if init_weights: + self._initialize_weights() + + def make_layers(self, cfg, batch_norm=False): + layers = [] + in_channels = 3 + for v in cfg: + if v == 'M': + layers += [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, bias=False) + if batch_norm: + layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] + else: + layers += [conv2d, nn.ReLU(inplace=True)] + in_channels = v + return nn.Sequential(*layers) + + def forward(self, x): + x = self.feature(x) + x = nn.AvgPool2d(2)(x) + x = x.view(x.size(0), -1) + y = self.classifier(x) + return y + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(0.5) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.weight.data.normal_(0, 0.01) + m.bias.data.zero_() + + +def updateBN(model): + for m in model.modules(): + if isinstance(m, nn.BatchNorm2d): + m.weight.grad.data.add_(0.0001 * torch.sign(m.weight.data)) # L1 + + +def train(model, device, train_loader, optimizer, sparse_bn=False): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = F.cross_entropy(output, target) + loss.backward() + # L1 regularization on BN layer + if sparse_bn: + updateBN(model) + optimizer.step() + if batch_idx % 100 == 0: + print('{:2.0f}% Loss {}'.format(100 * batch_idx / len(train_loader), loss.item())) + + +def test(model, device, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction='sum').item() + pred = output.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() + test_loss /= len(test_loader.dataset) + acc = 100 * correct / len(test_loader.dataset) + + print('Loss: {} Accuracy: {}%)\n'.format( + test_loss, acc)) + return acc + + +def main(): + torch.manual_seed(0) + device = torch.device('cuda') + train_loader = torch.utils.data.DataLoader( + datasets.CIFAR10('./data.cifar10', train=True, download=True, + transform=transforms.Compose([ + transforms.Pad(4), + transforms.RandomCrop(32), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) + ])), + batch_size=64, shuffle=True) + test_loader = torch.utils.data.DataLoader( + datasets.CIFAR10('./data.cifar10', train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) + ])), + batch_size=200, shuffle=False) + + model = vgg() + model.to(device) + + # Train the base VGG-19 model + # epochs = 160 + # optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) + # for epoch in range(epochs): + # if epoch in [epochs * 0.5, epochs * 0.75]: + # for param_group in optimizer.param_groups: + # param_group['lr'] *= 0.1 + # train(model, device, train_loader, optimizer, True) + # test(model, device, test_loader) + # torch.save(model.state_dict(), 'vgg19_cifar10.pth') + + # Pretrained model 'vgg19_cifar10.pth' can also be downloaded from + # https://drive.google.com/open?id=1eaIBg_hu4T0JTqIMpg_51dIWbAvvm5ya + + model.load_state_dict(torch.load('vgg19_cifar10.pth')) + + # Test base model accuracy + # top1 = 93.93% + test(model, device, test_loader) + + # Pruning Configuration, in paper 'Learning efficient convolutional networks through network slimming', + configure_list = [{ + 'sparsity': 0.7, + 'op_types': ['BatchNorm2d'], + }] + + # Prune model and test accuracy without fine tuning. + # top1 = 93.91% + pruner = SlimPruner(model, configure_list) + model = pruner.compress() + test(model, device, test_loader) + + # Fine tune the pruned model for 40 epochs and test accuracy + optimizer_finetune = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4) + best_top1 = 0 + for epoch in range(40): + pruner.update_epoch(epoch) + print('# Epoch {} #'.format(epoch)) + train(model, device, train_loader, optimizer_finetune) + top1 = test(model, device, test_loader) + if top1 > best_top1: + best_top1 = top1 + # Export the best model, 'model_path' stores state_dict of the pruned model, + # mask_path stores mask_dict of the pruned model + pruner.export_model(model_path='pruned_vgg19_cifar10.pth', mask_path='mask_vgg19_cifar10.pth') + + # Test the exported model + new_model = vgg() + new_model.to(device) + new_model.load_state_dict(torch.load('pruned_vgg19_cifar10.pth')) + test(new_model, device, test_loader) + + +if __name__ == '__main__': + main() diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 012bda9d94..ebbc81527b 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -303,7 +303,7 @@ def __init__(self, model, config_list): op_types = config.get('op_types') op_names = config.get('op_names') if op_types is not None: - assert op_types == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + assert op_types == ['BatchNorm2d'], 'SlimPruner only supports 2d batch normalization layer pruning' for name, m in model.named_modules(): if type(m).__name__ == 'BatchNorm2d': weight_list.append(m.weight.data.clone()) From b0471e1976fb68f02c34a31fdceab62d6bd10c11 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Fri, 8 Nov 2019 15:59:49 +0800 Subject: [PATCH 10/20] update --- examples/model_compress/main_filter_pruner.py | 30 ++++++++-------- examples/model_compress/main_slim_pruner.py | 34 ++++++++++--------- .../nni/compression/torch/builtin_pruners.py | 8 ++--- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/examples/model_compress/main_filter_pruner.py b/examples/model_compress/main_filter_pruner.py index b45dde0a17..57a0112ccd 100644 --- a/examples/model_compress/main_filter_pruner.py +++ b/examples/model_compress/main_filter_pruner.py @@ -115,22 +115,20 @@ def main(): model.to(device) # Train the base VGG-16 model - # optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) - # lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 300, 0) - # for epoch in range(300): - # train(model, device, train_loader, optimizer) - # test(model, device, test_loader) - # lr_scheduler.step(epoch) - # torch.save(model.state_dict(), 'vgg16_cifar10.pth') - - # Pretrained model 'vgg16_cifar10.pth' can also be downloaded from - # https://drive.google.com/open?id=1eaIBg_hu4T0JTqIMpg_51dIWbAvvm5ya - - model.load_state_dict(torch.load('vgg16_cifar10.pth')) + print('=' * 10 + 'Train the unpruned base model' + '=' * 10) + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) + lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 300, 0) + for epoch in range(300): + train(model, device, train_loader, optimizer) + test(model, device, test_loader) + lr_scheduler.step(epoch) + torch.save(model.state_dict(), 'vgg16_cifar10.pth') # Test base model accuracy - # top1 = 93.47% + print('=' * 10 + 'Test on the original model' + '=' * 10) + model.load_state_dict(torch.load('vgg16_cifar10.pth')) test(model, device, test_loader) + # top1 = 93.47% # Pruning Configuration, in paper 'PRUNING FILTERS FOR EFFICIENT CONVNETS', # Conv_1, Conv_8, Conv_9, Conv_10, Conv_11, Conv_12 are pruned with 50% sparsity, as 'VGG-16-pruned-A' @@ -141,12 +139,14 @@ def main(): }] # Prune model and test accuracy without fine tuning. - # top1 = 47.22% + print('=' * 10 + 'Test on the pruned model before fine tune' + '=' * 10) pruner = FilterPruner(model, configure_list) model = pruner.compress() test(model, device, test_loader) + # top1 = 47.22% # Fine tune the pruned model for 40 epochs and test accuracy + print('=' * 10 + 'Fine tuning' + '=' * 10) optimizer_finetune = torch.optim.SGD(model.parameters(), lr=0.004, momentum=0.9, weight_decay=1e-4) best_top1 = 0 for epoch in range(40): @@ -161,10 +161,12 @@ def main(): pruner.export_model(model_path='pruned_vgg16_cifar10.pth', mask_path='mask_vgg16_cifar10.pth') # Test the exported model + print('=' * 10 + 'Test on the pruned model after fine tune' + '=' * 10) new_model = vgg() new_model.to(device) new_model.load_state_dict(torch.load('pruned_vgg16_cifar10.pth')) test(new_model, device, test_loader) + # top1 = 93.18% if __name__ == '__main__': diff --git a/examples/model_compress/main_slim_pruner.py b/examples/model_compress/main_slim_pruner.py index f90db0e4f7..e50ad26210 100644 --- a/examples/model_compress/main_slim_pruner.py +++ b/examples/model_compress/main_slim_pruner.py @@ -118,24 +118,22 @@ def main(): model.to(device) # Train the base VGG-19 model - # epochs = 160 - # optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) - # for epoch in range(epochs): - # if epoch in [epochs * 0.5, epochs * 0.75]: - # for param_group in optimizer.param_groups: - # param_group['lr'] *= 0.1 - # train(model, device, train_loader, optimizer, True) - # test(model, device, test_loader) - # torch.save(model.state_dict(), 'vgg19_cifar10.pth') - - # Pretrained model 'vgg19_cifar10.pth' can also be downloaded from - # https://drive.google.com/open?id=1eaIBg_hu4T0JTqIMpg_51dIWbAvvm5ya - - model.load_state_dict(torch.load('vgg19_cifar10.pth')) + print('=' * 10 + 'Train the unpruned base model' + '=' * 10) + epochs = 160 + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) + for epoch in range(epochs): + if epoch in [epochs * 0.5, epochs * 0.75]: + for param_group in optimizer.param_groups: + param_group['lr'] *= 0.1 + train(model, device, train_loader, optimizer, True) + test(model, device, test_loader) + torch.save(model.state_dict(), 'vgg19_cifar10.pth') # Test base model accuracy - # top1 = 93.93% + print('=' * 10 + 'Test the original model' + '=' * 10) + model.load_state_dict(torch.load('vgg19_cifar10.pth')) test(model, device, test_loader) + # top1 = 93.93% # Pruning Configuration, in paper 'Learning efficient convolutional networks through network slimming', configure_list = [{ @@ -144,12 +142,14 @@ def main(): }] # Prune model and test accuracy without fine tuning. - # top1 = 93.91% + print('=' * 10 + 'Test the pruned model before fine tune' + '=' * 10) pruner = SlimPruner(model, configure_list) model = pruner.compress() test(model, device, test_loader) + # top1 = 93.91% # Fine tune the pruned model for 40 epochs and test accuracy + print('=' * 10 + 'Fine tuning' + '=' * 10) optimizer_finetune = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4) best_top1 = 0 for epoch in range(40): @@ -164,10 +164,12 @@ def main(): pruner.export_model(model_path='pruned_vgg19_cifar10.pth', mask_path='mask_vgg19_cifar10.pth') # Test the exported model + print('=' * 10 + 'Test the export pruned model after fine tune' + '=' * 10) new_model = vgg() new_model.to(device) new_model.load_state_dict(torch.load('pruned_vgg19_cifar10.pth')) test(new_model, device, test_loader) + # top1 = 93.91% if __name__ == '__main__': diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index ebbc81527b..35b0900650 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -13,7 +13,7 @@ class LevelPruner(Pruner): Attributes ---------- - mask_list : dict + mask_dict : dict Dictionary for saving masks. """ @@ -81,7 +81,7 @@ class AGP_Pruner(Pruner): Attributes ---------- - mask_list : dict + mask_dict : dict Dictionary for saving masks. """ @@ -208,7 +208,7 @@ class FilterPruner(Pruner): Attributes ---------- - mask_list : dict + mask_dict : dict Dictionary for saving masks. """ @@ -278,7 +278,7 @@ class SlimPruner(Pruner): Attributes ---------- - mask_list : dict + mask_dict : dict Dictionary for saving masks. """ From 3074f6b242008e5be2ab89d19977aabe26ff17c9 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Sat, 9 Nov 2019 15:30:04 +0800 Subject: [PATCH 11/20] updates --- docs/en_US/Compressor/Pruner.md | 26 ++++++---- ...pruner.py => filter_pruner_torch_vgg16.py} | 0 ...m_pruner.py => slim_pruner_torch_vgg19.py} | 0 .../nni/compression/torch/builtin_pruners.py | 50 ++++--------------- 4 files changed, 25 insertions(+), 51 deletions(-) rename examples/model_compress/{main_filter_pruner.py => filter_pruner_torch_vgg16.py} (100%) rename examples/model_compress/{main_slim_pruner.py => slim_pruner_torch_vgg19.py} (100%) diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 2452bd312a..c4ff62632b 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -98,14 +98,17 @@ In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), ![](../../img/filter_pruner.png) +> Filter Pruner prunes filters in the **convolution layers** +> > The procedure of pruning m filters from the ith convolutional layer is as follows: +> > 1. For each filter $F_{i,j}$ , calculate the sum of its absolute kernel weights $s_j = \sum_{l=1}^{n_i}\sum|K_l|$ > 2. Sort the filters by $s_j$. > 3. Prune $m$ filters with the smallest sum values and their corresponding feature maps. The -> kernels in the next convolutional layer corresponding to the pruned feature maps are also -> removed. +> kernels in the next convolutional layer corresponding to the pruned feature maps are also +> removed. > 4. A new kernel matrix is created for both the $i$th and $i+1$th layers, and the remaining kernel -> weights are copied to the new model. +> weights are copied to the new model. ### Usage @@ -114,13 +117,14 @@ PyTorch code ``` from nni.compression.torch import FilterPruner config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'] }] -pruner = FilterPruner(config_list) -pruner(model) +pruner = FilterPruner(model, config_list) +pruner.compress() ``` #### User configuration for Filter Pruner - **sparsity:** This is to specify the sparsity operations to be compressed to +- **op_types:** Only Conv2d is supported in Filter Pruner ## Slim Pruner @@ -128,7 +132,7 @@ In ['Learning Efficient Convolutional Networks through Network Slimming'](https: ![](../../img/slim_pruner.png) -> Slim Pruner prunes channels in the convolution layers by masking corresponding scaling factors in the later BN layers, L1 regularization on the scaling factors should be applied in batch normalization (BN) layers while training, scaling factors of BN layers are **globally ranked** while pruning, so the sparse model can be automatically found given sparsity. +> Slim Pruner **prunes channels in the convolution layers by masking corresponding scaling factors in the later BN layers**, L1 regularization on the scaling factors should be applied in batch normalization (BN) layers while training, scaling factors of BN layers are **globally ranked** while pruning, so the sparse model can be automatically found given sparsity. ### Usage @@ -136,11 +140,13 @@ PyTorch code ``` from nni.compression.torch import SlimPruner -config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'] }] -pruner = SlimPruner(config_list) -pruner(model) +config_list = [{ 'sparsity': 0.8, 'op_types': ['BatchNorm2d'] }] +pruner = SlimPruner(model, config_list) +pruner.compress() ``` #### User configuration for Filter Pruner -- **sparsity:** This is to specify the sparsity operations to be compressed to \ No newline at end of file +- **sparsity:** This is to specify the sparsity operations to be compressed to +- **op_types:** Only BatchNorm2d is supported in Slim Pruner + diff --git a/examples/model_compress/main_filter_pruner.py b/examples/model_compress/filter_pruner_torch_vgg16.py similarity index 100% rename from examples/model_compress/main_filter_pruner.py rename to examples/model_compress/filter_pruner_torch_vgg16.py diff --git a/examples/model_compress/main_slim_pruner.py b/examples/model_compress/slim_pruner_torch_vgg19.py similarity index 100% rename from examples/model_compress/main_slim_pruner.py rename to examples/model_compress/slim_pruner_torch_vgg19.py diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 35b0900650..aa5e04b32c 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -11,18 +11,10 @@ class LevelPruner(Pruner): """ Prune to an exact pruning level specification - Attributes - ---------- - mask_dict : dict - Dictionary for saving masks. - """ def __init__(self, model, config_list): """ - config_list: supported keys: - - sparsity - Parameters ---------- model : torch.nn.module @@ -79,22 +71,10 @@ class AGP_Pruner(Pruner): Learning of Phones and other Consumer Devices, https://arxiv.org/pdf/1710.01878.pdf - Attributes - ---------- - mask_dict : dict - Dictionary for saving masks. - """ def __init__(self, model, config_list): """ - config_list: supported keys: - - initial_sparsity - - final_sparsity: you should make sure initial_sparsity <= final_sparsity - - start_epoch: start epoch number begin update mask - - end_epoch: end epoch number stop update mask, you should make sure start_epoch <= end_epoch - - frequency: if you want update every 2 epoch, you can set it 2 - Parameters ---------- model : torch.nn.module @@ -162,6 +142,7 @@ def compute_target_sparsity(self, config): Target sparsity to be pruned """ + end_epoch = config.get('end_epoch', 1) start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) @@ -191,6 +172,7 @@ def update_epoch(self, epoch): current training epoch """ + if epoch > 0: self.now_epoch = epoch for k in self.if_init_list.keys(): @@ -206,19 +188,10 @@ class FilterPruner(Pruner): "PRUNING FILTERS FOR EFFICIENT CONVNETS", 2017 ICLR https://arxiv.org/abs/1608.08710 - Attributes - ---------- - mask_dict : dict - Dictionary for saving masks. - """ def __init__(self, model, config_list): """ - Initiate `FilterPruner` from `config_list` - config_list: supported keys: - - sparsity - Parameters ---------- model : torch.nn.module @@ -233,7 +206,8 @@ def __init__(self, model, config_list): def calc_mask(self, layer, config): """ - Calculate the mask of given layer + Calculate the mask of given layer. + Filters with the smallest sum of its absolute kernel weights are masked. Parameters ---------- @@ -276,19 +250,10 @@ class SlimPruner(Pruner): "Learning Efficient Convolutional Networks through Network Slimming", 2017 ICCV https://arxiv.org/pdf/1708.06519.pdf - Attributes - ---------- - mask_dict : dict - Dictionary for saving masks. - """ def __init__(self, model, config_list): """ - Initiate `FilterPruner` from `config_list` - config_list: supported keys: - - sparsity - Parameters ---------- config_list : list @@ -299,7 +264,9 @@ def __init__(self, model, config_list): super().__init__(model, config_list) self.if_init_list = {} weight_list = [] - config = self.config_list[0] + if len(config_list) > 1: + logger.warning('Slim pruner only supports 1 configuration') + config = config_list[0] op_types = config.get('op_types') op_names = config.get('op_names') if op_types is not None: @@ -319,7 +286,8 @@ def __init__(self, model, config_list): def calc_mask(self, layer, config): """ - Calculate the mask of given layer + Calculate the mask of given layer. + Scale factors with the smallest absolute value in the BN layer are masked. Parameters ---------- From e7bfba79f97841ac2e1fd06e3f80b17219b9959c Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Tue, 12 Nov 2019 20:30:54 +0800 Subject: [PATCH 12/20] refactor slim pruner --- .../model_compress/slim_pruner_torch_vgg19.py | 6 +++--- .../nni/compression/torch/builtin_pruners.py | 18 ++++-------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/examples/model_compress/slim_pruner_torch_vgg19.py b/examples/model_compress/slim_pruner_torch_vgg19.py index e50ad26210..e555b2ce72 100644 --- a/examples/model_compress/slim_pruner_torch_vgg19.py +++ b/examples/model_compress/slim_pruner_torch_vgg19.py @@ -133,7 +133,7 @@ def main(): print('=' * 10 + 'Test the original model' + '=' * 10) model.load_state_dict(torch.load('vgg19_cifar10.pth')) test(model, device, test_loader) - # top1 = 93.93% + # top1 = 93.44% # Pruning Configuration, in paper 'Learning efficient convolutional networks through network slimming', configure_list = [{ @@ -146,7 +146,7 @@ def main(): pruner = SlimPruner(model, configure_list) model = pruner.compress() test(model, device, test_loader) - # top1 = 93.91% + # top1 = 93.43% # Fine tune the pruned model for 40 epochs and test accuracy print('=' * 10 + 'Fine tuning' + '=' * 10) @@ -169,7 +169,7 @@ def main(): new_model.to(device) new_model.load_state_dict(torch.load('pruned_vgg19_cifar10.pth')) test(new_model, device, test_loader) - # top1 = 93.91% + # top1 = 93.64% if __name__ == '__main__': diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index aa5e04b32c..fdf18d5cff 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -111,7 +111,7 @@ def calc_mask(self, layer, config): start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: + self.now_epoch - start_epoch) % freq == 0: mask = self.mask_dict.get(op_name, torch.ones(weight.shape).type_as(weight)) target_sparsity = self.compute_target_sparsity(config) k = int(weight.numel() * target_sparsity) @@ -267,19 +267,9 @@ def __init__(self, model, config_list): if len(config_list) > 1: logger.warning('Slim pruner only supports 1 configuration') config = config_list[0] - op_types = config.get('op_types') - op_names = config.get('op_names') - if op_types is not None: - assert op_types == ['BatchNorm2d'], 'SlimPruner only supports 2d batch normalization layer pruning' - for name, m in model.named_modules(): - if type(m).__name__ == 'BatchNorm2d': - weight_list.append(m.weight.data.clone()) - else: - for name, m in model.named_modules(): - if name in op_names: - assert type( - m).__name__ == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' - weight_list.append(m.weight.data.clone()) + for (layer, config) in self.detect_modules_to_compress(): + assert layer.type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' + weight_list.append(layer.module.weight.data.clone()) all_bn_weights = torch.cat(weight_list) k = int(all_bn_weights.shape[0] * config['sparsity']) self.global_threshold = torch.topk(all_bn_weights.view(-1), k, largest=False).values.max() From dba53a23736ebd7955eff3e37f0f26b89a761874 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Tue, 12 Nov 2019 20:50:17 +0800 Subject: [PATCH 13/20] resolve pylint --- src/sdk/pynni/nni/compression/torch/builtin_pruners.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index fdf18d5cff..fc0fe1b151 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -110,8 +110,8 @@ def calc_mask(self, layer, config): op_name = layer.name start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) - if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: + if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True)\ + and (self.now_epoch - start_epoch) % freq == 0: mask = self.mask_dict.get(op_name, torch.ones(weight.shape).type_as(weight)) target_sparsity = self.compute_target_sparsity(config) k = int(weight.numel() * target_sparsity) From 600814ea03129b8b64d82084384e44b615b6f94c Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Tue, 19 Nov 2019 13:56:32 +0800 Subject: [PATCH 14/20] update docs --- docs/en_US/Compressor/FilterPruner.md | 54 +++++++++++++++++++ docs/en_US/Compressor/SlimPruner.md | 39 ++++++++++++++ .../filter_pruner_torch_vgg16.py | 14 ++--- .../model_compress/slim_pruner_torch_vgg19.py | 6 +-- 4 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 docs/en_US/Compressor/FilterPruner.md create mode 100644 docs/en_US/Compressor/SlimPruner.md diff --git a/docs/en_US/Compressor/FilterPruner.md b/docs/en_US/Compressor/FilterPruner.md new file mode 100644 index 0000000000..a61cf58bdc --- /dev/null +++ b/docs/en_US/Compressor/FilterPruner.md @@ -0,0 +1,54 @@ +FilterPruner on NNI Compressor +=== + +## 1. Introduction + +FilterPruner is a general structured pruning algorithm for pruning filters in the convolutional layers. + +In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. + +![](../../img/filter_pruner.png) + +> Filter Pruner prunes filters in the **convolution layers** +> +> The procedure of pruning m filters from the ith convolutional layer is as follows: +> +> 1. For each filter $F_{i,j}$ , calculate the sum of its absolute kernel weights $s_j = \sum_{l=1}^{n_i}\sum|K_l|$ +> 2. Sort the filters by $s_j$. +> 3. Prune $m$ filters with the smallest sum values and their corresponding feature maps. The +> kernels in the next convolutional layer corresponding to the pruned feature maps are also +> removed. +> 4. A new kernel matrix is created for both the $i$th and $i+1$th layers, and the remaining kernel +> weights are copied to the new model. + +## 2. Usage + +PyTorch code + +``` +from nni.compression.torch import FilterPruner +config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'], 'op_names': ['conv1', 'conv2'] }] +pruner = FilterPruner(model, config_list) +pruner.compress() +``` + +#### User configuration for Filter Pruner + +- **sparsity:** This is to specify the sparsity operations to be compressed to +- **op_types:** Only Conv2d is supported in Filter Pruner + +## 3. Experiment + +We implemented one of the experiments in ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), we pruned **VGG-16** for CIFAR-10 to **VGG-16-pruned-A** in the paper, in which $64\%$ parameters are pruned. Our experiments results are as follows: + +| Model | Error(paper/ours) | Parameters | Pruned | +| --------------- | ----------------- | --------------- | -------- | +| VGG-16 | $6.75$/$6.49$ | $1.5\times10^7$ | | +| VGG-16-pruned-A | $6.60$/$6.47$ | $5.4\times10^6$ | $64.0\%$ | + +The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/) + + + + + diff --git a/docs/en_US/Compressor/SlimPruner.md b/docs/en_US/Compressor/SlimPruner.md new file mode 100644 index 0000000000..c2e0325a8a --- /dev/null +++ b/docs/en_US/Compressor/SlimPruner.md @@ -0,0 +1,39 @@ +SlimPruner on NNI Compressor +=== + +## 1. Slim Pruner + +SlimPruner is a structured pruning algorithm for pruning channels in the convolutional layers by pruning corresponding scaling factors in the later BN layers. + +In ['Learning Efficient Convolutional Networks through Network Slimming'](https://arxiv.org/pdf/1708.06519.pdf), authors Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang. + +![](../../img/slim_pruner.png) + +> Slim Pruner **prunes channels in the convolution layers by masking corresponding scaling factors in the later BN layers**, L1 regularization on the scaling factors should be applied in batch normalization (BN) layers while training, scaling factors of BN layers are **globally ranked** while pruning, so the sparse model can be automatically found given sparsity. + +## 2. Usage + +PyTorch code + +``` +from nni.compression.torch import SlimPruner +config_list = [{ 'sparsity': 0.8, 'op_types': ['BatchNorm2d'] }] +pruner = SlimPruner(model, config_list) +pruner.compress() +``` + +#### User configuration for Filter Pruner + +- **sparsity:** This is to specify the sparsity operations to be compressed to +- **op_types:** Only BatchNorm2d is supported in Slim Pruner + +## 3. Experiment + +We implemented one of the experiments in ['Learning Efficient Convolutional Networks through Network Slimming'](https://arxiv.org/pdf/1708.06519.pdf), we pruned $70\%$ channels in the **VGGNet** for CIFAR-10 in the paper, in which $88.5\%$ parameters are pruned. Our experiments results are as follows: + +| Model | Error(paper/ours) | Parameters | Pruned | +| ------------- | ----------------- | ---------- | --------- | +| VGGNet | $6.34$/$6.40$ | $20.04M$ | | +| Pruned-VGGNet | $6.20/6.39$ | $2.03M$ | $88.5 \%$ | + +The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/) diff --git a/examples/model_compress/filter_pruner_torch_vgg16.py b/examples/model_compress/filter_pruner_torch_vgg16.py index 57a0112ccd..4cd3d5d2ff 100644 --- a/examples/model_compress/filter_pruner_torch_vgg16.py +++ b/examples/model_compress/filter_pruner_torch_vgg16.py @@ -103,7 +103,7 @@ def main(): transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) ])), - batch_size=256, shuffle=True) + batch_size=64, shuffle=True) test_loader = torch.utils.data.DataLoader( datasets.CIFAR10('./data.cifar10', train=False, transform=transforms.Compose([ transforms.ToTensor(), @@ -117,8 +117,8 @@ def main(): # Train the base VGG-16 model print('=' * 10 + 'Train the unpruned base model' + '=' * 10) optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4) - lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 300, 0) - for epoch in range(300): + lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 160, 0) + for epoch in range(160): train(model, device, train_loader, optimizer) test(model, device, test_loader) lr_scheduler.step(epoch) @@ -128,7 +128,7 @@ def main(): print('=' * 10 + 'Test on the original model' + '=' * 10) model.load_state_dict(torch.load('vgg16_cifar10.pth')) test(model, device, test_loader) - # top1 = 93.47% + # top1 = 93.51% # Pruning Configuration, in paper 'PRUNING FILTERS FOR EFFICIENT CONVNETS', # Conv_1, Conv_8, Conv_9, Conv_10, Conv_11, Conv_12 are pruned with 50% sparsity, as 'VGG-16-pruned-A' @@ -143,11 +143,11 @@ def main(): pruner = FilterPruner(model, configure_list) model = pruner.compress() test(model, device, test_loader) - # top1 = 47.22% + # top1 = 88.19% # Fine tune the pruned model for 40 epochs and test accuracy print('=' * 10 + 'Fine tuning' + '=' * 10) - optimizer_finetune = torch.optim.SGD(model.parameters(), lr=0.004, momentum=0.9, weight_decay=1e-4) + optimizer_finetune = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-4) best_top1 = 0 for epoch in range(40): pruner.update_epoch(epoch) @@ -166,7 +166,7 @@ def main(): new_model.to(device) new_model.load_state_dict(torch.load('pruned_vgg16_cifar10.pth')) test(new_model, device, test_loader) - # top1 = 93.18% + # top1 = 93.53% if __name__ == '__main__': diff --git a/examples/model_compress/slim_pruner_torch_vgg19.py b/examples/model_compress/slim_pruner_torch_vgg19.py index e50ad26210..a227e7a731 100644 --- a/examples/model_compress/slim_pruner_torch_vgg19.py +++ b/examples/model_compress/slim_pruner_torch_vgg19.py @@ -133,7 +133,7 @@ def main(): print('=' * 10 + 'Test the original model' + '=' * 10) model.load_state_dict(torch.load('vgg19_cifar10.pth')) test(model, device, test_loader) - # top1 = 93.93% + # top1 = 93.60% # Pruning Configuration, in paper 'Learning efficient convolutional networks through network slimming', configure_list = [{ @@ -146,7 +146,7 @@ def main(): pruner = SlimPruner(model, configure_list) model = pruner.compress() test(model, device, test_loader) - # top1 = 93.91% + # top1 = 93.55% # Fine tune the pruned model for 40 epochs and test accuracy print('=' * 10 + 'Fine tuning' + '=' * 10) @@ -169,7 +169,7 @@ def main(): new_model.to(device) new_model.load_state_dict(torch.load('pruned_vgg19_cifar10.pth')) test(new_model, device, test_loader) - # top1 = 93.91% + # top1 = 93.61% if __name__ == '__main__': From 8df1dac3d7b0ebc1c0e1fc6aea003c58247935a9 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Wed, 20 Nov 2019 15:02:47 +0800 Subject: [PATCH 15/20] update doc --- docs/en_US/Compressor/Pruner.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 9e3da506bb..cf9002e071 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -3,7 +3,7 @@ Pruner on NNI Compressor ## Level Pruner -This is one basic pruner: you can set a target sparsity level (expressed as a fraction, 0.6 means we will prune 60%). +This is one basic one-shot pruner: you can set a target sparsity level (expressed as a fraction, 0.6 means we will prune 60%). We first sort the weights in the specified layer by their absolute values. And then mask to zero the smallest magnitude weights until the desired sparsity level is reached. @@ -31,7 +31,7 @@ pruner.compress() *** ## AGP Pruner -In [To prune, or not to prune: exploring the efficacy of pruning for model compression](https://arxiv.org/abs/1710.01878), authors Michael Zhu and Suyog Gupta provide an algorithm to prune the weight gradually. +This is an iterative pruner, In [To prune, or not to prune: exploring the efficacy of pruning for model compression](https://arxiv.org/abs/1710.01878), authors Michael Zhu and Suyog Gupta provide an algorithm to prune the weight gradually. >We introduce a new automated gradual pruning algorithm in which the sparsity is increased from an initial sparsity value si (usually 0) to a final sparsity value sf over a span of n pruning steps, starting at training step t0 and with pruning frequency ∆t: ![](../../img/agp_pruner.png) @@ -94,7 +94,7 @@ You can view example for more information ## Filter Pruner -In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. +This is an one-shot pruner, In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. ![](../../img/filter_pruner.png) @@ -128,7 +128,7 @@ pruner.compress() ## Slim Pruner -In ['Learning Efficient Convolutional Networks through Network Slimming'](https://arxiv.org/pdf/1708.06519.pdf), authors Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang. +This is an one-shot pruner, In ['Learning Efficient Convolutional Networks through Network Slimming'](https://arxiv.org/pdf/1708.06519.pdf), authors Zhuang Liu, Jianguo Li, Zhiqiang Shen, Gao Huang, Shoumeng Yan and Changshui Zhang. ![](../../img/slim_pruner.png) @@ -151,7 +151,7 @@ pruner.compress() - **op_types:** Only BatchNorm2d is supported in Slim Pruner ## FPGM Pruner -FPGM Pruner is an implementation of paper [Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration](https://arxiv.org/pdf/1811.00250.pdf) +This is an one-shot pruner, FPGM Pruner is an implementation of paper [Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration](https://arxiv.org/pdf/1811.00250.pdf) >Previous works utilized “smaller-norm-less-important” criterion to prune filters with smaller norm values in a convolutional neural network. In this paper, we analyze this norm-based criterion and point out that its effectiveness depends on two requirements that are not always met: (1) the norm deviation of the filters should be large; (2) the minimum norm of the filters should be small. To solve this problem, we propose a novel filter pruning method, namely Filter Pruning via Geometric Median (FPGM), to compress the model regardless of those two requirements. Unlike previous methods, FPGM compresses CNN models by pruning filters with redundancy, rather than those with “relatively less” importance. From 56d3dea6c2a54cb841bc2896cc48c8630c816271 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 21 Nov 2019 14:12:37 +0800 Subject: [PATCH 16/20] refactor and rename --- .../{FilterPruner.md => L1FilterPruner.md} | 16 +++--- docs/en_US/Compressor/Overview.md | 2 +- docs/en_US/Compressor/Pruner.md | 14 +++--- ...{filter_pruner.PNG => l1filter_pruner.PNG} | Bin ...g16.py => L1_filter_pruner_torch_vgg16.py} | 4 +- .../nni/compression/torch/builtin_pruners.py | 46 +++++++++++------- 6 files changed, 46 insertions(+), 36 deletions(-) rename docs/en_US/Compressor/{FilterPruner.md => L1FilterPruner.md} (79%) rename docs/img/{filter_pruner.PNG => l1filter_pruner.PNG} (100%) rename examples/model_compress/{filter_pruner_torch_vgg16.py => L1_filter_pruner_torch_vgg16.py} (98%) diff --git a/docs/en_US/Compressor/FilterPruner.md b/docs/en_US/Compressor/L1FilterPruner.md similarity index 79% rename from docs/en_US/Compressor/FilterPruner.md rename to docs/en_US/Compressor/L1FilterPruner.md index a61cf58bdc..9a6e1c8e22 100644 --- a/docs/en_US/Compressor/FilterPruner.md +++ b/docs/en_US/Compressor/L1FilterPruner.md @@ -1,15 +1,15 @@ -FilterPruner on NNI Compressor +L1FilterPruner on NNI Compressor === ## 1. Introduction -FilterPruner is a general structured pruning algorithm for pruning filters in the convolutional layers. +L1FilterPruner is a general structured pruning algorithm for pruning filters in the convolutional layers. In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. -![](../../img/filter_pruner.png) +![](../../img/l1filter_pruner.png) -> Filter Pruner prunes filters in the **convolution layers** +> L1Filter Pruner prunes filters in the **convolution layers** > > The procedure of pruning m filters from the ith convolutional layer is as follows: > @@ -26,16 +26,16 @@ In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), PyTorch code ``` -from nni.compression.torch import FilterPruner +from nni.compression.torch import L1FilterPruner config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'], 'op_names': ['conv1', 'conv2'] }] -pruner = FilterPruner(model, config_list) +pruner = L1FilterPruner(model, config_list) pruner.compress() ``` -#### User configuration for Filter Pruner +#### User configuration for L1Filter Pruner - **sparsity:** This is to specify the sparsity operations to be compressed to -- **op_types:** Only Conv2d is supported in Filter Pruner +- **op_types:** Only Conv2d is supported in L1Filter Pruner ## 3. Experiment diff --git a/docs/en_US/Compressor/Overview.md b/docs/en_US/Compressor/Overview.md index 39616e50e5..279cbe6f89 100644 --- a/docs/en_US/Compressor/Overview.md +++ b/docs/en_US/Compressor/Overview.md @@ -12,7 +12,7 @@ We have provided two naive compression algorithms and three popular ones for use |---|---| | [Level Pruner](./Pruner.md#level-pruner) | Pruning the specified ratio on each weight based on absolute values of weights | | [AGP Pruner](./Pruner.md#agp-pruner) | Automated gradual pruning (To prune, or not to prune: exploring the efficacy of pruning for model compression) [Reference Paper](https://arxiv.org/abs/1710.01878)| -| [Filter Pruner](./Pruner.md#filter-pruner) | Pruning least important filters in convolution layers(PRUNING FILTERS FOR EFFICIENT CONVNETS)[Reference Paper](https://arxiv.org/abs/1608.08710) | +| [L1Filter Pruner](./Pruner.md#l1filter-pruner) | Pruning least important filters in convolution layers(PRUNING FILTERS FOR EFFICIENT CONVNETS)[Reference Paper](https://arxiv.org/abs/1608.08710) | | [Slim Pruner](./Pruner.md#slim-pruner) | Pruning channels in convolution layers by pruning scaling factors in BN layers(Learning Efficient Convolutional Networks through Network Slimming)[Reference Paper](https://arxiv.org/abs/1708.06519) | | [Lottery Ticket Pruner](./Pruner.md#agp-pruner) | The pruning process used by "The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks". It prunes a model iteratively. [Reference Paper](https://arxiv.org/abs/1803.03635)| | [FPGM Pruner](./Pruner.md#fpgm-pruner) | Filter Pruning via Geometric Median for Deep Convolutional Neural Networks Acceleration [Reference Paper](https://arxiv.org/pdf/1811.00250.pdf)| diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 921c84c03d..b758ddc79f 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -180,13 +180,13 @@ You can view example for more information *** -## Filter Pruner +## L1Filter Pruner This is an one-shot pruner, In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. -![](../../img/filter_pruner.png) +![](../../img/l1filter_pruner.png) -> Filter Pruner prunes filters in the **convolution layers** +> L1Filter Pruner prunes filters in the **convolution layers** > > The procedure of pruning m filters from the ith convolutional layer is as follows: > @@ -199,16 +199,16 @@ This is an one-shot pruner, In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https: > weights are copied to the new model. ``` -from nni.compression.torch import FilterPruner +from nni.compression.torch import L1FilterPruner config_list = [{ 'sparsity': 0.8, 'op_types': ['Conv2d'] }] -pruner = FilterPruner(model, config_list) +pruner = L1FilterPruner(model, config_list) pruner.compress() ``` -#### User configuration for Filter Pruner +#### User configuration for L1Filter Pruner - **sparsity:** This is to specify the sparsity operations to be compressed to -- **op_types:** Only Conv2d is supported in Filter Pruner +- **op_types:** Only Conv2d is supported in L1Filter Pruner ## Slim Pruner diff --git a/docs/img/filter_pruner.PNG b/docs/img/l1filter_pruner.PNG similarity index 100% rename from docs/img/filter_pruner.PNG rename to docs/img/l1filter_pruner.PNG diff --git a/examples/model_compress/filter_pruner_torch_vgg16.py b/examples/model_compress/L1_filter_pruner_torch_vgg16.py similarity index 98% rename from examples/model_compress/filter_pruner_torch_vgg16.py rename to examples/model_compress/L1_filter_pruner_torch_vgg16.py index 4cd3d5d2ff..c54fc12119 100644 --- a/examples/model_compress/filter_pruner_torch_vgg16.py +++ b/examples/model_compress/L1_filter_pruner_torch_vgg16.py @@ -3,7 +3,7 @@ import torch.nn as nn import torch.nn.functional as F from torchvision import datasets, transforms -from nni.compression.torch import FilterPruner +from nni.compression.torch import L1FilterPruner class vgg(nn.Module): @@ -140,7 +140,7 @@ def main(): # Prune model and test accuracy without fine tuning. print('=' * 10 + 'Test on the pruned model before fine tune' + '=' * 10) - pruner = FilterPruner(model, configure_list) + pruner = L1FilterPruner(model, configure_list) model = pruner.compress() test(model, device, test_loader) # top1 = 88.19% diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index b8c09cac79..1c4807c202 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -2,7 +2,7 @@ import torch from .compressor import Pruner -__all__ = ['LevelPruner', 'AGP_Pruner', 'FPGMPruner', 'FilterPruner', 'SlimPruner'] +__all__ = ['LevelPruner', 'AGP_Pruner', 'FPGMPruner', 'L1FilterPruner', 'SlimPruner'] logger = logging.getLogger('torch pruner') @@ -99,7 +99,7 @@ def calc_mask(self, layer, config): op_name = layer.name start_epoch = config.get('start_epoch', 0) freq = config.get('frequency', 1) - if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True)\ + if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) \ and (self.now_epoch - start_epoch) % freq == 0: mask = self.mask_dict.get(op_name, torch.ones(weight.shape).type_as(weight)) target_sparsity = self.compute_target_sparsity(config) @@ -282,7 +282,7 @@ def update_epoch(self, epoch): self.epoch_pruned_layers = set() -class FilterPruner(Pruner): +class L1FilterPruner(Pruner): """ A structured pruning algorithm that prunes the filters of smallest magnitude weights sum in the convolution layers to achieve a preset level of network sparsity. @@ -298,11 +298,12 @@ def __init__(self, model, config_list): model : torch.nn.module Model to be pruned config_list : list - List on pruning configs + support key for each list item: + - sparsity: percentage of convolutional filters to be pruned. """ super().__init__(model, config_list) - self.if_init_list = {} + self.mask_calculated_ops = set() def calc_mask(self, layer, config): """ @@ -323,8 +324,12 @@ def calc_mask(self, layer, config): weight = layer.module.weight.data op_name = layer.name op_type = layer.type - assert op_type == 'Conv2d', 'FilterPruner only supports 2d convolution layer pruning' - if self.if_init_list.get(op_name, True): + assert op_type == 'Conv2d', 'L1FilterPruner only supports 2d convolution layer pruning' + if op_name in self.mask_calculated_ops: + assert op_name in self.mask_dict + return self.mask_dict.get(op_name) + mask = torch.ones(weight.size()).type_as(weight) + try: kernels = weight.shape[0] w_abs = weight.abs() k = int(kernels * config['sparsity']) @@ -333,10 +338,10 @@ def calc_mask(self, layer, config): w_abs_structured = w_abs.view(kernels, -1).sum(dim=1) threshold = torch.topk(w_abs_structured.view(-1), k, largest=False).values.max() mask = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight) - self.mask_dict.update({op_name: mask}) - self.if_init_list.update({op_name: False}) - else: - mask = self.mask_dict[op_name] + finally: + self.mask_dict.update({layer.name: mask}) + self.mask_calculated_ops.add(layer.name) + return mask @@ -353,11 +358,12 @@ def __init__(self, model, config_list): Parameters ---------- config_list : list - List of pruning configs + support key for each list item: + - sparsity: percentage of convolutional filters to be pruned. """ super().__init__(model, config_list) - self.if_init_list = {} + self.mask_calculated_ops = set() weight_list = [] if len(config_list) > 1: logger.warning('Slim pruner only supports 1 configuration') @@ -389,11 +395,15 @@ def calc_mask(self, layer, config): op_name = layer.name op_type = layer.type assert op_type == 'BatchNorm2d', 'SlimPruner only supports 2d batch normalization layer pruning' - if self.if_init_list.get(op_name, True): + if op_name in self.mask_calculated_ops: + assert op_name in self.mask_dict + return self.mask_dict.get(op_name) + mask = torch.ones(weight.size()).type_as(weight) + try: w_abs = weight.abs() mask = torch.gt(w_abs, self.global_threshold).type_as(weight) - self.mask_dict.update({op_name: mask}) - self.if_init_list.update({op_name: False}) - else: - mask = self.mask_dict[op_name] + finally: + self.mask_dict.update({layer.name: mask}) + self.mask_calculated_ops.add(layer.name) + return mask From ed813fac1da95e50919fff03e723464ad643dbee Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 21 Nov 2019 14:39:53 +0800 Subject: [PATCH 17/20] rename --- src/sdk/pynni/nni/compression/torch/builtin_pruners.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py index 1c4807c202..6a080e488c 100644 --- a/src/sdk/pynni/nni/compression/torch/builtin_pruners.py +++ b/src/sdk/pynni/nni/compression/torch/builtin_pruners.py @@ -330,12 +330,12 @@ def calc_mask(self, layer, config): return self.mask_dict.get(op_name) mask = torch.ones(weight.size()).type_as(weight) try: - kernels = weight.shape[0] + filters = weight.shape[0] w_abs = weight.abs() - k = int(kernels * config['sparsity']) + k = int(filters * config['sparsity']) if k == 0: return torch.ones(weight.shape).type_as(weight) - w_abs_structured = w_abs.view(kernels, -1).sum(dim=1) + w_abs_structured = w_abs.view(filters, -1).sum(dim=1) threshold = torch.topk(w_abs_structured.view(-1), k, largest=False).values.max() mask = torch.gt(w_abs_structured, threshold)[:, None, None, None].expand_as(weight).type_as(weight) finally: From eb1cff6d213307c7d7d33ff3ed86ddcc8889abc6 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 21 Nov 2019 15:45:17 +0800 Subject: [PATCH 18/20] fix latex equation to codecogs --- docs/en_US/Compressor/L1FilterPruner.md | 14 +++++++------- docs/en_US/Compressor/Pruner.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/en_US/Compressor/L1FilterPruner.md b/docs/en_US/Compressor/L1FilterPruner.md index 9a6e1c8e22..20448366c8 100644 --- a/docs/en_US/Compressor/L1FilterPruner.md +++ b/docs/en_US/Compressor/L1FilterPruner.md @@ -13,13 +13,13 @@ In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), > > The procedure of pruning m filters from the ith convolutional layer is as follows: > -> 1. For each filter $F_{i,j}$ , calculate the sum of its absolute kernel weights $s_j = \sum_{l=1}^{n_i}\sum|K_l|$ -> 2. Sort the filters by $s_j$. -> 3. Prune $m$ filters with the smallest sum values and their corresponding feature maps. The -> kernels in the next convolutional layer corresponding to the pruned feature maps are also -> removed. -> 4. A new kernel matrix is created for both the $i$th and $i+1$th layers, and the remaining kernel -> weights are copied to the new model. +> 1. For each filter ![](http://latex.codecogs.com/gif.latex?F_{i,j}), calculate the sum of its absolute kernel weights![](http://latex.codecogs.com/gif.latex?s_j = \sum_{l=1}^{n_i}\sum|K_l|) +> 2. Sort the filters by ![](http://latex.codecogs.com/gif.latex?s_j). +> 3. Prune ![](http://latex.codecogs.com/gif.latex?m) filters with the smallest sum values and their corresponding feature maps. The +> kernels in the next convolutional layer corresponding to the pruned feature maps are also +> removed. +> 4. A new kernel matrix is created for both the ![](http://latex.codecogs.com/gif.latex?i)th and ![](http://latex.codecogs.com/gif.latex?i+1)th layers, and the remaining kernel +> weights are copied to the new model. ## 2. Usage diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index b758ddc79f..a4b9cfb4bf 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -190,13 +190,13 @@ This is an one-shot pruner, In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https: > > The procedure of pruning m filters from the ith convolutional layer is as follows: > -> 1. For each filter $F_{i,j}$ , calculate the sum of its absolute kernel weights $s_j = \sum_{l=1}^{n_i}\sum|K_l|$ -> 2. Sort the filters by $s_j$. -> 3. Prune $m$ filters with the smallest sum values and their corresponding feature maps. The -> kernels in the next convolutional layer corresponding to the pruned feature maps are also -> removed. -> 4. A new kernel matrix is created for both the $i$th and $i+1$th layers, and the remaining kernel -> weights are copied to the new model. +> 1. For each filter ![](http://latex.codecogs.com/gif.latex?F_{i,j}), calculate the sum of its absolute kernel weights![](http://latex.codecogs.com/gif.latex?s_j = \sum_{l=1}^{n_i}\sum|K_l|) +> 2. Sort the filters by ![](http://latex.codecogs.com/gif.latex?s_j). +> 3. Prune ![](http://latex.codecogs.com/gif.latex?m) filters with the smallest sum values and their corresponding feature maps. The +> kernels in the next convolutional layer corresponding to the pruned feature maps are also +> removed. +> 4. A new kernel matrix is created for both the ![](http://latex.codecogs.com/gif.latex?i)th and ![](http://latex.codecogs.com/gif.latex?i+1)th layers, and the remaining kernel +> weights are copied to the new model. ``` from nni.compression.torch import L1FilterPruner From b6a1a7f19a4916ad1035806c1e316ef3ed096443 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 21 Nov 2019 15:48:24 +0800 Subject: [PATCH 19/20] remove space --- docs/en_US/Compressor/L1FilterPruner.md | 2 +- docs/en_US/Compressor/Pruner.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en_US/Compressor/L1FilterPruner.md b/docs/en_US/Compressor/L1FilterPruner.md index 20448366c8..c096a2bcf5 100644 --- a/docs/en_US/Compressor/L1FilterPruner.md +++ b/docs/en_US/Compressor/L1FilterPruner.md @@ -13,7 +13,7 @@ In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), > > The procedure of pruning m filters from the ith convolutional layer is as follows: > -> 1. For each filter ![](http://latex.codecogs.com/gif.latex?F_{i,j}), calculate the sum of its absolute kernel weights![](http://latex.codecogs.com/gif.latex?s_j = \sum_{l=1}^{n_i}\sum|K_l|) +> 1. For each filter ![](http://latex.codecogs.com/gif.latex?F_{i,j}), calculate the sum of its absolute kernel weights![](http://latex.codecogs.com/gif.latex?s_j=\sum_{l=1}^{n_i}\sum|K_l|) > 2. Sort the filters by ![](http://latex.codecogs.com/gif.latex?s_j). > 3. Prune ![](http://latex.codecogs.com/gif.latex?m) filters with the smallest sum values and their corresponding feature maps. The > kernels in the next convolutional layer corresponding to the pruned feature maps are also diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index a4b9cfb4bf..298ade1d1f 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -190,7 +190,7 @@ This is an one-shot pruner, In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https: > > The procedure of pruning m filters from the ith convolutional layer is as follows: > -> 1. For each filter ![](http://latex.codecogs.com/gif.latex?F_{i,j}), calculate the sum of its absolute kernel weights![](http://latex.codecogs.com/gif.latex?s_j = \sum_{l=1}^{n_i}\sum|K_l|) +> 1. For each filter ![](http://latex.codecogs.com/gif.latex?F_{i,j}), calculate the sum of its absolute kernel weights![](http://latex.codecogs.com/gif.latex?s_j=\sum_{l=1}^{n_i}\sum|K_l|) > 2. Sort the filters by ![](http://latex.codecogs.com/gif.latex?s_j). > 3. Prune ![](http://latex.codecogs.com/gif.latex?m) filters with the smallest sum values and their corresponding feature maps. The > kernels in the next convolutional layer corresponding to the pruned feature maps are also From 1bdaa7db2b215240d87f5ff23a43a1b628c928a9 Mon Sep 17 00:00:00 2001 From: tanglang96 Date: Thu, 21 Nov 2019 15:53:17 +0800 Subject: [PATCH 20/20] fix experiments equation --- docs/en_US/Compressor/L1FilterPruner.md | 4 ++-- docs/en_US/Compressor/SlimPruner.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en_US/Compressor/L1FilterPruner.md b/docs/en_US/Compressor/L1FilterPruner.md index c096a2bcf5..2906fde271 100644 --- a/docs/en_US/Compressor/L1FilterPruner.md +++ b/docs/en_US/Compressor/L1FilterPruner.md @@ -43,8 +43,8 @@ We implemented one of the experiments in ['PRUNING FILTERS FOR EFFICIENT CONVNET | Model | Error(paper/ours) | Parameters | Pruned | | --------------- | ----------------- | --------------- | -------- | -| VGG-16 | $6.75$/$6.49$ | $1.5\times10^7$ | | -| VGG-16-pruned-A | $6.60$/$6.47$ | $5.4\times10^6$ | $64.0\%$ | +| VGG-16 | 6.75/6.49 | 1.5x10^7 | | +| VGG-16-pruned-A | 6.60/6.47 | 5.4x10^6 | 64.0% | The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/) diff --git a/docs/en_US/Compressor/SlimPruner.md b/docs/en_US/Compressor/SlimPruner.md index c2e0325a8a..e15112711a 100644 --- a/docs/en_US/Compressor/SlimPruner.md +++ b/docs/en_US/Compressor/SlimPruner.md @@ -33,7 +33,7 @@ We implemented one of the experiments in ['Learning Efficient Convolutional Netw | Model | Error(paper/ours) | Parameters | Pruned | | ------------- | ----------------- | ---------- | --------- | -| VGGNet | $6.34$/$6.40$ | $20.04M$ | | -| Pruned-VGGNet | $6.20/6.39$ | $2.03M$ | $88.5 \%$ | +| VGGNet | 6.34/6.40 | 20.04M | | +| Pruned-VGGNet | 6.20/6.39 | 2.03M | 88.5% | The experiments code can be found at [examples/model_compress]( https://github.com/microsoft/nni/tree/master/examples/model_compress/)