From 3d9045bf88d20ea5ab0baeb3a598a4a5c3927a7f Mon Sep 17 00:00:00 2001 From: Ashok Srirama Date: Fri, 14 Jul 2023 15:53:55 -0400 Subject: [PATCH] Network Policy updates --- .../docs/images/allow-ingress-app-one.png | Bin 0 -> 31635 bytes content/security/docs/network.md | 378 ++++++++++-------- 2 files changed, 222 insertions(+), 156 deletions(-) create mode 100644 content/security/docs/images/allow-ingress-app-one.png diff --git a/content/security/docs/images/allow-ingress-app-one.png b/content/security/docs/images/allow-ingress-app-one.png new file mode 100644 index 0000000000000000000000000000000000000000..0e1e5fa498d2f4882f723db94fbf8a2f1c1a1c16 GIT binary patch literal 31635 zcmeFZWmH|w5;lk&0>L36xVs)Ckl>aC4ess)cXx*bmmoodySux)y9amo+2pQMPbWx9LI?>S2Oa_f0!c*pgA4=&C?B}4fO`S_R4px@ zgn)olH4+q*5)l+6lCriiFf!GLfS`)73+EPV*Lv+IS%F@tiI0b0M+GM@N&>?3txAPM zPGfcV4yRz1pTCX#@O1^n(iVN8b1~N&u53swjs;~b*azLzsqC*r%z z*2XaPDD}GFT5-28#Mu=c9!1F2*<)&;$xz8Zb zB1l(Zv?3@a@YN05qS3mw(emvrzbmg)4i>vrfb(nW4K1neG*ToOoC=To@y{MFT(sVM zelvSd)s{W@ni`S(bs(mcmmf7*Acm;C7mf8}lBU=^H_Y2Ul>{>*d;FTOknKYBW{IcR z8Fc1&26ewA^JGv@s^rKw9&tu&+HU6Ii6kLHV274X65kDZ9SWrqxI(?L*1n?OCOiRp zTR~q*#NhL12pZrT4gxa32m%CLK>}YK;0pl(9qA7N3;ae0z8_Md{{0k`p9=l&Ylw>H zg8Z_AA|k+VSzT*=eRG>H7Pfwc`$9lf(?)Viwo0GHIdm<|7_{{)bo3b<%`Bf^f#7oF z04~k+ZMBIU%}mX0I2^f2ewW|?uAgs%Nr-+Iu{Gf)QTi-JBxqr+PsGN+$iPU#15ZRm z#AU5#z#;SD)4yH^{^BP2Vry&30R}rbI50S{Fj!a{f|=Oa*};s=U}k1|pai{*lew+7 zBfYr|>0g!nyPgmFHoDeEmbOL~=0wl+YU^0o*>aPRJPZ2!-(Py_I~x5*lDW;lnguiv z{QLyW#J~vtyKdlBuIIZPQbvyYrph0T%z*9zWbiP(XXN@_{{MLLABq3?4HqyK$X-bUYA(83ImY0LAUq50R#|NZb^FLHsOoBm&2@mD|pz6*3T4?Gw6??K~% zH`taTgMi?J5c$9_=Lor<1k>>5BW4#0_2S|p5xrd|KN*hlsaAhP_-~o2Wo05O@|u1 zg6}^I7xwI;S%Utf^8S1u3|ZM7k5kL_vLgPVVK|>BYEKMp?6`FzPJa0K;Naj6vN#ze z`gO@m)47VE+tv0a7{^OMggY7E!;X;ULdb7bh(4gL+|hjj3n~2yu{jfck2pfhEKUiD zPV`S7IBTF6VA!n*!vD~W7FwWy(?|cEKQ)D4!m}%nNN5N;@(;~${0MYKaSvnCA0!a< z06mSd(c{GT&ngl@7Q>AwZDL?&Mg2hnh#eSdZ#;Gs;eR$2vK0)VKm$p;?H?p~5mEB# z%7>F~L5logz#B;TbYm3t;w1hc0d)iPr2yV~gNx{&tu5SB5h!r_6>3Lge{F z6Jdc7{>Py@{>t~EnC-7_g!;-CwuZDZUQF9}S!=TK7{VEuZ%UjycfuKj&1=tx@4fkd z8nrLlv%yCo^o|pGg^RB&RnC`L-O5nU*JP3PhRa;i&(kd~HpRMPEJly+ss_aqg=X(+ zvO8(8JyohTZBKT{awtAev=u=L%Y*Zev$ZRlwU!&awXXXGpCY(SR77I98(-%se?rgv z{7AgMTKo4{`4T^iP5ag>OALrjT^eDFOw+Pg?+k{q5sm=&^(F+JWh&-=ip>0CE)v0= z)PoyQeizo?YZgGJy_-RNW2^M;xjW#Pp^6tYd%gK(_4*59WO6>zpkNq0_8RlVs zWX0a3-;7u=A@eoY*0JzYAQo}YZWr`_<`tjT^K50zDHBI0^16cia7z`LUGaK$thbtU zt~b81Td#j0#@e}CyOGksPYy!)|4;#f;EWgA-}vV-iv6tInuzedWf*r&4|ygGnEauO6lQSaZq8IYT2~ zhTMrniEu{~$bX#xrrSo2=~9_IG9f#O^v@3wc(dgqxsl}L`hBk%_6Oo4b`(=$)nV>e%@~|298(KP=tJ4C?0o zf>|3fZEs&@n=U!okB`rD(^D2o)u1?<@`|umk_A>YBaEbfOxmNj%Seo@80Ql{kE^3H zk8@T{Zk%U~`>kk#_%;J+(>~cuwo=k?;&(&IL=LlC7Y^H@WfGr&N4n-vY;tnZ*VTqw z%>OZX5ccytL}W$z07bTf59x?RC!18HidiI@exj~^3Ll8W`gJ1jW(az3*$#K(@?ajF z+u8P}LBMIxvQ($D?7NT4>Qtes_3ngGt<9FXrJ`Rq;I-gjK&a`v9tV3Sf|EN*LV9-% zm1Fv|N(b70yRjxRvi_61L{7A?WAx(3g=4BN^loO9d2sTJ)EbSk^LU10#CF}4Rg<}0 zDAqlGP$?FxM`h8g^tL{@km-PssS#nSU*`YYWoU;2(@*;_^3+$D=p3zuF_u>aC5CKA zR`eldzRnd2Ywn$Cj~1=R!;we$#ojyNEOBh^%=%_$@>i6fzi0L<_B6TOk;~^rN75Qf z>@1Y}WC*XKh1b|G7H{U_C|v6&fj1_0#>-4+;i)y763-83)p27V<2dA}iqt8s);c06 zLU4C0KL4l|9!i|e^jPMpLk=V0jy!oq(c2aBW8>W8Nn}+ZrN?lyU-d=h3nbkBssm12 z4XnvLxkBR3uWurd57c=L_w;IkrSg~`wZ?hZ8zW`+xGt#=PJW454w7%bG$3hcGVKJ}>rHHKo%=I_)V_tYbW_4Wtf^?D=wC}9wU@2bpV4ll6! zLQrtW4erTJp479?^adk8^2pnHX*_PHhZU?_|Gmf)*}ej1vaKfXR5GXyE&+IP2{ZZKwO|Qa0vQOzWpd zH!sT9h{}u%{3})?d3LG?jef4u=^rO|hEsTB{X>5AO_|M~b3Ce+Uur%$Jhi&rtlk}g zO0~sjXWUm-_`Ccd;oyN86$?s5S#D4$m*kVfqrtkuhfz)8bt{UBp%>CeIEveBwAbCsqd&sGd&NTbU4RgIYhAYQ0$-NNDbKfwi39ogB6<@^{Ldn<0P?7t6tr58kO{%oojyVp zkvSFCESa41t+N}(Q3jvas*PFBZf{O`Z&sOwD_E6Qu_)3*o&Xr}zcwIzGemsDc_mi( zPH)Ii*Iv?Lo!6^z*ngU4G4OsbQ#<%B^6E@Sp%G25akJu)xv*UrTcNzMBAHcK#z4}- z>}pS-BM|K({IzYvP%<~A%OqpVaTONbadeqjOkYffUH*54F(Otym`G9~d!x~YT#c@h zMvOdN+h6<2jiFDMN4#3}nzH%~%4KqEo!<;ITW#_`oJf*8Bcc>T57g;%d{v-ecvjOB z+RL=_tUs9?m)vQpB9iJevsbGzU+0_1;<0Rwt>nxf9S8ZtW;aAl<;2;FnZF=*83VQ+C(n3y18`bszJZWXDZv>{AZL-{;x zN3@f-sk(95Oqu4xA&!Qkk@_dZl=zC@ojKIH93^LnM5#En5R{&hN%^yFE7$X(#L}vE zC3oomdynHKB?{~xrJPrl%+TS?KqT(3MVt!=z@_M2F|wSk8 zj#THGU;-aV2z#8|juhRK%iV!+R)w{-B% zmMIRB(JYzH`DQ5QN14qk2ZpCxuvmbnhEMiRqsx{4@KWPk`mV($IlOx9mlO0{>BfWx z$CEZ1T3`*Ivfh=;EzsY7w+!!HhbxG2$?Z0mj)+yNmBZ}qDEH~{#;{a5I%Te+{QF>A zP!#1LiO1i2LV@RKK-4dc~mE_WDM)8EinSt zC=9OTP>WeGIGHQzQ?ngHxGbm3VIsklciRfJ)X5n<2gGr#=A?}Z52CbvZvyK#Hc@;o zTVdFjp&3vJx?KfsY?3q^K-VON7D8F%E>lXDeB~1YTW|Cj_lV2ThWz#*D>Pn8FV{qO z2=kt|3L=FBA?N0~yhCKyHqp%-VG+zBCQ5ZvR`qTV$E&0q6oeFcJyDd^(G2o3Pm=}G zpK0WCv&xg-TgkwJ-`y=XxpmX^g;X=;goks*cYnZ8@=xpE=B`gem~VI`?h4uZ7zp{r zJCjs{$dbRFtU1 zG9OrjJ3rSIU4PW-`I_p#*D58a783=FLZD*A zL;spCT+@($AUbPMb(feqB;jr4no|73*{`8O%+XxYI3lkW6C$TFy)wNW>v&FwrWjtG z4q*;L4lQ80W2--leU4`u$fQo|X{;|coMx+azQsCKJv6F(mRe=V#Fe{X4s&bIV>PiI zInQG?-cgnrtnz_H`SFqUp8ZFttUfx0MDkCALzm1Dp(x6yqA2ZD$w;zNGKs_bwAcaXPG$(o0H`XiO*~v*k$?H0LzK$rqEa5R14oz89aP z#=SC4Lv-7`H?_Ajr&y>#dhJ6r!f24oER*gmAJ1fym8@T4S-J*AI*q1JU}c(7DdBIeGi{Oky%Q2yo=Dq?5bPN4kO zG|yJ)y#VgRGHr{9z$A^l-o$pPGXkQ*h%953pdmjhAGb%7t9OluI;TVQBQz|60n9)Q zXG#q04XP%*&O5S7qiO!*EnF|J=Id;8mFdepUa78-6fNu<>7gwQo>z`u#1t6kj0Th) zJnzgGMoO!~v+HoGgo*zhPr>1Wwv{p1ib#jOLBf1!a28#@Ghc`(`bS!!FQT`$eTl1H z8^F9>d%r(y#r+c};l1{K@#$8y$f<7Br?Db4li3FCB!bHT=Ch-HqPE(+dM#Mc{dD_nqCi@G&twADyO*BvPRo^5 z8|M5ZPMUP1pQEzU3h5S)e^nX`(ix7rTX*PQNB?_0%b`I14Y%M8ucSKP)YsOsQ`r0t z*Y(CJSkI!t7lQo*@|m0GkCADf-KO%bUMGV;-dS%A;RoSUh8`Z*s08TuD^d;T+4a>E zMcSG!ORRN;kYa*0B6T{&c1cvKzEGH{K5gW_M~td<-pnHtkK<$9OHuT8-FB4*`iBP* z%6&8?RIezdmh4pmq7*@H=jKKJv~yO=r}a9~wR38`^16n{?NL$yFvSWU%fEKHz7PfU zrCgoV!#cLMrWi4qF4BksyostowRq96!bW_RaL|Ata8X=m4%bw_%>(;ra)yk}HYSX_*X1Ov}ak1`RWx zCW-G8cOukkz3cKf|G{kFZl1%yo9RGSGGH2~9(DDmUE(q84^RP{!0+1aLfhC7+Bt2< zu?vJ-k${UkcgUuj@Ye1GRu%6Mh+tW}+qXzn>t`*0J5Msyndb^k+vrk&Kz8C)53o9j zCtv^IK^d@wv3T+KKIgC94)Bw21yw7I*0QEd4v3!_MLqrV7@f79RQ^-pZ;X!E9|)m{ zf60`P{Lh2G|9q=(Lhz5+#&d+v^sEz8tT)Er6ZU_Krj~xa`dg*{x!?;V1`wRU+Sdhe z|F-J?DFNu#0Q-z!{L@DO@p;`H$kh7ykB@S7%I+YS9stBAZ%1%Vw9stvUg+P^nN|63HmF;V_cKPS_Ip%_^nL@E#Q<>GXf#eCf(MS3O~re${Tkub z-y5X=86IyRU|3uEle+%|nppz7OPXw@xPOQF{~5Hu1``$U+3JvYxhVc%HUMh!R)M{< z?ZY3&#%qNLFg#1CEv7%|n24TQ+DH9{;!pS}2=*B@@!<#)`2)7N5)Cv`qLeY>pKVu| z=>4<#;Y~OJ=+^&Q&R=)3fks+?^$Ysn*8kg}TloPCTuB|${gaOG$pQFNmhKsS{znmT z$UrZ_z}gW+{XxeTYM_ycI@%}r`?HaBMlx2k3UXVLF*Gq7y$_`wq4@LN4+q%ai03R`gIS3px?^-ZKJyjX)`^HM`oHgVWTfbkU2vVj7IX|xCsKY zDw)$kL-ad!w#^U_4Zo&U>F{fe^8>Z9L2|kLs5Xsh7YY@QrjjNTx-^x^(-RSiXZ9@IWD<%AahWt<(uWGC>@-hh}lex$r zZk@M9%Y}}b9yU+byT_W;adGkyqZv=XUHt+O(Y^AWcUs}u<$Vd{-v;6yvpd2thyTK~ z?}fB>D|WsQv6AOnKrt(Y{JgOnprV%kStEj6+B||=ZZI6-`x3ox09#g2j@zf~^Gv}< zT6U|`jA*c`LNlr0YdD22awte1#N*Cym~&DX0H`UFKo*3@X|JY8X0gP@AWv1o!g`

kUk=p+*vISN!6$q`ZPu4O~ez`jRJc0xH zBfBed#0nFczKUa82P@ZF%g^wLBqzy2RT?YA-d;3C$mPoH9(0Ei#sa7y=V1S{muf~U z@xb%U-d*u}$aEM!tdsl}2K71Sg!C<9Y@^-Y_e2gm6}k5mY8DSCz0)jC;d#(3iMTBlA>EnEj1tGX`gULM zoGBl6fy<`UZ+T<7Sk<^eFC`3*VKv?^%4plmc~iGkEz{VAIp=M5nuZ41Z`C`tNv1lq zAagW-q(6I#;mp#PFH`CNc;~R_T>cFYfLug{xbgIl!?m4V!Zh6UP9ig%+3o6rW$F*7 zoi(EfMQztIdSxm<(YM-Hqi4Ro6N_c|L8k-WOX(EbC?BTXbAFu2O5$=n=6g)$T+X|5 zK*H&Zp;at!TL!>++3ex)+vTU!t6b^)&|n;H1r)&&W-BjH12M$I$XfQ4cj^67Z$jl0 zd@`6z@E7}B*~Mb{YRVl}cW3zKcizfT{a9*Ee&>rF+E{S2i-cwIdcR!ZoaQdq$V&jS zjV|9BQl9KO3)sTG>AFA7cQ-#?K*4&eG@`_~)@Kvcu(X3LJLU0|6miJyykm%42hU-$ zRkUbe6Zn8cg^!5u_ui|Ll5}+7k_t_U%EqPXKZ+5ay5Hr0$KBELU4JFLm$l%}`iLFvNv82;dHoI5BBj#teoRcv8X@AYcZlL?bneNzc0-@l!_j`p6rRb0j<520yLW93gsou$P}=b=0GM4cjtij96S)t5 zuBy$+_a}E`y(!?7mh_Y6r^hJ402Eb@iN}*l%_lF8MvG=h1$Kkg`x^um@cOJ^v*pl& zlR;EVj*jhVmDxPZ0~1&G$m+IkV_9d^nMfFa@#iEq!C0gRPwP+CL~DQgtQ zvL&+%H=gqaq(Wgk#;Is9(821z?zt6@-+3IG_Qt*|c8jWPP-J~Ga>JmWmm%T9kBlUk zfjyfVP57vlCPcd4((2rscT|B&0|d^tk58>C@?22JGeE&m>~jSg}_H@z!3rmB!zr-eOvn?(PE z*=Sy6w%!@u;GN5N^S9SqJoc_jqx|iJ8un*T$@DK@QA~VV1_JIqjuzS1w@^u3_XkU* z@>}e-+a>N(Cu(ykH_H^_W5c5Z*yCh69f5#d??fH)rN8{NrDC_Mg8y0$nd$SfVU45r z8v(`pJRj{Z#`8_-*~E(@LVzQ_vd!8yjiG+3wfkIxe}(s`^&|RmWdn@*;@B?|BHPs zO;^W-TC;BiKM%elx7oaC#{?x`qNX=LB{GXJ26xduAgzy|)d<{pzcS5N|R& z_Q|gL=Cs6QYkl6cdh4Uo-t}33uga{B=Njb`r0Jy`lD?Q z_@>FnrnDwrr3;18C%=cu@M%AocX(@3h2pc__M$}_YM`9PPrPfI2=;g z%fl%oqylgvucC!e%SAZRu?AoFt4Kj?{}D)U6G}4OF2pkJMI7Zu>v;Jm`vcA$v!eTV zmdB_MQmI}qNT==fiD(q7f|o90t=8=S#S;verUj8PbLb9SdH%LyLei`jJ%e|i0o&vE-2{WyqkoFc^T z`>3uvTElTtFTXyh%`F(YqqH4XfBm*nfsN~({mFrwsmBtU4f3YNe3|q+;gM0?r}+=+ zTn)&?lX;JlyxlGiMLP?-*%GCyN;$ApV7Dg-e!G^3J(_FmW|q@#L)@N$e7IFX;lqw{D;g^;O!OB0%UP5VCrq7w;;t}zRO8*H(PEH&H;|OtTxnXYi}Gps7R%sff!Tr5 z!J^Y(Hxp~}hXxr=; zFIV<KLLRS8Dh!Teq)WKe2KG+prju}O7KrkUOOgPFzo_8AWhm~bgD5Zf z0%;^!ZKS-$e%o`ohW}xwCwNag03ul6*E%Lev8Q1bwvkT>lRXOMalsqe z?>Ne8*uW=`xK*mN9TmW4G4`$O?NGD-7*?oM^0=7_RSAQ~sR%#+vDI5t zEl}%#ODp|)mn3S(1c%FI;!MDD42`^i$0`f{59EV6(S(UyJ39~KNdZ(%*pPP(E$R_w z7M#l$lluW(xcQnXOENoVd`54vM&5Uf2)|`uya&sRPM)~dYXUi$25v--T%@Kag{@X4 z&G9enuGfLCy9(l2k`IEkvWb)pO>OgBNx@!2co2sLfj$uY0ry*Iq!-rZgg>^G4V!Z0 z#}oYEx=I8CLbngD!CuDubkgl;2W1{lko}B?l0g1^=4FM^qG*-rVpg+ti!j&Ybm`@w zGq#{T@ireGr@J@-k7LgC$uh8;Db%OCzT|Pg-w4w05eC3wW_L&10*xle+z@dcWl_POAYWvE%;JcY0OZkW80R+(H!OyOY(Vm~z(G@p)Bf-$$C^A@bpriC zKZmC9^)cjSQWpmSrZ5o|9N(}O7M=GNGA@t6*VokNOWI8HG>;<&?CEomz^WTiel6sb>9a(hN@_bYT)LVIyjmpi%I>M z`{V8Pp(^cC1-tAVa|b1Oj~0S%-3g|gGtf?5<2VT_o}AY?!@!3J_R{gE-lnvxZoV&!@u;^ zhjL6l7b@WwEV%ZZn|ZL$K)U@x>Q1mr33|A@EH#+lpR0{Z_C$0WHksx+FQ=~fc$W2? z(R+BO`-^UoccIoMc4>p|I;Uxm$UdGr z6eadXHkr6KxN|0Bx4&_@*fE@JuPp z=Xi+)#i&R6hVLX&zoK;t5tC{D1OcU)jfM^3JV0`7{tl~MAo%?*DC0*cN(21mVSauJ zx$EXT&0FmmzpbB*(kZMa3bGjz-P3qjVT=_hYva$x>DfBt$}Kc|iR{n*-f^=pe!RfH zMpJ01X|i=|C~15P^k8%~F_>i#Q6Y91#{0dv`Bnxc$R88_k?8omg{~w`n~m`9#xfp+ z=!JYRZ>-DP^g*%s=Fzlkx_CJpxsT7JL3hAiA7Fb7vderGZ4^Cc9s zq)&|7Tw7KPU}fkontl8zjxl!#q{JO#k@J03)8*Cjc`hW~CEq zK0w3sWm+$!PJzIRRyRaU=teL-PWw=jh`PRI^`hz`;n*%vN#yF<;u zV~YK7C9_SshK(pVMp$bgn81}>Na7WP4IqE_S}cq;kr3xSlWxPA0pkq-(hR|+GIKR2$_#hPX9sRh#u#Nxg(`VfmudV0$nn=#r4I-bMtKWO07jSskU; zvc53s06B#A;yxdlg@-thp^pzyAPQE^J%y_kZ)ptRj5XX{8=4{VBQVNzg~B|Jp2G%J z7>$1cOE*{yu=3ZuAk7*KvTHbtqyn>(9&hwSGo@?To!1DFEP*%szE71(tfqxUK#bJF zjRh0`q}T|c4r?YuRe;0eas6;LJ%ZT}q-#Tkg5x@{cocF)S=|ySB@!pxgl&cboeV+A z#TvbK7Yo>w(2fo`UI9<9CX=l`3e*$Wa0vmncC=J)f?;%Bh3%#Yl}8^N9#!16i?*IVz$Ezn0kB=#?VP8YL_I<7sgxmD)%lHC)iF zj~Fkg9r}S(X!*rAA#C1CQ{t2uaJD~U11`^IN?EDJ5FUF0$ssiu1q&S&kzh&NW^1Ur z$yX)5h7W29l`j$f6=i2KlE21TCRTgZF$QhsZVf1UCgn;tTXOFX>*}=gTkN@8upzoX zt*bOa5yk7k9In>6Y7+isIg zvD2;o{C;ki%LanSKxe^6<%$o}>czezPf!A|^NkReZY@qJI7_+>O9Q#-<)H^>Z|K6o z##7;%UBQv=ZZ1CtovC6f+hEY)^g+jd1O(ngAjnbTg-Ukt0YBe`@I6AO8ha(ujt`K_Ix`^TVr#KHfR=99i`|9 zVDlaiunzg+NK1?daUS>)NWFs@T15!B9A`Kbw<)k%Lg>_Lg!eE83XB_S2HEb8Qq$UP z=ifxh1!J+#%|(+qS)0se!zf-Tbp%G(N{^UahUN0a5Jt}f3*@$}8&f2-w-g!$D$jfX zk2HevJak7ne75g3XnULgD+Ybu^l%e?lTR@tF{}UAqLN!M5V6k9_{TR}zS`;|jg+y{<%7k|?D9USOPX-!uty-6Wq(HM5pplj;V4Cw>X{iRr>R@i7dQ1VJ#3qq2=VPPfYc<*}IwnU}I7nf9S zmN|2obMSVV^*Gv}*f3KpCer2VNO@<8JGN*Wxr5o02?>8p;3IELh0)lyJ|4u0>B=Z7 zPNzJ6@T?`{J&Y#rYP@j@_N4m~)5Hjc+PfYS{T@NX!48YoX&D#6@g~C9-@KAC39~|k z0l9^&B^ZR)Bkk3wN_c|FOlh8tN!q737tuM|PD!@!2$j5~|L(1fX0hc`p3>Bz`VPZ$b% zntZ~$Ge__W#}=*9%xOj24Us>Bzj~WZp?jl6#ZZUTPjA0(9Ndc?yY2faeKT+j*PR1C z+rGHeDjn;B70O%Sm%ijs^75|z(&^l5IXLQ2qya{EY3i`UZE$(`MqD`AgV$omTTw0b zZ!Nq8lFPJ7;2dvjXaKz2P9!Xb>d;_t0ojMU?O3bLcZ8&<>ZKb8Va24jt?!oEFH`ml z)U665S=A%TDik*otovs}>PzhM@*x117tkz0aD5&5!e@CAu9E{#xhgK7^vGs>?W5(- zIWbF>8H~-c9WW0f?#84e4c)}Xw09l+fCp)tL-d7@RVNvJEC&UGZ-Cx}QF1bgxUv_L;I5CJH>UNOVD0C^%7Y3t@YH zd*4Di&p8@6DQF+WpOKuF`YnU-gk#7foxssj_2}68(U=jFo$<_=qoxZy_?C@ImXd6C z&aBzPugC(P_O@-QvT5ln^nROf27id;zZpCTn6x{XSY5QVtf*%kKzl{968G%75}#Pz z^7t_Mf~}B;?&R%UuTBZ(w?cp7iS}6?$I1%|i;DA` z0}Ttg13P6ULD0f>dig z*iR1#tMDS6$M2kU@T#w`=lEr3k>ap@o^Pz6ynWx|TKzs*P^w_PPH=0MGS!lcVTjOd z9Ke^}o1a#M&gL_1aMM|z+WrLi9F!o&dS)F+T`FZ~49sCfYMX(ts};w_4xQl$Nsfw@ z^Qce5<~`PQ<1u!M7g(n-pXMp4mJN$4Ub^Y{U;a$TRQ+O1_=4ObMd`)4eWGenwf1Mc ze&iBu&rq0vOS=?}m7-)qj)EbM!a&-QOjU<`EZ5m~%U(}(tgeCSDNma5QarqN5WMX| zmmNAB6%@ZVT&G;;(o2m~YMGihvMoG7J}17(iJ@@}|B_j~S9h2&;k){1>r!b|3hmmf zIo;QD)mHJKHX6t<;#MzCx*1A*P|}uYgQObt_UwDBhGeZSU~}jnMs`K~0x?uSg&s_| z2wii?g9O`sQ~y)rM(LwDVk*~gJxAhg~>ceoN-$_kpI07fLLqe zcDf~N!6AIiMYtjdQoheXt>6{oMp#$)h;^ruGt) zeJwD+3sU@Aqis%UKa&f{o-@9_WKcjFG{WI=lQ`F)0O8+BU_RmJ5Exd4#mU10$d%&rCn?(SQ_nAm16|pG5Eg-1I)fPXqjaNaUk_cqSW=jBp|TL4@ZpP~bEk&E`)! zg3zA10&s6p1pgr7{~O%^_m@+ft-~xhitDiZXU+gXBmjt5ZVrz-6?D%3cPSQu+jNyv z8C7e)e*H3iH4d;ucr2gY-%t1JmLtey9)J(M3@7Dr8-ja9@|(f{rwqF0L73Y^`|h;9 z(17mz{~vJ!{l9UByO*71mD{wlfJC^;8{0LE|uX;Ut^|}diLGBKrdIYuE!sJ@CpKosV zM(_oav_R|;?^>5IKi6}tjp}%L9JEktndF~`vThAKtK4=RbO(Qh2{1WGfVWE=bi5?{3;Wz>8&-+E)&$rr9zD$Xd{$r5+-R zDYEP*B3ziw)PCoBg$`%*?7WN=eJ`H*d0vd-5E_su6C;4?AU4Pa!}Ah&(>^-h95^Y5 z7Z^qa-C$6T#e+b|9U$cz@VqoBa9027Jx?J1#&DK{ffIeRMgqBDQFBB^hMMqF%EA9v z{?+P4((Pq??G%?RIvmaO>H^di@XRR!<}+ND1hH2tMavySPSBYnvYg!3$^EB|tJDjty3 zc*$jh4u|+G>4p6FzZh6rf0<3wS&PaR)JqU6H3N@f3&^G_9{LVTYsEKNS&@XF0-3<${2tICP1WGSYGQLIZ@VCke zQfR47NHk-FLso~&&0r>-j&L|M63~vwa(#9vE{p$%^ItW^8jX>L!^t8*F2$$99MY)- zU_A5!`TEVC2vYA&Ai&;Tdqp8mMJ<~Zy+2zn<^=^q2B6oFPD-(Tw_3sS78@Zo&F<8( zgozP$TaOXkitl<3vZvo>TuVm@Kk9X6>LnOQ(iz|cVev%z%0AuGz#GVLTPYQUR}>-I ze!0SfXwpT~ygdYP(R=GIaLJ9v$Yct>L_pM{y5|ac;=K>X6j;P&1|O4d=HA@*vIM1& zNcR+I!R5#+O;i&@nB)|o+(wE`$W?zFQ|lJG5xIgG9_26S!?|Qtm&&yd;oJ=UMswRoD4+m#HHK_j}s=~Q_A6EBdg zXd?V@`D+jt`)VFk>$X?LHnh;_AZfl(Lu}@5OfC{hHr3tg0VFD8G`k-tex7y*W6PQ^ z*OCGl+RKuwJiAmrwV7s*=79uOF=Rp>x(JewksU#=@~ik|dXqS-4IFMx^=mCxz4sSt zWho>QscNI$(#;p^0K7kxuOMdYC-j1n-N_oc(bUhF)K%uABjre5O`c&PME^#hhU9cP>fVo5##C+|0{s_(0 z`Y8k_^OpOE+(VS`w>R_|wmV}`x@jy{4i2%ECewY1>^2#)*%H7Bf|rfMLWFJ`bS|D* z?8Om;D5VAm&10w8Y~KFxpzpmavG1(+dgYZJbt2WA95wRvZ}^?0W1_bsxSE6ev-9ca z!G)0*eS5pDXPh$JCZr-*^g($|%iIOtXA9+q!xZu5BZ6e_iy_1C9YMg6Tkv24TZAXG z9DDvwVy#W4Sa#?T8`B^rNQ2Ai@Y@1`vbvj%=}gtfxvSkrjhh>rt%%k|7m)$1=k3rG-Wh(!cyK-Kjo|SZo0*txB-m*B&yDe z#=)aI@^6Xtj{GVPM|6yr^W@UNndGu%zK67ysqFsGaH8lNzqPwAFfJVD-M zXhBN)(`@*xnoYzpjpcXWnaP@>yo7?*t~OuX;Jrxu3FIYZ(ga`#d5_xy<(pic09PAK zr`e=N@i~DEU^&GCM}9>Cf+W;M#r-dijSNvvhlB3`4Nq(+kzH>)oL|BkBpG>U;=sCZ zk)|fo{?ycSnO>8bcb=RE&-x9Gy((IcKwk5gDY;^dv`PDo$3zLgg9Yt@x?syZ{^X~p z{XN-PgL=hv6X@L;NcqrQu=|5*mdDjQyAuocqQfI&9@*lO)4g3Mml%|#NJnks(1X{geEQ$gqvxsBDSqA02we@8>uB70-8pGjN*?I77d_I!=0YF|s>a^CRQECjy{P z*Wwx==D#-DUUh+i(P_IrS+7hpUTN`?$(11|M3TSEYe5Zkdx3~Wlr51INi67pU4LW_ zUY@n~bW-4!UXpu0*I7l@cEOMV7c4i=P#oEZy(d!Td(((FcQx=3q~`9Z;3P{P!@OhM z2ela#!)qiW@@mWNC<8d(-4h&NPG-NaWGx$FNT0v(%kr?O2cTMk*6lPKo#`E^GsS)= z;INveTI@%pcl7{=VjXF+jLyD0QxO)_e0o%F_+agkC?=CFsvo&@_mQYB#yDpJlhtI~ z>=lJXB#9qFf1Agfmuk(x8E=Zo{0GUVLe1ehOhF5G>~|VP8jV?dFFuZxx^c#{x`gJf zqbRUj)_=e!$tf0Fc72E;V5`*KVo|{n%oqTz~Ul(Sxd0eYZ%smBeIaGNPJOxrN?+`e$oBN0crO zZV|ZMy=mSP+C3c@7aw1(m#(VJW-Cf!v@rO|6LrB!-aJTpmHD5*T$nEH3we1X4_X7q zoLes(a~2$}Br`t$=)(D7&RfjxF+8bBZ+TcV%u{?E|C)XcO;sO&n#;7?DfX1j$LnGU z3=bA4^DxP-Yhw)V6WEG-f_xEHhE+2eU!V-PzU2(cU`?;V^sr3oGP2h?v?58;w$NCu+^IXa;m9lmt6t%Gxsh*tQ#l9t+v0aN6l2 zkrR@SnF^avBB2FG=L13G*&8v=yfrGkcq^56w6koINMVc4e1~ea#3zyDk>Y;VqNQ`2 zO?Cw6w=zXT$In|E|8ISF4+^d#LR*P(2%Z}s{!%-h*xeM)|>qQ{<*p%E1 znlpUbJ;*b4=vCX$rB*91=m^GC-8$Q8#*|FSzb*J)eka*8c&12hc1RaFNFSzmH$nQ= zvyq3oJTnR(VS2}zayU|E7{vw2}F(7krO5#@cmcNWiLxcSVF zDTf)(2V8;3^iWcT#>bnH*(@Y2-sJLgkLg0iqw38%Gu!(aE7pQky+Alv?P7;7%E&n` z7nROo$b|0{E(y4ZV)_-s!TFOT?OCm9S!2L_M_$j7ugwci;*t(c<}xQ1a547AQK%6? z!fYt=-_#S2mB^DXp!>gSJL{mhx~d)~y1J@+?X}L?=j?T!=eKiGn5_0LZMS|$u(F@u4cFdH z!L$@DyK1nTCL9ClWtX z-PN2K*+ds|c>R8C>YQYVj5RUoCay(tkNA1T4v(5m0<-1la(n^baEheQZ*KN6)2W)6 z*WwVVS7h&-JcsB6Z{50nU2X2`lucDMR^&8_Gm#53IN;F5;oQdkB9G^j#A&KfWhMp^KTnKnyl+|CAU!Gg_ z=2@ko7U*irl{DY$BH5JN3tomG8yMa1=f^=HHA!Zzc?vxN*V9z8iG@-nYN!FZJSR5i zs}|S8c^lZ7q&z5m?&WgOmua+w6mCpCTuNtqLazq`V%r%B67Js2-a{}FI+KSA6$ zsV4~himx`LiT4-e{m5sTs*j@KVj5%JS$!jDG{-u3ADCy_d#{IJ8)D88L-88Eqvt0?sBSK6l_`8Wf?cmS6aw$JwjVx#aorCPO5ajwaYc{E+X?iE%6^Kod}Xx9%wFFVhu;jDywv8)SxX|p6S zcmTIt8|OM6a9~0{1Dc}ged7uodS?4xBU$^Q3dp8yyH(X&tM%H3u}oHgnOHY}6yFAxt z?sQYA-;FN+!z9t6`LG@=oe1m!h3O8mXhzkMLHQ2K6*bO8eE9Yr66tnp8ZJ=HhA=^i z=iOP!d15xsn+@Ct+xm{Dn?&>J4bG%}X7$n>hD-){#PaNpCm9mFSme+?$k}pfia&JC znyxVUDZ7AfMDh6Z`G|eMBE4(CH>|8~wJJRizEDBT6dk$?8A7FJ2K*}RR9u92}U>XK7o@9!>!jbDNQls2nbbwdR;u@e~Zt>-7E(JE4h*=!_0Rm7o-TI(~* zrB1ID#%HHc8rcMugX?2qf=<3W@Ym^l$wXKXob&0dv_`FENG7kF>MH^J53uXwD&bkt zJEIRYX*2jxK!y^~>!u_m^3XF=$V-yAbe~y=u}rxzj=*w1bq~Zu+-R#*(?WaPy4vcb z)oC%Wo>I3NOPHYTL+C7_DxXOQ6;3BW&G6sTW7$Vr;OLBZqD=U73HbP{0DVUTEu+gK z4Q>ynzfuI^#eb0jFsfFH_-K)ewiYKqJ0JA21wHexE7Lgi^zCJmSsPYco#kD>E>o!e z$WF0CY)n8tE(mlBv=Xx0H0CYf^PS%5R|&1IcQjmqc?q-OmBsU}68e(VgF47x&Zl!& zJ$H0lAk-hyS7Ku77E6GAIJMSpFV7{X2b$PvRI$`hi=Ms9^70c|ja|?3(zhXIJ2m^F z_q(EgXhnq_Hu?g}HIZHBD615EykQ!M3qd=Uy@kD-SFisCakGoJcp6>G#pR>Mt`$We#{>p+cCJ zi?+{xFbS00@Gf7$W5*x+yF#B*t`De>eH5v-eCg6$*dM>Gd2aJ4xFKRwY?>PZUa<6Q zrICJ|60%(^F49(Hm+iU0(0QNLe6Z>Ji|lLbhcAkXWCEU_ry{BSh)Ztyud;sj#9umW z4*l9;^F2iF2@``(#26dxPF}oTTkSXJQyTQm=88cu7M=&w%ek04A8zue)uMfms8D`( zRR!7UHq%4t1iieWDahLq*{u%e5vxB#^9gn4_~m@$y6ue*cX@q$6JxH!WD7sX17Bmx zNB^|+%Mc7EVxw40z_+d+4$bnikNCI<@qc_R7yd9yyecE4d|-6*Xj!yC$x|{qXm=F* z4n>(JUX)Yjt561SGC@ z9VI-Rmq_I+NP|2KdIAw}6nI6vUt8{6;nhGIHHH#t+;-WgYi-<$(SR+TvM2_RvGGlr z_1v+#w`0@c`08Bo*wg7FHvvdF_j&E@j7*x3b47o2f+E|%Xi*r7shY0(Qf`mb@zz`dAoW<`Z_$3=21?P~t$$gq!HI(BsB7aJ; zFqi8QWf*0FYV>h(9x7fSB|N$#$3Er7htUk~Sd&swq9_HHv9B7fXyuyv!R3IuR*gB- z-)fWRmfGhZ7{=QWa(-7W;g5_GfZ~#=Bph7?`z)&1EBfpgY9-i56+<}26jNl;YeE3R z^&_hu?+Xorz))TF1hv9>>?iqjjzIrkSgV*TZl_6n4P8?w_n4EKY9=h1yjG3C{pQV_ zgP9t}-$;xwat3GQEoIrGOmMFuGx8 zt5?(0;P6T2qR7*p9cS`z+)dc4+ds;^cxiK9$Ow%hHJQwxEYPV-W^=nz(O@%bgjoBB zEheS0J9iRH8+~8paj17-RFyjr(-FkGcm0&lM&YB}P{={Jp=0S{RT{0Z7|eVKoy<$U z6E!`NWg~?jWjZ5lm%rDWKxDkQvLI{Wu7O=S#h6oMh$*o-iAEWMDF+rMO$mqObOQ;<;B&Adck$uPqnuL1n+gR8n$nQ&o4eTJYomDJ03FM?br1Py_ipwT;O z{%|r+_Rfr4F+_!Oh7zV!L+#gF@XO&<04xi2l;Y*~Kq3^dpjkxaE2*^yXoYOJsHQW3 z4H>Ri5adSJs_dAdg)?@q$rc8CHO81Yt@**jZtG_8nH9@Dc{O6xua_99m7}Y7x$0Ql zyZAg9&Fd)_8uEU5_Rt`FJNQs`wRCW0LuvxA8aP1!mB7!&l5;8ZHQq~K)VyoO1#9Q* z?w@ImbmypPa#`o2KNUP-CJT|a9jqRQ^ElK|7Rm z=lV85&R5itYUyl|yKu>x=Q>HN+$>4halYmkDJ8d%(WT-EISKe$v{aa#&>`+?gB+BS zq@y2h#Z{J4^LQrK)s zNB+Yd&}LRXO+e5PEF6VM#N}Oh-j>&Oj{i+tI||Zs>Nh@HnymUYoxw z(SNv9PL`{H`f;|+u!2cI>@<+VYyvGz0!WD_$<#(8n=GeL_p2W2>XOM5CClRX%7%To zW|;IZRmBnTA?b*+7clFYV^P6IUx#+7P9@aV;j!3%!fGS&yB>i?$J6Kf zu2Eg&6Zsk@kTB`o_ZC0>1BPklLL+pI6>?Wv1xSJpS@TZ}FwCd=MAYOwUU^YF*^Caw z=H39)88RT|-k88GH}h)30JEfm{D|TSq);yCOmvyb@cheZf({_EEa2fd3Y8!x;j7a+ zeLulpFa&q{c!n1N2_ioJcyrVAO5#;*QD>cF_2iEs`8&Sd>F}Ywp31#I{pDJlk1X

8hOn3pCX*bqz3PV%LNTqs{YUAn6y`uk?KS6Qek4ZzkDv)@5egcOB&aW!I zhd55+1=&Xqi=sH9Ks6Qx-50>d2QG^Nl~H}m8Po&j&pErJ%yV;b_{aivECQHfY4VEO zD{f%*K9#rn)~d$_8=uo16rWIo@4HW&-^&PU08rdO#LdS+PU&|StX zRw*fo@ZE92D)yCEhU*}<`f|4!DCKK!_>YvO=52{5L+9=a0%`0sP^cjxV(vlC%UFl^ zXo)9&?rRkUBr56tuG!}y4U*jYTL8xKJIpsE@PkpTQEKnZ-YD(Sm2tdPiQou*ekeK> z0V_P&dyU91lN=^Pfo>`{JY=CfXm{*lSX(f0c_J9Rtq!r;Pv8Wis}wmeDt$k+@qjy6 z8dcHQRN0{*kwF8U3)bOE_367GxP52QDfC>eS4^&q@M4$a_TJWgEQekr3@hH|j$@Oz zQ)2q*>G4n9@qN&>Mzxhf%}Jt|RP4~|r?8%_&$@q-W~E@BH`Lhnaz+xBYd7nn36_T$ZK1fA_1IcZ41aQ#EQ_JW5$zeAaONqHv`4a7IX zLiDSh-a0tNGL}V70r0SqA_HiI#@Sczt}OZ!c!h*^H+`V@YHc6BD7!oQvpi@&{JH3W zs(y#;c6@8l_zv%S)8MGfu_h4mq)#zE#_Ulcx9wDf#dri-kwPMxL3SjHfwizJEcpA; zfGgU}Fo$oP+iQ9qaq{@drAx2pApH~hy`ktVHdi;bVI1LDj){ezI1&oyN6Wm#T;cDp zPMSA%H}k|2RDF7W6~LDDcA!7#o8H3>yGyOGcuPvjev;?pDp7~AoMaT-}sri z{Awfw`yBf4;i+=H({YZz$DXV0!%&Wb{;YixJc_`;3x6>+6W;FFQk`;(vc%x{a<@rr zk?~bV6i=1(PhU4n3VslpeHwe;VS*}=_gf_-l~BkqN#EN+*JQ5pvFptu2sdyVoeVmQ zetLFVcD9)+*KCL=a^K7|xjXNz$>WKl;>jHtdF3)lU}7jEV#DSBp>49a#=r>z{GAKV zgA3)A`u4~^2g3&S*Avwp9&sD?BYU1Y@BsOI2ILr;vxkETWA*!;ZXfyH+gfo$JpSNH z)|Dz1r1pijZY)&IG4dT}yM7RmlDyzF8X@Q}rdz#-!;A(wg#gBVaDVb$QfLF=qVea&%x zgnSB1b{K}_7=9ap5m5}B7!TTo@58o>^{B4~uWbwjt2zbT9F?lz`U`turz3dlGm9&s zbR1QU#%$O9GDgo7{?IE>%Y{z5s{Mu8a_q3R&-tZ-UKS-3Z~2q0ax9yzUaKu(x#2*p z(WK!Yb(lzgdfKRXi!8}Bi4*+2dsTbkP_CXhWOs6Y)N`O5A^Wkh+Qy$46MfPeM*MPh z)#14*gnF}!7}4-|G8Z@(Rp-HZsN0vUUjtp(Sel!TrUAr|OYC=^7=c?J7lt--4T%-$ zbt32;v0vd@A#vGah(q~!qc`MN20WZ-s*9aY8@ddnD-_}QA?oTZ%JSJz(=2N@opd4_ zu!K=ECA;PinK)viE!&Dq1*@yUVM95r9pxbhfsH;RH!MW*AdRES*QdNBss?#a!mr$q zR5)-sHUtwh{OI6w+Iv67*z@&;;fwxS*;8?{)&*}KmX$Fn*a|T}3e%6y!ZCib=r+OH ze4dh-M_T>USCuvN9LHMd0dEKlFy10PEPP;Ns4-f#lpv~@6$5MiY-gnLZ;67{0lL{) zwWEp5=w@DDeGRHh@YdjK&rnjV()Z3wH1;QiKXzlOta~x|VoxuQ-u;mfg;NH#7((cI zDs5P(NT!URq>RBFqMVe7UzIc3ZiH)m`kTpd8{#`mBHUE}xiplAhXC|jh>v;LsK_IB zE4N1sL02X`-Z;)7t>QVYXicqKRO9?mMb67i+evzy<7*#9K7>9sa zfc+WW3>880+f$0ZVA8*JrIBM-`lP~k#_8v9)-Ru1CjY;|IuW6+g zmxD#FB{wy~@&XC+kBA-iTKatsKHGB^!!yCa_qh69E>9wTW6|$dKFT%BdJ#Orlr9a$ z5_2WLAc#?wKtSVyu8sVP4AHvPLnvu3%zGDH-K7AGKYw752r{;*92|os-iDI8imj^3 zHR+*pYW&Vx55;_v{_jD~#|yH=nJmVL0n@C!eceiZw^2cSL1$eV^y^6-ObfI1opHgJ zfUFIoKqhU-y{C;Udp2UYe8@zQ-tJb`6PwwOoRvIurOPDu{JmD~fbKWiP^ z4}BwbP3uanMyC9)FBxtEXHVZr;c)ol!%B!kGk6B5S+lB}rEx*28gu}ue z#=>w?(#g)HV!aOvg>6H5L?A0=26(;G74HOgPyK}OQN#dt9CXg4JqcUykAeoFF3$VY zKomxb%b&D0bY5*F40&O;gHsoxb21tp)180Xh6&~?(3u#6C4EkMEm&o-u)577pb!N9 zvy`pu;4s{6o?cuYici0;hPsM-2YC{qs*f*bD45`0Z;+jB?3?*jfr3%y4>`^6j}rPf zl_Qr$4m$(ckK?{&)ROFHKdF6>=POTA)^yhCZhQ=#A7%yE_J3&0#jLJ=QqU7h4&S1d zxts+Qn$8bg&A!ycn9SvGikOtjY?aLpGLx11xQ+};M|j8#qN2&h*#1gwLFcY^gO!M- z^qSxsyC4`&_*Az}$_}wvYL!RAXMS`&B&ujbhD0GzENK-ayA58AauJ|lF$^hTQKPVa zZO6&PN3)tBlS8BP7wvIvLn06$WN~J0B6@W@U-b~|gS{dQXlQjBgw%SDYbNgQfuS&h z!WcH|)VvVd9WuPrapX{Js<@_tOdj9f#81$ED`|{|vp~%qP8LnGr%%@Dqb^Q+w;WP` zCw5d1TfGdb3pm^9UM9k6sp@w$Zxs}CIGQIGmX%F)PFK}ma=;c(6jMDet<5Xfb`0#s zjoS8u#h&ZO_QgXx6MHUeowGWgfL_{szLKVk>--#2!uyL^r$g9dkBi)1a)A{48J;67 zOtnYgp>XLAy~wGYi*NFw$>S%;fWcz6=G58iw(?>4O{cZrg9ivP6^d^j z>Y$)c?@!W?AFfs-4-m2l#`sKjH`C8NSuG?dv(X2ECG%pO?c$N!Z3-UB-^NbWS-yW# zB&B@Pt5+xMlap~J^jw5Awhlgc|a zbi{5pmVXJ2oghYkIP-K1Tq`XAns(ipX5rFL}R}V5_*6H zQWV->E+QzsSqQ$l*)G-SU{05pM=5uq5f#@N4G1DTXXw}&@=8--uA#x<7akt~XQz6% z4cl{_^ZWeXY?sYFn(_sa?bMK?Y+Ns;2SvLK=f(QFoB?AjyVS%G^)`J-d%T`oJZB=l zx`pmn_`>X7C$yeH7+h*pM9#Ks&QX>MgYO*9H0^m&3k;A!ERpYQNFBd%JMNjK`C>b< z1dI$Q*dHOG>En2hkUjPdztwB7;Hd;4?HLB^uuy0@_Q51Gn`F$ll_gGJv^pY3t(xws z8Lh(*Y~|SiM^Mtc@$d$jfa3$v zNAKvFTT&v{oB&MXuQyz}~ zo{Gr24N1^5X7QsAo=yZu5%{hvk>7)TV$6$uSdNzc&Fcr--r8f^-5NL8ORweH^$9T3 z@$utFA_m|m+2o`aJFEE$R!z;XGIA`V+6Jkp%DQ!+@h>{NdRhykNE=;W5K6S^0qh`9 zH)!g<7?e^*4^9hEAdP`uZHV96R4J-0E3x5&aP#-{4Y;fi*^9d6xk!F5edg&RG^L{;#&1@d+Rzn$pxWTSMmho1)VTK>EEld3i31e$p3PR;4%nN2H&fD^HLV*yvZJZ&R&qTtm6}297-~2ct|bp`F%y z?8U#V-dC}_9$)mCj+_$Gg2y{NM^TmrM-$9cQw0vP*XRx1HjS{9o+zq`1SNckxh&|b zD!kT^Gez;dx64dut@-5Q7k+#783!nUHaXmPUubWhUW1Tk11|WS2!2GyV-e&J`giWX z2;+a{^lcgoLw-Bvgy_Ty>AdB&Ybx03&i1#s1KbVX5{>rL!o4qk8fKkjYb`S6BHVx0 zV7Z5}(6vwA+(+OR3qM_W4orr55cLts-Ue2U6gLo$C;EwJ2Nt-^E@{4~Z|#pf&Y}X= zA9TVpekbA2wL45#rzKQ%>SCM=e+CP@L=3|obWPLNkrjMZ>3P7`Hi}Ol>YL1uW(wXQ zGrCQmsxdkeI&%Gru{T+(QTJEQKb2uNbB*>s#ksiH^Q-S&$K4CjL^By>pp7XqdFJPo z@@1Aup|)iWgpEOwTt+Xt^q_e@s*9eohbdNSllx$YTJKlU0Q+@G+GA;DQhW%m=Ch!!8}6SM1>hWN$NVf(LH*u+CPg2_2@$NK@HJh#y<= zX82Xa%VWil^K0Y56emZYfGSaKsgG9uqT+fX@2vlp6Gv~39Gy%hJ9t$_Jyouxy0?3y z1Uq5|e!OV&?OkNu*Sszl@(xPlTOG3g`M%)9)A<^SWH=5j2hwkLn%=#BNVMw3#;RGXqsBGjWB|tc>%?Q$b3}g zhT&Pf$>OhM)%0Y#&>#r!c9JlCNC5gY*8hRb^=mWj8 z=F=Q%nUA)Ph3=g-bv0kdxgED3V`HqJw}S4IC$Qf3-i-Il$?S%Tu=QaFo9i?DHxB;Q zq36OFI=j|ONsY=GiHyu0*G{Qm(oExKgRmuHZ#fmE>wX6#$qaJTMuB5NB?SyK%kuXp z%zaYu$ZDB`K^r@}E{$j2;4!PNL56%l5?t_osWM@kqgUINW zo3Lm4e?5H1-yiS#0=F;|M=?le^A6L5wz9PJ1jefzG5$<>t6hns$1H-IZE`it22Y|n zxvY^PzuWVxXHY@i7zblIkM%Ht&8HRCZ@6nptz7$pSEc04RFpyjX1^=EoH{rZzfsTQ z%o|Q$JiT}X5_3*Nd92kzIl(IE<-GQBZxPMj*)rM z>y1?wa*o;TkajtwomRI=zR$n=g~+=~d+`zWns-K+=af{)q2gBrf?2N)2Z<;3Z58a# z)J@+X01%nMwtY!E%iWiy=kr16 zszLq{wbd=bl{*Ppg5*F`D?!Xl@v9}3xeZ#SGF>%E}CV*>W}!W>4qD#ue)NX4wF z7-Ru_I<-Q!IpahlQk(;fp0#fZAUN_;6=; zGwiyQgeh#`NLn%5z`BI2mh}R!{Zs+T_^>6;U{@Ov8qtGnG#KIf9;ukea_#bI&ny@g zYrTi#9~AQqmIu9^hB<9{T9~+a_WgyISBLc)wEBPg-jf06n7ej2A}OA`K-VU0yQ&+v z7%>wEf-K{yg3MpBFQ_r$PnvEo1@)sjA|g?j2`KSLsAec2oaPO^QG7g+FaSl7L*o%R=OR4p#AKF(yaiI(PqSe`z7_xPG2J(y9&B}(fJY1*+y zMv}pa*#SpTF1)8g^i^$_*3~-9crVV_?hHN09Dg3)jU_vIMZ2K4&5iDqiEX|b+=Y;$ zec+M&_~85WrVzxonLWO)wk+Cgc9lOA@I+sQa#t*I_x8s)j0*0zMbFtbh4w@g39I(k z5{xu|E8yN1B==j;Y~TU0Myqp~2b8Z_ML#-w9U%qR)xlI}Ivh*y3%gxC-EHL4VH^-2lzgcF?^UF~M2oy6-O|{|d ztn*elv;p$Kkf#fYQQ6Q!RhXALs*!vqum!&M28@>GH;bSy?6=RH8u4yR5J8 z=?;42=oEZJ#!bJV56RK6gxpEuets@x<&jw!T= zy`Ccuuz7pv5huP%=6}@YG3#1+3^JapFv)xkEjLIJqUg@-_3*4mF{!~2|C z)+~0|l#HFT%TQDt7g0{c9>N9asqCODY-Anb`hYquk3}suN)#!o1CNmIw-ITR;eRIU zxq@Hh`DuN5<1wTw-prs$pkd-{J1o zO*tXEdd}Az)~%8{WU%$G3^lgTcs~6>vF=6Z>2L10h>!hHrBaH$$Zf7+&;=^TB<|Zo zVN`lVIeGVVfYQ3adndl@UrY~gK9wBOjanQ&lmjSNwLo~B7$viMpIe1h|0f~$#RL8+ z3-yb`l3}{7QpnHUk9^o&Y8Fq)2Nbx5TZ4jP<2VEIDewy?Fnpc zNTvh(cz(^{bt=Bp3$b#Z%oZ$?iX}6+BVKLuyP1}%^}MxoDZh%Tw_5l?apTw)d(B5o zpyli9J9Mh_ii4pcmEe~~=@L5%EKk2nRO|AT%zm2VhD^^%XBn6;)G)27XZNJGQMDNNYO{xiu2bdlKJcN z;O^CEtNS@b3d_UB4>i$Pi7JY8~1 zd)}2ksVXG6Vavov+zY)I4yFlOWOqE3fVvc4RmelQ+uKfuTmBoCa?2-VKSR8C2BfoP zIxCeZP6D%i{+ofeUe*;a;`}U1Ojh(KLMAJN>x4Fes#YyD9myg_O^jdgAS<;|IVRE+ zX(%EF#iga+BO)VB@>>6Dw7b`6f;oVym72o~mM`#hr(8YUv?OI>TX3*NP(^tN9^(?* z-a|TKFMVm3YW&8WHHFS3!Dj?IH!dO-@)s3GX5Sh)2+6&Mqe|)==o`O9j#~rA(8y1z z(>Z_~o~|Mw8#j*8Wouj5Yu`ETZBf)i)W=ArN_U*)aIL(AbzC;tcVGsSxwFdA$thYP zL;cu{5Ng&Mg2(z~3x?<~nHy>csyhYBmUBP7Ngswv4wJaQ#3e-6H=ZneUiD zP&rT%=IRg5!x!ZaJRda5S)2Oai1YU=bTC!$%0A$+dGpFed?ytn2IZ^TK}q^aqhFu8 zU$|=8mxxgMW!|A;v($`@j4ay7dZ z&wCg5nJ1WZ$s^6<@k;7W;>dQjdsJz+%pe!*BgTaLYBgH3F8{Pgn5)tg_7ed?O@l_V zM(o|<6sE-gc4);Tizrs>`Oy5qdF4~?5Rinmh1VZRMGK^F^1b3g38w1qVE=&Un^FPa zyg_?1R;@>)D5Jm9Xj5zW2lZOQCwendMNyF=97BDjV5#v-lE0sY0b=UA7;ET$5>Tpq zcZi5;$8xRDsA`I~fE?_e?7NegU-0zoFy~o756Db8c-8K&RVVNAh%^NqomBEyE!8i4 z~lCajmS_?)15qxgK)(U?Xt z9{zI$==-B}#y@{H6W8rPL>fn=YQb}fLc%tWKTn8@QiuZ$wu~dAjQ!(zQyF^-2?`1g zG0bCyOe3Lz-#=rdKt~pl&9#iStbH0Jb#Bc@yv{16>}_H8E}pt!R_b5+k&j{zmg8_} z#E*G*w_lKm^#fKp_UT*ZUk$H1vmuO@IpfmDn*AUjRE8)v9g zS;BID*mPF-eQ?zmd4Df683nr;%$O&e83kUf`^dyxFgWgN~7liECWpy*9l z=L-FDmKu6|-R>t*|KmXpaHj1x_dgPNBKfy3^8a$1zeAxRgh4G&uohkXuNwb5%K!cT z-3%Ik(e3_Ko|0m7;zdptpL(45wi(=sRPvQRG7ey8{R^UY4WX=EDtMUJ^or@$u z%SJy;8@%@4Z^!^00Q4B-n-m)S51V32OG3*F9YZQ8^KK4yAZ(t-vqF^Lfgw6 z%?eP@8|rIB&VNGmf6rY--aR-7t6(UsrZ@iK`=3bsBCrm$BJABfaHhBaea@kDDQHsn zvhbY#6G^}SpFukyjvfD}SlV%Ot4H`5_Wu)VKb#$USZt%E&i(gQdxzpqCt?6}Qyu$H fgTOL_eM1uSZIH)u68~rW=XcT)Kg55F>IeQ0TkYUY literal 0 HcmV?d00001 diff --git a/content/security/docs/network.md b/content/security/docs/network.md index f9c687b3d..73cf9e8f9 100644 --- a/content/security/docs/network.md +++ b/content/security/docs/network.md @@ -7,23 +7,25 @@ Network security has several facets. The first involves the application of rule #### Encryption in transit + Service Mesh + Container Network Interfaces (CNIs) ++ Ingress Controllers and Load Balancers + Nitro Instances + ACM Private CA with cert-manager ## Network policy -Within a Kubernetes cluster, all Pod to Pod communication is allowed by default. While this flexibility may help promote experimentation, it is not considered secure. Kubernetes network policies give you a mechanism to restrict network traffic between Pods (often referred to as East/West traffic) and between Pods and external services. Kubernetes network policies operate at layers 3 and 4 of the OSI model. Network policies use pod selectors and labels to identify source and destination pods, but can also include IP addresses, port numbers, protocol number, or a combination of these. [Calico](https://docs.projectcalico.org/introduction/), is an open source policy engine from [Tigera](https://tigera.io) that works well with EKS. In addition to implementing the full set of Kubernetes network policy features, Calico supports extended network polices with a richer set of features, including support for layer 7 rules, e.g. HTTP, when integrated with Istio. Isovalent, the maintainers of [Cilium](https://cilium.readthedocs.io/en/stable/intro/), have also extended the network policies to include partial support for layer 7 rules, e.g. HTTP. Cilium also has support for DNS hostnames which can be useful for restricting traffic between Kubernetes Services/Pods and resources that run within or outside of your VPC. By contrast, Calico Enterprise includes a feature that allows you to map a Kubernetes network policy to an AWS security group, as well as DNS hostnames. +Within a Kubernetes cluster, all Pod to Pod communication is allowed by default. While this flexibility may help promote experimentation, it is not considered secure. Kubernetes network policies give you a mechanism to restrict network traffic between Pods (often referred to as East/West traffic) as well as between Pods and external services. Kubernetes network policies operate at layers 3 and 4 of the OSI model. Network policies use pod, namespace selectors and labels to identify source and destination pods, but can also include IP addresses, port numbers, protocols, or a combination of these. Network Policies can be applied to both Inbound or Outbound connections to the pod, often called Ingress and Egress rules. + +With native network policy support of Amazon VPC CNI Plugin, you can implement network policies to secure network traffic in kubernetes clusters. This integrates with the upstream Kubernetes Network Policy API, ensuring compatibility and adherence to Kubernetes standards. You can define policies using different [identifiers](https://kubernetes.io/docs/concepts/services-networking/network-policies/) supported by the upstream API. By default, all ingress and egress traffic is allowed to a pod. When a network policy with a policyType Ingress is specified, only allowed connections into the pod are those from the pod's node and those allowed by the ingress rules. Same applies for egress rules. If multiple rules are defined, then union of all rules are taken into account when making the decision. Thus, order of evaluation does not affect the policy result. !!! attention - When you first provision an EKS cluster, the Calico policy engine is not installed by default. The instructions for installing Calico can be found in the AWS EKS documentation at [https://docs.aws.amazon.com/eks/latest/userguide/calico.html](https://docs.aws.amazon.com/eks/latest/userguide/calico.html). + When you first provision an EKS cluster, VPC CNI Network Policy functionality is not enabled by default. Ensure you deployed supported VPC CNI Add-on version and set `ENABLE_NETWORK_POLICY` flag to `true` on the vpc-cni add-on to enable this. Refer [Amazon EKS User guide](https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html) for detailed instructions. -Calico policies can be scoped to Namespaces, Pods, service accounts, or globally. When policies are scoped to a service account, it associates a set of ingress/egress rules with that service account. With the proper RBAC rules in place, you can prevent teams from overriding these rules, allowing IT security professionals to safely delegate administration of namespaces. +## Recommendations -You can find a list of common Kubernetes network policies at [https://github.com/ahmetb/kubernetes-network-policy-recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes). A similar set of rules for Calico are available at [https://docs.projectcalico.org/security/calico-network-policy](https://docs.projectcalico.org/security/calico-network-policy). +### Getting Started with Network Policies - Follow Principle of Least Privilege -## Recommendations +#### Create a default deny policy -### Create a default deny policy -As with RBAC policies, network policies should adhere to the policy of least privileged access. Start by creating a deny all policy that restricts all inbound and outbound traffic from a namespace or create a global policy using Calico. +As with RBAC policies, it is recommended to follow least privileged access principles with network policies. Start by creating a deny all policy that restricts all inbound and outbound traffic with in a namespace. _Kubernetes network policy_ ```yaml @@ -44,28 +46,11 @@ spec: !!! tip The image above was created by the network policy viewer from [Tufin](https://orca.tufin.io/netpol/). -_Calico global network policy_ -```yaml -apiVersion: projectcalico.org/v3 -kind: GlobalNetworkPolicy -metadata: - name: default-deny -spec: - selector: all() - types: - - Ingress - - Egress -``` +#### Create a rule to allow DNS queries -### Create a rule to allow DNS queries -Once you have the default deny all rule in place, you can begin layering on additional rules, such as a global rule that allows pods to query CoreDNS for name resolution. You begin by labeling the namespace: - -``` -kubectl label namespace kube-system name=kube-system -``` - -Then add the network policy: +Once you have the default deny all rule in place, you can begin layering on additional rules, such as a rule that allows pods to query CoreDNS for name resolution. +_Kubernetes network policy_ ```yaml apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -81,7 +66,10 @@ spec: - to: - namespaceSelector: matchLabels: - name: kube-system + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns ports: - protocol: UDP port: 53 @@ -89,144 +77,87 @@ spec: ![](./images/allow-dns-access.jpg) -_Calico global policy equivalent_ - -```yaml -apiVersion: projectcalico.org/v3 -kind: GlobalNetworkPolicy -metadata: - name: allow-dns-egress -spec: - selector: all() - types: - - Egress - egress: - - action: Allow - protocol: UDP - destination: - namespaceSelector: name == "kube-system" - ports: - - 53 -``` +#### Incrementally add rules to selectively allow the flow of traffic between namespaces/pods -The following is an example of how to associate a network policy with a service account while preventing users associated with the readonly-sa-group from editing the service account my-sa in the default namespace: +Understand the application requirements and create fine-grained ingress and egress rules as needed. Below example shows how to restrict ingress traffic on port 80 to `app-one` from `client-one`. This helps minimize the attack surface and reduces the risk of unauthorized access. +_Kubernetes network policy_ ```yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: my-sa - namespace: default - labels: - name: my-sa ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: default - name: readonly-sa-role -rules: -# Allows the subject to read a service account called my-sa -- apiGroups: [""] - resources: ["serviceaccounts"] - resourceNames: ["my-sa"] - verbs: ["get", "watch", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - namespace: default - name: readonly-sa-rolebinding -# Binds the readonly-sa-role to the RBAC group called readonly-sa-group. -subjects: -- kind: Group - name: readonly-sa-group - apiGroup: rbac.authorization.k8s.io -roleRef: - kind: Role - name: readonly-sa-role - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: projectcalico.org/v3 +apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: - name: netpol-sa-demo + name: allow-ingress-app-one namespace: default -# Allows all ingress traffic to services in the default namespace that reference -# the service account called my-sa spec: + podSelector: + matchLabels: + k8s-app: app-one + policyTypes: + - Ingress ingress: - - action: Allow - source: - serviceAccounts: - selector: 'name == "my-sa"' - selector: all() + - from: + - podSelector: + matchLabels: + k8s-app: client-one + ports: + - protocol: TCP + port: 80 ``` -### Incrementally add rules to selectively allow the flow of traffic between namespaces/pods -Start by allowing Pods within a Namespace to communicate with each other and then add custom rules that further restrict Pod to Pod communication within that Namespace. +![](./images/allow-ingress-app-one.png) + +### Monitoring network policy enforcement + ++ **Use Network Policy editor** + + [Network policy editor](https://networkpolicy.io/) helps with visualizations, security score, autogenerates from network flow logs + + Build network policies in an interactive way ++ **Audit Logs** + + Regularly review audit logs of your EKS cluster + + Audit logs provide wealth of information about what actions have been performed on your cluster including changes to network policies + + Use this information to track changes to your network policies over time and detect any unauthorized or unexpected changes ++ **Automated testing** + + Implement automated testing by creating a test environment that mirrors your production environment and periodically deploy workloads that attempt to violate your network policies. ++ **Monitoring metrics** + + Configure your observability agents to scrape the prometheus metrics from the VPC CNI node agents, that allows to monitor the agent health, and sdk errors. ++ **Audit Network Policies regularly** + + Periodically audit your Network Policies to make sure that they meet your current application requirements. As your application evolves, an audit gives you the opportunity to remove redundant ingress, egress rules and make sure that your applications don’t have excessive permissions. ++ **Ensure Network Policies exists using Open Policy Agent (OPA)** + + Use OPA Policy like shown below to ensure Network Policy always exists before onboarding application pods. This policy denies onboarding k8s pods with a label `k8s-app: sample-app` if corresponding network policy does not exist. + +```javascript +package kubernetes.admission +import data.kubernetes.networkpolicies + +deny[msg] { + input.request.kind.kind == "Pod" + pod_label_value := {v["k8s-app"] | v := input.request.object.metadata.labels} + contains_label(pod_label_value, "sample-app") + np_label_value := {v["k8s-app"] | v := networkpolicies[_].spec.podSelector.matchLabels} + not contains_label(np_label_value, "sample-app") + msg:= sprintf("The Pod %v could not be created because it is missing an associated Network Policy.", [input.request.object.metadata.name]) +} +contains_label(arr, val) { + arr[_] == val +} +``` -### Log network traffic metadata -[AWS VPC Flow Logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html) captures metadata about the traffic flowing through a VPC, such as source and destination IP address and port along with accepted/dropped packets. This information could be analyzed to look for suspicious or unusual activity between resources within the VPC, including Pods. However, since the IP addresses of pods frequently change as they are replaced, Flow Logs may not be sufficient on its own. Calico Enterprise extends the Flow Logs with pod labels and other metadata, making it easier to decipher the traffic flows between pods. +### Troubleshooting -### Use encryption with AWS load balancers -The [AWS Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) (ALB) and [Network Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) (NLB) both have support for transport encryption (SSL and TLS). The `alb.ingress.kubernetes.io/certificate-arn` annotation for the ALB lets you to specify which certificates to add to the ALB. If you omit the annotation the controller will attempt to add certificates to listeners that require it by matching the available [AWS Certificate Manager (ACM)](https://docs.aws.amazon.com/acm/latest/userguide/acm-overview.html) certificates using the host field. Starting with EKS v1.15 you can use the service.beta.kubernetes.io/aws-load-balancer-ssl-cert annotation with the NLB as shown in the example below. +#### Monitor the vpc-network-policy-controller, node-agent logs -```yaml -apiVersion: v1 -kind: Service -metadata: - name: demo-app - namespace: default - labels: - app: demo-app - annotations: - service.beta.kubernetes.io/aws-load-balancer-type: "nlb" - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "" - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" -spec: - type: LoadBalancer - ports: - - port: 443 - targetPort: 80 - protocol: TCP - selector: - app: demo-app ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: nginx - namespace: default - labels: - app: demo-app -spec: - replicas: 1 - selector: - matchLabels: - app: demo-app - template: - metadata: - labels: - app: demo-app - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 443 - protocol: TCP - - containerPort: 80 - protocol: TCP +Enable the EKS Control plane controller manager logs to diagnose the network policy functionality. You can stream the control plane logs to a CloudWatch log group and use [CloudWatch Log insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AnalyzingLogData.html) to perform advanced queries. From the logs, you can view what pod endpoint objects are resolved to a Network Policy, reconcilation status of the policies, and debug if the policy is working as expected. + +In addition, Amazon VPC CNI allows you to enable the collection and export of policy enforcement logs to [Amazon Cloudwatch](https://aws.amazon.com/cloudwatch/) from the EKS worker nodes. Once enabled, you can leverage [CloudWatch Container Insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/ContainerInsights.html) to provide insights on your usage related to Network Policies. + +Amazon VPC CNI also ships an SDK that provides an interface to interact with eBPF programs on the node. The SDK is installed when the `aws-node` is deployed onto the nodes. You can find the SDK binary installed under `/opt/cni/bin` directory on the node. At launch, the SDK provides support for fundamental functionalities such as inspecting eBPF programs and maps. + +```shell +sudo /opt/cni/bin/aws-eks-na-cli ebpf progs ``` -### Additional Resources -+ [Kubernetes & Tigera: Network Policies, Security, and Audit](https://youtu.be/lEY2WnRHYpg) -+ [Calico Enterprise](https://www.tigera.io/tigera-products/calico-enterprise/) -+ [Cilium](https://cilium.readthedocs.io/en/stable/intro/) -+ [NetworkPolicy Editor](https://cilium.io/blog/2021/02/10/network-policy-editor) an interactive policy editor from Cilium -+ [Kinvolk's Network Policy Advisor](https://kinvolk.io/blog/2020/03/writing-kubernetes-network-policies-with-inspektor-gadgets-network-policy-advisor/) Suggests network policies based on an analysis of network traffic +#### Log network traffic metadata +[AWS VPC Flow Logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html) captures metadata about the traffic flowing through a VPC, such as source and destination IP address and port along with accepted/dropped packets. This information could be analyzed to look for suspicious or unusual activity between resources within the VPC, including Pods. However, since the IP addresses of pods frequently change as they are replaced, Flow Logs may not be sufficient on its own. Calico Enterprise extends the Flow Logs with pod labels and other metadata, making it easier to decipher the traffic flows between pods. + ## Security groups EKS uses [AWS VPC Security Groups](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) (SGs) to control the traffic between the Kubernetes control plane and the cluster's worker nodes. Security groups are also used to control the traffic between worker nodes, and other VPC resources, and external IP addresses. When you provision an EKS cluster (with Kubernetes version 1.14-eks.3 or greater), a cluster security group is automatically created for you. This security group allows unfettered communication between the EKS control plane and the nodes from managed node groups. For simplicity, it is recommended that you add the cluster SG to all node groups, including unmanaged node groups. @@ -249,14 +180,97 @@ You can control which pods are assigned to a security group by creating a `Secur !!! important You **must** create rules for inbound traffic from the cluster security group (kubelet) for all of the probes configured for pod. -!!! warning - There is a [bug](https://github.com/aws/amazon-vpc-cni-k8s/pull/1212) that currently prevents the kubelet from communicating with pods that are assigned to SGs. The current workaround involves running `sudo sysctl net.ipv4.tcp_early_demux=0` on the affected worker nodes. This is fixed in CNI v1.7.3, [https://github.com/aws/amazon-vpc-cni-k8s/releases/tag/v1.7.3](https://github.com/aws/amazon-vpc-cni-k8s/releases/tag/v1.7.3). - !!! important Security groups for pods relies on a feature known as [ENI trunking](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/container-instance-eni.html) which was created to increase the ENI density of an EC2 instance. When a pod is assigned to an SG, a VPC controller associates a branch ENI from the node group with the pod. If there aren't enough branch ENIs available in a node group at the time the pod is scheduled, the pod will stay in pending state. The number of branch ENIs an instance can support varies by instance type/family. See [https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types) for further details. While security groups for pods offers an AWS-native way to control network traffic within and outside of your cluster without the overhead of a policy daemon, other options are available. For example, the Cilium policy engine allows you to reference a DNS name in a network policy. Calico Enterprise includes an option for mapping network policies to AWS security groups. If you've implemented a service mesh like Istio, you can use an egress gateway to restrict network egress to specific, fully qualified domains or IP addresses. For further information about this option, read the three part series on [egress traffic control in Istio](https://istio.io/blog/2019/egress-traffic-control-in-istio-part-1/). + +## When to use Network Policy vs Security Group for Pods? + +### When to use Kubernetes network policy: + ++ **Controlling pod-to-pod traffic** + + Suitable for controlling network traffic between pods inside a cluster (east-west traffic) ++ **Control traffic at the IP address or port level (OSI layer 3 or 4)** + +### When to use AWS Security groups for pods (SGP): + ++ **Leverage existing AWS configurations** + + If you already have complex set of EC2 security groups that manage access to AWS services and you are migrating applications from EC2 instances to EKS, SGPs can be a very good choice allowing you to reuse security group resources and apply them to your pods. ++ **Control access to AWS services** + + Your applications running within an EKS cluster wants to communicate with other AWS services (RDS database), use SGPs as an efficient mechanism to control the traffic from the pods to AWS services. ++ **Isolation of Pod & Node traffic** + + If you want to completely separate pod traffic from the rest of the node traffic, use SGP in `POD_SECURITY_GROUP_ENFORCING_MODE=strict` mode. + +### Best practices using `Security groups for pods` and `Network Policy` + ++ **Layered security** + + Use a combination of SGP and kubernetes network policy for a layered security approach + + Use SGPs to limit network level access to AWS services that are not part of a cluster, while kubernetes network policies can restrict network traffic between pods inside the cluster ++ **Principle of least privilege** + + Only allow necessary traffic between pods or namespaces ++ **Segment your applications** + + Wherever possible, segment applications by the network policy to reduce the blast radius if an application is compromised ++ **Keep policies simple and clear** + + Kubernetes network policies can be quite granular and complex, its best to keep them as simple as possible to reduce the risk of misconfiguration and ease the management overhead ++ **Reduce the attack surface** + + Minimize the attack surface by limiting the exposure of your applications + +!!! attention + Security Groups for pods provides two enforcing modes: `strict` and `standard`. You must use `standard` mode when using both Network Policy and Security Groups for pods features in an EKS cluster. + +When it comes to network security, a layered approach is often the most effective solution. Using kubernetes network policy and SGP in combination can provide a robust defense-in-depth strategy for your applications running in EKS. + +## Service Mesh Policy Enforcement or Kubernetes network policy + +A `service mesh` is a dedicated infrastructure layer that you can add to your applications. It allows you to transparently add capabilities like observability, traffic management, and security, without adding them to your own code. + +Service mesh enforces policies at Layer 7 (application) of OSI model whereas kubernetes network policies operate at Layer 3 (network) and Layer 4 (transport). There are many offerings in this space like AWS AppMesh, Istio, Linkerd, etc., + +### When to use Service mesh for policy enforcement: + ++ Have existing investment in a service mesh ++ Need more advanced capabilities like traffic management, observability & security + + Traffic control, load balancing, circuit breaking, rate limiting, timeouts etc. + + Detailed insights into how your services are performing (latency, error rates, requests per second, request volumes etc.) + + You want to implement and leverage service mesh for security features like mTLS + +### Choose Kubernetes network policy for simpler use cases + ++ Limit which pods can communicate with each other ++ Network policies require fewer resources than a service mesh making them a good fit for simpler use cases or for smaller clusters where the overhead of running and managing a service mesh might not be justified + +!!! tip + Network policies and Service mesh can also be used together. Use network policies to provide a baseline level of security and isolation between your pods and then use a service mesh to add additional capabilities like traffic management, observability and security. + + +## ThirdParty Network Policy Engines + +Consider a Third Party Network Policy Engine when you have advanced policy requirements like Global Network Policies, support for DNS Hostname based rules, Layer 7 rules, ServiceAccount based rules, and explicit deny/log actions, etc., [Calico](https://docs.projectcalico.org/introduction/), is an open source policy engine from [Tigera](https://tigera.io) that works well with EKS. In addition to implementing the full set of Kubernetes network policy features, Calico supports extended network polices with a richer set of features, including support for layer 7 rules, e.g. HTTP, when integrated with Istio. Calico policies can be scoped to Namespaces, Pods, service accounts, or globally. When policies are scoped to a service account, it associates a set of ingress/egress rules with that service account. With the proper RBAC rules in place, you can prevent teams from overriding these rules, allowing IT security professionals to safely delegate administration of namespaces. Isovalent, the maintainers of [Cilium](https://cilium.readthedocs.io/en/stable/intro/), have also extended the network policies to include partial support for layer 7 rules, e.g. HTTP. Cilium also has support for DNS hostnames which can be useful for restricting traffic between Kubernetes Services/Pods and resources that run within or outside of your VPC. By contrast, Calico Enterprise includes a feature that allows you to map a Kubernetes network policy to an AWS security group, as well as DNS hostnames. + +You can find a list of common Kubernetes network policies at [https://github.com/ahmetb/kubernetes-network-policy-recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes). A similar set of rules for Calico are available at [https://docs.projectcalico.org/security/calico-network-policy](https://docs.projectcalico.org/security/calico-network-policy). + +### Migration to Amazon VPC CNI Network Policy Engine + +To maintain consistency and avoid unexpected pod communication behavior, it is recommended to deploy only one Network Policy Engine in your cluster. If you want to migrate from 3P to VPC CNI Network Policy Engine, we recommend converting your existing 3P NetworkPolicy CRDs to the Kubernetes NetworkPolicy resources before enabling VPC CNI network policy support. And, test the migrated policies in a separate test cluster before applying them in you production environment. This allows you to identify and address any potential issues or inconsistencies in pod communication behavior. + +#### Migration Tool +To assist in your migration process, we have developed a tool called [K8s Network Policy Migrator](https://github.com/awslabs/k8s-network-policy-migrator) that converts your existing Calico/Cilium network policy CRDs to Kubernetes native network policies. After conversion you can directly test the converted network policies on your new clusters running VPC CNI network policy controller. The tool is designed to help you streamline the migration process and ensure a smooth transition. + +!!! Important + Migration tool will only convert 3P policies that are compatible with native kubernetes network policy api. If you are using advanced network policy features offered by 3P plugins, Migration tool will skip and report them. + +Please note that migration tool is currently not supported by AWS VPC CNI Network policy engineering team, it is made available to customers on a best-effort basis. We encourage you to utilize this tool to facilitate your migration process. In the event that you encounter any issues or bugs with the tool, we kindly ask you create a [Github issue](https://github.com/awslabs/k8s-network-policy-migrator/issues). Your feedback is invaluable to us and will assist in the continuous improvement of our services. + +### Additional Resources ++ [Kubernetes & Tigera: Network Policies, Security, and Audit](https://youtu.be/lEY2WnRHYpg) ++ [Calico Enterprise](https://www.tigera.io/tigera-products/calico-enterprise/) ++ [Cilium](https://cilium.readthedocs.io/en/stable/intro/) ++ [NetworkPolicy Editor](https://cilium.io/blog/2021/02/10/network-policy-editor) an interactive policy editor from Cilium ++ [Kinvolk's Network Policy Advisor](https://kinvolk.io/blog/2020/03/writing-kubernetes-network-policies-with-inspektor-gadgets-network-policy-advisor/) Suggests network policies based on an analysis of network traffic + + ## Encryption in transit Applications that need to conform to PCI, HIPAA, or other regulations may need to encrypt data while it is in transit. Nowadays TLS is the de facto choice for encrypting traffic on the wire. TLS, like it's predecessor SSL, provides secure communications over a network using cryptographic protocols. TLS uses symmetric encryption where the keys to encrypt the data are generated based on a shared secret that is negotiated at the beginning of the session. The following are a few ways that you can encrypt data in a Kubernetes environment. @@ -283,13 +297,65 @@ The [aws-app-mesh-examples](https://github.com/aws/aws-app-mesh-examples) GitHub ### Ingress Controllers and Load Balancers Ingress controllers are a way for you to intelligently route HTTP/S traffic that emanates from outside the cluster to services running inside the cluster. Oftentimes, these Ingresses are fronted by a layer 4 load balancer, like the Classic Load Balancer or the Network Load Balancer (NLB). Encrypted traffic can be terminated at different places within the network, e.g. at the load balancer, at the ingress resource, or the Pod. How and where you terminate your SSL connection will ultimately be dictated by your organization's network security policy. For instance, if you have a policy that requires end-to-end encryption, you will have to decrypt the traffic at the Pod. This will place additional burden on your Pod as it will have to spend cycles establishing the initial handshake. Overall SSL/TLS processing is very CPU intensive. Consequently, if you have the flexibility, try performing the SSL offload at the Ingress or the load balancer. -An ingress controller can be configured to terminate SSL/TLS connections. An example for how to terminate SSL/TLS connections at the NLB appears [above](#use-encryption-with-aws-load-balancers). Additional examples for SSL/TLS termination appear below. +#### Use encryption with AWS Elastic load balancers +The [AWS Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html) (ALB) and [Network Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) (NLB) both have support for transport encryption (SSL and TLS). The `alb.ingress.kubernetes.io/certificate-arn` annotation for the ALB lets you to specify which certificates to add to the ALB. If you omit the annotation the controller will attempt to add certificates to listeners that require it by matching the available [AWS Certificate Manager (ACM)](https://docs.aws.amazon.com/acm/latest/userguide/acm-overview.html) certificates using the host field. Starting with EKS v1.15 you can use the `service.beta.kubernetes.io/aws-load-balancer-ssl-cert` annotation with the NLB as shown in the example below. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: demo-app + namespace: default + labels: + app: demo-app + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443" + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http" +spec: + type: LoadBalancer + ports: + - port: 443 + targetPort: 80 + protocol: TCP + selector: + app: demo-app +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nginx + namespace: default + labels: + app: demo-app +spec: + replicas: 1 + selector: + matchLabels: + app: demo-app + template: + metadata: + labels: + app: demo-app + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 443 + protocol: TCP + - containerPort: 80 + protocol: TCP +``` + +Following are additional examples for SSL/TLS termination. + [Securing EKS Ingress With Contour And Let’s Encrypt The GitOps Way](https://aws.amazon.com/blogs/containers/securing-eks-ingress-contour-lets-encrypt-gitops/) + [How do I terminate HTTPS traffic on Amazon EKS workloads with ACM?](https://aws.amazon.com/premiumsupport/knowledge-center/terminate-https-traffic-eks-acm/) !!! attention - Some Ingresses, like the ALB ingress controller, implement the SSL/TLS using Annotations instead of as part of the Ingress Spec. + Some Ingresses, like the AWS LB controller, implement the SSL/TLS using Annotations instead of as part of the Ingress Spec. ### ACM Private CA with cert-manager You can enable TLS and mTLS to secure your EKS application workloads at the ingress, on the pod, and between pods using ACM Private Certificate Authority (CA) and [cert-manager](https://cert-manager.io/), a popular Kubernetes add-on to distribute, renew, and revoke certificates. ACM Private CA is a highly-available, secure, managed CA without the upfront and maintenance costs of managing your own CA. If you are using the default Kubernetes certificate authority, there is an opportunity to improve your security and meet compliance requirements with ACM Private CA. ACM Private CA secures private keys in FIPS 140-2 Level 3 hardware security modules (very secure), compared with the default CA storing keys encoded in memory (less secure). A centralized CA also gives you more control and improved auditability for private certificates both inside and outside of a Kubernetes environment.