From 69df30cdc9216efd6038a865747a3ac410284310 Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Sat, 24 Apr 2021 17:20:58 +0000 Subject: [PATCH 01/17] add setup for k8s external e2e tests --- Makefile | 6 +++- test/external-e2e/run.sh | 49 +++++++++++++++++++++++++++++++ test/external-e2e/testdriver.yaml | 12 ++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 test/external-e2e/run.sh create mode 100644 test/external-e2e/testdriver.yaml diff --git a/Makefile b/Makefile index 0a0f1d54..480ba557 100644 --- a/Makefile +++ b/Makefile @@ -124,4 +124,8 @@ e2e-teardown: .PHONY: e2e-test e2e-test: - go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS} + if [ ! -z "$(EXTERNAL_E2E_TEST)" ]; then \ + bash ./test/external-e2e/run.sh;\ + else \ + go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS};\ + fi diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh new file mode 100644 index 00000000..f1d1b3d6 --- /dev/null +++ b/test/external-e2e/run.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +PROJECT_ROOT=$(git rev-parse --show-toplevel) + +install_ginkgo () { + apt update -y + apt install -y golang-ginkgo-dev +} + +setup_e2e_binaries() { + # download k8s external e2e binary for kubernetes v1.19 + curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.19.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz + tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz + + # install the csi driver nfs + mkdir -p /tmp/csi-nfs && cp deploy/example/storageclass-nfs.yaml /tmp/csi-nfs/storageclass.yaml + make e2e-bootstrap + make install-nfs-server +} + +print_logs() { + echo "print out driver logs ..." + bash ./test/utils/nfs_log.sh +} + +install_ginkgo +setup_e2e_binaries +trap print_logs EXIT + +ginkgo -p --progress --v -focus='External.Storage.*nfs.csi.k8s.io' \ + -skip='\[Disruptive\]|\[Slow\]' kubernetes/test/bin/e2e.test -- \ + -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \ + --kubeconfig=$KUBECONFIG diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml new file mode 100644 index 00000000..b51d970e --- /dev/null +++ b/test/external-e2e/testdriver.yaml @@ -0,0 +1,12 @@ +# Manifest for Kubernetes external tests. +# See https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external + +StorageClass: + FromFile: /tmp/csi-nfs/storageclass.yaml +DriverInfo: + Name: nfs.csi.k8s.io + Capabilities: + persistence: true + exec: true + multipods: true + RWX: true From 2b35ab661294cdd04b3a49fbdc45d12642ce73a6 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Fri, 30 Apr 2021 14:18:56 +0000 Subject: [PATCH 02/17] fix: remove unnecessary podInfoOnMount --- charts/latest/csi-driver-nfs-v3.0.0.tgz | Bin 3151 -> 3137 bytes .../templates/csi-nfs-driverinfo.yaml | 1 - deploy/csi-nfs-driverinfo.yaml | 1 - 3 files changed, 2 deletions(-) diff --git a/charts/latest/csi-driver-nfs-v3.0.0.tgz b/charts/latest/csi-driver-nfs-v3.0.0.tgz index b554a07d1025846d16f2caa1aae1f65302a505e9..6be56befb7a68491cd2170e298f05643960d38cb 100644 GIT binary patch delta 3042 zcmV<83mx>&7{M5jJb&wR+cpx(QJ%56n)b?8WL@z2}$|0 zhik1?t97uq=l{1_t?K_)`=Ilzz1P~?ZM9pwyPapP_Fk*qet!n72b5uD%Zw%ZS?k_& zRTuY_G@{ri)P_kB!?+y~rDk6_v1n|NprM*6>z^HUPvPwQE*yK4fxgu6W7b10(4q+h zNil}9Nd@B>``xG&wSq6b;%!@O75^U-o}qc*0IdINW~?T+*MEfLF?D1;<#IQX4C*nX_=ZxM z3F{=G8i~aitj;jdJwlfrQYoyKoagUQqvDJb6GO*+%fx%}ju?wNhCboOsfx?NTO1>g zVLu3*@NxCw&JKLQK57&c!Ag(*3kYhq#m-l~mbp|Kys z&{}2UW`A=u+p3%j6G=_e5;HPQDn*|SGL4~9&0uFI z@LlH_=iOYO7=HdG*ntr8ov=|1bzE|h8-KR#uM+<)rpo(o^X&b{lWt@$?fvIim;dc{ z_A2pzf4{wr|4-83;}EM^9WyXMYn>w{sJNV9;(t7CA~Ow?XU>tMAowVO4v~(kqm@hy z7)xwsUV`FC47v~y2LXU{Kx#RJp>q(U2*tyzD~LXzr;E;}PvQF7$&fP(LxUWtmHM!u zbIPSraDCl?F#Jp`9e!z@eobwq|6dG|E7T^k>b|Mqy8N&8YQN(D2Yc<-*8iWPU0pR_ zz<-#fF&MPaXB_QB;bH33LWdZ`i{|xp5O}haOGQKic8Gp|1KIbVsmBaF-PKhH3HF&l zXo$Xv99aXxc@N3Bg(ycEyxT^f@`}Szz8|C3EJ66gB^83g!LI+v(r(u8vrA}%b7m+K z|2Y?plOA&8>(L?d6f_!{X^IAF<;ux(W`Fr-mwphXqH>DBgt0_Wx0TlgoZ2xd5oCZ# z*qc;Fb@RMx<-j6mxVnO=7cMF>S~+f3VEfr}-M*RC@x?HwmBa(~Wpsm^pFS@{!5`RdU#*|2nnFVgdLa|+mR_rC|$BN7X5t;|nLjxnU3s4K)0MKT+t zG?SXy$q^^Up_5+)6(%RYWrQ#Gc?dU# zhI^Td&#~V}>bCwf*`-5FGQQBgPJ}{viW7m4lbQt^E544`QCl)7a;KSF>%oFtxonMa zeAsC$Jq(?+BuO+HbNJ2kH^;{xPP*N1li&p(e;rk|biM%kty91pAY6=X0AtCs6yG`5 zT~db+5a+}Wb2%y!=F&U4(1!E40UqWo6817j&QeyL1%Rq7hIXr!R-UJrdYp8x)p}Q+ zTjOsTnw6YZG3qSxnQDKE`i5oL_XayGARE6wgf?W+B#b|v~|6Jv4-$Ql!J zB+G_lkY;|flEUJpEsLx)8#9grObSH5v>KvFk8x&MRgVB9Nn-f9(fz4=@#gs5=~?5K z5&_{{emBY&}@%j7Hvyb0y6u%3;b@BUqEz_!s0s0Ji@&WtHJw;z` z`!2VPob)agshxRI*81%u6YMf>HdnIbw{eW^S|tPsNsU)RS3vyAX#-m!|j<)Fr){<+f$e{4V z#&;}XI?q4KDclop>7CV~xAO?0e>c~AO{%$4-L6{lswltP4RvoP3S&6-s+M}PG;OUG zF;#YQ%ybM_*N<>`xEBA(JDKRq$HISFo!#9k{@_{qlhFDAJ@B{vqG41k->DhnocW+rA>4b88ea{2i!a#bXc2DV`0OfH&u-AbYPXqAWct`Ul$Y=S2=voMz<@5mNV8rx5Bse>A77b&+72SrpsA z4Sh~Y-o#6oQoVbWE%vr9QkgNzIxm_jOIpWyCHnK@zrQ=V@DR}1n|CKq z1_EsWu+`Yillt;}aOp+d%wmyr4W{}PAfOM5_&QgCb+D*U@V9{IV*{eQTR?PUK=giR z0goJ-z62J#5~-|Rf647o%DVmRCfM@5KJphozC3DrmvvrDf4{NO9I$FT`qE|lgSD0T zkM0o~KFlS+HTX~G)j{?CUwf~;#ebfpRZxWT&8(=n;uT(B2Vocnl^+Y|$W6C2WTSh6 z`OU6}#B7{&m0RJZbvbKZTup&#=di*D$+5IHPfapAl$!nSf1Q4giPyv5AzoV4g7C#F zO8z3%#!Ju{&m(*YM6n;V%#?9?Wsy$K>sh```N*Me=7NH9V$6o?+;B}eMq%%uxneahp`lhy zw|-gFFqfA;f9Qhl#gIy=6DAg|y>1TgqYt9A`OV`6H~fx?go(j@y|HpvFzgj<- z9M8_(u{O@llY`YaS2u}pE=ClE%yLlLEW*I-P2QlaQh1!l|H}R!4Uwm8Af(2J+hf+9 z|L%7VmVW==+3x>O(stmSSZ7%NN_YO|^WReUGRB?Le}E!%L4V{*>HbW=nz3lk z8TAMqP-V)WewpA!WIn6lMbsms;6-Fp6}EydVFGnNNMT9LlCh9Ni(O@ k^4B058!8iQ{&mWOZQHhO`wO@K1pom5|7+=fS^!J{00$`rwg3PC delta 3095 zcmV+y4CwR07|$4xJb&GD+qSaLGyjUca_3H(8&V&(m0ZsFA+_CJPZQe{<>pMM)A2yy zN8Mb7R%km?hg<(EKD>TqYlNuG!I8a+h{^k z{^IdktJP|C_xAk%R;yM0-)eU|FWP&pz4qSzn^t@OMXSBn>VNFLfYu|*u(D;w68)m} z;JK=c`$ig3>@#Y^q=;eM4v12-ubfykHb~G=&6M@e4tpnX`eP4{yve{oYWO+pqZVk< z1cIa(L)oN)@r?a$)QVcc*Ix0qEw+mPj|tDvJaPb7;CE;*c4u$$#B7;do3PSx>p#Pb7nS%qYI2 zRA$0DNvK9*F$SwM40NB+rH51st0m|8JJhH+qr}9}ao;j=CO!~jQO7VK+&EQnIed>} z46n;%i48dEqaV#_r zVi;MgOn=;LUe2~Er@};1)3n4~nkJQEz=oN|P{|~8iy=&)ZaRLUXBH~W#>C=OxsztF zvlIBP^NjOeE>H}={ub;&2>DLfD26&NxyX%wx9zVI|1GA<`)~8~?9*{CvRC%ubF9n% zb~}5O_`kp3*~b58Y4B-;)vS&g7^1b#krGr~PJb|Qo;H!02Ff$%$Wah{l0ZjDFRP=K zObi%HY-V1A;z$g-5D*6efO9}C?L^^0>eNC<7{jaP%}o$^vg0d7L;`k%et!cw@Smy23_RWSbqEO# zm_TTVfr%Vh1HyR^$+(3mM;W}^MxXPF!%?1%QEQeU{N<7gL7}_rKeDu!4F>EA8sVH7 zio}1;MdPH8-1vHQggga}MrN9#fm*q8@_(FJ{@JA;M5(BpA~0br5!7wvbpfY#OiBb9 zViNWzl~KJsuUa{<$QiD$Vd{m8N{m*Hn-$o8wp_RGW_5fq%xNVuqo#jZCDLLs;rz*T zH{JbiIsSKA`+wW` z|2!=W!{CyMB!(lO>ZPs%Tm%J)Ql>d>2Ps;T5KCO3oa20D3n{vM1{cGs#4fnL1|}%a zl1em=;4fezL1Cfe=!`;IE+-w!V#8Rl%W1zG12D}M^Ab9B3E}%$ha?OQOHkd7`T7ON>SuWL?P9!UTAt_%ydM4Z3IiqRto;?^zV^7h3 zk@_+(P| z4N{s(&Ftim6Jzpqn2E8NhDDteQq3%*gck#>I?AzY$M7CeUI=3UAt9IM8R z9zZI`Md}zD$A4smFZOu|H-?6LnTyYHFhJ_I{!`hbBTO>B(7jHCLV1c4flnyN40$PO zXC7|q@V>%3ikxQA8KN?}7%$DCG`P8m*HK$CEOMurTkGM1T)AwGaD32dEIkaJv?NJ1 z8gua7%XdddACG&z?~~pIAAh~9YUz9d^joKZIY77=+W^LrXDNPguDhfTA0W<&9p!RV zB+R9Ea-j|9aRWTeStRUbj+~{eItu_*Sq$w~E3G_FG4(jX*OmYhnN(IerYvCkv`+hvZ@{dNRq_xYoqr|@8aFjhm+IBZzTf4x%_fIWqxcZ z;UqswuFSIDirlJJ*?&%bXb#5*^%>*zprN9nVwBqiS#t2*%hR)?CySw$?2!>H;Uf{-@5qyqn2q^#Q+0_Jo$)&<({H1w|$peMoxN{iqy`$ zC~N)pkqLGcH=CHqKnx1Xj=Th4kmV?GL^yi`F&q|-t(tk=Sc?_Qp&lmbu9L)Je z6UH-*pGF$ZNb)3x{ZjwPTJ;m!s=ibnB36jug|F*lf>}oR5ppu=A(bLAG3>TVG(|w2 zb+^mi3N@2S{owV1PszlDKFF;Jeq*AWob~-l7f0K418d2(RAg9qV&eyvFrDWg(1izgMy zJY0+a7J;i;3=<;ojE4TY-NxOiD4tpp00bdOg zEAS$ zGf=%u1pf`xu;Q_YYD0F&Q3xhtG}A9(mSr13uu;Mv^*JkJC0+H##Tt96lJnpqUv zz%6}FN#4Xum{Pralr8qIE>o0j;i4>|E9(xHt2!^5DN9<%c_sSu^Z);FeBmLW({~?^ zpA7`s0AQ=JnP>Io`QXxvxSho!=^9M+8$duG74dbh0_$K=pWtr+(WeGPcejA(#(?O< z&VK@)I5d3?EOsSQS-Xb-r8a~b?z%}?!=S{bI|F6B*+2TLX(kduI`DRvBT=5ETZh|ljgUXMEbL6I18nV$n z!Te^|V`4T=y2`Ea(z={AFRrG*v~yVDqkrUBTAQaPnH@>Z{`5{i$HeR5?+~vnYC-to z6(xU>YU3s7jOP(P1ftkaT4u_)ys}6q=XEap+E6o;yx18EqA}{eCm5o2^%rNx|Mx$O zaUXIOxKD|Tis;APV?xRaOE;c1Qwgb`t9;~8H*-NjIWcC#b#AyO9HX%J&|I;a*MHDZ ztEOAOENYm`OCNPX4`N8A)Cm)d)?PP<_t8gD+WhA6f*bz8M8d@Iq25@zD;V|*wi`ph zS&nDt?pPb==E=e8o2#2dI2R*|LS{KAZ5Cl*_9kypRw+Er<9}uUk4DH-HWX6h z4nr>c`SA!7!`GmZ6U)Zv6Y+)TL?pou2+S)Tc&RiF*cB%JBmn;F`v}eipZK47t6U^k zBLZgvqaZr!UG%Kf80^5IOj9Y~^I;DXrcDqHnQi*Nx%ffU|5G>p-|2(Vuw3c>Ouw43 zXwDh+3BAmeKm9VntH^v&!K lPRrkeXl$rVu=)2X54LUFw(W1+{uKZK|Ns15g{=Tg000SVA`k!o diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml index bf248494..6c7c462f 100755 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml @@ -6,4 +6,3 @@ spec: attachRequired: false volumeLifecycleModes: - Persistent - podInfoOnMount: true diff --git a/deploy/csi-nfs-driverinfo.yaml b/deploy/csi-nfs-driverinfo.yaml index fd592311..269c9a9d 100644 --- a/deploy/csi-nfs-driverinfo.yaml +++ b/deploy/csi-nfs-driverinfo.yaml @@ -7,4 +7,3 @@ spec: attachRequired: false volumeLifecycleModes: - Persistent - podInfoOnMount: true From fa61d6ee15312adc246f59ce66bf25722f4d209f Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Mon, 3 May 2021 06:18:32 +0000 Subject: [PATCH 03/17] add a new helm parameter for livenessProbe health port --- charts/README.md | 1 + charts/latest/csi-driver-nfs-v3.0.0.tgz | Bin 3137 -> 3161 bytes .../templates/csi-nfs-node.yaml | 2 +- charts/latest/csi-driver-nfs/values.yaml | 2 ++ 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index f1e86f70..2f0a0c2b 100644 --- a/charts/README.md +++ b/charts/README.md @@ -50,6 +50,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | `controller.runOnMaster` | run controller on master node | false | | `controller.logLevel` | controller driver log level |`5` | | `node.logLevel` | node driver log level |`5` | +| `node.livenessProbe.healthPort ` | the health check port for liveness probe |`29653` | ## troubleshooting - Add `--wait -v=5 --debug` in `helm install` command to get detailed error diff --git a/charts/latest/csi-driver-nfs-v3.0.0.tgz b/charts/latest/csi-driver-nfs-v3.0.0.tgz index 6be56befb7a68491cd2170e298f05643960d38cb..25f59f4c7224707fc262e3b985bfea7be5bc4991 100644 GIT binary patch delta 3132 zcmV-C48!xm7}*$*Jb!O*+c=Z&Yki72XL0EED%o)ow~v6l=qB0cf^E~F?)9)(ECQMu z+uTs3mZaQxv$@ZHASv6DEXU5@qFD1qVvED!a5OWV`H`lYg^6MlRG}E^_R*Lq6HQ6V zUp!vxbUK~h{=WO)>2&JK<5!`8vozz?RDz@-|g*f{r@={OUVf1 z002M?wf|t7!hcL^W~7?NkWv+;j4G++&_qdQ45!00Y0edD6rO-;cCO@v*$9EEdHTz@IvPe-7g`kltG}=BM zAmkbo_vh!w8MB9+F_j-OEe zjxUB;twegI8wdvIGB|6NgG%YVCh zW`A!~U`_mg(>v%@pGn^q2 zdIl&Q`7+^WrnLEAX2$>Rrzr@l2*cS5YYH_f=!HzMUwN&zRd+$DW8h5WSAX;Vns&`EtNsck3j;O*CMKT|hDJB&&(<4r__Uq77Z7>asIw_=z8Ab^&1{ldv5yPY? zVQnE?qD>YYhGXYl$|x+W)|?(dW|oUoF|&`L{^M4{G)KfHW*eEjL8-~X|JkRwS@h0eN}6ct5@Nqsu6ih^CG zZ?10#%u>xFE$L-lOXnrfZ=C`b0AV?{0Zb&%Q~Y3EcS#*CK%5gZ_J4BJB=pi-xzL1* zxB(94EE4uAN6u1Kp9O$sc?`RqPFj1OV(M_x{Z8jYb#8@!=BQV4TE(ba<}=g&43`Y6 zu2lY7?Vvv)|ikbSvDMvGk)t?NesWW`oH!s-W`89J!}15At0Q~uje!7Cz=vY{84gkmi1O- zSGCG^nnQCqIc&}tX9o>64Hcv8CdiV*A6}k)JU+QNJ#GryD1VYnGGPvXczM)6y*N2L zKL2=n_W8$+;&;KfDSrQ~WLj4-z>pzNKH+e=r|9#xZ*$AYN^et<#+l2qHg6xfU{`Uw zy()&-sIa8z`L?|{ndvr$eji%?s`X_`8c8LO;q%dXsc*%>oLe+uJXiR6tWb|7PyC6| zqK%Eo{*9)tuYb&kh!tWu;hXxHV1^NXjGRpSNTo=0411jlO%YJ%-MiK93>A|}^WgP? zFO!K5U65N7{3b-TIUBemERLpa2R4!`nUPW9iLIYl!j#WHswvzvZ|U9Dp||@4q1WrZ zA=O-|Zd)xmRaD>YhPpS?3}ZNUs+MZHG;O06G0n{Mn187lu5X^;RB$c+<2#uc%BRAA zI^Dg!I{vfYd$YxVo}-nap5i`1botif%DV$JWVe8aj{2wmfUf}wvOBZgEBwGYZbOVA zbo>eb%$c%z#q8|g``vpsL^`D$KgeX}gdv<;VYESEwim^oqpkISn@&9NKG-_^_h7Gn z{`aQa+kf5q|8uk*_&jDBetq}hy*vGz<#HP8mx*TI*@_pdp+mOb8 z62McE&PDpFaEtqP)pC0a13e)Obn8vd+km0vYou#|qJ|;M2FYicS{*XNzL8md&3Ltl zQ6){tG$q18MeP9-Z9Q(ihE_=JUzI|rhR~j^*2arj{!wfexAa*hIY+1a_qMR<#<1zbP7IznG<^%ib|s$KxRSeJ znN9oIZQ$ldb>yyxe0yy5KI^=g{d{AiIe%c)cJ!@_`wiMk{6`H44Ik$c;2Qj=`=(dF z|F^r}J=o$u&(UfqLiJ`=R9x{2Z*GDx41?N_g*|f9uMFAfo}jqpD7lMN8!tiS+{f+^NPouu ztz@2Qn=uxN;-Z3u-&)xmB`;!rK{P=%a0DYXwzA?(yZ`QIG4_L3f&G-&c!zG>Jtm}_ zuyo^D^DH6Fb5)NVnr1F2C?{HPxXumNgcB6z9-1pw^BP*&s_E7*i)v;w&PQF)gBVgN zRl-Es+MDKZKKdw1TiiS@x#3StB!5hd9_o#?yMp1MV7oB{oO?XmyJKUVTWtHQZ?0|< z;cSd3N|fcGv{{6K`J22&S(Wg(i2t?yKN=%X*+@u*kGIFHJO4fC_LhGCzqj50pQY`< zIWg9-+?8(s=KYVP2RY-`X&@u?l8jJCL2+25^UT$=pvTBLjJO>5;}IrCuYW-yCx%VX zrP!tCL?pou2=p}!yv!61*%c=4Bmn;7#|S}m!m!;Oz9vRG1GQ`z4$>i_)E3j-`RulsBQnuzUql+FBlC7z05Or`lW+ck^Y(m zuc86D3|>Vh&4O3|7wo_nqBWSzHJl!w=pf3JwBaF$Sc0VO>6H8(QJ%56n)b?8WL@z2}$|0 zhik1?t97uq=l{1_t?K_)`=Ilzz1P~?ZM9pwyPapP_Fk*qet!n72b5uD%Zw%ZS?k_& zRTuY_G@{ri)P_kB!?+y~rDk6_v1n|NprM*6>z^HUPvPwQE*yK4fxgu6W7b10(4q+h zNil}9Nd@B>``xG&wSq6b;%!@O75^U-o}qc*0IdINW~?T+*MEfLF?D1;<#IQX4C*nX_=ZxM z3F{=G8i~aitj;jdJwlfrQYoyKoagUQqvDJb6GO*+%fx%}ju?wNhCboOsfx?NTO1>g zVLu3*@NxCw&JKLQK57&c!Ag(*3kYhq#m-l~mbp|Kys z&{}2UW`A=u+p3%j6G=_e5;HPQDn*|SGL4~9&0uFI z@LlH_=iOYO7=HdG*ntr8ov=|1bzE|h8-KR#uM+<)rpo(o^X&b{lWt@$?fvIim;dc{ z_A2pzf4{wr|4-83;}EM^9WyXMYn>w{sJNV9;(t7CA~Ow?XU>tMAowVO4v~(kqm@hy z7)xwsUV`FC47v~y2LXU{Kx#RJp>q(U2*tyzD~LXzr;E;}PvQF7$&fP(LxUWtmHM!u zbIPSraDCl?F#Jp`9e!z@eobwq|6dG|E7T^k>b|Mqy8N&8YQN(D2Yc<-*8iWPU0pR_ zz<-#fF&MPaXB_QB;bH33LWdZ`i{|xp5O}haOGQKic8Gp|1KIbVsmBaF-PKhH3HF&l zXo$Xv99aXxc@N3Bg(ycEyxT^f@`}Szz8|C3EJ66gB^83g!LI+v(r(u8vrA}%b7m+K z|2Y?plOA&8>(L?d6f_!{X^IAF<;ux(W`Fr-mwphXqH>DBgt0_Wx0TlgoZ2xd5oCZ# z*qc;Fb@RMx<-j6mxVnO=7cMF>S~+f3VEfr}-M*RC@x?HwmBa(~Wpsm^pFS@{!5`RdU#*|2nnFVgdLa|+mR_rC|$BN7X5t;|nLjxnU3sDH!~MKT+t zG?SXy$q^^Up-UD#hU4a4$|xMG#*7|7 zD#u0Y7#b(PWrQ#Gc?dU#hI^Td&#~V}>bCwf*`-5FGQQBgPJ}{viW7m4D98+XDQRaO zZtC#9!aItbX3-gjYy*l8?144t$jNi-UB z_|5Y-$HyN|y4`PU2sx4jb?B{|Nl{Xi7+0suRZ*}j_09C{fmv!;WF#F`wRFA!`mIyI z93WhbZ2)7*vlQPs*IiPF4-n_X4s$sw66Vr7xzL95xB(vKEE4uIM}N*zR-FZasw{?f ztCd!sr&&oByqOi2?QN9> z7BE`iyaU&`{A( zG0JU%EIIt<`Puv9lZ(^iy0DERQIZLJ_|5a9?&-zJ+41@N(|@y%-)G%kPywrKOcr@)$lIoiFsQIGFQ`CX8nqKMpmTq2x&n`=$P&wdx17Reh;EM63|Q z3t!jA1hb6rV}Imi(nTsoVq(~Bm1v59I_qwiyA^6CllsBy1D}$K34M@T6a2ar`TK{+H#69nWt;2u!4qjEx|6X4VZv8=cgchzaq6J*a15v7Kla=NYNOg zCtxM$O!#{O3&Z+S`;w}T=xI4Z0iT&2%FHhE0F&Q3xqmI3d>455CZzHI4dB_`0z6Lx@Z5Mu z^Crk=`GV+L2&iVrvZ?WD4ps$&uxq8Nt{E@q(Pu#uGEIr_AW*Z%MAHl#FQE}q_gAM7 z>OM55t96lJnpqUvzzuy)N#4Xum{Pralr8qQE>o0j;i4>|E9(xIt2!^5DN9<%c_sSu z(Ak@JCr<_fZ2+*<*vymq@_caVMcm9{k#r5F`V}Cc4~qCYSAlh~s88^> zfaqfbqPtr_bYnpDerExX9Gboa7P}IutX;|NP|CXf>?YXqy*~06KfXL_dY5%xOn<+z z(HyX9JNnXP`-8QW_>b-p8a~V=z%}?!=YQ2f_5NRbuf4^8o}^V!g!0X-sJP-4US9`c z7zULe3+Kp9w=`s*_DgjQ{U{7UMqTDsZ0?7ZuTuyTgQ(6P9i~Yo-!XKUew4p>F1af^uTahU?sL zO*lqj@1VJ2H7}u|R!z5lS=2C>mp#&3}`F)i+l+iEu7P6ot%kP}(fQ!0b)lpsZ4OoX7vl z{vQpIr)(gk#)sQu)}80?p}1Q)EMl*kxWx5;L}kT5~fWM4VZ2Ezq$B9)cajG z{om<>;h^dMOuw43XwDh+2pwh0pMII(MPxp!;6>CUqu@nkQx&}UZ?FTOh-NY~aC&@V yf=FrU!b1?T1W7Ze)AH9K8XX%d6Kwu<%7bm&wr%?hxBmqI0RR7M>3&)OOaK7x-zK;K diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml index 8441184d..a7d50baf 100755 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml @@ -29,7 +29,7 @@ spec: args: - --csi-address=/csi/csi.sock - --probe-timeout=3s - - --health-port=29653 + - --health-port={{ .Values.node.livenessProbe.healthPort }} - --v=2 imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} volumeMounts: diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index 372bc893..949a5946 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -26,6 +26,8 @@ controller: node: logLevel: 5 + livenessProbe: + healthPort: 29653 ## Reference to one or more secrets to be used when pulling images ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ From eb1bf794e9ec3e250ff13b6d823e8ce626bd1d80 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 5 May 2021 14:11:03 +0200 Subject: [PATCH 04/17] Squashed 'release-tools/' changes from a1e11275..6616a6b5 https://github.com/kubernetes-csi/csi-release-tools/commit/6616a6b5 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/146 from pohly/kubernetes-1.21 https://github.com/kubernetes-csi/csi-release-tools/commit/510fb0f9 prow.sh: support Kubernetes 1.21 https://github.com/kubernetes-csi/csi-release-tools/commit/c63c61b3 prow.sh: add CSI_PROW_DEPLOYMENT_SUFFIX https://github.com/kubernetes-csi/csi-release-tools/commit/51ac11c3 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/144 from pohly/pull-jobs https://github.com/kubernetes-csi/csi-release-tools/commit/dd54c926 pull-test.sh: test importing csi-release-tools into other repo https://github.com/kubernetes-csi/csi-release-tools/commit/7d2643a5 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/143 from pohly/path-setup https://github.com/kubernetes-csi/csi-release-tools/commit/6880b0c8 prow.sh: avoid creating paths unless really running tests https://github.com/kubernetes-csi/csi-release-tools/commit/bc0504ad Merge https://github.com/kubernetes-csi/csi-release-tools/pull/140 from jsafrane/remove-unused-k8s-libs https://github.com/kubernetes-csi/csi-release-tools/commit/5b1de1ad go-get-kubernetes.sh: remove unused k8s libs https://github.com/kubernetes-csi/csi-release-tools/commit/49b42693 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/120 from pohly/add-kubernetes-release https://github.com/kubernetes-csi/csi-release-tools/commit/f7e7ee49 docs: steps for adding testing against new Kubernetes release git-subtree-dir: release-tools git-subtree-split: 6616a6b5294b6df39cfce37f4fce7cdce0a77583 --- SIDECAR_RELEASE_PROCESS.md | 44 ++++++++++++++++++++++++++++++++++ go-get-kubernetes.sh | 6 +++++ prow.sh | 49 ++++++++++++++++++++++++-------------- pull-test.sh | 32 +++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 18 deletions(-) create mode 100755 pull-test.sh diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 3aee7a81..f5ec71b9 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -104,3 +104,47 @@ naming convention `-on-`. CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + +## Adding support for a new Kubernetes release + +1. Add the new release to `k8s_versions` in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25 + to enable generating a job for it. Set `experimental_k8s_version` + in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40 + to ensure that the new jobs aren't run for PRs unless explicitly + requested. Generate and submit the new jobs. +1. Create a test PR to try out the new job in some repo with `/test + pull-kubernetes-csi---on-kubernetes-` where x.y + matches the Kubernetes release. Alternatively, run .prow.sh in that + repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`. +1. Optional: update to a [new + release](https://github.com/kubernetes-sigs/kind/tags) of kind with + pre-built images for the new Kubernetes release. This is optional + if the current version of kind is able to build images for the new + Kubernetes release. However, jobs require less resources when they + don't need to build those images from the Kubernetes source code. + This change needs to be tried out in a PR against a component + first, then get submitted against csi-release-tools. +1. Optional: propagate the updated csi-release-tools to all components + with the script from + https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402 +1. Once it is likely to work in all components, unset + `experimental_k8s_version` and submit the updated jobs. +1. Once all sidecars for the new Kubernetes release are released, + either bump the version number of the images in the existing + [csi-driver-host-path + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and/or create a new deployment, depending on what Kubernetes + release an updated sidecar is compatible with. If no new deployment + is needed, then add a symlink to document that there intentionally + isn't a separate deployment. This symlink is not needed for Prow + testing because that will use "kubernetes-latest" as fallback. + Update that link when creating a new deployment. +1. Create a new csi-driver-host-path release. +1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and + (eventually) roll that change out to all repos by updating + `release-tools` in them. This is used when testing manually. The + Prow jobs override that value, so also update + `hostpath_driver_version` in + https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47 diff --git a/go-get-kubernetes.sh b/go-get-kubernetes.sh index 8c4e3024..cbbbb7c3 100755 --- a/go-get-kubernetes.sh +++ b/go-get-kubernetes.sh @@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do + if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + echo "Kubernetes module $mod is not used, skipping" + # Remove the module from go.mod "replace" that was added by an older version of this script. + (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" + continue + fi # The presence of a potentially incomplete go.mod file affects this command, # so move elsewhere. modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || diff --git a/prow.sh b/prow.sh index b09c2920..fe23cf51 100755 --- a/prow.sh +++ b/prow.sh @@ -136,6 +136,9 @@ kind_version_default () { case "${CSI_PROW_KUBERNETES_VERSION}" in latest|master) echo main;; + 1.21*|release-1.21) + # TODO: replace this special case once the next KinD release supports 1.21. + echo main;; *) echo v0.10.0;; esac @@ -159,11 +162,6 @@ kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" -# Work directory. It has to allow running executables, therefore /tmp -# is avoided. Cleaning up after the script is intentionally left to -# the caller. -configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" - # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on # a deployment script that it searches for in several places: @@ -190,8 +188,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # CSI_PROW_DEPLOYMENT variable can be set in the # .prow.sh of each component when there are breaking changes # that require using a non-default deployment. The default -# is a deployment named "kubernetes-x.yy" (if available), -# otherwise "kubernetes-latest". +# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available), +# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}". # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, @@ -203,6 +201,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" +configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" # The install_csi_driver function may work also for other CSI drivers, # as long as they follow the conventions of the CSI hostpath driver. @@ -361,10 +360,23 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # to all the K8s versions we test against configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" -# This is the directory for additional result files. Usually set by Prow, but -# if not (for example, when invoking manually) it defaults to the work directory. -configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" -mkdir -p "${ARTIFACTS}" +# This creates directories that are required for testing. +ensure_paths () { + # Work directory. It has to allow running executables, therefore /tmp + # is avoided. Cleaning up after the script is intentionally left to + # the caller. + configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + + # This is the directory for additional result files. Usually set by Prow, but + # if not (for example, when invoking manually) it defaults to the work directory. + configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" + mkdir -p "${ARTIFACTS}" + + # For additional tools. + CSI_PROW_BIN="${CSI_PROW_WORK}/bin" + mkdir -p "${CSI_PROW_BIN}" + PATH="${CSI_PROW_BIN}:$PATH" +} run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 @@ -384,11 +396,6 @@ die () { exit 1 } -# For additional tools. -CSI_PROW_BIN="${CSI_PROW_WORK}/bin" -mkdir -p "${CSI_PROW_BIN}" -PATH="${CSI_PROW_BIN}:$PATH" - # Ensure that PATH has the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. @@ -647,9 +654,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy.sh" + file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -1098,6 +1105,9 @@ main () { local images ret ret=0 + # Set up work directory. + ensure_paths + images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. @@ -1259,6 +1269,9 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + # Might not be needed here, but call it just in case. + ensure_paths + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then # Needed for "RUN" steps on non-linux/amd64 platforms. # See https://github.com/multiarch/qemu-user-static#getting-started diff --git a/pull-test.sh b/pull-test.sh new file mode 100755 index 00000000..b019c177 --- /dev/null +++ b/pull-test.sh @@ -0,0 +1,32 @@ +#! /bin/sh + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is called by pull Prow jobs for the csi-release-tools +# repo to ensure that the changes in the PR work when imported into +# some other repo. + +set -ex + +# It must be called inside the updated csi-release-tools repo. +CSI_RELEASE_TOOLS_DIR="$(pwd)" + +# Update the other repo. +cd "$PULL_TEST_REPO_DIR" +git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master +git log -n2 + +# Now fall through to testing. +exec ./.prow.sh From 98ec1a8aaf7b4cb62632b9f7b87a8ff75f461022 Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Wed, 5 May 2021 15:42:48 +0000 Subject: [PATCH 05/17] Remove capacity setting in create Volume response --- pkg/nfs/controllerserver.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index 76967bab..7b8187d7 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -104,9 +104,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if err = os.Mkdir(internalVolumePath, 0777); err != nil && !os.IsExist(err) { return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err.Error()) } - // Remove capacity setting when provisioner 1.4.0 is available with fix for - // https://github.com/kubernetes-csi/external-provisioner/pull/271 - return &csi.CreateVolumeResponse{Volume: cs.nfsVolToCSI(nfsVol, reqCapacity)}, nil + return &csi.CreateVolumeResponse{Volume: cs.nfsVolToCSI(nfsVol)}, nil } // DeleteVolume delete a volume @@ -338,9 +336,9 @@ func (cs *ControllerServer) getVolumeSharePath(vol *nfsVolume) string { } // Convert into nfsVolume into a csi.Volume -func (cs *ControllerServer) nfsVolToCSI(vol *nfsVolume, reqCapacity int64) *csi.Volume { +func (cs *ControllerServer) nfsVolToCSI(vol *nfsVolume) *csi.Volume { return &csi.Volume{ - CapacityBytes: reqCapacity, + CapacityBytes: 0, // by setting it to zero, Provisioner will use PVC requested size as PV size VolumeId: vol.id, VolumeContext: map[string]string{ paramServer: vol.server, From 83ad09b7aa3cf383f10a8c69c7f6bc0d168d3f63 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Wed, 12 May 2021 10:45:03 +0000 Subject: [PATCH 06/17] chore: upgrade base image to fix CVE issue --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 085f66b5..af10df3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG ARCH=amd64 -FROM k8s.gcr.io/build-image/debian-base-${ARCH}:v2.1.3 +FROM k8s.gcr.io/build-image/debian-base:buster-v1.6.0 # Copy nfsplugin from build _output directory COPY bin/nfsplugin /nfsplugin From f5abcf764a0fc5bb1122fbaf028863cfd7f80692 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Wed, 19 May 2021 13:25:53 +0000 Subject: [PATCH 07/17] fix: remove unnecessary lock fix golint --- pkg/nfs/nodeserver.go | 10 ---------- pkg/nfs/nodeserver_test.go | 26 -------------------------- pkg/nfs/utils.go | 4 ---- 3 files changed, 40 deletions(-) diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go index 9a4891ff..f2e68587 100644 --- a/pkg/nfs/nodeserver.go +++ b/pkg/nfs/nodeserver.go @@ -65,11 +65,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return &csi.NodePublishVolumeResponse{}, nil } - if acquired := ns.Driver.volumeLocks.TryAcquire(volumeID); !acquired { - return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volumeID) - } - defer ns.Driver.volumeLocks.Release(volumeID) - mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags() if req.GetReadonly() { mountOptions = append(mountOptions, "ro") @@ -122,11 +117,6 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Error(codes.NotFound, "Volume not mounted") } - if acquired := ns.Driver.volumeLocks.TryAcquire(volumeID); !acquired { - return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volumeID) - } - defer ns.Driver.volumeLocks.Release(volumeID) - klog.V(2).Infof("NodeUnpublishVolume: CleanupMountPoint %s on volumeID(%s)", targetPath, volumeID) err = mount.CleanupMountPoint(targetPath, ns.mounter, false) if err != nil { diff --git a/pkg/nfs/nodeserver_test.go b/pkg/nfs/nodeserver_test.go index 5bd2a01d..03fce0af 100644 --- a/pkg/nfs/nodeserver_test.go +++ b/pkg/nfs/nodeserver_test.go @@ -19,7 +19,6 @@ package nfs import ( "context" "errors" - "fmt" "os" "reflect" "testing" @@ -69,19 +68,6 @@ func TestNodePublishVolume(t *testing.T) { VolumeId: "vol_1"}, expectedErr: status.Error(codes.InvalidArgument, "Target path not provided"), }, - { - desc: "[Error] Volume operation in progress", - setup: func() { - ns.Driver.volumeLocks.TryAcquire("vol_1") - }, - req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap}, - VolumeId: "vol_1", - TargetPath: targetTest}, - expectedErr: status.Error(codes.Aborted, fmt.Sprintf(volumeOperationAlreadyExistsFmt, "vol_1")), - cleanup: func() { - ns.Driver.volumeLocks.Release("vol_1") - }, - }, { desc: "[Success] Stage target path missing", req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap}, @@ -148,7 +134,6 @@ func TestNodeUnpublishVolume(t *testing.T) { errorTarget := testutil.GetWorkDirPath("error_is_likely_target", t) targetTest := testutil.GetWorkDirPath("target_test", t) targetFile := testutil.GetWorkDirPath("abc.go", t) - alreadyMountedTarget := testutil.GetWorkDirPath("false_is_likely_exist_target", t) tests := []struct { desc string @@ -177,17 +162,6 @@ func TestNodeUnpublishVolume(t *testing.T) { req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"}, expectedErr: status.Error(codes.NotFound, "Volume not mounted"), }, - { - desc: "[Error] Volume operation in progress", - setup: func() { - ns.Driver.volumeLocks.TryAcquire("vol_1") - }, - req: csi.NodeUnpublishVolumeRequest{TargetPath: alreadyMountedTarget, VolumeId: "vol_1"}, - expectedErr: status.Error(codes.Aborted, fmt.Sprintf(volumeOperationAlreadyExistsFmt, "vol_1")), - cleanup: func() { - ns.Driver.volumeLocks.Release("vol_1") - }, - }, } // Setup diff --git a/pkg/nfs/utils.go b/pkg/nfs/utils.go index 3b3c68ef..1c20569f 100644 --- a/pkg/nfs/utils.go +++ b/pkg/nfs/utils.go @@ -95,10 +95,6 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h return resp, err } -const ( - volumeOperationAlreadyExistsFmt = "An operation with the given Volume ID %s already exists" -) - type VolumeLocks struct { locks sets.String mux sync.Mutex From 92775b0d25fb93091efe1eee81ba0b14328a3db1 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Thu, 20 May 2021 13:08:52 +0000 Subject: [PATCH 08/17] chore: upgrade liveness probe and registrar --- charts/README.md | 4 ++-- charts/latest/csi-driver-nfs-v3.0.0.tgz | Bin 3161 -> 3168 bytes charts/latest/csi-driver-nfs/values.yaml | 4 ++-- deploy/csi-nfs-controller.yaml | 2 +- deploy/csi-nfs-node.yaml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/README.md b/charts/README.md index 2f0a0c2b..982747c2 100644 --- a/charts/README.md +++ b/charts/README.md @@ -38,10 +38,10 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | `image.csiProvisioner.tag` | csi-provisioner docker image tag | v2.0.4 | | `image.csiProvisioner.pullPolicy` | csi-provisioner image pull policy | IfNotPresent | | `image.livenessProbe.repository` | liveness-probe docker image | k8s.gcr.io/sig-storage/livenessprobe | -| `image.livenessProbe.tag` | liveness-probe docker image tag | v2.1.0 | +| `image.livenessProbe.tag` | liveness-probe docker image tag | v2.3.0 | | `image.livenessProbe.pullPolicy` | liveness-probe image pull policy | IfNotPresent | | `image.nodeDriverRegistrar.repository` | csi-node-driver-registrar docker image | k8s.gcr.io/sig-storage/csi-node-driver-registrar | -| `image.nodeDriverRegistrar.tag` | csi-node-driver-registrar docker image tag | v2.0.1 | +| `image.nodeDriverRegistrar.tag` | csi-node-driver-registrar docker image tag | v2.2.0 | | `image.nodeDriverRegistrar.pullPolicy` | csi-node-driver-registrar image pull policy | IfNotPresent | | `imagePullSecrets` | Specify docker-registry secret names as an array | [] (does not add image pull secrets to deployed pods) | | `serviceAccount.create` | whether create service account of csi-nfs-controller | true | diff --git a/charts/latest/csi-driver-nfs-v3.0.0.tgz b/charts/latest/csi-driver-nfs-v3.0.0.tgz index 25f59f4c7224707fc262e3b985bfea7be5bc4991..86ff63a3bb105f35a54cd4e22cfad01ee8147ee6 100644 GIT binary patch delta 3139 zcmV-J47~H%7~mL?Jb!I(+qjeOYki72XL0EED%r8~(7%AZ=qB0!1>2@U-RohoSOhdR zwz;85ElIiYW^^-TMD?G?tPP z#sL6;7;68)Hh+bg)XYdVjUlBfN*PsB%b|&=Oxss)ben3{7DF9U&E}&F8O`Iy>{*s` zejz!d(-_W%=h9p#)F?aw)$Bsa39}IbRrB;$Z*(-GDq^y&*(lUTDl$Slf0!-CR%MuU zqur>pnGBpwP@vWqN)GT|BFsi+cI1)(y-fuWGC?0uKY!wgX`{&fHDP(o99hk{EQ!?J zL?n%B!YIC_ROZ51NvJ|%Fb1P?4Ag+orH51sqa^449V*N?qeREhwcm2_LA)p0po(Eg zxVEa|a`X--$YaF?qzZV2MJMa;Qs8CP@BS8uTsUVdK zK_gXYw10g(K*%*F@MVm`EuD!GIOErrg~DMBV`DNMx7%0q9nESwkyN)0(O0@nr5LhN zt}x7G652HrCQ!94ztAxYGsPyv;Oret+riFG;JPky&ih`V7=HUb*ntrIhS@5HS}tDX z*0b%BwfJu^&7A+X&p&)R?MLR?JbaFI`CqTQSAUED`}@7UZTx?Z2A{@Q&+3?l5gKDX zB*Bc!DJIs1rZQJRd2Ss!3W84(=osl$eRL)h4cZWs>(`KRBpPi9h=TyYdKNVt!q8g2 z&IrY$n;VEeqN9t>XHVhw*2>_Sg`q}{)JS#IQl4^Y6x`mnAPm0{L&x7*r{7as>Hn8w zIBgq9fU$da`nT=bBf8;Cs0;M_L)>?;mS z`C)=eu>|37n^Xu22R-+ZrTu(3WY^FN7k|u9B<`~pjgtX#?ds7n@)Q(mnJbDK8s+lj zo|*q_^9zz?QFZTULK`Be-J_cVPUD!A2r|MX987DY`aZ8(Ij~3uZf;=ag=Hm1E62?X zY&V>R|GS<2ZTx?p7KUMP#Y7UriA(iTTLG3qL6T*5GBQ<)0`V(7kj z000s;!x<8xXMn=7FB5)dN}K;>X8hlNnu4&3Fr2TjrcjfDUdROdmDg%pb$=I>ItI=} zem(E6X}4@J3q+#y)~fuJN&FKMTX1PceL+kX92R zyY-*TJ{@C{^HTRZ5ensHoPP*>LP6$`N<}-1aMOhM72Z+gG-YRqs_0_0fYRXhHeN?< z$f(Gj=5DPlL|V=qTd zLNC3Q3r)C)8{lBhB4Mv`vY~%=T`V7)(~OHtkMtA9Hz$GxwWX7{Lewfm%YC5CzvW3maz8WXZ4%Z8(o=5Di+!r-;3 zikvAnVH`)86o`IhH6$Yg#+hMtJpzy1cY<>1`^~ICELn z=ItXF>^g3@SH%z;6_zwT-?kSgGu_6}??cO9wZ2SABdO#ud^)};^{qIVbBiX7=L$cK z73#6%i9az~w12TN*}u`$^_BS$u|f{$dcEEoQq7g>w$+kTMfKfosCzTbFoqMSYN@76(|)9v-@_|M+q{ucjvj#h?xiu(l7)f6CJOFO!)QhHznp(FTRtUKD$dw$}e`I`P2! zVC(ST{eNEl{O_=PaJcpV=V&|dY0NbI`u6=hcltNW0&9{^zIDO9vF4zkPrDY*5h#FkFMp zJgYA+23KCh-7LzaYmnFP04IG^T-dq_tOHN|=CFlLpBOgnZDG@mVbh157(8)k`WA@o zN<6c1CHKQJoA$H2z|9Zp*j*3#_Sovztbg-z_VbO6=73e((YG${H)t#IA2lE}e4I;w zYw(}$;X(cW-|k+ox5a;+qt#G^>dmaExZ)Mw-UeY92DKjxd*r5H8M4tmL4ULBF)q%d06cZ4WDalpHH-bJQd^W2x9*&Yc&Scr*MR;<6y``zf*U z4&At~n2>V9(v4@$vxGFyRXuWOnz^8$oM^q_IyYPsPEeSy&|I;a*U-vVO}BnoR5P1# zKI(!V#E?p<5+=&l-ZY2v(MM6*;(z9G$qj#EB4J|mP;ac=6$}Rj+l?XM+~e8a9UJ4^ zV%uMRb9IvlXJbTBqAUlc%_0oU-{c+2s)WZy{IBi*(HMEkMnWolygg>!`R{)BVCnb& zd)xj0S=tU<5MvF?UFr63-v3B?kTY(b1~Njg$Ov^56o*AR&s;qVdW@XIh=0q0KOSLX z^coa$V%P*-id}k6L=xrU1Q=-0^mP>jNpUdQ}_Gr%!_2JKj2JY6htTe z%f68cgB>`QX(|PLKJG)pln$a1Gi~?RiyuUTzf{}(ojn+j+V;=vtDcDVg3*A`t2}e3 zUpjad=`UIEDjJZh;8kSOEIfGif58rXCW^^i!`aEH4x&s+8yCzG{@b=~+xCxd{~rJV|NpTTkWc_j001v~KW+d3 delta 3132 zcmV-C48!x_7}*$*Jb!O*+c=Z&Yki72XL0EED%o)ow~v6l=qB0cf^E~F?)9)(ECQMu z+uTs3mZaQxv$@ZHASv6DEXU5@qFD1qVvED!a5OWV`H`lYg^6MlRG}E^_R*Lq6HQ6V zUp!vxbUK~h{=WO)>2&JK<5!`8vozz?RDz@-|g*f{r@={OUVf1 z002M?wf|t7!hcL^W~7?NkWv+;j4G++&_qdQ45!00Y0edD6rO-;cCO@v*$9EEdHTz@IvPe-7g`kltG}=BM zAmkbo_vh!w8MB9+F_j-OEe zjxUB;twegI8wdvIGB|6NgG%YVCh zW`A!~U`_mg(>v%@pGn^q2 zdIl&Q`7+^WrnLEAX2$>Rrzr@l2*cS5YYH_f=!HzMUwN&zRd+$DW8h5WSAX;Vns&`EtNsck3j;O*CMKT|hDJB&&(<4r__Uq77Z7>asIw_=z8Ab^&1{ldv5yPY? zVQnE?qD>YYhGXYl$|x+W)|?(dW|oUoF|&`L{^M4{G)KfHW*eEjL8-~X|JkRwS@h0eN}6ct5@Nqsu6ih^CG zZ?10#%u>xFE$L-lOXnrfZ=C`b0AV?{0Zb&%Q~Y3EcS#*CK%5gZ_J4BJB=pi-xzL1* zxB(94EE4uAN6u1Kp9O$sc?`RqPFj1OV(M_x{Z8jYb#8@!=BQV4TE(ba<}=g&43`Y6 zu2lY7?Vvv)|ikbSvDMvGk)t?NesWW`oH!s-W`89J!}15At0Q~uje!7Cz=vY{84gkmi1O- zSGCG^nnQCqIc&}tX9o>64Hcv8CdiV*A6}k)JU+QNJ#GryD1VYnGGPvXczM)6y*N2L zKL2=n_W8$+;&;KfDSrQ~WLj4-z>pzNKH+e=r|9#xZ*$AYN^et<#+l2qHg6xfU{`Uw zy()&-sIa8z`L?|{ndvr$eji%?s`X_`8c8LO;q%dXsc*%>oLe+uJXiR6tWb|7PyC6| zqK%Eo{*9)tuYb&kh!tWu;hXxHV1^NXjGRpSNTo=0411jlO%YJ%-MiK93>A|}^WgP? zFO!K5U65N7{3b-TIUBemERLpa2R4!`nUPW9iLIYl!j#WHswvzvZ|U9Dp||@4q1WrZ zA=O-|Zd)xmRaD>YhPpS?3}ZNUs+MZHG;O06G0n{Mn187lu5X^;RB$c+<2#uc%BRAA zI^Dg!I{vfYd$YxVo}-nap5i`1botif%DV$JWVe8aj{2wmfUf}wvOBZgEBwGYZbOVA zbo>eb%$c%z#q8|g``vpsL^`D$KgeX}gdv<;VYESEwim^oqpkISn@&9NKG-_^_h7Gn z{`aQa+kf5q|8uk*_&jDBetq}hy*vGz<#HP8mx*TI*@_pdp+mOb8 z62McE&PDpFaEtqP)pC0a13e)Obn8vd+km0vYou#|qJ|;M2FYicS{*XNzL8md&3Ltl zQ6){tG$q18MeP9-Z9Q(ihE_=JUzI|rhR~j^*2arj{!wfexAa*hIY+1a_qMR<#<1zbP7IznG<^%ib|s$KxRSeJ znN9oIZQ$ldb>yyxe0yy5KI^=g{d{AiIe%c)cJ!@_`wiMk{6`H44Ik$c;2Qj=`=(dF z|F^r}J=o$u&(UfqLiJ`=R9x{2Z*GDx41?N_g*|f9uMFAfo}jqpD7lMN8!tiS+{f+^NPouu ztz@2Qn=uxN;-Z3u-&)xmB`;!rK{P=%a0DYXwzA?(yZ`QIG4_L3f&G-&c!zG>Jtm}_ zuyo^D^DH6Fb5)NVnr1F2C?{HPxXumNgcB6z9-1pw^BP*&s_E7*i)v;w&PQF)gBVgN zRl-Es+MDKZKKdw1TiiS@x#3StB!5hd9_o#?yMp1MV7oB{oO?XmyJKUVTWtHQZ?0|< z;cSd3N|fcGv{{6K`J22&S(Wg(i2t?yKN=%X*+@u*kGIFHJO4fC_LhGCzqj50pQY`< zIWg9-+?8(s=KYVP2RY-`X&@u?l8jJCL2+25^UT$=pvTBLjJO>5;}IrCuYW-yCx%VX zrP!tCL?pou2=p}!yv!61*%c=4Bmn;7#|S}m!m!;Oz9vRG1GQ`z4$>i_)E3j-`RulsBQnuzUql+FBlC7z05Or`lW+ck^Y(m zuc86D3|>Vh&4O3|7wo_nqBWSzHJl!w=pf3JwBaF$Sc0VO>6H8 Date: Fri, 21 May 2021 09:23:46 +0200 Subject: [PATCH 09/17] Squashed 'release-tools/' changes from 6616a6b5..f3255906 https://github.com/kubernetes-csi/csi-release-tools/commit/f3255906 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/149 from pohly/cluster-logs https://github.com/kubernetes-csi/csi-release-tools/commit/4b03b308 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/155 from pohly/owners https://github.com/kubernetes-csi/csi-release-tools/commit/a6453c86 owners: introduce aliases https://github.com/kubernetes-csi/csi-release-tools/commit/ad83def4 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/153 from pohly/fix-image-builds https://github.com/kubernetes-csi/csi-release-tools/commit/55617801 build.make: fix image publishng https://github.com/kubernetes-csi/csi-release-tools/commit/29bd39b3 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/152 from pohly/bump-csi-test https://github.com/kubernetes-csi/csi-release-tools/commit/bc427931 prow.sh: use csi-test v4.2.0 https://github.com/kubernetes-csi/csi-release-tools/commit/b546baaf Merge https://github.com/kubernetes-csi/csi-release-tools/pull/150 from mauriciopoppe/windows-multiarch-args https://github.com/kubernetes-csi/csi-release-tools/commit/bfbb6f35 add parameter base_image and addon_image to BUILD_PARAMETERS https://github.com/kubernetes-csi/csi-release-tools/commit/2d61d3bc Merge https://github.com/kubernetes-csi/csi-release-tools/pull/151 from humblec/cm https://github.com/kubernetes-csi/csi-release-tools/commit/48e71f06 Replace `which` command ( non standard) with `command -v` builtin https://github.com/kubernetes-csi/csi-release-tools/commit/feb20e26 prow.sh: collect cluster logs https://github.com/kubernetes-csi/csi-release-tools/commit/7b96bea3 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/148 from dobsonj/add-checkpathcmd-to-prow https://github.com/kubernetes-csi/csi-release-tools/commit/2d2e03b7 prow.sh: enable -csi.checkpathcmd option in csi-sanity https://github.com/kubernetes-csi/csi-release-tools/commit/09d41512 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/147 from pohly/mock-testing https://github.com/kubernetes-csi/csi-release-tools/commit/74cfbc97 prow.sh: support mock tests https://github.com/kubernetes-csi/csi-release-tools/commit/4a3f1103 prow.sh: remove obsolete test suppression git-subtree-dir: release-tools git-subtree-split: f3255906abf2e4134cb3db4ff79ddfb5ce1e91aa --- KUBERNETES_CSI_OWNERS_ALIASES | 44 ++++++++++++++ OWNERS | 7 +-- OWNERS_ALIASES | 1 + build.make | 49 ++++++++++++---- cloudbuild.yaml | 2 +- prow.sh | 107 +++++++++++++++++++++++++++------- verify-shellcheck.sh | 2 +- 7 files changed, 173 insertions(+), 39 deletions(-) create mode 100644 KUBERNETES_CSI_OWNERS_ALIASES create mode 120000 OWNERS_ALIASES diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES new file mode 100644 index 00000000..6fb858aa --- /dev/null +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -0,0 +1,44 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +aliases: + + # SIG-Storage chairs and leads should always have approval rights in all repos. + # Others may be added as needed here or in each repo. + kubernetes-csi-approvers: + - jsafrane + - msau42 + - saad-ali + - xing-yang + + # Reviewers are automatically assigned to new PRs. The following + # reviewers will be active in all repos. Other reviewers can be + # added in each repo. + # + # Reviewers are encouraged to set the "Busy" flag in their GitHub status + # when they are temporarily unable to review PRs. + kubernetes-csi-reviewers: + - andyzhangx + - chrishenzie + - ggriffiths + - gnufied + - j-griffith + - Jiawei0227 + - jingxu97 + - jsafrane + - pohly + - xing-yang + +# This documents who previously contributed to Kubernetes-CSI +# as approver. +emeritus_approver: +- lpabon +- sbezverk +- vladimirvivien + +# This documents who previously contributed to Kubernetes-CSI +# as reviewer. +emeritus_reviewer: +- lpabon +- saad-ali +- sbezverk +- vladimirvivien diff --git a/OWNERS b/OWNERS index 6d2f474e..1fb74587 100644 --- a/OWNERS +++ b/OWNERS @@ -1,11 +1,8 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: -- saad-ali -- msau42 +- kubernetes-csi-approvers - pohly reviewers: -- saad-ali -- msau42 -- pohly +- kubernetes-csi-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 00000000..7ec6c034 --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file diff --git a/build.make b/build.make index 1faaf3b9..0ed51e80 100644 --- a/build.make +++ b/build.make @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# force the usage of /bin/bash instead of /bin/sh +SHELL := /bin/bash + .PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be @@ -63,26 +66,35 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of triplets, +# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. BUILD_PLATFORMS = # Add go ldflags using LDFLAGS at the time of compilation. -IMPORTPATH_LDFLAGS = -X main.version=$(REV) +IMPORTPATH_LDFLAGS = -X main.version=$(REV) EXT_LDFLAGS = -extldflags "-static" -LDFLAGS = +LDFLAGS = FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + # os_arch_seen captures all of the $$os-$$arch seen for the current binary + # that we want to build, if we've seen an $$os-$$arch before it means that + # we don't need to build it again, this is done to avoid building + # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ + continue; \ + fi; \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ + os_arch_seen+=";$$os-$$arch"; \ done $(CMDS:%=container-%): container-%: build-% @@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?= # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ - DOCKER_CLI_EXPERIMENTAL=enabled; \ - export DOCKER_CLI_EXPERIMENTAL; \ + export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ + --build-arg BASE_IMAGE=$$base_image \ + --build-arg ADDON_IMAGE=$$addon_image \ --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + if [ $$os = "windows" ]; then \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \ + docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \ + fi; \ + done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ if [ $(PULL_BASE_REF) = "master" ]; then \ @@ -288,4 +316,3 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" - diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1e02ba6c..b39a6db3 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/prow.sh b/prow.sh index fe23cf51..55f2a91c 100755 --- a/prow.sh +++ b/prow.sh @@ -77,7 +77,10 @@ version_to_git () { esac } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +# the list of windows versions was matched from: +# - https://hub.docker.com/_/microsoft-windows-nanoserver +# - https://hub.docker.com/_/microsoft-windows-servercore +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -236,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 +configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -296,6 +299,17 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go +# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants). +# When testing canary images, those get used instead of the images specified +# in the e2e.test's normal YAML files. +# +# The default is to enable this for all jobs which use canary images +# because we want to know whether our release candidates will pass all +# existing tests (the storage testsuites and mock testing in +# Kubernetes). +configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" + # Regex for non-alpha, feature-tagged tests that should be run. # configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version" @@ -354,11 +368,7 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -# -# "different node" test skips can be removed once -# https://github.com/kubernetes/kubernetes/pull/82678 has been backported -# to all the K8s versions we test against -configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" # This creates directories that are required for testing. ensure_paths () { @@ -625,11 +635,16 @@ EOF # Deletes kind cluster inside a prow job delete_cluster_inside_prow_job() { + local name="$1" + # Inside a real Prow job it is better to clean up at runtime # instead of leaving that to the Prow job cleanup code # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + # + # This is also a good time to collect logs. if [ "$JOB_NAME" ]; then if kind get clusters | grep -q csi-prow; then + run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name" run kind delete cluster --name=csi-prow || die "kind delete failed" fi unset KUBECONFIG @@ -716,7 +731,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" @@ -763,7 +778,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" @@ -832,7 +847,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done } @@ -877,6 +892,29 @@ start_loggers () { done } +# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k +# source code, if needed. +patch_kubernetes () { + local source="$1" target="$2" + + if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because + # e2e.test does not support it (see test/utils/image/manifest.go). Instead we + # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that + # overrides that registry. + find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;' + cat >"$target/e2e-repo-list" <&2 <&1) + +EOF + fi +} + # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test". install_e2e () { if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then @@ -885,6 +923,7 @@ install_e2e () { git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then + patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" @@ -936,7 +975,7 @@ run_e2e () ( trap move_junit EXIT cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" ) # Run csi-sanity against installed CSI driver. @@ -948,7 +987,7 @@ run_sanity () ( kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@" EOF # Using "rm -rf" as fallback for "rmdir" is a workaround for: - # Node Service + # Node Service # should work # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624 # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896 @@ -975,6 +1014,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- exit 1 fi EOF + + cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" </dev/null; then +if command -v shellcheck &>/dev/null; then detected_version="$(shellcheck --version | grep 'version: .*')" if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then HAVE_SHELLCHECK=true From 5bdbb62e0f266a5f223834f328c0b72b837623b9 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 21 May 2021 09:24:27 +0200 Subject: [PATCH 10/17] OWNERS update with aliases --- OWNERS | 7 ++----- OWNERS_ALIASES | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) create mode 120000 OWNERS_ALIASES diff --git a/OWNERS b/OWNERS index a99a8301..aeddf629 100644 --- a/OWNERS +++ b/OWNERS @@ -1,10 +1,7 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: +- kubernetes-csi-approvers - andyzhangx -- msau42 -- saad-ali reviewers: -- andyzhangx -- msau42 -- saad-ali +- kubernetes-csi-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 00000000..3f60d461 --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +release-tools/KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file From 61fca129a0432efe8a5ce0038c87a299e4ec8521 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sun, 23 May 2021 09:47:55 +0000 Subject: [PATCH 11/17] test: run external e2e test using 1.20 suites run on 1.20 run on 1.21 run on 1.20 --- test/external-e2e/run.sh | 6 +++--- test/external-e2e/testdriver.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh index f1d1b3d6..c41c5840 100644 --- a/test/external-e2e/run.sh +++ b/test/external-e2e/run.sh @@ -24,12 +24,12 @@ install_ginkgo () { } setup_e2e_binaries() { - # download k8s external e2e binary for kubernetes v1.19 - curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.19.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz + # download k8s external e2e binary for kubernetes v1.20 + curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.20.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz # install the csi driver nfs - mkdir -p /tmp/csi-nfs && cp deploy/example/storageclass-nfs.yaml /tmp/csi-nfs/storageclass.yaml + mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml make e2e-bootstrap make install-nfs-server } diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml index b51d970e..1cf77cfd 100644 --- a/test/external-e2e/testdriver.yaml +++ b/test/external-e2e/testdriver.yaml @@ -2,7 +2,7 @@ # See https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external StorageClass: - FromFile: /tmp/csi-nfs/storageclass.yaml + FromFile: /tmp/csi/storageclass.yaml DriverInfo: Name: nfs.csi.k8s.io Capabilities: From 485a98a1116b1b61b1796c17b5b405adf4f938cd Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Mon, 24 May 2021 04:47:52 +0000 Subject: [PATCH 12/17] test: run externel tests with 1.21 suites --- deploy/example/storageclass-nfs.yaml | 2 +- test/external-e2e/run.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/example/storageclass-nfs.yaml b/deploy/example/storageclass-nfs.yaml index d3f7d748..9704c9dd 100644 --- a/deploy/example/storageclass-nfs.yaml +++ b/deploy/example/storageclass-nfs.yaml @@ -7,7 +7,7 @@ provisioner: nfs.csi.k8s.io parameters: server: nfs-server.default.svc.cluster.local share: / -reclaimPolicy: Retain +reclaimPolicy: Delete volumeBindingMode: Immediate mountOptions: - hard diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh index c41c5840..fffe607f 100644 --- a/test/external-e2e/run.sh +++ b/test/external-e2e/run.sh @@ -24,8 +24,8 @@ install_ginkgo () { } setup_e2e_binaries() { - # download k8s external e2e binary for kubernetes v1.20 - curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.20.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz + # download k8s external e2e binary + curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz # install the csi driver nfs From 5df685273dbde4904930cc5732dd21f44b832279 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Sat, 29 May 2021 04:07:34 +0000 Subject: [PATCH 13/17] feat: support fsGroupPolicy feature update chart file fix chart file fix helm options fix Makefile --- Makefile | 11 ++++++----- charts/README.md | 6 ++++++ charts/latest/csi-driver-nfs-v3.0.0.tgz | Bin 3168 -> 3210 bytes .../templates/csi-nfs-driverinfo.yaml | 3 +++ charts/latest/csi-driver-nfs/values.yaml | 3 +++ test/external-e2e/run.sh | 5 ++++- test/external-e2e/testdriver.yaml | 1 + 7 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 480ba557..c294a4d7 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ REGISTRY_NAME ?= $(shell echo $(REGISTRY) | sed "s/.azurecr.io//g") IMAGE_TAG = $(REGISTRY)/$(IMAGENAME):$(IMAGE_VERSION) IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGENAME):latest +E2E_HELM_OPTIONS ?= --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) --set image.nfs.tag=$(IMAGE_VERSION) --set image.nfs.pullPolicy=Always +E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS} + all: nfs .PHONY: verify @@ -112,11 +115,9 @@ install-helm: e2e-bootstrap: install-helm docker pull $(IMAGE_TAG) || make container push helm install csi-driver-nfs ./charts/latest/csi-driver-nfs --namespace kube-system --wait --timeout=15m -v=5 --debug \ - --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) \ - --set image.nfs.tag=$(IMAGE_VERSION) \ - --set image.nfs.pullPolicy=Always - --set controller.logLevel=8 - --set node.logLevel=8 + ${E2E_HELM_OPTIONS} \ + --set controller.logLevel=8 \ + --set node.logLevel=8 .PHONY: e2e-teardown e2e-teardown: diff --git a/charts/README.md b/charts/README.md index 982747c2..4d3ce5d1 100644 --- a/charts/README.md +++ b/charts/README.md @@ -3,6 +3,11 @@ ## Prerequisites - [install Helm](https://helm.sh/docs/intro/quickstart/#install-helm) +### Tips + - `--set controller.runOnMaster=true` could make csi-nfs-controller only run on master node + - `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster + - `--set controller.replicas=1` could set replica of csi-nfs-controller as `1` + ## install latest version ```console helm repo add csi-driver-nfs https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts @@ -31,6 +36,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | Parameter | Description | Default | |---------------------------------------------------|------------------------------------------------------------|-------------------------------------------------------------------| +| `feature.enableFSGroupPolicy` | enable `fsGroupPolicy` on a k8s 1.20+ cluster | `false` | | `image.nfs.repository` | csi-driver-nfs docker image | gcr.io/k8s-staging-sig-storage/nfsplugin | | `image.nfs.tag` | csi-driver-nfs docker image tag | amd64-linux-canary | | `image.nfs.pullPolicy` | csi-driver-nfs image pull policy | IfNotPresent | diff --git a/charts/latest/csi-driver-nfs-v3.0.0.tgz b/charts/latest/csi-driver-nfs-v3.0.0.tgz index 86ff63a3bb105f35a54cd4e22cfad01ee8147ee6..46cc8ca3e1ab6b8c8e10c4ada8819268db6c1353 100644 GIT binary patch delta 3010 zcmV;z3qADU7>XH?Jbzzv+c=WXGe1S2Jav`Hg_JD$PnK1F$k@)FYZJ#6<#IJOwdI1q zmV`A3Z~#!T#^d|!R{$u9q$o>{|724mJlGP^Xf&GLjqV>Hs#%yQHbE7Np>7|Ki89fY zr2NI>wN9tgIo#iO|2v&d{eNfop!;HXud~0`+u!ST_g-{%_kTLQ?hEKVq6{lrt_@Ky zIuD+!zPKNxk&JyorI{2lOm+j3W%I9`7}O?6P*cS+Gl|@wSy)&HpEa=cpe!0Ic!<-Gjr!y8rh&dt3j1j>b|l z!Z-i`5JT-h*ng%llbRW+rZJ>cMJc07YB@9!m1+Cxjc!xT+G40fs@Z&$A)|TRm_5sK z&MzcqbQ;6i@LZY;g&Kt?pqgDMIbk+JplY7}>Wz*@R7Fg-H5-N6NJU0y=MS^R*s2VZ zZnPV9Hj{y~2@2HuLdgN%ON80T%#K_Vptq?2LMG?~>VGF3F>MsNza}h?nIo$imnD(9 zn~0=QO&G+^S2E9L zOY;*3JAXUy35TdqPy{1E3IwSjl?p*4RcN$*JV3}bCh&EP!fA+!5jY>wmW9G$3}a(5 z9k<(8^Nr8GkVI15Hbh_PHkD$?M!CW;lSyb*LzqC-w){fJEX)*}5QDQ-O!ujxUB;twego11RSGZFh=W*8;B7+@qvMGTXogtdimg*I7m7>=EHDWkBgT61~;nOQDU z#n3waJtw@}=OEk|TJ~j@pMT?Uh}3TV=dw@7nB=_Fy-tKec^M}HpHPrFR#MTJWuh1b=@U(xBziM%-G9Olh8|Vuc8Q#z?b z!%~#FIqK;p0uQo=nSWH~sQb5r{q7RQTUnTkPr%%W>~(rP)2IUF|-pU5TOI z#F%UXvc`lg$+F>Sq`BLyq%e4Gsv>8KO&G@!CIzBjSq;g^fN^G6U5@}HNn-e|)&I4B z`S#?)*?H@q6#~M!{CY8CeyS0leVbcGR(hL?G|pU>wR!u<1-p*h?Nu?vMujC!&$sQx$xOE~^!w2AN3AbY z(nuC&`~TEsLn(-Wp*xVe3TyEAL?AK%HuP(Bs@)9LN+E#W_%E&lT?tqk=P_X(n_Hy&5s z9iSn*1w3@zKl2BC4M>pPneATT2hMRDVho|-yiZt@VGKPCW2F*gE`ozgIv1 zJM11DZvFo`+75gkGY!AK{qWwM{>^eZ4fV@J@Si|6D;_&YF5~%#33z*P26FH(q(%oj zV8+A%Q3wJlYE9G>j0BYncTb=+Y$%m0c?$HIV1Ft_AMXo3TZT^rJKqmTE(CcJ2)W?> z1AxanAQc-iLAXPSsT9KJmGD*OJb?kiK*=i<@Rgad%uSgGXn*VEzHsulz{7VTjejM8 zrzD+=^i|;&_wB0X_7(rzjTf{0qu4C&=(9?4j!;3Knw_X>vG;YX zqOc3gvV^wyJ6f*&oM^f#X%pv_7|}oe=aeyWm z`Tp4Ix2*GW_PdOY=73e((f5-y3Q~WP94l*c)Fd}!sn{RRofnvRGyEOmwLv8aFJDn| z7pXR0g37s%-64>SIV4-#jIl@*7Zoi0*2?B6c@gsqq6w;jBN(Bvl@({&{dYf$u^+q& z?5D)WJ9Oi|VM59YOE;c1&l1u+SM|uDY3720a-#Ky>)dcnI6+~)L371wUPFH?TQ%MK zWl_y+#`&lVdJsb@rAnA6TYJ+S&PN|bX^Wf3B{%$qiG+#KL%p$fS1=qDY&V90bB||x zcWjJvi*0}P&DBjJoQ)AhiLxA&Hj6MYf0K78s}dd;@xQkJM`PqE8wsiK@%ET?=fC^i zgZl6PyL;RH|5@4&To7Xo%UuWQ_HW+*`gxO#3qLWA;G^JE_rvVWi)5=m;7ni?L?`{r zzL5%p9XOV0Dg}Hw?nAU@Jb!I(+qjeOYki72XL0EED%r8~(7%AZ=qB0!1>2@U-RohoSOhdR zwz;85ElIiYW^^-TMD?G?tPP z#sL6;7;68)Hh+bg)XYdVjUlBfN*PsB%b|&=Oxss)ben3{7DF9U&E}&F8O`Iy>{*s` zejz!d(-_W%=h9p#)F?aw)$Bsa39}IbRrB;$Z*(-GDq^y&*(lUTDl$Slf0!-CR%MuU zqur>pnGBpwP@vWqN)GT|BFsi+cI1)(y-fuWGC?0uKY!wgX`{&fHDP(o99hk{EQ!?J zL?n%B!YIC_ROZ51NvJ|%Fb1P?4Ag+orH51sqa^449V*N?qeREhwcm2_LA)p0po(Eg zxVEa|a`X--$YaF?qzZV2MJMa;Qs8CP@BS8uTsUVdK zK_gXYw3AH&2Y+RZ!Y!SN5jf-5mW9G$3}a(59k<(8^Bv7ohFUIOdT2#A9KzuX%yVvwjd0@5JSh`TBqMrTj~FoW8@hs z9hvN*sejsk?(+rPNs}?%O7+$q+Z-c;* zonB`|Bw)tqHZG7u_nA7(z|q~@gplBn351py>d2C{AYAm2oEwNd%HZ5K`s^zXOZj1f zO0fjtZ<|yI3I{#+k){27IAqt*3Kz^!B<`~pjenB?a_#ETG4d1?YMCpF8XD#Dg$E&3LZ*9EW_TaWM{=1^W zmVbBi%>JuUfi>~}@L>O-8vnbU{cZezo)(5-aK%It!--4vQdJw3dYQOjxl}+} zk&OJ6qWC^VQ6%$0nPO5gGd<=+YrhUX)dthBsFOmfm|>LgVt|nx z6){YT64n;N720IMVK{c)rHsO|YR%~ZWM;WY6+`Rv&z$gbpM!8?XxW!pevZQ-QoHq^ z%RU`rlJipcIuQ!xWt<3nLP6$`N`FN=i*VC~_Z8kzMkTqtY^`u|)NL(246U>zNfc^*^ux=yCnq0I`~4pq2sx4jRp_jnNl{UhnAE59 zswmi1`sVs}z%11)(vn`)wRBMe{nja90T7mB8^A>JJjM6cb(hrP0>lL|V}CD4O+qid zl?zR{h#TNw&LUy2a^x&!^;rODmdCK$>7=#iDW(o5-RpGTSLasvXO4O$r&Wx)Wj-_Q z&v41G3j5w+hZQ7qof%e%H?zX3y{)nUPwAu%4NFnx=BTHa2t3FdW>T4>?!ON9yGs<4 zqwc*7tGcAGv8y{Q$GxwWW`FmncD4JYb|r>-6JxRo$Ql!}B+G`Qk>+l*lEUD%sfwH_ zHenn`m=uV9Wi=!t1IC$Qbv*)*B#Gg-R{z)j<=d0@XXmZoD+Gjd`Q>89{8UrIi9brN z&9dH#?5bATPIG7uCr8a03^ux>Z4=1OWXD3Zz8-GQTNhZwE4=<1V zXP2kvCl?>i&OiOQQT#6WHpTCsl}zg@1{gBr$wwS6_Y{5J_HAw%S?O&m(l~Qj*5>Ua z7wkH2w^zjw8x@u`J>Rw$Co|o~(C3{DD|y4m~)FJjOPkJjTP## zm9k{KBlp4j?{B~1DJqng4!^OoLS9eTS@ z5PH4d8&b`c>bBLAQ$_XNZm4@R%`k=&r)sIDOVc)L5!1{}Pk)$-;pX-UP6gNEKfaTR zp?oU*r_=5A>iEyz;r>#;}=OZTI?Zp|$ z!5>JC4tBtdi2Ke6|dq2zI_7kX#7z zBoK1J`#S)ScR(sOVuEmo5>qLJ&nw}J%y|L>gn^P*D1YD!Gh>;XG7r%H*2#V0e^m;h8bW)vS{pBB`A4x?+|g&1ClEbXC$O&MPsZU;gK((@O^woxgp5ll=)80q>I*3Lz*K zi0w)|vvDQ&!!n!pv%A2}59-)m5Bc`k>esCEa`y9$jpl$=+tIg^UkXwTlpHH-bJQd^ zW2x9*lf4QZfBjp@JkvH~EE2^<1q;8mvN=j##QcJ2f@Vh7`kV>f%Cd$^{G>7xiM^W11=5fgle_|qGUSjl6Z>-%F3YYgvUkvukHWQ7G%J8+x`Dp+74V0V-3q)>Gp5l|44e1whKQ9jNpUdQ}_Grll==Abw2Jx!jul85i@P~ z*NY!SgTGYU{hd7+kJ|Rn?5m!L_JYxX(5pOir(ZgF73nWo@G2URtKd~+(kyuOf58rX zCW^^i!`aEH4x&s+8yCzG{@b=~+xCxd{~rJV|NpTTkWc_j F0080;%@F_q diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml index 6c7c462f..a1bfac8d 100755 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml @@ -6,3 +6,6 @@ spec: attachRequired: false volumeLifecycleModes: - Persistent + {{- if .Values.feature.enableFSGroupPolicy}} + fsGroupPolicy: File + {{- end}} diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index 74d1bcf1..5cf553fa 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -29,6 +29,9 @@ node: livenessProbe: healthPort: 29653 +feature: + enableFSGroupPolicy: false + ## Reference to one or more secrets to be used when pulling images ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh index fffe607f..1d89d8ab 100644 --- a/test/external-e2e/run.sh +++ b/test/external-e2e/run.sh @@ -28,7 +28,10 @@ setup_e2e_binaries() { curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz - # install the csi driver nfs + # enable fsGroupPolicy (only available from k8s 1.20) + export EXTRA_HELM_OPTIONS="--set feature.enableFSGroupPolicy=true" + + # install csi driver mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml make e2e-bootstrap make install-nfs-server diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml index 1cf77cfd..c9386554 100644 --- a/test/external-e2e/testdriver.yaml +++ b/test/external-e2e/testdriver.yaml @@ -10,3 +10,4 @@ DriverInfo: exec: true multipods: true RWX: true + fsGroup: true From 742f7a55c07086fcba632e582bb205f176f0b8f7 Mon Sep 17 00:00:00 2001 From: Andy Zhang Date: Tue, 1 Jun 2021 11:44:29 +0800 Subject: [PATCH 14/17] Create README.md --- deploy/example/fsgroup/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 deploy/example/fsgroup/README.md diff --git a/deploy/example/fsgroup/README.md b/deploy/example/fsgroup/README.md new file mode 100644 index 00000000..8234d2f1 --- /dev/null +++ b/deploy/example/fsgroup/README.md @@ -0,0 +1,24 @@ +# fsGroup Support + +[fsGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) feature is supported from Kubernetes 1.20, default CSI driver installation does not have this feature, follow below steps to enable this feature. + +### Option#1: Enable fsGroupPolicy support in [driver helm installation](../../../charts) + +add `--set feature.enableFSGroupPolicy=true` in helm installation command. + +### Option#2: Enable fsGroupPolicy support on a cluster with CSI driver already installed + +```console +kubectl delete CSIDriver nfs.csi.k8s.io +cat < Date: Tue, 1 Jun 2021 11:45:07 +0800 Subject: [PATCH 15/17] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ef98c92d..c2cb48ae 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Please refer to [`nfs.csi.k8s.io` driver parameters](./docs/driver-parameters.md ### Examples - [Set up a NFS Server on a Kubernetes cluster](./deploy/example/nfs-provisioner/README.md) - [Basic usage](./deploy/example/README.md) + - [fsGroupPolicy](./deploy/example/fsgroup) ### Troubleshooting - [CSI driver troubleshooting guide](./docs/csi-debug.md) From 4d2d1194e624b96fe9ae61df1a49c59900341722 Mon Sep 17 00:00:00 2001 From: Andy Zhang Date: Wed, 2 Jun 2021 14:40:32 +0800 Subject: [PATCH 16/17] Update README.md --- deploy/example/fsgroup/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/example/fsgroup/README.md b/deploy/example/fsgroup/README.md index 8234d2f1..6c4640b9 100644 --- a/deploy/example/fsgroup/README.md +++ b/deploy/example/fsgroup/README.md @@ -1,6 +1,6 @@ # fsGroup Support -[fsGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) feature is supported from Kubernetes 1.20, default CSI driver installation does not have this feature, follow below steps to enable this feature. +[fsGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) feature is Beta from Kubernetes 1.20, and disabled by default, follow below steps to enable this feature. ### Option#1: Enable fsGroupPolicy support in [driver helm installation](../../../charts) From d1737d3ea1e70de903219b0f0b834e4131037e54 Mon Sep 17 00:00:00 2001 From: Andy Zhang Date: Wed, 2 Jun 2021 14:42:51 +0800 Subject: [PATCH 17/17] Update README.md --- charts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/README.md b/charts/README.md index 4d3ce5d1..1d20854f 100644 --- a/charts/README.md +++ b/charts/README.md @@ -5,7 +5,7 @@ ### Tips - `--set controller.runOnMaster=true` could make csi-nfs-controller only run on master node - - `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster + - `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster (this feature is in beta, check details [here](../deploy/example/fsgroup)) - `--set controller.replicas=1` could set replica of csi-nfs-controller as `1` ## install latest version