diff --git a/Dockerfile b/Dockerfile index 085f66b5..af10df3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG ARCH=amd64 -FROM k8s.gcr.io/build-image/debian-base-${ARCH}:v2.1.3 +FROM k8s.gcr.io/build-image/debian-base:buster-v1.6.0 # Copy nfsplugin from build _output directory COPY bin/nfsplugin /nfsplugin diff --git a/Makefile b/Makefile index 0a0f1d54..c294a4d7 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,9 @@ REGISTRY_NAME ?= $(shell echo $(REGISTRY) | sed "s/.azurecr.io//g") IMAGE_TAG = $(REGISTRY)/$(IMAGENAME):$(IMAGE_VERSION) IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGENAME):latest +E2E_HELM_OPTIONS ?= --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) --set image.nfs.tag=$(IMAGE_VERSION) --set image.nfs.pullPolicy=Always +E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS} + all: nfs .PHONY: verify @@ -112,11 +115,9 @@ install-helm: e2e-bootstrap: install-helm docker pull $(IMAGE_TAG) || make container push helm install csi-driver-nfs ./charts/latest/csi-driver-nfs --namespace kube-system --wait --timeout=15m -v=5 --debug \ - --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) \ - --set image.nfs.tag=$(IMAGE_VERSION) \ - --set image.nfs.pullPolicy=Always - --set controller.logLevel=8 - --set node.logLevel=8 + ${E2E_HELM_OPTIONS} \ + --set controller.logLevel=8 \ + --set node.logLevel=8 .PHONY: e2e-teardown e2e-teardown: @@ -124,4 +125,8 @@ e2e-teardown: .PHONY: e2e-test e2e-test: - go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS} + if [ ! -z "$(EXTERNAL_E2E_TEST)" ]; then \ + bash ./test/external-e2e/run.sh;\ + else \ + go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS};\ + fi diff --git a/README.md b/README.md index ef98c92d..c2cb48ae 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Please refer to [`nfs.csi.k8s.io` driver parameters](./docs/driver-parameters.md ### Examples - [Set up a NFS Server on a Kubernetes cluster](./deploy/example/nfs-provisioner/README.md) - [Basic usage](./deploy/example/README.md) + - [fsGroupPolicy](./deploy/example/fsgroup) ### Troubleshooting - [CSI driver troubleshooting guide](./docs/csi-debug.md) diff --git a/charts/README.md b/charts/README.md index f1e86f70..1d20854f 100644 --- a/charts/README.md +++ b/charts/README.md @@ -3,6 +3,11 @@ ## Prerequisites - [install Helm](https://helm.sh/docs/intro/quickstart/#install-helm) +### Tips + - `--set controller.runOnMaster=true` could make csi-nfs-controller only run on master node + - `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster (this feature is in beta, check details [here](../deploy/example/fsgroup)) + - `--set controller.replicas=1` could set replica of csi-nfs-controller as `1` + ## install latest version ```console helm repo add csi-driver-nfs https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts @@ -31,6 +36,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | Parameter | Description | Default | |---------------------------------------------------|------------------------------------------------------------|-------------------------------------------------------------------| +| `feature.enableFSGroupPolicy` | enable `fsGroupPolicy` on a k8s 1.20+ cluster | `false` | | `image.nfs.repository` | csi-driver-nfs docker image | gcr.io/k8s-staging-sig-storage/nfsplugin | | `image.nfs.tag` | csi-driver-nfs docker image tag | amd64-linux-canary | | `image.nfs.pullPolicy` | csi-driver-nfs image pull policy | IfNotPresent | @@ -38,10 +44,10 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | `image.csiProvisioner.tag` | csi-provisioner docker image tag | v2.0.4 | | `image.csiProvisioner.pullPolicy` | csi-provisioner image pull policy | IfNotPresent | | `image.livenessProbe.repository` | liveness-probe docker image | k8s.gcr.io/sig-storage/livenessprobe | -| `image.livenessProbe.tag` | liveness-probe docker image tag | v2.1.0 | +| `image.livenessProbe.tag` | liveness-probe docker image tag | v2.3.0 | | `image.livenessProbe.pullPolicy` | liveness-probe image pull policy | IfNotPresent | | `image.nodeDriverRegistrar.repository` | csi-node-driver-registrar docker image | k8s.gcr.io/sig-storage/csi-node-driver-registrar | -| `image.nodeDriverRegistrar.tag` | csi-node-driver-registrar docker image tag | v2.0.1 | +| `image.nodeDriverRegistrar.tag` | csi-node-driver-registrar docker image tag | v2.2.0 | | `image.nodeDriverRegistrar.pullPolicy` | csi-node-driver-registrar image pull policy | IfNotPresent | | `imagePullSecrets` | Specify docker-registry secret names as an array | [] (does not add image pull secrets to deployed pods) | | `serviceAccount.create` | whether create service account of csi-nfs-controller | true | @@ -50,6 +56,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv | `controller.runOnMaster` | run controller on master node | false | | `controller.logLevel` | controller driver log level |`5` | | `node.logLevel` | node driver log level |`5` | +| `node.livenessProbe.healthPort ` | the health check port for liveness probe |`29653` | ## troubleshooting - Add `--wait -v=5 --debug` in `helm install` command to get detailed error diff --git a/charts/latest/csi-driver-nfs-v3.0.0.tgz b/charts/latest/csi-driver-nfs-v3.0.0.tgz index b554a07d..46cc8ca3 100644 Binary files a/charts/latest/csi-driver-nfs-v3.0.0.tgz and b/charts/latest/csi-driver-nfs-v3.0.0.tgz differ diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml index bf248494..a1bfac8d 100755 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml @@ -6,4 +6,6 @@ spec: attachRequired: false volumeLifecycleModes: - Persistent - podInfoOnMount: true + {{- if .Values.feature.enableFSGroupPolicy}} + fsGroupPolicy: File + {{- end}} diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml index 8441184d..a7d50baf 100755 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml @@ -29,7 +29,7 @@ spec: args: - --csi-address=/csi/csi.sock - --probe-timeout=3s - - --health-port=29653 + - --health-port={{ .Values.node.livenessProbe.healthPort }} - --v=2 imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} volumeMounts: diff --git a/charts/latest/csi-driver-nfs/values.yaml b/charts/latest/csi-driver-nfs/values.yaml index 372bc893..5cf553fa 100755 --- a/charts/latest/csi-driver-nfs/values.yaml +++ b/charts/latest/csi-driver-nfs/values.yaml @@ -9,11 +9,11 @@ image: pullPolicy: IfNotPresent livenessProbe: repository: k8s.gcr.io/sig-storage/livenessprobe - tag: v2.1.0 + tag: v2.3.0 pullPolicy: IfNotPresent nodeDriverRegistrar: repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar - tag: v2.1.0 + tag: v2.2.0 pullPolicy: IfNotPresent serviceAccount: create: true @@ -26,6 +26,11 @@ controller: node: logLevel: 5 + livenessProbe: + healthPort: 29653 + +feature: + enableFSGroupPolicy: false ## Reference to one or more secrets to be used when pulling images ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ diff --git a/deploy/csi-nfs-controller.yaml b/deploy/csi-nfs-controller.yaml index 64cc148a..ca6bc443 100644 --- a/deploy/csi-nfs-controller.yaml +++ b/deploy/csi-nfs-controller.yaml @@ -50,7 +50,7 @@ spec: cpu: 10m memory: 20Mi - name: liveness-probe - image: k8s.gcr.io/sig-storage/livenessprobe:v2.1.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 args: - --csi-address=/csi/csi.sock - --probe-timeout=3s diff --git a/deploy/csi-nfs-driverinfo.yaml b/deploy/csi-nfs-driverinfo.yaml index fd592311..269c9a9d 100644 --- a/deploy/csi-nfs-driverinfo.yaml +++ b/deploy/csi-nfs-driverinfo.yaml @@ -7,4 +7,3 @@ spec: attachRequired: false volumeLifecycleModes: - Persistent - podInfoOnMount: true diff --git a/deploy/csi-nfs-node.yaml b/deploy/csi-nfs-node.yaml index 8167d702..2bc087c8 100644 --- a/deploy/csi-nfs-node.yaml +++ b/deploy/csi-nfs-node.yaml @@ -21,7 +21,7 @@ spec: kubernetes.io/os: linux containers: - name: liveness-probe - image: k8s.gcr.io/sig-storage/livenessprobe:v2.1.0 + image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0 args: - --csi-address=/csi/csi.sock - --probe-timeout=3s @@ -38,7 +38,7 @@ spec: cpu: 10m memory: 20Mi - name: node-driver-registrar - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0 + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0 lifecycle: preStop: exec: diff --git a/deploy/example/fsgroup/README.md b/deploy/example/fsgroup/README.md new file mode 100644 index 00000000..6c4640b9 --- /dev/null +++ b/deploy/example/fsgroup/README.md @@ -0,0 +1,24 @@ +# fsGroup Support + +[fsGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) feature is Beta from Kubernetes 1.20, and disabled by default, follow below steps to enable this feature. + +### Option#1: Enable fsGroupPolicy support in [driver helm installation](../../../charts) + +add `--set feature.enableFSGroupPolicy=true` in helm installation command. + +### Option#2: Enable fsGroupPolicy support on a cluster with CSI driver already installed + +```console +kubectl delete CSIDriver nfs.csi.k8s.io +cat <-on-`. CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) and [k/k in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) + +## Adding support for a new Kubernetes release + +1. Add the new release to `k8s_versions` in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25 + to enable generating a job for it. Set `experimental_k8s_version` + in + https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40 + to ensure that the new jobs aren't run for PRs unless explicitly + requested. Generate and submit the new jobs. +1. Create a test PR to try out the new job in some repo with `/test + pull-kubernetes-csi---on-kubernetes-` where x.y + matches the Kubernetes release. Alternatively, run .prow.sh in that + repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`. +1. Optional: update to a [new + release](https://github.com/kubernetes-sigs/kind/tags) of kind with + pre-built images for the new Kubernetes release. This is optional + if the current version of kind is able to build images for the new + Kubernetes release. However, jobs require less resources when they + don't need to build those images from the Kubernetes source code. + This change needs to be tried out in a PR against a component + first, then get submitted against csi-release-tools. +1. Optional: propagate the updated csi-release-tools to all components + with the script from + https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402 +1. Once it is likely to work in all components, unset + `experimental_k8s_version` and submit the updated jobs. +1. Once all sidecars for the new Kubernetes release are released, + either bump the version number of the images in the existing + [csi-driver-host-path + deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and/or create a new deployment, depending on what Kubernetes + release an updated sidecar is compatible with. If no new deployment + is needed, then add a symlink to document that there intentionally + isn't a separate deployment. This symlink is not needed for Prow + testing because that will use "kubernetes-latest" as fallback. + Update that link when creating a new deployment. +1. Create a new csi-driver-host-path release. +1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and + (eventually) roll that change out to all repos by updating + `release-tools` in them. This is used when testing manually. The + Prow jobs override that value, so also update + `hostpath_driver_version` in + https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47 diff --git a/release-tools/build.make b/release-tools/build.make index 1faaf3b9..0ed51e80 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# force the usage of /bin/bash instead of /bin/sh +SHELL := /bin/bash + .PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be @@ -63,26 +66,35 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of triplets, +# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. BUILD_PLATFORMS = # Add go ldflags using LDFLAGS at the time of compilation. -IMPORTPATH_LDFLAGS = -X main.version=$(REV) +IMPORTPATH_LDFLAGS = -X main.version=$(REV) EXT_LDFLAGS = -extldflags "-static" -LDFLAGS = +LDFLAGS = FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + # os_arch_seen captures all of the $$os-$$arch seen for the current binary + # that we want to build, if we've seen an $$os-$$arch before it means that + # we don't need to build it again, this is done to avoid building + # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ + continue; \ + fi; \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ + os_arch_seen+=";$$os-$$arch"; \ done $(CMDS:%=container-%): container-%: build-% @@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?= # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ - DOCKER_CLI_EXPERIMENTAL=enabled; \ - export DOCKER_CLI_EXPERIMENTAL; \ + export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ + --build-arg BASE_IMAGE=$$base_image \ + --build-arg ADDON_IMAGE=$$addon_image \ --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + if [ $$os = "windows" ]; then \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \ + docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \ + fi; \ + done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ if [ $(PULL_BASE_REF) = "master" ]; then \ @@ -288,4 +316,3 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" - diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml index 1e02ba6c..b39a6db3 100644 --- a/release-tools/cloudbuild.yaml +++ b/release-tools/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/release-tools/go-get-kubernetes.sh b/release-tools/go-get-kubernetes.sh index 8c4e3024..cbbbb7c3 100755 --- a/release-tools/go-get-kubernetes.sh +++ b/release-tools/go-get-kubernetes.sh @@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' ) || die "failed to determine Kubernetes staging modules" for mod in $mods; do + if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then + echo "Kubernetes module $mod is not used, skipping" + # Remove the module from go.mod "replace" that was added by an older version of this script. + (set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed" + continue + fi # The presence of a potentially incomplete go.mod file affects this command, # so move elsewhere. modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") || diff --git a/release-tools/prow.sh b/release-tools/prow.sh index b09c2920..55f2a91c 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -77,7 +77,10 @@ version_to_git () { esac } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +# the list of windows versions was matched from: +# - https://hub.docker.com/_/microsoft-windows-nanoserver +# - https://hub.docker.com/_/microsoft-windows-servercore +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -136,6 +139,9 @@ kind_version_default () { case "${CSI_PROW_KUBERNETES_VERSION}" in latest|master) echo main;; + 1.21*|release-1.21) + # TODO: replace this special case once the next KinD release supports 1.21. + echo main;; *) echo v0.10.0;; esac @@ -159,11 +165,6 @@ kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" -# Work directory. It has to allow running executables, therefore /tmp -# is avoided. Cleaning up after the script is intentionally left to -# the caller. -configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" - # By default, this script tests sidecars with the CSI hostpath driver, # using the install_csi_driver function. That function depends on # a deployment script that it searches for in several places: @@ -190,8 +191,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # CSI_PROW_DEPLOYMENT variable can be set in the # .prow.sh of each component when there are breaking changes # that require using a non-default deployment. The default -# is a deployment named "kubernetes-x.yy" (if available), -# otherwise "kubernetes-latest". +# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available), +# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}". # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, @@ -203,6 +204,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" +configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" # The install_csi_driver function may work also for other CSI drivers, # as long as they follow the conventions of the CSI hostpath driver. @@ -237,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 +configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -297,6 +299,17 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go +# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants). +# When testing canary images, those get used instead of the images specified +# in the e2e.test's normal YAML files. +# +# The default is to enable this for all jobs which use canary images +# because we want to know whether our release candidates will pass all +# existing tests (the storage testsuites and mock testing in +# Kubernetes). +configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" + # Regex for non-alpha, feature-tagged tests that should be run. # configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version" @@ -355,16 +368,25 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -# -# "different node" test skips can be removed once -# https://github.com/kubernetes/kubernetes/pull/82678 has been backported -# to all the K8s versions we test against -configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" -# This is the directory for additional result files. Usually set by Prow, but -# if not (for example, when invoking manually) it defaults to the work directory. -configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" -mkdir -p "${ARTIFACTS}" +# This creates directories that are required for testing. +ensure_paths () { + # Work directory. It has to allow running executables, therefore /tmp + # is avoided. Cleaning up after the script is intentionally left to + # the caller. + configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" + + # This is the directory for additional result files. Usually set by Prow, but + # if not (for example, when invoking manually) it defaults to the work directory. + configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts" + mkdir -p "${ARTIFACTS}" + + # For additional tools. + CSI_PROW_BIN="${CSI_PROW_WORK}/bin" + mkdir -p "${CSI_PROW_BIN}" + PATH="${CSI_PROW_BIN}:$PATH" +} run () { echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2 @@ -384,11 +406,6 @@ die () { exit 1 } -# For additional tools. -CSI_PROW_BIN="${CSI_PROW_WORK}/bin" -mkdir -p "${CSI_PROW_BIN}" -PATH="${CSI_PROW_BIN}:$PATH" - # Ensure that PATH has the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. @@ -618,11 +635,16 @@ EOF # Deletes kind cluster inside a prow job delete_cluster_inside_prow_job() { + local name="$1" + # Inside a real Prow job it is better to clean up at runtime # instead of leaving that to the Prow job cleanup code # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + # + # This is also a good time to collect logs. if [ "$JOB_NAME" ]; then if kind get clusters | grep -q csi-prow; then + run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name" run kind delete cluster --name=csi-prow || die "kind delete failed" fi unset KUBECONFIG @@ -647,9 +669,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy.sh" + file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -709,7 +731,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" @@ -756,7 +778,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" @@ -825,7 +847,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done } @@ -870,6 +892,29 @@ start_loggers () { done } +# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k +# source code, if needed. +patch_kubernetes () { + local source="$1" target="$2" + + if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because + # e2e.test does not support it (see test/utils/image/manifest.go). Instead we + # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that + # overrides that registry. + find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;' + cat >"$target/e2e-repo-list" <&2 <&1) + +EOF + fi +} + # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test". install_e2e () { if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then @@ -878,6 +923,7 @@ install_e2e () { git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then + patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" @@ -929,7 +975,7 @@ run_e2e () ( trap move_junit EXIT cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" ) # Run csi-sanity against installed CSI driver. @@ -941,7 +987,7 @@ run_sanity () ( kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@" EOF # Using "rm -rf" as fallback for "rmdir" is a workaround for: - # Node Service + # Node Service # should work # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624 # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896 @@ -968,6 +1014,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- exit 1 fi EOF + + cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" </dev/null; then +if command -v shellcheck &>/dev/null; then detected_version="$(shellcheck --version | grep 'version: .*')" if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then HAVE_SHELLCHECK=true diff --git a/test/external-e2e/run.sh b/test/external-e2e/run.sh new file mode 100644 index 00000000..1d89d8ab --- /dev/null +++ b/test/external-e2e/run.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +PROJECT_ROOT=$(git rev-parse --show-toplevel) + +install_ginkgo () { + apt update -y + apt install -y golang-ginkgo-dev +} + +setup_e2e_binaries() { + # download k8s external e2e binary + curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz + tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz + + # enable fsGroupPolicy (only available from k8s 1.20) + export EXTRA_HELM_OPTIONS="--set feature.enableFSGroupPolicy=true" + + # install csi driver + mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml + make e2e-bootstrap + make install-nfs-server +} + +print_logs() { + echo "print out driver logs ..." + bash ./test/utils/nfs_log.sh +} + +install_ginkgo +setup_e2e_binaries +trap print_logs EXIT + +ginkgo -p --progress --v -focus='External.Storage.*nfs.csi.k8s.io' \ + -skip='\[Disruptive\]|\[Slow\]' kubernetes/test/bin/e2e.test -- \ + -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \ + --kubeconfig=$KUBECONFIG diff --git a/test/external-e2e/testdriver.yaml b/test/external-e2e/testdriver.yaml new file mode 100644 index 00000000..c9386554 --- /dev/null +++ b/test/external-e2e/testdriver.yaml @@ -0,0 +1,13 @@ +# Manifest for Kubernetes external tests. +# See https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external + +StorageClass: + FromFile: /tmp/csi/storageclass.yaml +DriverInfo: + Name: nfs.csi.k8s.io + Capabilities: + persistence: true + exec: true + multipods: true + RWX: true + fsGroup: true