Merge remote-tracking branch 'upstream/master' into upstream-merge

Conflicts:
  pkg/nfs/nodeserver.go

https://github.com/kubernetes-csi/csi-driver-nfs/pull/201 removed locks
in NodePublishVolume and NodeUnpublishVolume which conflicted with a
downstream-only patch.

Dockerfile

@@ -14,7 +14,7 @@
 
 ARG ARCH=amd64
 
-FROM k8s.gcr.io/build-image/debian-base-${ARCH}:v2.1.3
+FROM k8s.gcr.io/build-image/debian-base:buster-v1.6.0
 
 # Copy nfsplugin from build _output directory
 COPY bin/nfsplugin /nfsplugin

Makefile

@@ -42,6 +42,9 @@ REGISTRY_NAME ?= $(shell echo $(REGISTRY) | sed "s/.azurecr.io//g")
 IMAGE_TAG = $(REGISTRY)/$(IMAGENAME):$(IMAGE_VERSION)
 IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGENAME):latest
 
+E2E_HELM_OPTIONS ?= --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) --set image.nfs.tag=$(IMAGE_VERSION) --set image.nfs.pullPolicy=Always
+E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS}
+
 all: nfs
 
 .PHONY: verify
@@ -112,11 +115,9 @@ install-helm:
 e2e-bootstrap: install-helm
 	docker pull $(IMAGE_TAG) || make container push
 	helm install csi-driver-nfs ./charts/latest/csi-driver-nfs --namespace kube-system --wait --timeout=15m -v=5 --debug \
-	--set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) \
+		${E2E_HELM_OPTIONS} \
-	--set image.nfs.tag=$(IMAGE_VERSION) \
+		--set controller.logLevel=8 \
-	--set image.nfs.pullPolicy=Always
+		--set node.logLevel=8
-	--set controller.logLevel=8
-	--set node.logLevel=8
 
 .PHONY: e2e-teardown
 e2e-teardown:
@@ -124,4 +125,8 @@ e2e-teardown:
 
 .PHONY: e2e-test
 e2e-test:
-	go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS}
+	if [ ! -z "$(EXTERNAL_E2E_TEST)" ]; then \
+		bash ./test/external-e2e/run.sh;\
+	else \
+		go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS};\
+	fi

README.md

@@ -31,6 +31,7 @@ Please refer to [`nfs.csi.k8s.io` driver parameters](./docs/driver-parameters.md
 ### Examples
  - [Set up a NFS Server on a Kubernetes cluster](./deploy/example/nfs-provisioner/README.md)
  - [Basic usage](./deploy/example/README.md)
+ - [fsGroupPolicy](./deploy/example/fsgroup)
 
 ### Troubleshooting
  - [CSI driver troubleshooting guide](./docs/csi-debug.md) 

charts/README.md

@@ -3,6 +3,11 @@
 ## Prerequisites
  - [install Helm](https://helm.sh/docs/intro/quickstart/#install-helm)
 
+### Tips
+ - `--set controller.runOnMaster=true` could make csi-nfs-controller only run on master node
+ - `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster (this feature is in beta, check details [here](../deploy/example/fsgroup))
+ - `--set controller.replicas=1` could set replica of csi-nfs-controller as `1`
+
 ## install latest version
 ```console
 helm repo add csi-driver-nfs https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts
@@ -31,6 +36,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv
 
 | Parameter                                         | Description                                                | Default                                                           |
 |---------------------------------------------------|------------------------------------------------------------|-------------------------------------------------------------------|
+| `feature.enableFSGroupPolicy`                     | enable `fsGroupPolicy` on a k8s 1.20+ cluster           | `false`                      |
 | `image.nfs.repository`                            | csi-driver-nfs docker image                                | gcr.io/k8s-staging-sig-storage/nfsplugin                          |
 | `image.nfs.tag`                                   | csi-driver-nfs docker image tag                            | amd64-linux-canary                                                |
 | `image.nfs.pullPolicy`                            | csi-driver-nfs image pull policy                           | IfNotPresent                                                      |
@@ -38,10 +44,10 @@ The following table lists the configurable parameters of the latest NFS CSI Driv
 | `image.csiProvisioner.tag`                        | csi-provisioner docker image tag                           | v2.0.4                                                            |
 | `image.csiProvisioner.pullPolicy`                 | csi-provisioner image pull policy                          | IfNotPresent                                                      |
 | `image.livenessProbe.repository`                  | liveness-probe docker image                                | k8s.gcr.io/sig-storage/livenessprobe                              |
-| `image.livenessProbe.tag`                         | liveness-probe docker image tag                            | v2.1.0                                                            |
+| `image.livenessProbe.tag`                         | liveness-probe docker image tag                            | v2.3.0                                                            |
 | `image.livenessProbe.pullPolicy`                  | liveness-probe image pull policy                           | IfNotPresent                                                      |
 | `image.nodeDriverRegistrar.repository`            | csi-node-driver-registrar docker image                     | k8s.gcr.io/sig-storage/csi-node-driver-registrar                  |
-| `image.nodeDriverRegistrar.tag`                   | csi-node-driver-registrar docker image tag                 | v2.0.1                                                            |
+| `image.nodeDriverRegistrar.tag`                   | csi-node-driver-registrar docker image tag                 | v2.2.0                                                            |
 | `image.nodeDriverRegistrar.pullPolicy`            | csi-node-driver-registrar image pull policy                | IfNotPresent                                                      |
 | `imagePullSecrets`                                | Specify docker-registry secret names as an array           | [] (does not add image pull secrets to deployed pods)                                                           |
 | `serviceAccount.create`                           | whether create service account of csi-nfs-controller       | true                                                              |
@@ -50,6 +56,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv
 | `controller.runOnMaster`                          | run controller on master node                              | false                                                             |
 | `controller.logLevel`                             | controller driver log level                                                          |`5`                                                           |
 | `node.logLevel`                                   | node driver log level                                                          |`5`                                                           |
+| `node.livenessProbe.healthPort `                  | the health check port for liveness probe                    |`29653`                                                           |
 
 ## troubleshooting
  - Add `--wait -v=5 --debug` in `helm install` command to get detailed error

charts/latest/csi-driver-nfs/templates/csi-nfs-driverinfo.yaml

@@ -6,4 +6,6 @@ spec:
   attachRequired: false
   volumeLifecycleModes:
     - Persistent
-  podInfoOnMount: true
+  {{- if .Values.feature.enableFSGroupPolicy}}
+  fsGroupPolicy: File
+  {{- end}}

charts/latest/csi-driver-nfs/templates/csi-nfs-node.yaml

@@ -29,7 +29,7 @@ spec:
           args:
             - --csi-address=/csi/csi.sock
             - --probe-timeout=3s
-            - --health-port=29653
+            - --health-port={{ .Values.node.livenessProbe.healthPort }}
             - --v=2
           imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }}
           volumeMounts:

charts/latest/csi-driver-nfs/values.yaml

@@ -9,11 +9,11 @@ image:
         pullPolicy: IfNotPresent
     livenessProbe:
         repository: k8s.gcr.io/sig-storage/livenessprobe
-        tag: v2.1.0
+        tag: v2.3.0
         pullPolicy: IfNotPresent
     nodeDriverRegistrar:
         repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar
-        tag: v2.1.0
+        tag: v2.2.0
         pullPolicy: IfNotPresent
 serviceAccount:
   create: true
@@ -26,6 +26,11 @@ controller:
 
 node:
   logLevel: 5
+  livenessProbe:
+    healthPort: 29653
+
+feature:
+  enableFSGroupPolicy: false
 
 ## Reference to one or more secrets to be used when pulling images
 ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/

deploy/csi-nfs-controller.yaml

@@ -50,7 +50,7 @@ spec:
               cpu: 10m
               memory: 20Mi
         - name: liveness-probe
-          image: k8s.gcr.io/sig-storage/livenessprobe:v2.1.0
+          image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0
           args:
             - --csi-address=/csi/csi.sock
             - --probe-timeout=3s

deploy/csi-nfs-driverinfo.yaml

@@ -7,4 +7,3 @@ spec:
   attachRequired: false
   volumeLifecycleModes:
     - Persistent
-  podInfoOnMount: true

deploy/csi-nfs-node.yaml

@@ -21,7 +21,7 @@ spec:
         kubernetes.io/os: linux
       containers:
         - name: liveness-probe
-          image: k8s.gcr.io/sig-storage/livenessprobe:v2.1.0
+          image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0
           args:
             - --csi-address=/csi/csi.sock
             - --probe-timeout=3s
@@ -38,7 +38,7 @@ spec:
               cpu: 10m
               memory: 20Mi
         - name: node-driver-registrar
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0
           lifecycle:
             preStop:
               exec:

deploy/example/fsgroup/README.md

@@ -0,0 +1,24 @@
+# fsGroup Support
+
+[fsGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) feature is Beta from Kubernetes 1.20, and disabled by default, follow below steps to enable this feature.
+
+### Option#1: Enable fsGroupPolicy support in [driver helm installation](../../../charts)
+
+add `--set feature.enableFSGroupPolicy=true` in helm installation command.
+
+### Option#2: Enable fsGroupPolicy support on a cluster with CSI driver already installed
+
+```console
+kubectl delete CSIDriver nfs.csi.k8s.io
+cat <<EOF | kubectl create -f -
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: nfs.csi.k8s.io
+spec:
+  attachRequired: false
+  volumeLifecycleModes:
+    - Persistent
+  fsGroupPolicy: File
+EOF
+```

deploy/example/storageclass-nfs.yaml

@@ -7,7 +7,7 @@ provisioner: nfs.csi.k8s.io
 parameters:
   server: nfs-server.default.svc.cluster.local
   share: /
-reclaimPolicy: Retain
+reclaimPolicy: Delete
 volumeBindingMode: Immediate
 mountOptions:
   - hard

pkg/nfs/controllerserver.go

@@ -104,9 +104,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	if err = os.Mkdir(internalVolumePath, 0777); err != nil && !os.IsExist(err) {
 		return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err.Error())
 	}
-	// Remove capacity setting when provisioner 1.4.0 is available with fix for
+	return &csi.CreateVolumeResponse{Volume: cs.nfsVolToCSI(nfsVol)}, nil
-	// https://github.com/kubernetes-csi/external-provisioner/pull/271
-	return &csi.CreateVolumeResponse{Volume: cs.nfsVolToCSI(nfsVol, reqCapacity)}, nil
 }
 
 // DeleteVolume delete a volume
@@ -338,9 +336,9 @@ func (cs *ControllerServer) getVolumeSharePath(vol *nfsVolume) string {
 }
 
 // Convert into nfsVolume into a csi.Volume
-func (cs *ControllerServer) nfsVolToCSI(vol *nfsVolume, reqCapacity int64) *csi.Volume {
+func (cs *ControllerServer) nfsVolToCSI(vol *nfsVolume) *csi.Volume {
 	return &csi.Volume{
-		CapacityBytes: reqCapacity,
+		CapacityBytes: 0, // by setting it to zero, Provisioner will use PVC requested size as PV size
 		VolumeId:      vol.id,
 		VolumeContext: map[string]string{
 			paramServer: vol.server,

pkg/nfs/nodeserver.go

@@ -72,11 +72,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 		return &csi.NodePublishVolumeResponse{}, nil
 	}
 
-	if acquired := ns.Driver.volumeLocks.TryAcquire(volumeID); !acquired {
-		return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volumeID)
-	}
-	defer ns.Driver.volumeLocks.Release(volumeID)
-
 	mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags()
 	if req.GetReadonly() {
 		mountOptions = append(mountOptions, "ro")
@@ -141,11 +136,6 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
-	if acquired := ns.Driver.volumeLocks.TryAcquire(volumeID); !acquired {
-		return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volumeID)
-	}
-	defer ns.Driver.volumeLocks.Release(volumeID)
-
 	klog.V(4).Infof("NodeUnpublishVolume: path %s is *not* a mount point: %t", targetPath, notMnt)
 	if !notMnt {
 

pkg/nfs/nodeserver_test.go

@@ -19,7 +19,6 @@ package nfs
 import (
 	"context"
 	"errors"
-	"fmt"
 	"os"
 	"reflect"
 	"testing"
@@ -69,19 +68,6 @@ func TestNodePublishVolume(t *testing.T) {
 				VolumeId: "vol_1"},
 			expectedErr: status.Error(codes.InvalidArgument, "Target path not provided"),
 		},
-		{
-			desc: "[Error] Volume operation in progress",
-			setup: func() {
-				ns.Driver.volumeLocks.TryAcquire("vol_1")
-			},
-			req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap},
-				VolumeId:   "vol_1",
-				TargetPath: targetTest},
-			expectedErr: status.Error(codes.Aborted, fmt.Sprintf(volumeOperationAlreadyExistsFmt, "vol_1")),
-			cleanup: func() {
-				ns.Driver.volumeLocks.Release("vol_1")
-			},
-		},
 		{
 			desc: "[Success] Stage target path missing",
 			req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap},
@@ -148,7 +134,6 @@ func TestNodeUnpublishVolume(t *testing.T) {
 	errorTarget := testutil.GetWorkDirPath("error_is_likely_target", t)
 	targetTest := testutil.GetWorkDirPath("target_test", t)
 	targetFile := testutil.GetWorkDirPath("abc.go", t)
-	alreadyMountedTarget := testutil.GetWorkDirPath("false_is_likely_exist_target", t)
 
 	tests := []struct {
 		desc        string
@@ -177,17 +162,6 @@ func TestNodeUnpublishVolume(t *testing.T) {
 			req:         csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"},
 			expectedErr: status.Error(codes.NotFound, "Volume not mounted"),
 		},
-		{
-			desc: "[Error] Volume operation in progress",
-			setup: func() {
-				ns.Driver.volumeLocks.TryAcquire("vol_1")
-			},
-			req:         csi.NodeUnpublishVolumeRequest{TargetPath: alreadyMountedTarget, VolumeId: "vol_1"},
-			expectedErr: status.Error(codes.Aborted, fmt.Sprintf(volumeOperationAlreadyExistsFmt, "vol_1")),
-			cleanup: func() {
-				ns.Driver.volumeLocks.Release("vol_1")
-			},
-		},
 	}
 
 	// Setup

pkg/nfs/utils.go

@@ -95,10 +95,6 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h
 	return resp, err
 }
 
-const (
-	volumeOperationAlreadyExistsFmt = "An operation with the given Volume ID %s already exists"
-)
-
 type VolumeLocks struct {
 	locks sets.String
 	mux   sync.Mutex

release-tools/KUBERNETES_CSI_OWNERS_ALIASES

@@ -0,0 +1,44 @@
+# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
+
+aliases:
+
+  # SIG-Storage chairs and leads should always have approval rights in all repos.
+  # Others may be added as needed here or in each repo.
+  kubernetes-csi-approvers:
+  - jsafrane
+  - msau42
+  - saad-ali
+  - xing-yang
+
+  # Reviewers are automatically assigned to new PRs. The following
+  # reviewers will be active in all repos. Other reviewers can be
+  # added in each repo.
+  #
+  # Reviewers are encouraged to set the "Busy" flag in their GitHub status
+  # when they are temporarily unable to review PRs.
+  kubernetes-csi-reviewers:
+  - andyzhangx
+  - chrishenzie
+  - ggriffiths
+  - gnufied
+  - j-griffith
+  - Jiawei0227
+  - jingxu97
+  - jsafrane
+  - pohly
+  - xing-yang
+
+# This documents who previously contributed to Kubernetes-CSI
+# as approver.
+emeritus_approver:
+- lpabon
+- sbezverk
+- vladimirvivien
+
+# This documents who previously contributed to Kubernetes-CSI
+# as reviewer.
+emeritus_reviewer:
+- lpabon
+- saad-ali
+- sbezverk
+- vladimirvivien

release-tools/OWNERS

@@ -1,11 +1,8 @@
 # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
 
 approvers:
-- saad-ali
+- kubernetes-csi-approvers
-- msau42
 - pohly
 
 reviewers:
-- saad-ali
+- kubernetes-csi-reviewers
-- msau42
-- pohly

release-tools/OWNERS_ALIASES

@@ -0,0 +1 @@
+KUBERNETES_CSI_OWNERS_ALIASES

release-tools/SIDECAR_RELEASE_PROCESS.md

@@ -104,3 +104,47 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
    CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
    and [k/k
    in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath)
+
+## Adding support for a new Kubernetes release
+
+1. Add the new release to `k8s_versions` in
+   https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25
+   to enable generating a job for it. Set `experimental_k8s_version`
+   in
+   https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40
+   to ensure that the new jobs aren't run for PRs unless explicitly
+   requested. Generate and submit the new jobs.
+1. Create a test PR to try out the new job in some repo with `/test
+   pull-kubernetes-csi-<repo>-<x.y>-on-kubernetes-<x.y>` where x.y
+   matches the Kubernetes release. Alternatively, run .prow.sh in that
+   repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`.
+1. Optional: update to a [new
+   release](https://github.com/kubernetes-sigs/kind/tags) of kind with
+   pre-built images for the new Kubernetes release. This is optional
+   if the current version of kind is able to build images for the new
+   Kubernetes release. However, jobs require less resources when they
+   don't need to build those images from the Kubernetes source code.
+   This change needs to be tried out in a PR against a component
+   first, then get submitted against csi-release-tools.
+1. Optional: propagate the updated csi-release-tools to all components
+   with the script from
+   https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402
+1. Once it is likely to work in all components, unset
+   `experimental_k8s_version` and submit the updated jobs.
+1. Once all sidecars for the new Kubernetes release are released,
+   either bump the version number of the images in the existing
+   [csi-driver-host-path
+   deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
+   and/or create a new deployment, depending on what Kubernetes
+   release an updated sidecar is compatible with. If no new deployment
+   is needed, then add a symlink to document that there intentionally
+   isn't a separate deployment. This symlink is not needed for Prow
+   testing because that will use "kubernetes-latest" as fallback.
+   Update that link when creating a new deployment.
+1. Create a new csi-driver-host-path release.
+1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and
+   (eventually) roll that change out to all repos by updating
+   `release-tools` in them. This is used when testing manually. The
+   Prow jobs override that value, so also update
+   `hostpath_driver_version` in
+   https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47

release-tools/build.make

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# force the usage of /bin/bash instead of /bin/sh
+SHELL := /bin/bash
+
 .PHONY: build-% build container-% container push-% push clean test
 
 # A space-separated list of all commands in the repository, must be
@@ -63,7 +66,7 @@ endif
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
 # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
 
-# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image]
 # separated by semicolon. An empty variable or empty entry (= just a
 # semicolon) builds for the default platform of the current Go
 # toolchain.
@@ -78,11 +81,20 @@ FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
 # defined by BUILD_PLATFORMS.
 $(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
-	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+	# os_arch_seen captures all of the $$os-$$arch seen for the current binary
+	# that we want to build, if we've seen an $$os-$$arch before it means that
+	# we don't need to build it again, this is done to avoid building
+	# the windows binary multiple times (see the default value of $$BUILD_PLATFORMS)
+	export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+		os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \
+		if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \
+			continue; \
+		fi; \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
 			exit 1; \
 		fi; \
+		os_arch_seen+=";$$os-$$arch"; \
 	done
 
 $(CMDS:%=container-%): container-%: build-%
@@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # the tag for the resulting multiarch image.
 $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
-	DOCKER_CLI_EXPERIMENTAL=enabled; \
+	export DOCKER_CLI_EXPERIMENTAL=enabled; \
-	export DOCKER_CLI_EXPERIMENTAL; \
 	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
 	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
 	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
 	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
 	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
 	if ! [ -f "$$dockerfile_windows" ]; then \
-		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \
 	fi; \
 	pushMultiArch () { \
 		tag=$$1; \
-		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
 			docker buildx build --push \
-				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \
 				--platform=$$os/$$arch \
 				--file $$(eval echo \$${dockerfile_$$os}) \
 				--build-arg binary=./bin/$*$$suffix \
 				--build-arg ARCH=$$arch \
+				--build-arg BASE_IMAGE=$$base_image \
+				--build-arg ADDON_IMAGE=$$addon_image \
 				--label revision=$(REV) \
 				.; \
 		done; \
-		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+			echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+		done); \
 		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			if [ $$os = "windows" ]; then \
+				escaped_base_image=$${base_image/:/-}; \
+				if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+				image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+				os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \
+				docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \
+			fi; \
+		done; \
 		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
 	if [ $(PULL_BASE_REF) = "master" ]; then \
@@ -288,4 +316,3 @@ test-spelling:
 test-boilerplate:
 	@ echo; echo "### $@:"
 	@ ./release-tools/verify-boilerplate.sh "$(pwd)"
-

release-tools/cloudbuild.yaml

@@ -27,7 +27,7 @@ steps:
   # The image must contain bash and curl. Ideally it should also contain
   # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
     entrypoint: ./.cloudbuild.sh
     env:
     - GIT_TAG=${_GIT_TAG}

release-tools/go-get-kubernetes.sh

@@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten
           sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
    ) || die "failed to determine Kubernetes staging modules"
 for mod in $mods; do
+    if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then
+        echo "Kubernetes module $mod is not used, skipping"
+        # Remove the module from go.mod "replace" that was added by an older version of this script.
+        (set -x;  env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed"
+        continue
+    fi
     # The presence of a potentially incomplete go.mod file affects this command,
     # so move elsewhere.
     modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") ||

release-tools/prow.sh

@@ -77,7 +77,10 @@ version_to_git () {
     esac
 }
 
-configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+# the list of windows versions was matched from:
+# - https://hub.docker.com/_/microsoft-windows-nanoserver
+# - https://hub.docker.com/_/microsoft-windows-servercore
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
 
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because
@@ -136,6 +139,9 @@ kind_version_default () {
     case "${CSI_PROW_KUBERNETES_VERSION}" in
         latest|master)
             echo main;;
+        1.21*|release-1.21)
+            # TODO: replace this special case once the next KinD release supports 1.21.
+            echo main;;
         *)
             echo v0.10.0;;
     esac
@@ -159,11 +165,6 @@ kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f
 # Use kind node-image --type=bazel by default, but allow to disable that.
 configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
 
-# Work directory. It has to allow running executables, therefore /tmp
-# is avoided. Cleaning up after the script is intentionally left to
-# the caller.
-configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
-
 # By default, this script tests sidecars with the CSI hostpath driver,
 # using the install_csi_driver function. That function depends on
 # a deployment script that it searches for in several places:
@@ -190,8 +191,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
 #   CSI_PROW_DEPLOYMENT variable can be set in the
 #   .prow.sh of each component when there are breaking changes
 #   that require using a non-default deployment. The default
-#   is a deployment named "kubernetes-x.yy" (if available),
+#   is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available),
-#   otherwise "kubernetes-latest".
+#   otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}".
 #   "none" disables the deployment of the hostpath driver.
 #
 # When no deploy script is found (nothing in `deploy` directory,
@@ -203,6 +204,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
 configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version"
 configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
 configvar CSI_PROW_DEPLOYMENT "" "deployment"
+configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files"
 
 # The install_csi_driver function may work also for other CSI drivers,
 # as long as they follow the conventions of the CSI hostpath driver.
@@ -237,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 # of the cluster. The alternative would have been to (cross-)compile csi-sanity
 # and install it inside the cluster, which is not necessarily easier.
 configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo"
-configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2
+configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version"
 configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package"
 configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock"
 configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver"
@@ -297,6 +299,17 @@ tests_need_alpha_cluster () {
     tests_enabled "parallel-alpha" "serial-alpha"
 }
 
+# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go
+# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants).
+# When testing canary images, those get used instead of the images specified
+# in the e2e.test's normal YAML files.
+#
+# The default is to enable this for all jobs which use canary images
+# because we want to know whether our release candidates will pass all
+# existing tests (the storage testsuites and mock testing in
+# Kubernetes).
+configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests"
+
 # Regex for non-alpha, feature-tagged tests that should be run.
 #
 configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
@@ -355,16 +368,25 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external
 # whether they can run with the current cluster provider, but until
 # they are, we filter them out by name. Like the other test selection
 # variables, this is again a space separated list of regular expressions.
-#
+configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped"
-# "different node" test skips can be removed once
-# https://github.com/kubernetes/kubernetes/pull/82678 has been backported
-# to all the K8s versions we test against
-configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped"
 
-# This is the directory for additional result files. Usually set by Prow, but
+# This creates directories that are required for testing.
-# if not (for example, when invoking manually) it defaults to the work directory.
+ensure_paths () {
-configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
+    # Work directory. It has to allow running executables, therefore /tmp
-mkdir -p "${ARTIFACTS}"
+    # is avoided. Cleaning up after the script is intentionally left to
+    # the caller.
+    configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
+
+    # This is the directory for additional result files. Usually set by Prow, but
+    # if not (for example, when invoking manually) it defaults to the work directory.
+    configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
+    mkdir -p "${ARTIFACTS}"
+
+    # For additional tools.
+    CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
+    mkdir -p "${CSI_PROW_BIN}"
+    PATH="${CSI_PROW_BIN}:$PATH"
+}
 
 run () {
     echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2
@@ -384,11 +406,6 @@ die () {
     exit 1
 }
 
-# For additional tools.
-CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
-mkdir -p "${CSI_PROW_BIN}"
-PATH="${CSI_PROW_BIN}:$PATH"
-
 # Ensure that PATH has the desired version of the Go tools, then run command given as argument.
 # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by
 # bumping the container image regularly.
@@ -618,11 +635,16 @@ EOF
 
 # Deletes kind cluster inside a prow job
 delete_cluster_inside_prow_job() {
+    local name="$1"
+
     # Inside a real Prow job it is better to clean up at runtime
     # instead of leaving that to the Prow job cleanup code
     # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872).
+    #
+    # This is also a good time to collect logs.
     if [ "$JOB_NAME" ]; then
         if kind get clusters | grep -q csi-prow; then
+            run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name"
             run kind delete cluster --name=csi-prow || die "kind delete failed"
         fi
         unset KUBECONFIG
@@ -647,9 +669,9 @@ find_deployment () {
 
     # Ignore: See if you can use ${variable//search/replace} instead.
     # shellcheck disable=SC2001
-    file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh"
+    file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh"
     if ! [ -e "$file" ]; then
-        file="$dir/kubernetes-latest/deploy.sh"
+        file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh"
         if ! [ -e "$file" ]; then
             return 1
         fi
@@ -870,6 +892,29 @@ start_loggers () {
     done
 }
 
+# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k
+# source code, if needed.
+patch_kubernetes () {
+    local source="$1" target="$2"
+
+    if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
+        # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because
+        # e2e.test does not support it (see test/utils/image/manifest.go). Instead we
+        # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that
+        # overrides that registry.
+        find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;'
+        cat >"$target/e2e-repo-list" <<EOF
+sigStorageRegistry: gcr.io/k8s-staging-sig-storage
+EOF
+        cat >&2 <<EOF
+
+Using a modified version of k/k/test/e2e:
+$(cd "$source" && git diff 2>&1)
+
+EOF
+    fi
+}
+
 # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test".
 install_e2e () {
     if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then
@@ -878,6 +923,7 @@ install_e2e () {
 
     git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 &&
     if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then
+        patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" &&
         go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" &&
         run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
         ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}"
@@ -929,7 +975,7 @@ run_e2e () (
     trap move_junit EXIT
 
     cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
-    run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
+    run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
 )
 
 # Run csi-sanity against installed CSI driver.
@@ -968,6 +1014,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" --
     exit 1
 fi
 EOF
+
+    cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" <<EOF
+#!/bin/sh
+CHECK_PATH=\$(cat <<SCRIPT
+if [ -f "\$@" ]; then
+    echo "file"
+elif [ -d "\$@" ]; then
+    echo "directory"
+elif [ -e "\$@" ]; then
+    echo "other"
+else
+    echo "not_found"
+fi
+SCRIPT
+)
+kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PATH}"
+EOF
+
     chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh
 
     # This cannot run in parallel, because -csi.junitfile output
@@ -983,6 +1047,7 @@ EOF
                      -csi.createmountpathcmd "${CSI_PROW_WORK}/mkdir_in_pod.sh" \
                      -csi.removestagingpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
                      -csi.removemountpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
+                     -csi.checkpathcmd "${CSI_PROW_WORK}/checkdir_in_pod.sh" \
 )
 
 ascii_to_xml () {
@@ -1098,6 +1163,9 @@ main () {
     local images ret
     ret=0
 
+    # Set up work directory.
+    ensure_paths
+
     images=
     if ${CSI_PROW_BUILD_JOB}; then
         # A successful build is required for testing.
@@ -1157,6 +1225,12 @@ main () {
             done
         fi
 
+        # Run the external driver tests and optionally also mock tests.
+        local focus="External.Storage"
+        if "$CSI_PROW_E2E_MOCK"; then
+            focus="($focus|CSI.mock.volume)"
+        fi
+
         if tests_need_non_alpha_cluster; then
             start_cluster || die "starting the non-alpha cluster failed"
 
@@ -1164,6 +1238,7 @@ main () {
             install_snapshot_crds
             install_snapshot_controller
 
+
             # Installing the driver might be disabled.
             if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
                 collect_cluster_info
@@ -1178,7 +1253,7 @@ main () {
                     # Ignore: Double quote to prevent globbing and word splitting.
                     # shellcheck disable=SC2086
                     if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \
-                         -focus="External.Storage" \
+                         -focus="$focus" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E parallel failed"
                         ret=1
@@ -1188,7 +1263,7 @@ main () {
                     # Ignore: Double quote to prevent globbing and word splitting.
                     # shellcheck disable=SC2086
                     if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \
-                         -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
+                         -focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then
                         warn "E2E parallel features failed"
                         ret=1
@@ -1197,14 +1272,14 @@ main () {
 
                 if tests_enabled "serial"; then
                     if ! run_e2e serial \
-                         -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
+                         -focus="$focus.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E serial failed"
                         ret=1
                     fi
                 fi
             fi
-            delete_cluster_inside_prow_job
+            delete_cluster_inside_prow_job non-alpha
         fi
 
         if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
@@ -1223,7 +1298,7 @@ main () {
                     # Ignore: Double quote to prevent globbing and word splitting.
                     # shellcheck disable=SC2086
                     if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \
-                         -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
+                         -focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E parallel alpha failed"
                         ret=1
@@ -1232,14 +1307,14 @@ main () {
 
                 if tests_enabled "serial-alpha"; then
                     if ! run_e2e serial-alpha \
-                         -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
+                         -focus="$focus.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E serial alpha failed"
                         ret=1
                     fi
                 fi
             fi
-            delete_cluster_inside_prow_job
+            delete_cluster_inside_prow_job alpha
         fi
     fi
 
@@ -1259,6 +1334,9 @@ gcr_cloud_build () {
     # Required for "docker buildx build --push".
     gcloud auth configure-docker
 
+    # Might not be needed here, but call it just in case.
+    ensure_paths
+
     if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
         # Needed for "RUN" steps on non-linux/amd64 platforms.
         # See https://github.com/multiarch/qemu-user-static#getting-started

release-tools/pull-test.sh

@@ -0,0 +1,32 @@
+#! /bin/sh
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is called by pull Prow jobs for the csi-release-tools
+# repo to ensure that the changes in the PR work when imported into
+# some other repo.
+
+set -ex
+
+# It must be called inside the updated csi-release-tools repo.
+CSI_RELEASE_TOOLS_DIR="$(pwd)"
+
+# Update the other repo.
+cd "$PULL_TEST_REPO_DIR"
+git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master
+git log -n2
+
+# Now fall through to testing.
+exec ./.prow.sh

release-tools/verify-shellcheck.sh

@@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \
 # detect if the host machine has the required shellcheck version installed
 # if so, we will use that instead.
 HAVE_SHELLCHECK=false
-if which shellcheck &>/dev/null; then
+if command -v shellcheck &>/dev/null; then
   detected_version="$(shellcheck --version | grep 'version: .*')"
   if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
     HAVE_SHELLCHECK=true

test/external-e2e/run.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+PROJECT_ROOT=$(git rev-parse --show-toplevel)
+
+install_ginkgo () {
+    apt update -y
+    apt install -y golang-ginkgo-dev
+}
+
+setup_e2e_binaries() {
+    # download k8s external e2e binary
+    curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
+    tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz
+
+    # enable fsGroupPolicy (only available from k8s 1.20)
+    export EXTRA_HELM_OPTIONS="--set feature.enableFSGroupPolicy=true"
+
+    # install csi driver
+    mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml
+    make e2e-bootstrap
+    make install-nfs-server
+}
+
+print_logs() {
+    echo "print out driver logs ..."
+    bash ./test/utils/nfs_log.sh
+}
+
+install_ginkgo
+setup_e2e_binaries
+trap print_logs EXIT
+
+ginkgo -p --progress --v -focus='External.Storage.*nfs.csi.k8s.io' \
+       -skip='\[Disruptive\]|\[Slow\]' kubernetes/test/bin/e2e.test  -- \
+       -storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \
+       --kubeconfig=$KUBECONFIG

test/external-e2e/testdriver.yaml

@@ -0,0 +1,13 @@
+# Manifest for Kubernetes external tests.
+# See https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external
+
+StorageClass:
+  FromFile: /tmp/csi/storageclass.yaml
+DriverInfo:
+  Name: nfs.csi.k8s.io
+  Capabilities:
+    persistence: true
+    exec: true
+    multipods: true
+    RWX: true
+    fsGroup: true