CI: Add K8s external storage E2E tests

Signed-off-by: Mayank Shah <mayankshah1614@gmail.com>

.github/workflows/e2e.yaml

@@ -0,0 +1,39 @@
+name: K8s external storage E2E Tests
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+jobs:
+  test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.13
+        id: go
+      - name: Build image 
+        run: |
+          make build
+          docker build -t nfsplugin:latest .
+      - uses: engineerd/setup-kind@v0.4.0
+        name: Installing KinD cluster
+        with:
+          version: "v0.9.0"
+      - name: Push image to KinD
+        run: |
+          kind load docker-image --name=kind nfsplugin:latest
+      - name: Install NFS Server on KinD
+        run: kubectl apply -f ./examples/kubernetes/nfs-server/nfs-server.yaml
+      - name: Install csi-nfs driver on KinD
+        run: make kind-install-driver
+      - name: Setup K8s E2E external storage tests
+        run: make setup-external-e2e
+      - name: Run tests
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          make run-external-e2e

Makefile

@@ -14,6 +14,7 @@
 CMDS=nfsplugin
 DEPLOY_FOLDER = ./deploy/kubernetes
 CMDS=nfsplugin
+LOCAL_REGISTRY=localhost:5000
 all: build
 
 include release-tools/build.make
@@ -44,3 +45,29 @@ local-k8s-uninstall:
 	kubectl delete -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml --ignore-not-found
 	kubectl delete -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml --ignore-not-found
 	echo "Uninstalled NFS driver"
+
+
+.PHONY: kind-install-driver
+kind-install-driver:
+	echo "Instlling locally"
+	kubectl apply -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml
+	kubectl apply -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml
+	kubectl apply -f ./test/e2e_external/manifests/csi-nfs-controller.yaml
+	kubectl apply -f ./test/e2e_external/manifests/csi-nfs-node.yaml
+	echo "Successfully installed"
+
+.PHONY: setup-external-e2e
+setup-external-e2e:
+	curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.19.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
+	tar -xvf e2e-tests.tar.gz
+	rm e2e-tests.tar.gz
+	mkdir /tmp/csi-nfs
+	cp ./kubernetes/test/bin/e2e.test /tmp/csi-nfs/e2e.test
+	rm -r kubernetes
+	cp ./examples/kubernetes/storageclass-nfs.yaml /tmp/csi-nfs/storageclass.yaml
+	cp ./test/e2e_external/testdriver.yaml /tmp/csi-nfs/testdriver.yaml
+
+.PHONY: run-external-e2e
+run-external-e2e:
+	bash ./test/e2e_external/run.sh
+	

test/e2e_external/manifests/csi-nfs-controller.yaml

@@ -0,0 +1,108 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: csi-nfs-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-nfs-controller
+  template:
+    metadata:
+      labels:
+        app: csi-nfs-controller
+    spec:
+      serviceAccountName: csi-nfs-controller-sa
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      tolerations:
+        - key: "node-role.kubernetes.io/master"
+          operator: "Equal"
+          value: "true"
+          effect: "NoSchedule"
+      containers:
+        - name: csi-provisioner
+          image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v1.4.0
+          args:
+            - "-v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--enable-leader-election"
+            - "--leader-election-type=leases"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: liveness-probe
+          image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v1.1.0
+          args:
+            - --csi-address=/csi/csi.sock
+            - --connection-timeout=3s
+            - --health-port=29642
+            - --v=5
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - name: nfs
+          image: nfsplugin:latest
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          imagePullPolicy: IfNotPresent
+          args:
+            - "-v=5"
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /plugin
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+            - mountPath: /csi
+              name: socket-dir
+          resources:
+            limits:
+              cpu: 200m
+              memory: 200Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+      volumes:
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi-nfsplugin
+            type: DirectoryOrCreate
+        - name: pods-mount-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: Directory
+        - name: socket-dir
+          emptyDir: {}

test/e2e_external/manifests/csi-nfs-node.yaml

@@ -0,0 +1,77 @@
+# This YAML file contains driver-registrar & csi driver nodeplugin API objects
+# that are necessary to run CSI nodeplugin for nfs
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-nfs-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-nfs-node
+  template:
+    metadata:
+      labels:
+        app: csi-nfs-node
+    spec:
+      hostNetwork: true # original nfs connection would be broken without hostNetwork setting
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: node-driver-registrar
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"]
+          args:
+            - --v=5
+            - --csi-address=/plugin/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /plugin
+            - name: registration-dir
+              mountPath: /registration
+        - name: nfs
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: nfsplugin:latest
+          args :
+            - "-v=5"
+            - "--nodeid=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix://plugin/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /plugin
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+      volumes:
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi-nfsplugin
+            type: DirectoryOrCreate
+        - name: pods-mount-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: Directory
+        - hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+          name: registration-dir

test/e2e_external/run.sh

@@ -0,0 +1,21 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+ginkgo -p --progress --v -focus='External.Storage.*csi-nfsplugin' \
+       -skip='\[Disruptive\]' \
+       /tmp/csi-nfs/e2e.test \
+       -- \
+       -storage.testdriver=/tmp/csi-nfs/testdriver.yaml \
+       --kubeconfig=$HOME/.kube/config

test/e2e_external/testdriver.yaml

@@ -0,0 +1,10 @@
+StorageClass:
+  FromName: false
+  FromFile: /tmp/csi-nfs/storageclass.yaml
+DriverInfo:
+  Name: csi-nfsplugin
+  Capabilities:
+    persistence: true
+    exec: true
+    multipods: true
+    RWX: true