CI: Add K8s external storage E2E tests
Signed-off-by: Mayank Shah <mayankshah1614@gmail.com>
This commit is contained in:
Mayank Shah
parent
108aef3ed2
commit
0ad6d94899
|
|
@ -0,0 +1,39 @@
|
||||||
|
name: K8s external storage E2E Tests
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ master ]
|
||||||
|
pull_request:
|
||||||
|
branches: [ master ]
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
name: E2E Test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
- name: Set up Go 1.x
|
||||||
|
uses: actions/setup-go@v2
|
||||||
|
with:
|
||||||
|
go-version: ^1.13
|
||||||
|
id: go
|
||||||
|
- name: Build image
|
||||||
|
run: |
|
||||||
|
make build
|
||||||
|
docker build -t nfsplugin:latest .
|
||||||
|
- uses: engineerd/setup-kind@v0.4.0
|
||||||
|
name: Installing KinD cluster
|
||||||
|
with:
|
||||||
|
version: "v0.9.0"
|
||||||
|
- name: Push image to KinD
|
||||||
|
run: |
|
||||||
|
kind load docker-image --name=kind nfsplugin:latest
|
||||||
|
- name: Install NFS Server on KinD
|
||||||
|
run: kubectl apply -f ./examples/kubernetes/nfs-server/nfs-server.yaml
|
||||||
|
- name: Install csi-nfs driver on KinD
|
||||||
|
run: make kind-install-driver
|
||||||
|
- name: Setup K8s E2E external storage tests
|
||||||
|
run: make setup-external-e2e
|
||||||
|
- name: Run tests
|
||||||
|
run: |
|
||||||
|
go get -u github.com/onsi/ginkgo/ginkgo
|
||||||
|
make run-external-e2e
|
||||||
27
Makefile
27
Makefile
|
|
@ -14,6 +14,7 @@
|
||||||
CMDS=nfsplugin
|
CMDS=nfsplugin
|
||||||
DEPLOY_FOLDER = ./deploy/kubernetes
|
DEPLOY_FOLDER = ./deploy/kubernetes
|
||||||
CMDS=nfsplugin
|
CMDS=nfsplugin
|
||||||
|
LOCAL_REGISTRY=localhost:5000
|
||||||
all: build
|
all: build
|
||||||
|
|
||||||
include release-tools/build.make
|
include release-tools/build.make
|
||||||
|
|
@ -44,3 +45,29 @@ local-k8s-uninstall:
|
||||||
kubectl delete -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml --ignore-not-found
|
kubectl delete -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml --ignore-not-found
|
||||||
kubectl delete -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml --ignore-not-found
|
kubectl delete -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml --ignore-not-found
|
||||||
echo "Uninstalled NFS driver"
|
echo "Uninstalled NFS driver"
|
||||||
|
|
||||||
|
|
||||||
|
.PHONY: kind-install-driver
|
||||||
|
kind-install-driver:
|
||||||
|
echo "Instlling locally"
|
||||||
|
kubectl apply -f $(DEPLOY_FOLDER)/rbac-csi-nfs-controller.yaml
|
||||||
|
kubectl apply -f $(DEPLOY_FOLDER)/csi-nfs-driverinfo.yaml
|
||||||
|
kubectl apply -f ./test/e2e_external/manifests/csi-nfs-controller.yaml
|
||||||
|
kubectl apply -f ./test/e2e_external/manifests/csi-nfs-node.yaml
|
||||||
|
echo "Successfully installed"
|
||||||
|
|
||||||
|
.PHONY: setup-external-e2e
|
||||||
|
setup-external-e2e:
|
||||||
|
curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.19.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
|
||||||
|
tar -xvf e2e-tests.tar.gz
|
||||||
|
rm e2e-tests.tar.gz
|
||||||
|
mkdir /tmp/csi-nfs
|
||||||
|
cp ./kubernetes/test/bin/e2e.test /tmp/csi-nfs/e2e.test
|
||||||
|
rm -r kubernetes
|
||||||
|
cp ./examples/kubernetes/storageclass-nfs.yaml /tmp/csi-nfs/storageclass.yaml
|
||||||
|
cp ./test/e2e_external/testdriver.yaml /tmp/csi-nfs/testdriver.yaml
|
||||||
|
|
||||||
|
.PHONY: run-external-e2e
|
||||||
|
run-external-e2e:
|
||||||
|
bash ./test/e2e_external/run.sh
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
---
|
||||||
|
kind: Deployment
|
||||||
|
apiVersion: apps/v1
|
||||||
|
metadata:
|
||||||
|
name: csi-nfs-controller
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: csi-nfs-controller
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: csi-nfs-controller
|
||||||
|
spec:
|
||||||
|
serviceAccountName: csi-nfs-controller-sa
|
||||||
|
nodeSelector:
|
||||||
|
kubernetes.io/os: linux
|
||||||
|
priorityClassName: system-cluster-critical
|
||||||
|
tolerations:
|
||||||
|
- key: "node-role.kubernetes.io/master"
|
||||||
|
operator: "Equal"
|
||||||
|
value: "true"
|
||||||
|
effect: "NoSchedule"
|
||||||
|
containers:
|
||||||
|
- name: csi-provisioner
|
||||||
|
image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v1.4.0
|
||||||
|
args:
|
||||||
|
- "-v=5"
|
||||||
|
- "--csi-address=$(ADDRESS)"
|
||||||
|
- "--enable-leader-election"
|
||||||
|
- "--leader-election-type=leases"
|
||||||
|
env:
|
||||||
|
- name: ADDRESS
|
||||||
|
value: /csi/csi.sock
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /csi
|
||||||
|
name: socket-dir
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 100Mi
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 20Mi
|
||||||
|
- name: liveness-probe
|
||||||
|
image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v1.1.0
|
||||||
|
args:
|
||||||
|
- --csi-address=/csi/csi.sock
|
||||||
|
- --connection-timeout=3s
|
||||||
|
- --health-port=29642
|
||||||
|
- --v=5
|
||||||
|
volumeMounts:
|
||||||
|
- name: socket-dir
|
||||||
|
mountPath: /csi
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 100m
|
||||||
|
memory: 100Mi
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 20Mi
|
||||||
|
- name: nfs
|
||||||
|
image: nfsplugin:latest
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
capabilities:
|
||||||
|
add: ["SYS_ADMIN"]
|
||||||
|
allowPrivilegeEscalation: true
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- "-v=5"
|
||||||
|
- "--nodeid=$(NODE_ID)"
|
||||||
|
- "--endpoint=$(CSI_ENDPOINT)"
|
||||||
|
env:
|
||||||
|
- name: NODE_ID
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
- name: CSI_ENDPOINT
|
||||||
|
value: unix:///csi/csi.sock
|
||||||
|
volumeMounts:
|
||||||
|
- name: plugin-dir
|
||||||
|
mountPath: /plugin
|
||||||
|
- name: pods-mount-dir
|
||||||
|
mountPath: /var/lib/kubelet/pods
|
||||||
|
mountPropagation: "Bidirectional"
|
||||||
|
- mountPath: /csi
|
||||||
|
name: socket-dir
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: 200m
|
||||||
|
memory: 200Mi
|
||||||
|
requests:
|
||||||
|
cpu: 10m
|
||||||
|
memory: 20Mi
|
||||||
|
volumes:
|
||||||
|
- name: plugin-dir
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/kubelet/plugins/csi-nfsplugin
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
- name: pods-mount-dir
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/kubelet/pods
|
||||||
|
type: Directory
|
||||||
|
- name: socket-dir
|
||||||
|
emptyDir: {}
|
||||||
|
|
@ -0,0 +1,77 @@
|
||||||
|
# This YAML file contains driver-registrar & csi driver nodeplugin API objects
|
||||||
|
# that are necessary to run CSI nodeplugin for nfs
|
||||||
|
kind: DaemonSet
|
||||||
|
apiVersion: apps/v1
|
||||||
|
metadata:
|
||||||
|
name: csi-nfs-node
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: csi-nfs-node
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: csi-nfs-node
|
||||||
|
spec:
|
||||||
|
hostNetwork: true # original nfs connection would be broken without hostNetwork setting
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
containers:
|
||||||
|
- name: node-driver-registrar
|
||||||
|
image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
|
||||||
|
lifecycle:
|
||||||
|
preStop:
|
||||||
|
exec:
|
||||||
|
command: ["/bin/sh", "-c", "rm -rf /registration/csi-nfsplugin /registration/csi-nfsplugin-reg.sock"]
|
||||||
|
args:
|
||||||
|
- --v=5
|
||||||
|
- --csi-address=/plugin/csi.sock
|
||||||
|
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi-nfsplugin/csi.sock
|
||||||
|
env:
|
||||||
|
- name: KUBE_NODE_NAME
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
volumeMounts:
|
||||||
|
- name: plugin-dir
|
||||||
|
mountPath: /plugin
|
||||||
|
- name: registration-dir
|
||||||
|
mountPath: /registration
|
||||||
|
- name: nfs
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
capabilities:
|
||||||
|
add: ["SYS_ADMIN"]
|
||||||
|
allowPrivilegeEscalation: true
|
||||||
|
image: nfsplugin:latest
|
||||||
|
args :
|
||||||
|
- "-v=5"
|
||||||
|
- "--nodeid=$(NODE_ID)"
|
||||||
|
- "--endpoint=$(CSI_ENDPOINT)"
|
||||||
|
env:
|
||||||
|
- name: NODE_ID
|
||||||
|
valueFrom:
|
||||||
|
fieldRef:
|
||||||
|
fieldPath: spec.nodeName
|
||||||
|
- name: CSI_ENDPOINT
|
||||||
|
value: unix://plugin/csi.sock
|
||||||
|
imagePullPolicy: "IfNotPresent"
|
||||||
|
volumeMounts:
|
||||||
|
- name: plugin-dir
|
||||||
|
mountPath: /plugin
|
||||||
|
- name: pods-mount-dir
|
||||||
|
mountPath: /var/lib/kubelet/pods
|
||||||
|
mountPropagation: "Bidirectional"
|
||||||
|
volumes:
|
||||||
|
- name: plugin-dir
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/kubelet/plugins/csi-nfsplugin
|
||||||
|
type: DirectoryOrCreate
|
||||||
|
- name: pods-mount-dir
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/kubelet/pods
|
||||||
|
type: Directory
|
||||||
|
- hostPath:
|
||||||
|
path: /var/lib/kubelet/plugins_registry
|
||||||
|
type: Directory
|
||||||
|
name: registration-dir
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
# Copyright 2020 The Kubernetes Authors.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
#!/bin/bash
|
||||||
|
ginkgo -p --progress --v -focus='External.Storage.*csi-nfsplugin' \
|
||||||
|
-skip='\[Disruptive\]' \
|
||||||
|
/tmp/csi-nfs/e2e.test \
|
||||||
|
-- \
|
||||||
|
-storage.testdriver=/tmp/csi-nfs/testdriver.yaml \
|
||||||
|
--kubeconfig=$HOME/.kube/config
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
StorageClass:
|
||||||
|
FromName: false
|
||||||
|
FromFile: /tmp/csi-nfs/storageclass.yaml
|
||||||
|
DriverInfo:
|
||||||
|
Name: csi-nfsplugin
|
||||||
|
Capabilities:
|
||||||
|
persistence: true
|
||||||
|
exec: true
|
||||||
|
multipods: true
|
||||||
|
RWX: true
|
||||||
Loading…
Reference in New Issue