Merge remote-tracking branch 'upstream/master' into upstream-merge
Conflicts: pkg/nfs/nodeserver.go https://github.com/kubernetes-csi/csi-driver-nfs/pull/201 removed locks in NodePublishVolume and NodeUnpublishVolume which conflicted with a downstream-only patch.
This commit is contained in:
Matthew Booth
commit
a7f5765049
|
|
@ -14,7 +14,7 @@
|
|||
|
||||
ARG ARCH=amd64
|
||||
|
||||
FROM k8s.gcr.io/build-image/debian-base-${ARCH}:v2.1.3
|
||||
FROM k8s.gcr.io/build-image/debian-base:buster-v1.6.0
|
||||
|
||||
# Copy nfsplugin from build _output directory
|
||||
COPY bin/nfsplugin /nfsplugin
|
||||
|
|
|
|||
17
Makefile
17
Makefile
|
|
@ -42,6 +42,9 @@ REGISTRY_NAME ?= $(shell echo $(REGISTRY) | sed "s/.azurecr.io//g")
|
|||
IMAGE_TAG = $(REGISTRY)/$(IMAGENAME):$(IMAGE_VERSION)
|
||||
IMAGE_TAG_LATEST = $(REGISTRY)/$(IMAGENAME):latest
|
||||
|
||||
E2E_HELM_OPTIONS ?= --set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) --set image.nfs.tag=$(IMAGE_VERSION) --set image.nfs.pullPolicy=Always
|
||||
E2E_HELM_OPTIONS += ${EXTRA_HELM_OPTIONS}
|
||||
|
||||
all: nfs
|
||||
|
||||
.PHONY: verify
|
||||
|
|
@ -112,11 +115,9 @@ install-helm:
|
|||
e2e-bootstrap: install-helm
|
||||
docker pull $(IMAGE_TAG) || make container push
|
||||
helm install csi-driver-nfs ./charts/latest/csi-driver-nfs --namespace kube-system --wait --timeout=15m -v=5 --debug \
|
||||
--set image.nfs.repository=$(REGISTRY)/$(IMAGENAME) \
|
||||
--set image.nfs.tag=$(IMAGE_VERSION) \
|
||||
--set image.nfs.pullPolicy=Always
|
||||
--set controller.logLevel=8
|
||||
--set node.logLevel=8
|
||||
${E2E_HELM_OPTIONS} \
|
||||
--set controller.logLevel=8 \
|
||||
--set node.logLevel=8
|
||||
|
||||
.PHONY: e2e-teardown
|
||||
e2e-teardown:
|
||||
|
|
@ -124,4 +125,8 @@ e2e-teardown:
|
|||
|
||||
.PHONY: e2e-test
|
||||
e2e-test:
|
||||
go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS}
|
||||
if [ ! -z "$(EXTERNAL_E2E_TEST)" ]; then \
|
||||
bash ./test/external-e2e/run.sh;\
|
||||
else \
|
||||
go test -v -timeout=0 ./test/e2e ${GINKGO_FLAGS};\
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ Please refer to [`nfs.csi.k8s.io` driver parameters](./docs/driver-parameters.md
|
|||
### Examples
|
||||
- [Set up a NFS Server on a Kubernetes cluster](./deploy/example/nfs-provisioner/README.md)
|
||||
- [Basic usage](./deploy/example/README.md)
|
||||
- [fsGroupPolicy](./deploy/example/fsgroup)
|
||||
|
||||
### Troubleshooting
|
||||
- [CSI driver troubleshooting guide](./docs/csi-debug.md)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@
|
|||
## Prerequisites
|
||||
- [install Helm](https://helm.sh/docs/intro/quickstart/#install-helm)
|
||||
|
||||
### Tips
|
||||
- `--set controller.runOnMaster=true` could make csi-nfs-controller only run on master node
|
||||
- `--set feature.enableFSGroupPolicy=true` could enable `fsGroupPolicy` on a k8s 1.20+ cluster (this feature is in beta, check details [here](../deploy/example/fsgroup))
|
||||
- `--set controller.replicas=1` could set replica of csi-nfs-controller as `1`
|
||||
|
||||
## install latest version
|
||||
```console
|
||||
helm repo add csi-driver-nfs https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/charts
|
||||
|
|
@ -31,6 +36,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv
|
|||
|
||||
| Parameter | Description | Default |
|
||||
|---------------------------------------------------|------------------------------------------------------------|-------------------------------------------------------------------|
|
||||
| `feature.enableFSGroupPolicy` | enable `fsGroupPolicy` on a k8s 1.20+ cluster | `false` |
|
||||
| `image.nfs.repository` | csi-driver-nfs docker image | gcr.io/k8s-staging-sig-storage/nfsplugin |
|
||||
| `image.nfs.tag` | csi-driver-nfs docker image tag | amd64-linux-canary |
|
||||
| `image.nfs.pullPolicy` | csi-driver-nfs image pull policy | IfNotPresent |
|
||||
|
|
@ -38,10 +44,10 @@ The following table lists the configurable parameters of the latest NFS CSI Driv
|
|||
| `image.csiProvisioner.tag` | csi-provisioner docker image tag | v2.0.4 |
|
||||
| `image.csiProvisioner.pullPolicy` | csi-provisioner image pull policy | IfNotPresent |
|
||||
| `image.livenessProbe.repository` | liveness-probe docker image | k8s.gcr.io/sig-storage/livenessprobe |
|
||||
| `image.livenessProbe.tag` | liveness-probe docker image tag | v2.1.0 |
|
||||
| `image.livenessProbe.tag` | liveness-probe docker image tag | v2.3.0 |
|
||||
| `image.livenessProbe.pullPolicy` | liveness-probe image pull policy | IfNotPresent |
|
||||
| `image.nodeDriverRegistrar.repository` | csi-node-driver-registrar docker image | k8s.gcr.io/sig-storage/csi-node-driver-registrar |
|
||||
| `image.nodeDriverRegistrar.tag` | csi-node-driver-registrar docker image tag | v2.0.1 |
|
||||
| `image.nodeDriverRegistrar.tag` | csi-node-driver-registrar docker image tag | v2.2.0 |
|
||||
| `image.nodeDriverRegistrar.pullPolicy` | csi-node-driver-registrar image pull policy | IfNotPresent |
|
||||
| `imagePullSecrets` | Specify docker-registry secret names as an array | [] (does not add image pull secrets to deployed pods) |
|
||||
| `serviceAccount.create` | whether create service account of csi-nfs-controller | true |
|
||||
|
|
@ -50,6 +56,7 @@ The following table lists the configurable parameters of the latest NFS CSI Driv
|
|||
| `controller.runOnMaster` | run controller on master node | false |
|
||||
| `controller.logLevel` | controller driver log level |`5` |
|
||||
| `node.logLevel` | node driver log level |`5` |
|
||||
| `node.livenessProbe.healthPort ` | the health check port for liveness probe |`29653` |
|
||||
|
||||
## troubleshooting
|
||||
- Add `--wait -v=5 --debug` in `helm install` command to get detailed error
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -6,4 +6,6 @@ spec:
|
|||
attachRequired: false
|
||||
volumeLifecycleModes:
|
||||
- Persistent
|
||||
podInfoOnMount: true
|
||||
{{- if .Values.feature.enableFSGroupPolicy}}
|
||||
fsGroupPolicy: File
|
||||
{{- end}}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ spec:
|
|||
args:
|
||||
- --csi-address=/csi/csi.sock
|
||||
- --probe-timeout=3s
|
||||
- --health-port=29653
|
||||
- --health-port={{ .Values.node.livenessProbe.healthPort }}
|
||||
- --v=2
|
||||
imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }}
|
||||
volumeMounts:
|
||||
|
|
|
|||
|
|
@ -9,11 +9,11 @@ image:
|
|||
pullPolicy: IfNotPresent
|
||||
livenessProbe:
|
||||
repository: k8s.gcr.io/sig-storage/livenessprobe
|
||||
tag: v2.1.0
|
||||
tag: v2.3.0
|
||||
pullPolicy: IfNotPresent
|
||||
nodeDriverRegistrar:
|
||||
repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar
|
||||
tag: v2.1.0
|
||||
tag: v2.2.0
|
||||
pullPolicy: IfNotPresent
|
||||
serviceAccount:
|
||||
create: true
|
||||
|
|
@ -26,6 +26,11 @@ controller:
|
|||
|
||||
node:
|
||||
logLevel: 5
|
||||
livenessProbe:
|
||||
healthPort: 29653
|
||||
|
||||
feature:
|
||||
enableFSGroupPolicy: false
|
||||
|
||||
## Reference to one or more secrets to be used when pulling images
|
||||
## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ spec:
|
|||
cpu: 10m
|
||||
memory: 20Mi
|
||||
- name: liveness-probe
|
||||
image: k8s.gcr.io/sig-storage/livenessprobe:v2.1.0
|
||||
image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0
|
||||
args:
|
||||
- --csi-address=/csi/csi.sock
|
||||
- --probe-timeout=3s
|
||||
|
|
|
|||
|
|
@ -7,4 +7,3 @@ spec:
|
|||
attachRequired: false
|
||||
volumeLifecycleModes:
|
||||
- Persistent
|
||||
podInfoOnMount: true
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ spec:
|
|||
kubernetes.io/os: linux
|
||||
containers:
|
||||
- name: liveness-probe
|
||||
image: k8s.gcr.io/sig-storage/livenessprobe:v2.1.0
|
||||
image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0
|
||||
args:
|
||||
- --csi-address=/csi/csi.sock
|
||||
- --probe-timeout=3s
|
||||
|
|
@ -38,7 +38,7 @@ spec:
|
|||
cpu: 10m
|
||||
memory: 20Mi
|
||||
- name: node-driver-registrar
|
||||
image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
|
||||
image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# fsGroup Support
|
||||
|
||||
[fsGroupPolicy](https://kubernetes-csi.github.io/docs/support-fsgroup.html) feature is Beta from Kubernetes 1.20, and disabled by default, follow below steps to enable this feature.
|
||||
|
||||
### Option#1: Enable fsGroupPolicy support in [driver helm installation](../../../charts)
|
||||
|
||||
add `--set feature.enableFSGroupPolicy=true` in helm installation command.
|
||||
|
||||
### Option#2: Enable fsGroupPolicy support on a cluster with CSI driver already installed
|
||||
|
||||
```console
|
||||
kubectl delete CSIDriver nfs.csi.k8s.io
|
||||
cat <<EOF | kubectl create -f -
|
||||
apiVersion: storage.k8s.io/v1beta1
|
||||
kind: CSIDriver
|
||||
metadata:
|
||||
name: nfs.csi.k8s.io
|
||||
spec:
|
||||
attachRequired: false
|
||||
volumeLifecycleModes:
|
||||
- Persistent
|
||||
fsGroupPolicy: File
|
||||
EOF
|
||||
```
|
||||
|
|
@ -7,7 +7,7 @@ provisioner: nfs.csi.k8s.io
|
|||
parameters:
|
||||
server: nfs-server.default.svc.cluster.local
|
||||
share: /
|
||||
reclaimPolicy: Retain
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
mountOptions:
|
||||
- hard
|
||||
|
|
|
|||
|
|
@ -104,9 +104,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
|
|||
if err = os.Mkdir(internalVolumePath, 0777); err != nil && !os.IsExist(err) {
|
||||
return nil, status.Errorf(codes.Internal, "failed to make subdirectory: %v", err.Error())
|
||||
}
|
||||
// Remove capacity setting when provisioner 1.4.0 is available with fix for
|
||||
// https://github.com/kubernetes-csi/external-provisioner/pull/271
|
||||
return &csi.CreateVolumeResponse{Volume: cs.nfsVolToCSI(nfsVol, reqCapacity)}, nil
|
||||
return &csi.CreateVolumeResponse{Volume: cs.nfsVolToCSI(nfsVol)}, nil
|
||||
}
|
||||
|
||||
// DeleteVolume delete a volume
|
||||
|
|
@ -338,9 +336,9 @@ func (cs *ControllerServer) getVolumeSharePath(vol *nfsVolume) string {
|
|||
}
|
||||
|
||||
// Convert into nfsVolume into a csi.Volume
|
||||
func (cs *ControllerServer) nfsVolToCSI(vol *nfsVolume, reqCapacity int64) *csi.Volume {
|
||||
func (cs *ControllerServer) nfsVolToCSI(vol *nfsVolume) *csi.Volume {
|
||||
return &csi.Volume{
|
||||
CapacityBytes: reqCapacity,
|
||||
CapacityBytes: 0, // by setting it to zero, Provisioner will use PVC requested size as PV size
|
||||
VolumeId: vol.id,
|
||||
VolumeContext: map[string]string{
|
||||
paramServer: vol.server,
|
||||
|
|
|
|||
|
|
@ -72,11 +72,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
|||
return &csi.NodePublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
if acquired := ns.Driver.volumeLocks.TryAcquire(volumeID); !acquired {
|
||||
return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volumeID)
|
||||
}
|
||||
defer ns.Driver.volumeLocks.Release(volumeID)
|
||||
|
||||
mountOptions := req.GetVolumeCapability().GetMount().GetMountFlags()
|
||||
if req.GetReadonly() {
|
||||
mountOptions = append(mountOptions, "ro")
|
||||
|
|
@ -141,11 +136,6 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
|
|||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if acquired := ns.Driver.volumeLocks.TryAcquire(volumeID); !acquired {
|
||||
return nil, status.Errorf(codes.Aborted, volumeOperationAlreadyExistsFmt, volumeID)
|
||||
}
|
||||
defer ns.Driver.volumeLocks.Release(volumeID)
|
||||
|
||||
klog.V(4).Infof("NodeUnpublishVolume: path %s is *not* a mount point: %t", targetPath, notMnt)
|
||||
if !notMnt {
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package nfs
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
|
@ -69,19 +68,6 @@ func TestNodePublishVolume(t *testing.T) {
|
|||
VolumeId: "vol_1"},
|
||||
expectedErr: status.Error(codes.InvalidArgument, "Target path not provided"),
|
||||
},
|
||||
{
|
||||
desc: "[Error] Volume operation in progress",
|
||||
setup: func() {
|
||||
ns.Driver.volumeLocks.TryAcquire("vol_1")
|
||||
},
|
||||
req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap},
|
||||
VolumeId: "vol_1",
|
||||
TargetPath: targetTest},
|
||||
expectedErr: status.Error(codes.Aborted, fmt.Sprintf(volumeOperationAlreadyExistsFmt, "vol_1")),
|
||||
cleanup: func() {
|
||||
ns.Driver.volumeLocks.Release("vol_1")
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "[Success] Stage target path missing",
|
||||
req: csi.NodePublishVolumeRequest{VolumeCapability: &csi.VolumeCapability{AccessMode: &volumeCap},
|
||||
|
|
@ -148,7 +134,6 @@ func TestNodeUnpublishVolume(t *testing.T) {
|
|||
errorTarget := testutil.GetWorkDirPath("error_is_likely_target", t)
|
||||
targetTest := testutil.GetWorkDirPath("target_test", t)
|
||||
targetFile := testutil.GetWorkDirPath("abc.go", t)
|
||||
alreadyMountedTarget := testutil.GetWorkDirPath("false_is_likely_exist_target", t)
|
||||
|
||||
tests := []struct {
|
||||
desc string
|
||||
|
|
@ -177,17 +162,6 @@ func TestNodeUnpublishVolume(t *testing.T) {
|
|||
req: csi.NodeUnpublishVolumeRequest{TargetPath: targetFile, VolumeId: "vol_1"},
|
||||
expectedErr: status.Error(codes.NotFound, "Volume not mounted"),
|
||||
},
|
||||
{
|
||||
desc: "[Error] Volume operation in progress",
|
||||
setup: func() {
|
||||
ns.Driver.volumeLocks.TryAcquire("vol_1")
|
||||
},
|
||||
req: csi.NodeUnpublishVolumeRequest{TargetPath: alreadyMountedTarget, VolumeId: "vol_1"},
|
||||
expectedErr: status.Error(codes.Aborted, fmt.Sprintf(volumeOperationAlreadyExistsFmt, "vol_1")),
|
||||
cleanup: func() {
|
||||
ns.Driver.volumeLocks.Release("vol_1")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Setup
|
||||
|
|
|
|||
|
|
@ -95,10 +95,6 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h
|
|||
return resp, err
|
||||
}
|
||||
|
||||
const (
|
||||
volumeOperationAlreadyExistsFmt = "An operation with the given Volume ID %s already exists"
|
||||
)
|
||||
|
||||
type VolumeLocks struct {
|
||||
locks sets.String
|
||||
mux sync.Mutex
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
|
||||
|
||||
aliases:
|
||||
|
||||
# SIG-Storage chairs and leads should always have approval rights in all repos.
|
||||
# Others may be added as needed here or in each repo.
|
||||
kubernetes-csi-approvers:
|
||||
- jsafrane
|
||||
- msau42
|
||||
- saad-ali
|
||||
- xing-yang
|
||||
|
||||
# Reviewers are automatically assigned to new PRs. The following
|
||||
# reviewers will be active in all repos. Other reviewers can be
|
||||
# added in each repo.
|
||||
#
|
||||
# Reviewers are encouraged to set the "Busy" flag in their GitHub status
|
||||
# when they are temporarily unable to review PRs.
|
||||
kubernetes-csi-reviewers:
|
||||
- andyzhangx
|
||||
- chrishenzie
|
||||
- ggriffiths
|
||||
- gnufied
|
||||
- j-griffith
|
||||
- Jiawei0227
|
||||
- jingxu97
|
||||
- jsafrane
|
||||
- pohly
|
||||
- xing-yang
|
||||
|
||||
# This documents who previously contributed to Kubernetes-CSI
|
||||
# as approver.
|
||||
emeritus_approver:
|
||||
- lpabon
|
||||
- sbezverk
|
||||
- vladimirvivien
|
||||
|
||||
# This documents who previously contributed to Kubernetes-CSI
|
||||
# as reviewer.
|
||||
emeritus_reviewer:
|
||||
- lpabon
|
||||
- saad-ali
|
||||
- sbezverk
|
||||
- vladimirvivien
|
||||
|
|
@ -1,11 +1,8 @@
|
|||
# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
|
||||
|
||||
approvers:
|
||||
- saad-ali
|
||||
- msau42
|
||||
- kubernetes-csi-approvers
|
||||
- pohly
|
||||
|
||||
reviewers:
|
||||
- saad-ali
|
||||
- msau42
|
||||
- pohly
|
||||
- kubernetes-csi-reviewers
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
KUBERNETES_CSI_OWNERS_ALIASES
|
||||
|
|
@ -104,3 +104,47 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
|
|||
CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
|
||||
and [k/k
|
||||
in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath)
|
||||
|
||||
## Adding support for a new Kubernetes release
|
||||
|
||||
1. Add the new release to `k8s_versions` in
|
||||
https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25
|
||||
to enable generating a job for it. Set `experimental_k8s_version`
|
||||
in
|
||||
https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40
|
||||
to ensure that the new jobs aren't run for PRs unless explicitly
|
||||
requested. Generate and submit the new jobs.
|
||||
1. Create a test PR to try out the new job in some repo with `/test
|
||||
pull-kubernetes-csi-<repo>-<x.y>-on-kubernetes-<x.y>` where x.y
|
||||
matches the Kubernetes release. Alternatively, run .prow.sh in that
|
||||
repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`.
|
||||
1. Optional: update to a [new
|
||||
release](https://github.com/kubernetes-sigs/kind/tags) of kind with
|
||||
pre-built images for the new Kubernetes release. This is optional
|
||||
if the current version of kind is able to build images for the new
|
||||
Kubernetes release. However, jobs require less resources when they
|
||||
don't need to build those images from the Kubernetes source code.
|
||||
This change needs to be tried out in a PR against a component
|
||||
first, then get submitted against csi-release-tools.
|
||||
1. Optional: propagate the updated csi-release-tools to all components
|
||||
with the script from
|
||||
https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402
|
||||
1. Once it is likely to work in all components, unset
|
||||
`experimental_k8s_version` and submit the updated jobs.
|
||||
1. Once all sidecars for the new Kubernetes release are released,
|
||||
either bump the version number of the images in the existing
|
||||
[csi-driver-host-path
|
||||
deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
|
||||
and/or create a new deployment, depending on what Kubernetes
|
||||
release an updated sidecar is compatible with. If no new deployment
|
||||
is needed, then add a symlink to document that there intentionally
|
||||
isn't a separate deployment. This symlink is not needed for Prow
|
||||
testing because that will use "kubernetes-latest" as fallback.
|
||||
Update that link when creating a new deployment.
|
||||
1. Create a new csi-driver-host-path release.
|
||||
1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and
|
||||
(eventually) roll that change out to all repos by updating
|
||||
`release-tools` in them. This is used when testing manually. The
|
||||
Prow jobs override that value, so also update
|
||||
`hostpath_driver_version` in
|
||||
https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# force the usage of /bin/bash instead of /bin/sh
|
||||
SHELL := /bin/bash
|
||||
|
||||
.PHONY: build-% build container-% container push-% push clean test
|
||||
|
||||
# A space-separated list of all commands in the repository, must be
|
||||
|
|
@ -63,26 +66,35 @@ endif
|
|||
# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
|
||||
# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
|
||||
|
||||
# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
|
||||
# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image]
|
||||
# separated by semicolon. An empty variable or empty entry (= just a
|
||||
# semicolon) builds for the default platform of the current Go
|
||||
# toolchain.
|
||||
BUILD_PLATFORMS =
|
||||
|
||||
# Add go ldflags using LDFLAGS at the time of compilation.
|
||||
IMPORTPATH_LDFLAGS = -X main.version=$(REV)
|
||||
IMPORTPATH_LDFLAGS = -X main.version=$(REV)
|
||||
EXT_LDFLAGS = -extldflags "-static"
|
||||
LDFLAGS =
|
||||
LDFLAGS =
|
||||
FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
|
||||
# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
|
||||
# defined by BUILD_PLATFORMS.
|
||||
$(CMDS:%=build-%): build-%: check-go-version-go
|
||||
mkdir -p bin
|
||||
echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
|
||||
# os_arch_seen captures all of the $$os-$$arch seen for the current binary
|
||||
# that we want to build, if we've seen an $$os-$$arch before it means that
|
||||
# we don't need to build it again, this is done to avoid building
|
||||
# the windows binary multiple times (see the default value of $$BUILD_PLATFORMS)
|
||||
export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
|
||||
os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \
|
||||
if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \
|
||||
continue; \
|
||||
fi; \
|
||||
if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
|
||||
echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
os_arch_seen+=";$$os-$$arch"; \
|
||||
done
|
||||
|
||||
$(CMDS:%=container-%): container-%: build-%
|
||||
|
|
@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?=
|
|||
# the tag for the resulting multiarch image.
|
||||
$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
|
||||
set -ex; \
|
||||
DOCKER_CLI_EXPERIMENTAL=enabled; \
|
||||
export DOCKER_CLI_EXPERIMENTAL; \
|
||||
export DOCKER_CLI_EXPERIMENTAL=enabled; \
|
||||
docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
|
||||
trap "docker buildx rm multiarchimage-buildertest" EXIT; \
|
||||
dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
|
||||
dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
|
||||
if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
|
||||
if ! [ -f "$$dockerfile_windows" ]; then \
|
||||
build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
|
||||
build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \
|
||||
fi; \
|
||||
pushMultiArch () { \
|
||||
tag=$$1; \
|
||||
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
|
||||
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
|
||||
escaped_base_image=$${base_image/:/-}; \
|
||||
if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
|
||||
docker buildx build --push \
|
||||
--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
|
||||
--tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \
|
||||
--platform=$$os/$$arch \
|
||||
--file $$(eval echo \$${dockerfile_$$os}) \
|
||||
--build-arg binary=./bin/$*$$suffix \
|
||||
--build-arg ARCH=$$arch \
|
||||
--build-arg BASE_IMAGE=$$base_image \
|
||||
--build-arg ADDON_IMAGE=$$addon_image \
|
||||
--label revision=$(REV) \
|
||||
.; \
|
||||
done; \
|
||||
images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
|
||||
images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
|
||||
escaped_base_image=$${base_image/:/-}; \
|
||||
if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
|
||||
echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
|
||||
done); \
|
||||
docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
|
||||
echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
|
||||
if [ $$os = "windows" ]; then \
|
||||
escaped_base_image=$${base_image/:/-}; \
|
||||
if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
|
||||
image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
|
||||
os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \
|
||||
docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \
|
||||
fi; \
|
||||
done; \
|
||||
docker manifest push -p $(IMAGE_NAME):$$tag; \
|
||||
}; \
|
||||
if [ $(PULL_BASE_REF) = "master" ]; then \
|
||||
|
|
@ -288,4 +316,3 @@ test-spelling:
|
|||
test-boilerplate:
|
||||
@ echo; echo "### $@:"
|
||||
@ ./release-tools/verify-boilerplate.sh "$(pwd)"
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ steps:
|
|||
# The image must contain bash and curl. Ideally it should also contain
|
||||
# the desired version of Go (currently defined in release-tools/prow.sh),
|
||||
# but that just speeds up the build and is not required.
|
||||
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
|
||||
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
|
||||
entrypoint: ./.cloudbuild.sh
|
||||
env:
|
||||
- GIT_TAG=${_GIT_TAG}
|
||||
|
|
|
|||
|
|
@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten
|
|||
sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
|
||||
) || die "failed to determine Kubernetes staging modules"
|
||||
for mod in $mods; do
|
||||
if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then
|
||||
echo "Kubernetes module $mod is not used, skipping"
|
||||
# Remove the module from go.mod "replace" that was added by an older version of this script.
|
||||
(set -x; env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed"
|
||||
continue
|
||||
fi
|
||||
# The presence of a potentially incomplete go.mod file affects this command,
|
||||
# so move elsewhere.
|
||||
modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") ||
|
||||
|
|
|
|||
|
|
@ -77,7 +77,10 @@ version_to_git () {
|
|||
esac
|
||||
}
|
||||
|
||||
configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
|
||||
# the list of windows versions was matched from:
|
||||
# - https://hub.docker.com/_/microsoft-windows-nanoserver
|
||||
# - https://hub.docker.com/_/microsoft-windows-servercore
|
||||
configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
|
||||
|
||||
# If we have a vendor directory, then use it. We must be careful to only
|
||||
# use this for "make" invocations inside the project's repo itself because
|
||||
|
|
@ -136,6 +139,9 @@ kind_version_default () {
|
|||
case "${CSI_PROW_KUBERNETES_VERSION}" in
|
||||
latest|master)
|
||||
echo main;;
|
||||
1.21*|release-1.21)
|
||||
# TODO: replace this special case once the next KinD release supports 1.21.
|
||||
echo main;;
|
||||
*)
|
||||
echo v0.10.0;;
|
||||
esac
|
||||
|
|
@ -159,11 +165,6 @@ kindest/node:v1.14.10@sha256:3fbed72bcac108055e46e7b4091eb6858ad628ec51bf693c21f
|
|||
# Use kind node-image --type=bazel by default, but allow to disable that.
|
||||
configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
|
||||
|
||||
# Work directory. It has to allow running executables, therefore /tmp
|
||||
# is avoided. Cleaning up after the script is intentionally left to
|
||||
# the caller.
|
||||
configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
|
||||
|
||||
# By default, this script tests sidecars with the CSI hostpath driver,
|
||||
# using the install_csi_driver function. That function depends on
|
||||
# a deployment script that it searches for in several places:
|
||||
|
|
@ -190,8 +191,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
|
|||
# CSI_PROW_DEPLOYMENT variable can be set in the
|
||||
# .prow.sh of each component when there are breaking changes
|
||||
# that require using a non-default deployment. The default
|
||||
# is a deployment named "kubernetes-x.yy" (if available),
|
||||
# otherwise "kubernetes-latest".
|
||||
# is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available),
|
||||
# otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}".
|
||||
# "none" disables the deployment of the hostpath driver.
|
||||
#
|
||||
# When no deploy script is found (nothing in `deploy` directory,
|
||||
|
|
@ -203,6 +204,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
|
|||
configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version"
|
||||
configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
|
||||
configvar CSI_PROW_DEPLOYMENT "" "deployment"
|
||||
configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files"
|
||||
|
||||
# The install_csi_driver function may work also for other CSI drivers,
|
||||
# as long as they follow the conventions of the CSI hostpath driver.
|
||||
|
|
@ -237,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
|
|||
# of the cluster. The alternative would have been to (cross-)compile csi-sanity
|
||||
# and install it inside the cluster, which is not necessarily easier.
|
||||
configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo"
|
||||
configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2
|
||||
configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version"
|
||||
configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package"
|
||||
configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock"
|
||||
configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver"
|
||||
|
|
@ -297,6 +299,17 @@ tests_need_alpha_cluster () {
|
|||
tests_enabled "parallel-alpha" "serial-alpha"
|
||||
}
|
||||
|
||||
# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go
|
||||
# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants).
|
||||
# When testing canary images, those get used instead of the images specified
|
||||
# in the e2e.test's normal YAML files.
|
||||
#
|
||||
# The default is to enable this for all jobs which use canary images
|
||||
# because we want to know whether our release candidates will pass all
|
||||
# existing tests (the storage testsuites and mock testing in
|
||||
# Kubernetes).
|
||||
configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests"
|
||||
|
||||
# Regex for non-alpha, feature-tagged tests that should be run.
|
||||
#
|
||||
configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
|
||||
|
|
@ -355,16 +368,25 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external
|
|||
# whether they can run with the current cluster provider, but until
|
||||
# they are, we filter them out by name. Like the other test selection
|
||||
# variables, this is again a space separated list of regular expressions.
|
||||
#
|
||||
# "different node" test skips can be removed once
|
||||
# https://github.com/kubernetes/kubernetes/pull/82678 has been backported
|
||||
# to all the K8s versions we test against
|
||||
configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped"
|
||||
configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped"
|
||||
|
||||
# This is the directory for additional result files. Usually set by Prow, but
|
||||
# if not (for example, when invoking manually) it defaults to the work directory.
|
||||
configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
|
||||
mkdir -p "${ARTIFACTS}"
|
||||
# This creates directories that are required for testing.
|
||||
ensure_paths () {
|
||||
# Work directory. It has to allow running executables, therefore /tmp
|
||||
# is avoided. Cleaning up after the script is intentionally left to
|
||||
# the caller.
|
||||
configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
|
||||
|
||||
# This is the directory for additional result files. Usually set by Prow, but
|
||||
# if not (for example, when invoking manually) it defaults to the work directory.
|
||||
configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
|
||||
mkdir -p "${ARTIFACTS}"
|
||||
|
||||
# For additional tools.
|
||||
CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
|
||||
mkdir -p "${CSI_PROW_BIN}"
|
||||
PATH="${CSI_PROW_BIN}:$PATH"
|
||||
}
|
||||
|
||||
run () {
|
||||
echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2
|
||||
|
|
@ -384,11 +406,6 @@ die () {
|
|||
exit 1
|
||||
}
|
||||
|
||||
# For additional tools.
|
||||
CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
|
||||
mkdir -p "${CSI_PROW_BIN}"
|
||||
PATH="${CSI_PROW_BIN}:$PATH"
|
||||
|
||||
# Ensure that PATH has the desired version of the Go tools, then run command given as argument.
|
||||
# Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by
|
||||
# bumping the container image regularly.
|
||||
|
|
@ -618,11 +635,16 @@ EOF
|
|||
|
||||
# Deletes kind cluster inside a prow job
|
||||
delete_cluster_inside_prow_job() {
|
||||
local name="$1"
|
||||
|
||||
# Inside a real Prow job it is better to clean up at runtime
|
||||
# instead of leaving that to the Prow job cleanup code
|
||||
# because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872).
|
||||
#
|
||||
# This is also a good time to collect logs.
|
||||
if [ "$JOB_NAME" ]; then
|
||||
if kind get clusters | grep -q csi-prow; then
|
||||
run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name"
|
||||
run kind delete cluster --name=csi-prow || die "kind delete failed"
|
||||
fi
|
||||
unset KUBECONFIG
|
||||
|
|
@ -647,9 +669,9 @@ find_deployment () {
|
|||
|
||||
# Ignore: See if you can use ${variable//search/replace} instead.
|
||||
# shellcheck disable=SC2001
|
||||
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh"
|
||||
file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh"
|
||||
if ! [ -e "$file" ]; then
|
||||
file="$dir/kubernetes-latest/deploy.sh"
|
||||
file="$dir/kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}/deploy.sh"
|
||||
if ! [ -e "$file" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
|
@ -709,7 +731,7 @@ install_csi_driver () {
|
|||
fi
|
||||
}
|
||||
|
||||
# Installs all nessesary snapshotter CRDs
|
||||
# Installs all nessesary snapshotter CRDs
|
||||
install_snapshot_crds() {
|
||||
# Wait until volumesnapshot CRDs are in place.
|
||||
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
|
||||
|
|
@ -756,7 +778,7 @@ install_snapshot_controller() {
|
|||
fi
|
||||
echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt"
|
||||
cnt=$((cnt + 1))
|
||||
sleep 10
|
||||
sleep 10
|
||||
done
|
||||
|
||||
SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
|
||||
|
|
@ -825,7 +847,7 @@ install_snapshot_controller() {
|
|||
fi
|
||||
echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt"
|
||||
cnt=$((cnt + 1))
|
||||
sleep 10
|
||||
sleep 10
|
||||
done
|
||||
}
|
||||
|
||||
|
|
@ -870,6 +892,29 @@ start_loggers () {
|
|||
done
|
||||
}
|
||||
|
||||
# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k
|
||||
# source code, if needed.
|
||||
patch_kubernetes () {
|
||||
local source="$1" target="$2"
|
||||
|
||||
if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
|
||||
# We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because
|
||||
# e2e.test does not support it (see test/utils/image/manifest.go). Instead we
|
||||
# invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that
|
||||
# overrides that registry.
|
||||
find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;'
|
||||
cat >"$target/e2e-repo-list" <<EOF
|
||||
sigStorageRegistry: gcr.io/k8s-staging-sig-storage
|
||||
EOF
|
||||
cat >&2 <<EOF
|
||||
|
||||
Using a modified version of k/k/test/e2e:
|
||||
$(cd "$source" && git diff 2>&1)
|
||||
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test".
|
||||
install_e2e () {
|
||||
if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then
|
||||
|
|
@ -878,6 +923,7 @@ install_e2e () {
|
|||
|
||||
git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 &&
|
||||
if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then
|
||||
patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" &&
|
||||
go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" &&
|
||||
run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
|
||||
ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}"
|
||||
|
|
@ -929,7 +975,7 @@ run_e2e () (
|
|||
trap move_junit EXIT
|
||||
|
||||
cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
|
||||
run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
|
||||
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
|
||||
)
|
||||
|
||||
# Run csi-sanity against installed CSI driver.
|
||||
|
|
@ -941,7 +987,7 @@ run_sanity () (
|
|||
kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
|
||||
EOF
|
||||
# Using "rm -rf" as fallback for "rmdir" is a workaround for:
|
||||
# Node Service
|
||||
# Node Service
|
||||
# should work
|
||||
# /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624
|
||||
# STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896
|
||||
|
|
@ -968,6 +1014,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" --
|
|||
exit 1
|
||||
fi
|
||||
EOF
|
||||
|
||||
cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" <<EOF
|
||||
#!/bin/sh
|
||||
CHECK_PATH=\$(cat <<SCRIPT
|
||||
if [ -f "\$@" ]; then
|
||||
echo "file"
|
||||
elif [ -d "\$@" ]; then
|
||||
echo "directory"
|
||||
elif [ -e "\$@" ]; then
|
||||
echo "other"
|
||||
else
|
||||
echo "not_found"
|
||||
fi
|
||||
SCRIPT
|
||||
)
|
||||
kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PATH}"
|
||||
EOF
|
||||
|
||||
chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh
|
||||
|
||||
# This cannot run in parallel, because -csi.junitfile output
|
||||
|
|
@ -983,6 +1047,7 @@ EOF
|
|||
-csi.createmountpathcmd "${CSI_PROW_WORK}/mkdir_in_pod.sh" \
|
||||
-csi.removestagingpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
|
||||
-csi.removemountpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
|
||||
-csi.checkpathcmd "${CSI_PROW_WORK}/checkdir_in_pod.sh" \
|
||||
)
|
||||
|
||||
ascii_to_xml () {
|
||||
|
|
@ -1086,7 +1151,7 @@ make_test_to_junit () {
|
|||
# version_gt 1.3.1 v1.2.0 (returns true)
|
||||
# version_gt 1.1.1 release-1.2.0 (returns false)
|
||||
# version_gt 1.2.0 1.2.2 (returns false)
|
||||
function version_gt() {
|
||||
function version_gt() {
|
||||
versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done)
|
||||
greaterVersion=${1#"release-"};
|
||||
greaterVersion=${greaterVersion#"kubernetes-"};
|
||||
|
|
@ -1098,6 +1163,9 @@ main () {
|
|||
local images ret
|
||||
ret=0
|
||||
|
||||
# Set up work directory.
|
||||
ensure_paths
|
||||
|
||||
images=
|
||||
if ${CSI_PROW_BUILD_JOB}; then
|
||||
# A successful build is required for testing.
|
||||
|
|
@ -1146,7 +1214,7 @@ main () {
|
|||
if [ "$rbac_file_path" == "" ]; then
|
||||
rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
|
||||
fi
|
||||
|
||||
|
||||
if [ -e "$rbac_file_path" ]; then
|
||||
# This is one of those components which has its own RBAC rules (like external-provisioner).
|
||||
# We are testing a locally built image and also want to test with the the current,
|
||||
|
|
@ -1157,6 +1225,12 @@ main () {
|
|||
done
|
||||
fi
|
||||
|
||||
# Run the external driver tests and optionally also mock tests.
|
||||
local focus="External.Storage"
|
||||
if "$CSI_PROW_E2E_MOCK"; then
|
||||
focus="($focus|CSI.mock.volume)"
|
||||
fi
|
||||
|
||||
if tests_need_non_alpha_cluster; then
|
||||
start_cluster || die "starting the non-alpha cluster failed"
|
||||
|
||||
|
|
@ -1164,6 +1238,7 @@ main () {
|
|||
install_snapshot_crds
|
||||
install_snapshot_controller
|
||||
|
||||
|
||||
# Installing the driver might be disabled.
|
||||
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
|
||||
collect_cluster_info
|
||||
|
|
@ -1178,7 +1253,7 @@ main () {
|
|||
# Ignore: Double quote to prevent globbing and word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \
|
||||
-focus="External.Storage" \
|
||||
-focus="$focus" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E parallel failed"
|
||||
ret=1
|
||||
|
|
@ -1188,7 +1263,7 @@ main () {
|
|||
# Ignore: Double quote to prevent globbing and word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \
|
||||
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
|
||||
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then
|
||||
warn "E2E parallel features failed"
|
||||
ret=1
|
||||
|
|
@ -1197,14 +1272,14 @@ main () {
|
|||
|
||||
if tests_enabled "serial"; then
|
||||
if ! run_e2e serial \
|
||||
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
|
||||
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E serial failed"
|
||||
ret=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
delete_cluster_inside_prow_job
|
||||
delete_cluster_inside_prow_job non-alpha
|
||||
fi
|
||||
|
||||
if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
|
||||
|
|
@ -1223,7 +1298,7 @@ main () {
|
|||
# Ignore: Double quote to prevent globbing and word splitting.
|
||||
# shellcheck disable=SC2086
|
||||
if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \
|
||||
-focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
|
||||
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E parallel alpha failed"
|
||||
ret=1
|
||||
|
|
@ -1232,14 +1307,14 @@ main () {
|
|||
|
||||
if tests_enabled "serial-alpha"; then
|
||||
if ! run_e2e serial-alpha \
|
||||
-focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
|
||||
-focus="$focus.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
|
||||
-skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then
|
||||
warn "E2E serial alpha failed"
|
||||
ret=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
delete_cluster_inside_prow_job
|
||||
delete_cluster_inside_prow_job alpha
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
@ -1259,6 +1334,9 @@ gcr_cloud_build () {
|
|||
# Required for "docker buildx build --push".
|
||||
gcloud auth configure-docker
|
||||
|
||||
# Might not be needed here, but call it just in case.
|
||||
ensure_paths
|
||||
|
||||
if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
|
||||
# Needed for "RUN" steps on non-linux/amd64 platforms.
|
||||
# See https://github.com/multiarch/qemu-user-static#getting-started
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
#! /bin/sh
|
||||
|
||||
# Copyright 2021 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script is called by pull Prow jobs for the csi-release-tools
|
||||
# repo to ensure that the changes in the PR work when imported into
|
||||
# some other repo.
|
||||
|
||||
set -ex
|
||||
|
||||
# It must be called inside the updated csi-release-tools repo.
|
||||
CSI_RELEASE_TOOLS_DIR="$(pwd)"
|
||||
|
||||
# Update the other repo.
|
||||
cd "$PULL_TEST_REPO_DIR"
|
||||
git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master
|
||||
git log -n2
|
||||
|
||||
# Now fall through to testing.
|
||||
exec ./.prow.sh
|
||||
|
|
@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \
|
|||
# detect if the host machine has the required shellcheck version installed
|
||||
# if so, we will use that instead.
|
||||
HAVE_SHELLCHECK=false
|
||||
if which shellcheck &>/dev/null; then
|
||||
if command -v shellcheck &>/dev/null; then
|
||||
detected_version="$(shellcheck --version | grep 'version: .*')"
|
||||
if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
|
||||
HAVE_SHELLCHECK=true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2021 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -xe
|
||||
|
||||
PROJECT_ROOT=$(git rev-parse --show-toplevel)
|
||||
|
||||
install_ginkgo () {
|
||||
apt update -y
|
||||
apt install -y golang-ginkgo-dev
|
||||
}
|
||||
|
||||
setup_e2e_binaries() {
|
||||
# download k8s external e2e binary
|
||||
curl -sL https://storage.googleapis.com/kubernetes-release/release/v1.21.0/kubernetes-test-linux-amd64.tar.gz --output e2e-tests.tar.gz
|
||||
tar -xvf e2e-tests.tar.gz && rm e2e-tests.tar.gz
|
||||
|
||||
# enable fsGroupPolicy (only available from k8s 1.20)
|
||||
export EXTRA_HELM_OPTIONS="--set feature.enableFSGroupPolicy=true"
|
||||
|
||||
# install csi driver
|
||||
mkdir -p /tmp/csi && cp deploy/example/storageclass-nfs.yaml /tmp/csi/storageclass.yaml
|
||||
make e2e-bootstrap
|
||||
make install-nfs-server
|
||||
}
|
||||
|
||||
print_logs() {
|
||||
echo "print out driver logs ..."
|
||||
bash ./test/utils/nfs_log.sh
|
||||
}
|
||||
|
||||
install_ginkgo
|
||||
setup_e2e_binaries
|
||||
trap print_logs EXIT
|
||||
|
||||
ginkgo -p --progress --v -focus='External.Storage.*nfs.csi.k8s.io' \
|
||||
-skip='\[Disruptive\]|\[Slow\]' kubernetes/test/bin/e2e.test -- \
|
||||
-storage.testdriver=$PROJECT_ROOT/test/external-e2e/testdriver.yaml \
|
||||
--kubeconfig=$KUBECONFIG
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# Manifest for Kubernetes external tests.
|
||||
# See https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external
|
||||
|
||||
StorageClass:
|
||||
FromFile: /tmp/csi/storageclass.yaml
|
||||
DriverInfo:
|
||||
Name: nfs.csi.k8s.io
|
||||
Capabilities:
|
||||
persistence: true
|
||||
exec: true
|
||||
multipods: true
|
||||
RWX: true
|
||||
fsGroup: true
|
||||
Loading…
Reference in New Issue