diff --git a/cmd/openshift-tests/csi.go b/cmd/openshift-tests/csi.go new file mode 100644 index 000000000000..da189ae3ed43 --- /dev/null +++ b/cmd/openshift-tests/csi.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/openshift/origin/test/extended/csi" + + "k8s.io/kubernetes/test/e2e/storage/external" +) + +const ( + manifestEnvVar = "TEST_CSI_DRIVER_FILES" + installDriversEnvVar = "TEST_INSTALL_CSI_DRIVERS" +) + +// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES. +func initCSITests(dryRun bool) error { + driverList := os.Getenv(installDriversEnvVar) + if driverList != "" { + drivers := strings.Split(driverList, ",") + for _, driver := range drivers { + manifestFile, err := csi.InstallCSIDriver(driver, dryRun) + if err != nil { + return fmt.Errorf("failed to install CSI driver from %q: %s", driver, err) + } + // Children processes need to see the newly introduced manifest, + // store it in TEST_CSI_DRIVER_FILES env. var for them. + manifestList := os.Getenv(manifestEnvVar) + if len(manifestList) > 0 { + manifestList += "," + } + manifestList += manifestFile + os.Setenv(manifestEnvVar, manifestList) + } + } + + // Clear TEST_INSTALL_CSI_DRIVERS, we don't want the driver installed by children too. + os.Setenv(installDriversEnvVar, "") + + manifestList := os.Getenv(manifestEnvVar) + if manifestList != "" { + manifests := strings.Split(manifestList, ",") + for _, manifest := range manifests { + if err := external.AddDriverDefinition(manifest); err != nil { + return fmt.Errorf("failed to load manifest from %q: %s", manifest, err) + } + } + } + + return nil +} diff --git a/cmd/openshift-tests/openshift-tests.go b/cmd/openshift-tests/openshift-tests.go index 86606783e818..2aa7272c40f1 100644 --- a/cmd/openshift-tests/openshift-tests.go +++ b/cmd/openshift-tests/openshift-tests.go @@ -8,7 +8,6 @@ import ( "io" "math/rand" "os" - "strings" "time" "github.com/onsi/ginkgo" @@ -22,7 +21,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" reale2e "k8s.io/kubernetes/test/e2e" e2e "k8s.io/kubernetes/test/e2e/framework" - "k8s.io/kubernetes/test/e2e/storage/external" "github.com/openshift/library-go/pkg/serviceability" "github.com/openshift/origin/pkg/monitor" @@ -103,6 +101,7 @@ func newRunCommand() *cobra.Command { opt := &testginkgo.Options{ Suites: staticSuites, } + cmd := &cobra.Command{ Use: "run SUITE", Short: "Run a test suite", @@ -208,6 +207,7 @@ func newRunTestCommand() *cobra.Command { Out: os.Stdout, ErrOut: os.Stderr, } + cmd := &cobra.Command{ Use: "run-test NAME", Short: "Run a single test by name", @@ -283,7 +283,7 @@ func initProvider(provider string, dryRun bool) error { exutil.TestContext.MaxNodesToGather = 0 reale2e.SetViperConfig(os.Getenv("VIPERCONFIG")) - if err := initCSITests(); err != nil { + if err := initCSITests(dryRun); err != nil { return err } @@ -337,19 +337,3 @@ func decodeProviderTo(provider string, testContext *e2e.TestContextType) error { klog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig) return nil } - -// Initialize openshift/csi suite, i.e. define CSI tests from TEST_CSI_DRIVER_FILES. -func initCSITests() error { - // TODO: replace with cmdline argument - driverList := os.Getenv("TEST_CSI_DRIVER_FILES") - if driverList == "" { - return nil - } - drivers := strings.Split(driverList, ",") - for _, driver := range drivers { - if err := external.AddDriverDefinition(driver); err != nil { - return fmt.Errorf("failed to load driver from %q: %s", driver, err) - } - } - return nil -} diff --git a/test/extended/csi/install.go b/test/extended/csi/install.go new file mode 100644 index 000000000000..f6f9ec90ddde --- /dev/null +++ b/test/extended/csi/install.go @@ -0,0 +1,94 @@ +package csi + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/openshift/origin/test/extended/testdata" + exutil "github.com/openshift/origin/test/extended/util" +) + +const ( + csiBasePath = "test/extended/testdata/csi" + defaultImageFormat = "registry.svc.ci.openshift.org/origin/4.3:${component}" + imageFormatVariable = "IMAGE_FORMAT" +) + +// InstallCSIDriver installs a CSI driver and defines its tests. +// It applies "test/extended/csi//install-template.yaml" and +// returns path to test manifest "test/extended/csi//manifest.yaml" +func InstallCSIDriver(driverName string, dryRun bool) (string, error) { + // The driver name comes from an user and we want a nice error message instead + // of panic in FixturePath(). + templatePath := filepath.Join(csiBasePath, driverName, "install-template.yaml") + if _, err := testdata.AssetInfo(templatePath); err != nil { + return "", fmt.Errorf("failed to install CSI driver %q: %s", driverName, err) + } + manifestPath := filepath.Join(csiBasePath, driverName, "manifest.yaml") + if _, err := testdata.AssetInfo(manifestPath); err != nil { + return "", fmt.Errorf("failed to install CSI driver %q: %s", driverName, err) + } + + // Convert to array and cut "test/extended" for FixturePath() + templateFixturePath := strings.Split(templatePath, string(os.PathSeparator))[2:] + yamlPath, err := executeTemplate(exutil.FixturePath(templateFixturePath...)) + defer os.Remove(yamlPath) + if err != nil { + return "", err + } + + if !dryRun { + // Install the driver + oc := exutil.NewCLIWithoutNamespace("csi-install") + if err := oc.Run("apply").Args("-f", yamlPath).Execute(); err != nil { + return "", fmt.Errorf("failed to apply %s: %s", yamlPath, err) + } + } + + // Cut "test/extended" for FixturePath() + manifestFixturePath := strings.Split(manifestPath, string(os.PathSeparator))[2:] + return exutil.FixturePath(manifestFixturePath...), nil +} + +// ListCSIDrivers returns list of hardcoded CSI drivers, i.e. list of directories in "test/extended/csi". +func ListCSIDrivers() ([]string, error) { + return testdata.AssetDir(csiBasePath) +} + +// Executes given golang template file and returns path to resulting file. +func executeTemplate(templatePath string) (string, error) { + tmpl, err := template.ParseFiles(templatePath) + if err != nil { + return "", err + } + yamlFile, err := ioutil.TempFile("", "openshift-tests-csi-*") + if err != nil { + return "", err + } + yamlPath := yamlFile.Name() + + imageFormat := os.Getenv(imageFormatVariable) + if imageFormat == "" { + imageFormat = defaultImageFormat + } + + variables := struct{ AttacherImage, ProvisionerImage, NodeDriverRegistrarImage, LivenessProbeImage, ImageFormat string }{ + AttacherImage: strings.ReplaceAll(imageFormat, "${component}", "csi-external-attacher"), + ProvisionerImage: strings.ReplaceAll(imageFormat, "${component}", "csi-external-provisioner"), + NodeDriverRegistrarImage: strings.ReplaceAll(imageFormat, "${component}", "csi-node-driver-registrar"), + LivenessProbeImage: strings.ReplaceAll(imageFormat, "${component}", "csi-livenessprobe"), + ImageFormat: imageFormat, + } + + err = tmpl.Execute(yamlFile, variables) + yamlFile.Close() + if err != nil { + os.Remove(yamlPath) + return "", err + } + return yamlPath, nil +} diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index 36b778d4f2e4..b464a4dc3066 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -307,6 +307,8 @@ // test/extended/testdata/cmd/test/cmd/volumes.sh // test/extended/testdata/cmd/test/cmd/whoami.sh // test/extended/testdata/config-map-jenkins-slave-pods.yaml +// test/extended/testdata/csi/aws-ebs/install-template.yaml +// test/extended/testdata/csi/aws-ebs/manifest.yaml // test/extended/testdata/custom-secret-builder/Dockerfile // test/extended/testdata/custom-secret-builder/build.sh // test/extended/testdata/deployments/custom-deployment.yaml @@ -46951,6 +46953,341 @@ func testExtendedTestdataConfigMapJenkinsSlavePodsYaml() (*asset, error) { return a, nil } +var _testExtendedTestdataCsiAwsEbsInstallTemplateYaml = []byte(`# Installation of AWS EBS CSI driver into OpenShift. +# Source: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/e727882eae38614ec3246b243255e536aeef5a0f/deploy/kubernetes/manifest.yaml +# Tailored for OCP 4.2: +# - Removed features we don't support in 4.2 (snapshots) +# - Run controllers with hostNetwork: true to access AWS instance metadata (https://bugzilla.redhat.com/show_bug.cgi?id=1734600) +# - Removed livenessprobe for this +# - Scaled controllers to 1 +# - Run in kube-system and use aws-creds secrets there + +# Controller Service +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + serviceName: ebs-csi-controller + replicas: 1 + template: + metadata: + labels: + app: ebs-csi-controller + spec: + hostNetwork: true + serviceAccount: ebs-csi-controller-sa + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: ebs-plugin + image: amazon/aws-ebs-csi-driver:latest + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds + key: aws_access_key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds + key: aws_secret_access_key + optional: true + ports: + - name: healthz + containerPort: 19808 + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-provisioner + image: {{.ProvisionerImage}} + args: + - --provisioner=ebs.csi.aws.com + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: {{.AttacherImage}} + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} + +--- +# Node Service +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + template: + metadata: + labels: + app: ebs-csi-node + spec: + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: amazon/aws-ebs-csi-driver:latest + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: node-driver-registrar + securityContext: + privileged: true + image: {{.NodeDriverRegistrarImage}} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{.LivenessProbeImage}} + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + +--- + +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false +`) + +func testExtendedTestdataCsiAwsEbsInstallTemplateYamlBytes() ([]byte, error) { + return _testExtendedTestdataCsiAwsEbsInstallTemplateYaml, nil +} + +func testExtendedTestdataCsiAwsEbsInstallTemplateYaml() (*asset, error) { + bytes, err := testExtendedTestdataCsiAwsEbsInstallTemplateYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/csi/aws-ebs/install-template.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataCsiAwsEbsManifestYaml = []byte(`# Test manifest for https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external +ShortName: ebs +StorageClass: + FromName: true +SupportedMountOption: + dirsync: true +DriverInfo: + Name: ebs.csi.aws.com + SnapshotClass: + FromName: true + SupportedFsType: + # xfs: {} xfs is broken: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/326 + ext4: {} + SupportedMountOption: + dirsync: {} + Capabilities: + persistence: true + fsGroup: true + block: true + exec: true + volumeLimits: true +`) + +func testExtendedTestdataCsiAwsEbsManifestYamlBytes() ([]byte, error) { + return _testExtendedTestdataCsiAwsEbsManifestYaml, nil +} + +func testExtendedTestdataCsiAwsEbsManifestYaml() (*asset, error) { + bytes, err := testExtendedTestdataCsiAwsEbsManifestYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/csi/aws-ebs/manifest.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _testExtendedTestdataCustomSecretBuilderDockerfile = []byte(`FROM openshift/origin-custom-docker-builder # Override the default build script ADD build.sh /tmp/build.sh @@ -57936,6 +58273,8 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/cmd/test/cmd/volumes.sh": testExtendedTestdataCmdTestCmdVolumesSh, "test/extended/testdata/cmd/test/cmd/whoami.sh": testExtendedTestdataCmdTestCmdWhoamiSh, "test/extended/testdata/config-map-jenkins-slave-pods.yaml": testExtendedTestdataConfigMapJenkinsSlavePodsYaml, + "test/extended/testdata/csi/aws-ebs/install-template.yaml": testExtendedTestdataCsiAwsEbsInstallTemplateYaml, + "test/extended/testdata/csi/aws-ebs/manifest.yaml": testExtendedTestdataCsiAwsEbsManifestYaml, "test/extended/testdata/custom-secret-builder/Dockerfile": testExtendedTestdataCustomSecretBuilderDockerfile, "test/extended/testdata/custom-secret-builder/build.sh": testExtendedTestdataCustomSecretBuilderBuildSh, "test/extended/testdata/deployments/custom-deployment.yaml": testExtendedTestdataDeploymentsCustomDeploymentYaml, @@ -58585,6 +58924,12 @@ var _bintree = &bintree{nil, map[string]*bintree{ }}, }}, "config-map-jenkins-slave-pods.yaml": &bintree{testExtendedTestdataConfigMapJenkinsSlavePodsYaml, map[string]*bintree{}}, + "csi": &bintree{nil, map[string]*bintree{ + "aws-ebs": &bintree{nil, map[string]*bintree{ + "install-template.yaml": &bintree{testExtendedTestdataCsiAwsEbsInstallTemplateYaml, map[string]*bintree{}}, + "manifest.yaml": &bintree{testExtendedTestdataCsiAwsEbsManifestYaml, map[string]*bintree{}}, + }}, + }}, "custom-secret-builder": &bintree{nil, map[string]*bintree{ "Dockerfile": &bintree{testExtendedTestdataCustomSecretBuilderDockerfile, map[string]*bintree{}}, "build.sh": &bintree{testExtendedTestdataCustomSecretBuilderBuildSh, map[string]*bintree{}}, diff --git a/test/extended/testdata/csi/README.md b/test/extended/testdata/csi/README.md new file mode 100644 index 000000000000..09a68c65bbf5 --- /dev/null +++ b/test/extended/testdata/csi/README.md @@ -0,0 +1,37 @@ +# CSI driver installer manifests + +Each CSI driver is represented as a directory with two files: + +``` +/install-template.yaml +/manifest.yaml +``` + +## Driver template +`install-template.yaml` is a [golang template](https://golang.org/pkg/text/template/) of YAML file with all Kubernetes objects of a CSI driver. +It will be instantiated at the beginning of the test via `oc apply -f `, where `` is result of the template evaluation. + +It is expected that the YAML file creates also a hardcoded namespace, so multiple `oc apply -f ` are idempotent and don't install the driver multiple times. + +Following variables are available in the template: + +* Name of sidecar image to test with. It is either the last build in the appropriate 4.x branch or image build from PR that's being tested. + * `{{.AttacherImage}}` + * `{{.ProvisionerImage}}` + * `{{.NodeDriverRegistrarImage}}` + * `{{.LivenessProbeImage}}` + +* `{{.ImageFormat}}`: Generic format of image names for the test, provided in case the template wants to use a different image than the listed above. E.g. `registry.svc.ci.openshift.org/ci-op-pthpkjbt/stable:${component}`. + +## Manifest +`manifest.yaml` describes features of the CSI driver. See [upstream documentation](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/external/README.md) for its format and usage. + +## Usage + +CSI driver defined here can be installed and tested using: + +``` +TEST_INSTALL_CSI_DRIVERS= openshift-tests run openshift/csi +``` + +Multiple CSI drivers can be installed & tested in one run, `TEST_INSTALL_CSI_DRIVERS` is comma-separated list. diff --git a/test/extended/testdata/csi/aws-ebs/install-template.yaml b/test/extended/testdata/csi/aws-ebs/install-template.yaml new file mode 100644 index 000000000000..4ecc93ead60e --- /dev/null +++ b/test/extended/testdata/csi/aws-ebs/install-template.yaml @@ -0,0 +1,280 @@ +# Installation of AWS EBS CSI driver into OpenShift. +# Source: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/e727882eae38614ec3246b243255e536aeef5a0f/deploy/kubernetes/manifest.yaml +# Tailored for OCP 4.2: +# - Removed features we don't support in 4.2 (snapshots) +# - Run controllers with hostNetwork: true to access AWS instance metadata (https://bugzilla.redhat.com/show_bug.cgi?id=1734600) +# - Removed livenessprobe for this +# - Scaled controllers to 1 +# - Run in kube-system and use aws-creds secrets there + +# Controller Service +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + serviceName: ebs-csi-controller + replicas: 1 + template: + metadata: + labels: + app: ebs-csi-controller + spec: + hostNetwork: true + serviceAccount: ebs-csi-controller-sa + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: ebs-plugin + image: amazon/aws-ebs-csi-driver:latest + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-creds + key: aws_access_key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-creds + key: aws_secret_access_key + optional: true + ports: + - name: healthz + containerPort: 19808 + protocol: TCP + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-provisioner + image: {{.ProvisionerImage}} + args: + - --provisioner=ebs.csi.aws.com + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: {{.AttacherImage}} + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + volumes: + - name: socket-dir + emptyDir: {} + +--- +# Node Service +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + template: + metadata: + labels: + app: ebs-csi-node + spec: + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: amazon/aws-ebs-csi-driver:latest + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: node-driver-registrar + securityContext: + privileged: true + image: {{.NodeDriverRegistrarImage}} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{.LivenessProbeImage}} + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory + +--- + +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/test/extended/testdata/csi/aws-ebs/manifest.yaml b/test/extended/testdata/csi/aws-ebs/manifest.yaml new file mode 100644 index 000000000000..de48a887a0f0 --- /dev/null +++ b/test/extended/testdata/csi/aws-ebs/manifest.yaml @@ -0,0 +1,21 @@ +# Test manifest for https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external +ShortName: ebs +StorageClass: + FromName: true +SupportedMountOption: + dirsync: true +DriverInfo: + Name: ebs.csi.aws.com + SnapshotClass: + FromName: true + SupportedFsType: + # xfs: {} xfs is broken: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/326 + ext4: {} + SupportedMountOption: + dirsync: {} + Capabilities: + persistence: true + fsGroup: true + block: true + exec: true + volumeLimits: true