add e2e test for PVC and resize

Signed-off-by: Andrei Kvapil <[email protected]>

charts/etcd-operator/templates/rbac/clusterrole-manager-role.yml

@@ -72,6 +72,7 @@ rules:
     - storageclasses
     verbs:
     - get
+    - list
   - apiGroups:
       - etcd.aenix.io
     resources:

config/rbac/role.yaml

@@ -27,10 +27,19 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - secrets
+  - persistentvolumeclaims
   verbs:
+  - get
   - list
+  - patch
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
   - get
+  - list
   - watch
 - apiGroups:
   - ""
@@ -56,21 +65,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  verbs:
-  - get
-  - list
-  - patch
-  - watch
-- apiGroups:
-  - "storage.k8s.io"
-  resources:
-  - storageclasses
-  verbs:
-  - get
 - apiGroups:
   - etcd.aenix.io
   resources:
@@ -109,3 +103,10 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list

examples/manifests/etcdcluster-persistent.yaml

@@ -3,14 +3,13 @@ apiVersion: etcd.aenix.io/v1alpha1
 kind: EtcdCluster
 metadata:
   name: test
-  namespace: default
 spec:
   replicas: 3
   storage:
     volumeClaimTemplate:
       spec:
-        storageClassName: gp3
+        storageClassName: standard-with-expansion
         accessModes: [ "ReadWriteOnce" ]
         resources:
           requests:
-            storage: 10Gi
+            storage: 4Gi

internal/controller/etcdcluster_controller.go

@@ -65,11 +65,11 @@ type EtcdClusterReconciler struct {
 // +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;watch;delete;patch
 // +kubebuilder:rbac:groups="",resources=services,verbs=get;create;delete;update;patch;list;watch
-// +kubebuilder:rbac:groups="",resources=secrets,verbs=view;list;watch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
 // +kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;create;delete;update;patch;list;watch
 // +kubebuilder:rbac:groups="policy",resources=poddisruptionbudgets,verbs=get;create;delete;update;patch;list;watch
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;patch;watch
-// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get
+// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list
 
 // Reconcile checks CR and current cluster state and performs actions to transform current state to desired.
 func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {

internal/controller/factory/pvc.go

@@ -41,7 +41,7 @@ func GetPVCName(cluster *etcdaenixiov1alpha1.EtcdCluster) string {
 // UpdatePersistentVolumeClaims checks and updates the sizes of PVCs in an EtcdCluster if the specified storage size is larger than the current.
 func UpdatePersistentVolumeClaims(ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster, rclient client.Client) error {
 	labelSelector := labels.SelectorFromSet(labels.Set{
-		"app.kubernetes.io/name": cluster.Name,
+		"app.kubernetes.io/instance": cluster.Name,
 	})
 	listOptions := &client.ListOptions{
 		Namespace:     cluster.Namespace,

internal/controller/factory/pvc_test.go

@@ -118,7 +118,9 @@ var _ = Describe("UpdatePersistentVolumeClaims", func() {
 					Name:      "data-test-cluster-0",
 					Namespace: ns.Name,
 					Labels: map[string]string{
-						"app.kubernetes.io/name": cluster.Name,
+						"app.kubernetes.io/instance":   cluster.Name,
+						"app.kubernetes.io/managed-by": "etcd-operator",
+						"app.kubernetes.io/name":       "etcd",
 					},
 				},
 				Spec: corev1.PersistentVolumeClaimSpec{

test/e2e/e2e_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strings"
 	"sync"
 	"time"
 
@@ -60,7 +61,6 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			_, err = utils.Run(cmd)
 			ExpectWithOffset(1, err).NotTo(HaveOccurred())
 		})
-
 	})
 
 	if os.Getenv("DO_CLEANUP_AFTER_E2E") == "true" {
@@ -72,59 +72,89 @@ var _ = Describe("etcd-operator", Ordered, func() {
 		})
 	}
 
-	Context("Simple", func() {
-		It("should deploy etcd cluster", func() {
+	Context("With PVC and resize", func() {
+		const namespace = "test-pvc-and-resize-etcd-cluster"
+		const storageClass = `
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard-with-expansion
+provisioner: rancher.io/local-path
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+`
+
+		It("should resize PVCs of the etcd cluster", func() {
 			var err error
-			const namespace = "test-simple-etcd-cluster"
-			var wg sync.WaitGroup
-			wg.Add(1)
-
 			By("create namespace", func() {
-				cmd := exec.Command("sh", "-c",
-					fmt.Sprintf("kubectl create namespace %s --dry-run=client -o yaml | kubectl apply -f -", namespace)) // nolint:lll
+				cmd := exec.Command("kubectl", "create", "namespace", namespace)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred())
 			})
 
-			By("apply simple etcd cluster manifest", func() {
+			By("create StorageClass", func() {
+				cmd := exec.Command("kubectl", "apply", "-f", "-")
+				cmd.Stdin = strings.NewReader(storageClass)
+				_, err = utils.Run(cmd)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			By("deploying etcd cluster with initial PVC size", func() {
 				dir, _ := utils.GetProjectDir()
 				cmd := exec.Command("kubectl", "apply",
-					"--filename", dir+"/examples/manifests/etcdcluster-simple.yaml",
+					"--filename", dir+"/examples/manifests/etcdcluster-persistent.yaml",
 					"--namespace", namespace,
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
+				Expect(err).NotTo(HaveOccurred())
 			})
 
-			Eventually(func() error {
-				cmd := exec.Command("kubectl", "wait",
-					"statefulset/test",
-					"--for", "jsonpath={.status.readyReplicas}=3",
-					"--namespace", namespace,
-					"--timeout", "5m",
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
+			By("waiting for statefulset to be ready", func() {
+				Eventually(func() error {
+					cmd := exec.Command("kubectl", "wait",
+						"statefulset/test",
+						"--for", "jsonpath={.status.readyReplicas}=3",
+						"--namespace", namespace,
+						"--timeout", "5m",
+					)
+					_, err = utils.Run(cmd)
+					return err
+				}, 5*time.Minute, 10*time.Second).Should(Succeed())
+			})
 
-			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
-			Expect(err).NotTo(HaveOccurred())
-			defer func() {
-				err := client.Close()
+			By("updating the storage request", func() {
+				// Patch the EtcdCluster to increase storage size
+				patch := `{"spec": {"storage": {"volumeClaimTemplate": {"spec": {"resources": {"requests": {"storage": "8Gi"}}}}}}}`
+				cmd := exec.Command("kubectl", "patch", "etcdcluster", "test", "--namespace", namespace, "--type", "merge", "--patch", patch) //nolint:lll
+				_, err = utils.Run(cmd)
 				Expect(err).NotTo(HaveOccurred())
-			}()
-
-			By("check etcd cluster is healthy", func() {
-				Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue())
 			})
 
+			By("checking that PVC sizes have been updated", func() {
+				Eventually(func() bool {
+					cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, "-o", "jsonpath={.items[*].spec.resources.requests.storage}") //nolint:lll
+					output, err := utils.Run(cmd)
+					if err != nil {
+						return false
+					}
+					// Split the output into individual sizes and check each one
+					sizes := strings.Fields(string(output))
+					for _, size := range sizes {
+						if size != "8Gi" {
+							return false
+						}
+					}
+					return true
+				}, 5*time.Minute, 10*time.Second).Should(BeTrue(), "PVCs should be resized to 8Gi")
+			})
 		})
 	})
 
 	Context("With emptyDir", func() {
 		It("should deploy etcd cluster", func() {
 			var err error
-			const namespace = "test-emtydir-etcd-cluster"
+			const namespace = "test-emptydir-etcd-cluster"
 			var wg sync.WaitGroup
 			wg.Add(1)
 
@@ -166,7 +196,6 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			By("check etcd cluster is healthy", func() {
 				Expect(utils.IsEtcdClusterHealthy(ctx, client)).To(BeTrue())
 			})
-
 		})
 	})
 
@@ -235,8 +264,6 @@ var _ = Describe("etcd-operator", Ordered, func() {
 				Expect(err).NotTo(HaveOccurred())
 				Expect(authStatus.Enabled).To(BeTrue())
 			})
-
 		})
 	})
-
 })