Retry list operations

Signed-off-by: Raul Sevilla <[email protected]>

pkg/burner/delete.go

@@ -55,10 +55,11 @@ func setupDeleteJob(jobConfig config.Job) Executor {
 
 // RunDeleteJob executes a deletion job
 func (ex *Executor) RunDeleteJob() {
-	log.Infof("Triggering job: %s", ex.Config.Name)
-	ex.Start = time.Now().UTC()
 	var wg sync.WaitGroup
 	var err error
+	var resp *unstructured.UnstructuredList
+	log.Infof("Triggering job: %s", ex.Config.Name)
+	ex.Start = time.Now().UTC()
 	RestConfig, err = config.GetRestConfig(ex.Config.QPS, ex.Config.Burst)
 	if err != nil {
 		log.Fatalf("Error creating restConfig for kube-burner: %s", err)
@@ -73,9 +74,16 @@ func (ex *Executor) RunDeleteJob() {
 		listOptions := metav1.ListOptions{
 			LabelSelector: labelSelector,
 		}
-		resp, err := dynamicClient.Resource(obj.gvr).List(context.TODO(), listOptions)
+		err = RetryWithExponentialBackOff(func() (done bool, err error) {
+			resp, err = dynamicClient.Resource(obj.gvr).List(context.TODO(), listOptions)
+			if err != nil {
+				log.Errorf("Error found listing %s labeled with %s: %s", obj.gvr.Resource, labelSelector, err)
+				return false, nil
+			}
+			return true, nil
+		})
 		if err != nil {
-			log.Errorf("Error found listing %s labeled with %s: %s", obj.gvr.Resource, labelSelector, err)
+			continue
 		}
 		for _, item := range resp.Items {
 			wg.Add(1)

pkg/burner/utils.go

@@ -19,15 +19,25 @@ import (
 	"context"
 	"fmt"
 	"text/template"
+	"time"
 
 	"github.com/cloud-bulldozer/kube-burner/log"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubectl/pkg/scheme"
 )
 
+const (
+	// Parameters for retrying with exponential backoff.
+	retryBackoffInitialDuration = 1 * time.Second
+	retryBackoffFactor          = 3
+	retryBackoffJitter          = 0
+	retryBackoffSteps           = 3
+)
+
 func renderTemplate(original []byte, data interface{}) []byte {
 	var rendered bytes.Buffer
 	t, err := template.New("").Parse(string(original))
@@ -58,15 +68,25 @@ func (ex *Executor) Cleanup() {
 
 // Verify verifies the number of created objects
 func (ex *Executor) Verify() bool {
+	var objList *unstructured.UnstructuredList
 	success := true
 	log.Info("Verifying created objects")
 	for objectIndex, obj := range ex.objects {
 		listOptions := metav1.ListOptions{
 			LabelSelector: fmt.Sprintf("kube-burner-uuid=%s,kube-burner-job=%s,kube-burner-index=%d", ex.uuid, ex.Config.Name, objectIndex),
 		}
-		objList, err := dynamicClient.Resource(obj.gvr).Namespace(metav1.NamespaceAll).List(context.TODO(), listOptions)
+		err := RetryWithExponentialBackOff(func() (done bool, err error) {
+			objList, err = dynamicClient.Resource(obj.gvr).Namespace(metav1.NamespaceAll).List(context.TODO(), listOptions)
+			if err != nil {
+				log.Errorf("Error verifying object: %s", err)
+				return false, nil
+			}
+			return true, nil
+		})
+		// Mark success to false if we found an error
 		if err != nil {
-			log.Errorf("Error verifying object: %s", err)
+			success = false
+			continue
 		}
 		objectsExpected := ex.Config.JobIterations * obj.replicas
 		if len(objList.Items) != objectsExpected {
@@ -78,3 +98,14 @@ func (ex *Executor) Verify() bool {
 	}
 	return success
 }
+
+// RetryWithExponentialBackOff a utility for retrying the given function with exponential backoff.
+func RetryWithExponentialBackOff(fn wait.ConditionFunc) error {
+	backoff := wait.Backoff{
+		Duration: retryBackoffInitialDuration,
+		Factor:   retryBackoffFactor,
+		Jitter:   retryBackoffJitter,
+		Steps:    retryBackoffSteps,
+	}
+	return wait.ExponentialBackoff(backoff, fn)
+}