addon support rollout configs (#340)

Signed-off-by: haoqing0110 <[email protected]>

pkg/addon/controllers/addonconfiguration/addon_configuration_reconciler.go

@@ -8,6 +8,7 @@ import (
 	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
 
+	"open-cluster-management.io/ocm/pkg/common/helpers"
 	"open-cluster-management.io/ocm/pkg/common/patcher"
 )
 
@@ -30,7 +31,15 @@ func (d *managedClusterAddonConfigurationReconciler) reconcile(
 		}
 	}
 
-	return cma, reconcileContinue, utilerrors.NewAggregate(errs)
+	if len(errs) > 0 {
+		return cma, reconcileContinue, utilerrors.NewAggregate(errs)
+	}
+
+	if graph.getRequeueTime() < maxRequeueTime {
+		return cma, reconcileContinue, helpers.NewRequeueError("Rollout requeue", graph.getRequeueTime())
+	}
+
+	return cma, reconcileContinue, nil
 }
 
 func (d *managedClusterAddonConfigurationReconciler) mergeAddonConfig(

pkg/addon/controllers/addonconfiguration/controller.go

@@ -2,10 +2,12 @@ package addonconfiguration
 
 import (
 	"context"
+	"errors"
+	"time"
 
 	"github.com/openshift/library-go/pkg/controller/factory"
 	"github.com/openshift/library-go/pkg/operator/events"
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
@@ -23,6 +25,11 @@ import (
 	"open-cluster-management.io/ocm/pkg/common/queue"
 )
 
+const (
+	// maxRequeueTime is the minimum informer resync period
+	maxRequeueTime = 10 * time.Minute
+)
+
 // addonConfigurationController is a controller to update configuration of mca with the following order
 // 1. use configuration in mca spec if it is set
 // 2. use configuration in install strategy
@@ -104,7 +111,7 @@ func (c *addonConfigurationController) sync(ctx context.Context, syncCtx factory
 
 	cma, err := c.clusterManagementAddonLister.Get(addonName)
 	switch {
-	case errors.IsNotFound(err):
+	case apierrors.IsNotFound(err):
 		return nil
 	case err != nil:
 		return err
@@ -129,17 +136,31 @@ func (c *addonConfigurationController) sync(ctx context.Context, syncCtx factory
 
 	var state reconcileState
 	var errs []error
+	minRequeue := maxRequeueTime
 	for _, reconciler := range c.reconcilers {
 		cma, state, err = reconciler.reconcile(ctx, cma, graph)
-		if err != nil {
+		var rqe helpers.RequeueError
+		if err != nil && errors.As(err, &rqe) {
+			if minRequeue > rqe.RequeueTime {
+				minRequeue = rqe.RequeueTime
+			}
+		} else if err != nil {
 			errs = append(errs, err)
 		}
 		if state == reconcileStop {
 			break
 		}
 	}
 
-	return utilerrors.NewAggregate(errs)
+	if len(errs) > 0 {
+		return utilerrors.NewAggregate(errs)
+	}
+
+	if minRequeue < maxRequeueTime {
+		syncCtx.Queue().AddAfter(key, minRequeue)
+	}
+
+	return nil
 }
 
 func (c *addonConfigurationController) buildConfigurationGraph(logger klog.Logger, cma *addonv1alpha1.ClusterManagementAddOn) (*configurationGraph, error) {

pkg/addon/controllers/addonconfiguration/graph.go

@@ -3,6 +3,7 @@ package addonconfiguration
 import (
 	"fmt"
 	"sort"
+	"time"
 
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -271,6 +272,19 @@ func (g *configurationGraph) getAddonsToUpdate() []*addonNode {
 	return addons
 }
 
+func (g *configurationGraph) getRequeueTime() time.Duration {
+	minRequeue := maxRequeueTime
+
+	for _, node := range g.nodes {
+		nodeRecheckAfter := node.rolloutResult.RecheckAfter
+		if nodeRecheckAfter != nil && *nodeRecheckAfter < minRequeue {
+			minRequeue = *nodeRecheckAfter
+		}
+	}
+
+	return minRequeue
+}
+
 func (n *installStrategyNode) addNode(addon *addonv1alpha1.ManagedClusterAddOn) {
 	n.children[addon.Namespace] = &addonNode{
 		mca:            addon,
@@ -372,26 +386,47 @@ func (n *installStrategyNode) getAddonsToUpdate() []*addonNode {
 	return addons
 }
 
+// Return the number of succeed addons.
+// Including the addons with status Succeed after MinSuccessTime.
 func (n *installStrategyNode) countAddonUpgradeSucceed() int {
 	count := 0
 	for _, addon := range n.children {
-		if desiredConfigsEqual(addon.desiredConfigs, n.desiredConfigs) && addon.status.Status == clusterv1alpha1.Succeeded {
+		if desiredConfigsEqual(addon.desiredConfigs, n.desiredConfigs) &&
+			addon.status.Status == clusterv1alpha1.Succeeded &&
+			!rolloutStatusHasCluster(n.rolloutResult.ClustersToRollout, addon.mca.Namespace) {
 			count += 1
 		}
 	}
 	return count
 }
 
+// Return the number of failed addons after ProgressDeadline.
+func (n *installStrategyNode) countAddonUpgradeFailed() int {
+	count := 0
+	for _, addon := range n.children {
+		if desiredConfigsEqual(addon.desiredConfigs, n.desiredConfigs) &&
+			addon.status.Status == clusterv1alpha1.Failed &&
+			!rolloutStatusHasCluster(n.rolloutResult.ClustersToRollout, addon.mca.Namespace) {
+			count += 1
+		}
+	}
+	return count
+}
+
+// Return the number of exiting addons in rolloutResult.ClustersToRollout.
+// Including the addons with status ToApply, Progressing within ProgressDeadline, Failed within ProgressDeadline and Succeed within MinSuccessTime.
 func (n *installStrategyNode) countAddonUpgrading() int {
 	count := 0
 	for _, addon := range n.children {
-		if desiredConfigsEqual(addon.desiredConfigs, n.desiredConfigs) && addon.status.Status == clusterv1alpha1.Progressing {
+		if rolloutStatusHasCluster(n.rolloutResult.ClustersToRollout, addon.mca.Namespace) {
 			count += 1
 		}
 	}
 	return count
 }
 
+// Return the number of addons in rolloutResult.ClustersTimeOut.
+// Including the addons with status Progressing after ProgressDeadline, Failed after ProgressDeadline.
 func (n *installStrategyNode) countAddonTimeOut() int {
 	return len(n.rolloutResult.ClustersTimeOut)
 }
@@ -416,3 +451,12 @@ func desiredConfigsEqual(a, b addonConfigMap) bool {
 
 	return true
 }
+
+func rolloutStatusHasCluster(clusterRolloutStatus []clusterv1alpha1.ClusterRolloutStatus, clusterName string) bool {
+	for _, s := range clusterRolloutStatus {
+		if s.ClusterName == clusterName {
+			return true
+		}
+	}
+	return false
+}

pkg/addon/controllers/addonconfiguration/mgmt_addon_progressing_reconciler.go

@@ -45,6 +45,7 @@ func (d *clusterManagementAddonProgressingReconciler) reconcile(
 			isUpgrade,
 			placementNode.countAddonUpgrading(),
 			placementNode.countAddonUpgradeSucceed(),
+			placementNode.countAddonUpgradeFailed(),
 			placementNode.countAddonTimeOut(),
 			len(placementNode.clusters),
 		)
@@ -60,7 +61,7 @@ func (d *clusterManagementAddonProgressingReconciler) reconcile(
 func setAddOnInstallProgressionsAndLastApplied(
 	installProgression *addonv1alpha1.InstallProgression,
 	isUpgrade bool,
-	progressing, done, timeout, total int) {
+	progressing, done, failed, timeout, total int) {
 	// always update progressing condition when there is no config
 	// skip update progressing condition when last applied config already the same as desired
 	skip := len(installProgression.ConfigReferences) > 0
@@ -80,10 +81,10 @@ func setAddOnInstallProgressionsAndLastApplied(
 		condition.Status = metav1.ConditionTrue
 		if isUpgrade {
 			condition.Reason = addonv1alpha1.ProgressingReasonUpgrading
-			condition.Message = fmt.Sprintf("%d/%d upgrading..., %d timeout.", progressing+done, total, timeout)
+			condition.Message = fmt.Sprintf("%d/%d upgrading..., %d failed %d timeout.", progressing+done, total, failed, timeout)
 		} else {
 			condition.Reason = addonv1alpha1.ProgressingReasonInstalling
-			condition.Message = fmt.Sprintf("%d/%d installing..., %d timeout.", progressing+done, total, timeout)
+			condition.Message = fmt.Sprintf("%d/%d installing..., %d failed %d timeout.", progressing+done, total, failed, timeout)
 		}
 	} else {
 		for i, configRef := range installProgression.ConfigReferences {
@@ -93,10 +94,10 @@ func setAddOnInstallProgressionsAndLastApplied(
 		condition.Status = metav1.ConditionFalse
 		if isUpgrade {
 			condition.Reason = addonv1alpha1.ProgressingReasonUpgradeSucceed
-			condition.Message = fmt.Sprintf("%d/%d upgrade completed with no errors, %d timeout.", done, total, timeout)
+			condition.Message = fmt.Sprintf("%d/%d upgrade completed with no errors, %d failed %d timeout.", done, total, failed, timeout)
 		} else {
 			condition.Reason = addonv1alpha1.ProgressingReasonInstallSucceed
-			condition.Message = fmt.Sprintf("%d/%d install completed with no errors, %d timeout.", done, total, timeout)
+			condition.Message = fmt.Sprintf("%d/%d install completed with no errors, %d failed %d timeout.", done, total, failed, timeout)
 		}
 	}
 	meta.SetStatusCondition(&installProgression.Conditions, condition)

pkg/addon/controllers/addonconfiguration/mgmt_addon_progressing_reconciler_test.go

@@ -95,7 +95,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonInstalling {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions[0].Reason)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "0/2 installing..., 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "0/2 installing..., 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions[0].Message)
 				}
 			},
@@ -188,7 +188,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonInstalling {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions[0].Reason)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/2 installing..., 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/2 installing..., 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions[0].Message)
 				}
 			},
@@ -271,7 +271,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonInstallSucceed {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/1 install completed with no errors, 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/1 install completed with no errors, 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
 			},
@@ -347,7 +347,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonUpgrading {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/2 upgrading..., 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/2 upgrading..., 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
 			},
@@ -434,7 +434,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonUpgradeSucceed {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/1 upgrade completed with no errors, 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/1 upgrade completed with no errors, 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
 			},
@@ -520,7 +520,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonUpgrading {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "0/1 upgrading..., 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "0/1 upgrading..., 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
 			},
@@ -595,7 +595,7 @@ func TestMgmtAddonProgressingReconcile(t *testing.T) {
 				if cma.Status.InstallProgressions[0].Conditions[0].Reason != addonv1alpha1.ProgressingReasonInstalling {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions)
 				}
-				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/2 installing..., 0 timeout." {
+				if cma.Status.InstallProgressions[0].Conditions[0].Message != "1/2 installing..., 0 failed 0 timeout." {
 					t.Errorf("InstallProgressions condition is not correct: %v", cma.Status.InstallProgressions[0].Conditions[0].Message)
 				}
 			},