From 10539cff4301db36b71b6b59ba2d787a9792702b Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Sat, 1 Oct 2022 17:50:59 +0530 Subject: [PATCH 01/14] Add Prometheus metrics for TriggerAuthentication totals Signed-off-by: Vighnesh Shenoy --- .../keda/triggerauthentication_controller.go | 43 +++++++++++++++++++ pkg/metrics/operator_prom_metrics.go | 26 +++++++++++ 2 files changed, 69 insertions(+) diff --git a/controllers/keda/triggerauthentication_controller.go b/controllers/keda/triggerauthentication_controller.go index 0b66e5780c0..054ee0c517d 100644 --- a/controllers/keda/triggerauthentication_controller.go +++ b/controllers/keda/triggerauthentication_controller.go @@ -18,6 +18,7 @@ package keda import ( "context" + "sync" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -31,6 +32,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/eventreason" + "github.com/kedacore/keda/v2/pkg/metrics" ) // TriggerAuthenticationReconciler reconciles a TriggerAuthentication object @@ -40,6 +42,20 @@ type TriggerAuthenticationReconciler struct { Recorder record.EventRecorder } +type triggerAuthMetricsData struct { + namespace string +} + +var ( + triggerAuthMetricsMap map[string]triggerAuthMetricsData + triggerAuthMetricsLock *sync.Mutex +) + +func init() { + triggerAuthMetricsMap = make(map[string]triggerAuthMetricsData) + triggerAuthMetricsLock = &sync.Mutex{} +} + // +kubebuilder:rbac:groups=keda.sh,resources=triggerauthentications;triggerauthentications/status,verbs="*" // Reconcile performs reconciliation on the identified TriggerAuthentication resource based on the request information passed, returns the result and an error (if any). @@ -50,6 +66,7 @@ func (r *TriggerAuthenticationReconciler) Reconcile(ctx context.Context, req ctr err := r.Client.Get(ctx, req.NamespacedName, triggerAuthentication) if err != nil { if errors.IsNotFound(err) { + r.updateMetricsOnDelete(req.NamespacedName.String()) return ctrl.Result{}, nil } reqLogger.Error(err, "Failed ot get TriggerAuthentication") @@ -57,9 +74,11 @@ func (r *TriggerAuthenticationReconciler) Reconcile(ctx context.Context, req ctr } if triggerAuthentication.GetDeletionTimestamp() != nil { + r.updateMetricsOnDelete(req.NamespacedName.String()) r.Recorder.Event(triggerAuthentication, corev1.EventTypeNormal, eventreason.TriggerAuthenticationDeleted, "TriggerAuthentication was deleted") return ctrl.Result{}, nil } + r.updateMetrics(triggerAuthentication, req.NamespacedName.String()) if triggerAuthentication.ObjectMeta.Generation == 1 { r.Recorder.Event(triggerAuthentication, corev1.EventTypeNormal, eventreason.TriggerAuthenticationAdded, "New TriggerAuthentication configured") @@ -74,3 +93,27 @@ func (r *TriggerAuthenticationReconciler) SetupWithManager(mgr ctrl.Manager) err For(&kedav1alpha1.TriggerAuthentication{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Complete(r) } + +func (r *TriggerAuthenticationReconciler) updateMetrics(triggerAuth *kedav1alpha1.TriggerAuthentication, namespacedName string) { + triggerAuthMetricsLock.Lock() + defer triggerAuthMetricsLock.Unlock() + + if metricsData, ok := triggerAuthMetricsMap[namespacedName]; ok { + metrics.DecrementCRDTotal("trigger_authentication", metricsData.namespace) + } + + metrics.IncrementCRDTotal("trigger_authentication", triggerAuth.Namespace) + triggerAuthMetricsMap[namespacedName] = triggerAuthMetricsData{namespace: triggerAuth.Namespace} +} + +// this method is idempotent, so it can be called multiple times without side-effects +func (r *TriggerAuthenticationReconciler) updateMetricsOnDelete(namespacedName string) { + triggerAuthMetricsLock.Lock() + defer triggerAuthMetricsLock.Unlock() + + if metricsData, ok := triggerAuthMetricsMap[namespacedName]; ok { + metrics.DecrementCRDTotal("trigger_authentication", metricsData.namespace) + } + + delete(triggerAuthMetricsMap, namespacedName) +} diff --git a/pkg/metrics/operator_prom_metrics.go b/pkg/metrics/operator_prom_metrics.go index ede0c1f4e7e..b964dc9b756 100644 --- a/pkg/metrics/operator_prom_metrics.go +++ b/pkg/metrics/operator_prom_metrics.go @@ -30,10 +30,20 @@ var ( }, []string{"type"}, ) + + crdTotalsGaugeVec = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "keda_operator", + Subsystem: "crd", + Name: "totals", + }, + []string{"type", "namespace"}, + ) ) func init() { metrics.Registry.MustRegister(triggerTotalsGaugeVec) + metrics.Registry.MustRegister(crdTotalsGaugeVec) } func IncrementTriggerTotal(triggerType string) { @@ -47,3 +57,19 @@ func DecrementTriggerTotal(triggerType string) { triggerTotalsGaugeVec.WithLabelValues(triggerType).Dec() } } + +func IncrementCRDTotal(crdType, namespace string) { + if namespace == "" { + namespace = "default" + } + + crdTotalsGaugeVec.WithLabelValues(crdType, namespace).Inc() +} + +func DecrementCRDTotal(crdType, namespace string) { + if namespace == "" { + namespace = "default" + } + + crdTotalsGaugeVec.WithLabelValues(crdType, namespace).Dec() +} From e8d8677a703afba522c718b2be062c7fd6cbffbb Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Mon, 3 Oct 2022 00:34:55 +0530 Subject: [PATCH 02/14] Add Prometheus metrics for ClusterTriggerAuthentication totals Signed-off-by: Vighnesh Shenoy --- ...clustertriggerauthentication_controller.go | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/controllers/keda/clustertriggerauthentication_controller.go b/controllers/keda/clustertriggerauthentication_controller.go index 7f4a4c104ae..4e34e166650 100644 --- a/controllers/keda/clustertriggerauthentication_controller.go +++ b/controllers/keda/clustertriggerauthentication_controller.go @@ -18,6 +18,7 @@ package keda import ( "context" + "sync" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -31,6 +32,7 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/eventreason" + "github.com/kedacore/keda/v2/pkg/metrics" ) // ClusterTriggerAuthenticationReconciler reconciles a ClusterTriggerAuthentication object @@ -40,6 +42,20 @@ type ClusterTriggerAuthenticationReconciler struct { Recorder record.EventRecorder } +type clusterTriggerAuthMetricsData struct { + namespace string +} + +var ( + clusterTriggerAuthMetricsMap map[string]clusterTriggerAuthMetricsData + clusterTriggerAuthMetricsLock *sync.Mutex +) + +func init() { + clusterTriggerAuthMetricsMap = make(map[string]clusterTriggerAuthMetricsData) + clusterTriggerAuthMetricsLock = &sync.Mutex{} +} + // +kubebuilder:rbac:groups=keda.sh,resources=clustertriggerauthentications;clustertriggerauthentications/status,verbs="*" // Reconcile performs reconciliation on the identified TriggerAuthentication resource based on the request information passed, returns the result and an error (if any). @@ -50,6 +66,7 @@ func (r *ClusterTriggerAuthenticationReconciler) Reconcile(ctx context.Context, err := r.Client.Get(ctx, req.NamespacedName, clusterTriggerAuthentication) if err != nil { if errors.IsNotFound(err) { + r.updateMetricsOnDelete(req.NamespacedName.String()) return ctrl.Result{}, nil } reqLogger.Error(err, "Failed ot get ClusterTriggerAuthentication") @@ -57,9 +74,11 @@ func (r *ClusterTriggerAuthenticationReconciler) Reconcile(ctx context.Context, } if clusterTriggerAuthentication.GetDeletionTimestamp() != nil { + r.updateMetricsOnDelete(req.NamespacedName.String()) r.Recorder.Event(clusterTriggerAuthentication, corev1.EventTypeNormal, eventreason.ClusterTriggerAuthenticationDeleted, "ClusterTriggerAuthentication was deleted") return ctrl.Result{}, nil } + r.updateMetrics(clusterTriggerAuthentication, req.NamespacedName.String()) if clusterTriggerAuthentication.ObjectMeta.Generation == 1 { r.Recorder.Event(clusterTriggerAuthentication, corev1.EventTypeNormal, eventreason.ClusterTriggerAuthenticationAdded, "New ClusterTriggerAuthentication configured") @@ -73,3 +92,27 @@ func (r *ClusterTriggerAuthenticationReconciler) SetupWithManager(mgr ctrl.Manag For(&kedav1alpha1.ClusterTriggerAuthentication{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Complete(r) } + +func (r *ClusterTriggerAuthenticationReconciler) updateMetrics(clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication, namespacedName string) { + clusterTriggerAuthMetricsLock.Lock() + defer clusterTriggerAuthMetricsLock.Unlock() + + if metricsData, ok := clusterTriggerAuthMetricsMap[namespacedName]; ok { + metrics.DecrementCRDTotal("cluster_trigger_authentication", metricsData.namespace) + } + + metrics.IncrementCRDTotal("cluster_trigger_authentication", clusterTriggerAuth.Namespace) + clusterTriggerAuthMetricsMap[namespacedName] = clusterTriggerAuthMetricsData{namespace: clusterTriggerAuth.Namespace} +} + +// this method is idempotent, so it can be called multiple times without side-effects +func (r *ClusterTriggerAuthenticationReconciler) updateMetricsOnDelete(namespacedName string) { + clusterTriggerAuthMetricsLock.Lock() + defer clusterTriggerAuthMetricsLock.Unlock() + + if metricsData, ok := clusterTriggerAuthMetricsMap[namespacedName]; ok { + metrics.DecrementCRDTotal("cluster_trigger_authentication", metricsData.namespace) + } + + delete(clusterTriggerAuthMetricsMap, namespacedName) +} From 913ef5b48239d435e2d9b0daac550d4e4fa43957 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Mon, 3 Oct 2022 00:37:46 +0530 Subject: [PATCH 03/14] Update CHANGELOG. Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 036febe38e0..a8067cdf3af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General**: Expand Prometheus metric with label "ScalerName" to distinguish different triggers. The scaleName is defined per Trigger.Name ([#3588](https://github.com/kedacore/keda/issues/3588) - **General:** Introduce new Loki Scaler ([#3699](https://github.com/kedacore/keda/issues/3699)) - **General**: Add ratelimitting parameters to keda manager to allow override of client defaults ([#3730](https://github.com/kedacore/keda/issues/2920)) +- **General**: Provide Prometheus metric with indication of total number of custom resources per namespace for each custom resource type (CRD). ([#2637](https://github.com/kedacore/keda/issues/2637)|[#2638](https://github.com/kedacore/keda/issues/2638)|[#2639](https://github.com/kedacore/keda/issues/2639)) - **General**: Provide Prometheus metric with indication of total number of triggers per trigger type in `ScaledJob`/`ScaledObject`. ([#3663](https://github.com/kedacore/keda/issues/3663)) - **AWS Scalers**: Add setting AWS endpoint url. ([#3337](https://github.com/kedacore/keda/issues/3337)) - **Azure Service Bus Scaler**: Add support for Shared Access Signature (SAS) tokens for authentication. ([#2920](https://github.com/kedacore/keda/issues/2920)) From 1cdb181c53fec3d9d6b4ab33b29f36483d9dd888 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 7 Oct 2022 18:07:49 +0530 Subject: [PATCH 04/14] Add finalizers for (Cluster)TriggerAuthentication custom resources Signed-off-by: Vighnesh Shenoy --- ...clustertriggerauthentication_controller.go | 49 +++++++++++++++++-- .../keda/triggerauthentication_controller.go | 49 +++++++++++++++++-- tests/clean-crds.sh | 4 +- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/controllers/keda/clustertriggerauthentication_controller.go b/controllers/keda/clustertriggerauthentication_controller.go index 4e34e166650..3ddfacbecdf 100644 --- a/controllers/keda/clustertriggerauthentication_controller.go +++ b/controllers/keda/clustertriggerauthentication_controller.go @@ -20,6 +20,7 @@ import ( "context" "sync" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -31,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/controllers/keda/util" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metrics" ) @@ -46,6 +48,10 @@ type clusterTriggerAuthMetricsData struct { namespace string } +const ( + clusterTriggerAuthenticationFinalizer = "finalizer.keda.sh" +) + var ( clusterTriggerAuthMetricsMap map[string]clusterTriggerAuthMetricsData clusterTriggerAuthMetricsLock *sync.Mutex @@ -66,17 +72,18 @@ func (r *ClusterTriggerAuthenticationReconciler) Reconcile(ctx context.Context, err := r.Client.Get(ctx, req.NamespacedName, clusterTriggerAuthentication) if err != nil { if errors.IsNotFound(err) { - r.updateMetricsOnDelete(req.NamespacedName.String()) return ctrl.Result{}, nil } - reqLogger.Error(err, "Failed ot get ClusterTriggerAuthentication") + reqLogger.Error(err, "Failed to get ClusterTriggerAuthentication") return ctrl.Result{}, err } if clusterTriggerAuthentication.GetDeletionTimestamp() != nil { - r.updateMetricsOnDelete(req.NamespacedName.String()) - r.Recorder.Event(clusterTriggerAuthentication, corev1.EventTypeNormal, eventreason.ClusterTriggerAuthenticationDeleted, "ClusterTriggerAuthentication was deleted") - return ctrl.Result{}, nil + return ctrl.Result{}, r.finalizeClusterTriggerAuthentication(ctx, reqLogger, clusterTriggerAuthentication, req.NamespacedName.String()) + } + + if err := r.ensureFinalizer(ctx, reqLogger, clusterTriggerAuthentication); err != nil { + return ctrl.Result{}, err } r.updateMetrics(clusterTriggerAuthentication, req.NamespacedName.String()) @@ -93,6 +100,38 @@ func (r *ClusterTriggerAuthenticationReconciler) SetupWithManager(mgr ctrl.Manag Complete(r) } +func (r *ClusterTriggerAuthenticationReconciler) ensureFinalizer(ctx context.Context, logger logr.Logger, clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication) error { + if !util.Contains(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer) { + logger.Info("Adding Finalizer for the ClusterTriggerAuthentication") + clusterTriggerAuth.SetFinalizers(append(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer)) + + // Update CR + err := r.Client.Update(ctx, clusterTriggerAuth) + if err != nil { + logger.Error(err, "Failed to update ClusterTriggerAuthentication with a finalizer", "finalizer", clusterTriggerAuthenticationFinalizer) + return err + } + } + return nil +} + +func (r *ClusterTriggerAuthenticationReconciler) finalizeClusterTriggerAuthentication(ctx context.Context, logger logr.Logger, + clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication, namespacedName string) error { + if util.Contains(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer) { + clusterTriggerAuth.SetFinalizers(util.Remove(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer)) + if err := r.Client.Update(ctx, clusterTriggerAuth); err != nil { + logger.Error(err, "Failed to update ClusterTriggerAuthentication after removing a finalizer", "finalizer", clusterTriggerAuthenticationFinalizer) + return err + } + + r.updateMetricsOnDelete(namespacedName) + } + + logger.Info("Successfully finalized ClusterTriggerAuthentication") + r.Recorder.Event(clusterTriggerAuth, corev1.EventTypeNormal, eventreason.TriggerAuthenticationDeleted, "ClusterTriggerAuthentication was deleted") + return nil +} + func (r *ClusterTriggerAuthenticationReconciler) updateMetrics(clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication, namespacedName string) { clusterTriggerAuthMetricsLock.Lock() defer clusterTriggerAuthMetricsLock.Unlock() diff --git a/controllers/keda/triggerauthentication_controller.go b/controllers/keda/triggerauthentication_controller.go index 054ee0c517d..52884664cc8 100644 --- a/controllers/keda/triggerauthentication_controller.go +++ b/controllers/keda/triggerauthentication_controller.go @@ -20,6 +20,7 @@ import ( "context" "sync" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -31,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/controllers/keda/util" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metrics" ) @@ -46,6 +48,10 @@ type triggerAuthMetricsData struct { namespace string } +const ( + triggerAuthenticationFinalizer = "finalizer.keda.sh" +) + var ( triggerAuthMetricsMap map[string]triggerAuthMetricsData triggerAuthMetricsLock *sync.Mutex @@ -66,17 +72,18 @@ func (r *TriggerAuthenticationReconciler) Reconcile(ctx context.Context, req ctr err := r.Client.Get(ctx, req.NamespacedName, triggerAuthentication) if err != nil { if errors.IsNotFound(err) { - r.updateMetricsOnDelete(req.NamespacedName.String()) return ctrl.Result{}, nil } - reqLogger.Error(err, "Failed ot get TriggerAuthentication") + reqLogger.Error(err, "Failed to get TriggerAuthentication") return ctrl.Result{}, err } if triggerAuthentication.GetDeletionTimestamp() != nil { - r.updateMetricsOnDelete(req.NamespacedName.String()) - r.Recorder.Event(triggerAuthentication, corev1.EventTypeNormal, eventreason.TriggerAuthenticationDeleted, "TriggerAuthentication was deleted") - return ctrl.Result{}, nil + return ctrl.Result{}, r.finalizeTriggerAuthentication(ctx, reqLogger, triggerAuthentication, req.NamespacedName.String()) + } + + if err := r.ensureFinalizer(ctx, reqLogger, triggerAuthentication); err != nil { + return ctrl.Result{}, err } r.updateMetrics(triggerAuthentication, req.NamespacedName.String()) @@ -94,6 +101,38 @@ func (r *TriggerAuthenticationReconciler) SetupWithManager(mgr ctrl.Manager) err Complete(r) } +func (r *TriggerAuthenticationReconciler) ensureFinalizer(ctx context.Context, logger logr.Logger, triggerAuth *kedav1alpha1.TriggerAuthentication) error { + if !util.Contains(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer) { + logger.Info("Adding Finalizer for the TriggerAuthentication") + triggerAuth.SetFinalizers(append(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer)) + + // Update CR + err := r.Client.Update(ctx, triggerAuth) + if err != nil { + logger.Error(err, "Failed to update TriggerAuthentication with a finalizer", "finalizer", triggerAuthenticationFinalizer) + return err + } + } + return nil +} + +func (r *TriggerAuthenticationReconciler) finalizeTriggerAuthentication(ctx context.Context, logger logr.Logger, + triggerAuth *kedav1alpha1.TriggerAuthentication, namespacedName string) error { + if util.Contains(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer) { + triggerAuth.SetFinalizers(util.Remove(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer)) + if err := r.Client.Update(ctx, triggerAuth); err != nil { + logger.Error(err, "Failed to update TriggerAuthentication after removing a finalizer", "finalizer", triggerAuthenticationFinalizer) + return err + } + + r.updateMetricsOnDelete(namespacedName) + } + + logger.Info("Successfully finalized TriggerAuthentication") + r.Recorder.Event(triggerAuth, corev1.EventTypeNormal, eventreason.TriggerAuthenticationDeleted, "TriggerAuthentication was deleted") + return nil +} + func (r *TriggerAuthenticationReconciler) updateMetrics(triggerAuth *kedav1alpha1.TriggerAuthentication, namespacedName string) { triggerAuthMetricsLock.Lock() defer triggerAuthMetricsLock.Unlock() diff --git a/tests/clean-crds.sh b/tests/clean-crds.sh index bcaf925e09a..d092d1b077b 100755 --- a/tests/clean-crds.sh +++ b/tests/clean-crds.sh @@ -1,9 +1,9 @@ #! /bin/bash -echo "Cleaning up scaled objects and jobs before undeploying KEDA" +echo "Cleaning up CRDs before undeploying KEDA" while read -r namespace do - resources=$(kubectl get so,sj -n $namespace -o name) + resources=$(kubectl get so,sj,ta,cta -n $namespace -o name) if [[ -n "$resources" ]] then kubectl delete $resources -n $namespace From c380f34061b763f32e29bc6a348d67f9c8c909f8 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 7 Oct 2022 22:00:04 +0530 Subject: [PATCH 05/14] Add Prometheus metrics for ScaledObject totals Signed-off-by: Vighnesh Shenoy --- controllers/keda/scaledobject_controller.go | 53 +++++++++++---------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/controllers/keda/scaledobject_controller.go b/controllers/keda/scaledobject_controller.go index 5ffbe9f19f1..3a92a3cc49b 100644 --- a/controllers/keda/scaledobject_controller.go +++ b/controllers/keda/scaledobject_controller.go @@ -77,12 +77,17 @@ type ScaledObjectReconciler struct { kubeVersion kedautil.K8sVersion } +type scaledObjectMetricsData struct { + namespace string + triggerTypes []string +} + var ( // A cache mapping "resource.group" to true or false if we know if this resource is scalable. isScalableCache *sync.Map - scaledObjectTriggers map[string][]string - scaledObjectTriggersLock *sync.Mutex + scaledObjectMetricsMap map[string]scaledObjectMetricsData + scaledObjectMetricsLock *sync.Mutex ) func init() { @@ -91,8 +96,8 @@ func init() { isScalableCache.Store("deployments.apps", true) isScalableCache.Store("statefulsets.apps", true) - scaledObjectTriggers = make(map[string][]string) - scaledObjectTriggersLock = &sync.Mutex{} + scaledObjectMetricsMap = make(map[string]scaledObjectMetricsData) + scaledObjectMetricsLock = &sync.Mutex{} } // SetupWithManager initializes the ScaledObjectReconciler instance and starts a new controller managed by the passed Manager instance. @@ -177,7 +182,7 @@ func (r *ScaledObjectReconciler) Reconcile(ctx context.Context, req ctrl.Request if scaledObject.GetDeletionTimestamp() != nil { return ctrl.Result{}, r.finalizeScaledObject(ctx, reqLogger, scaledObject, req.NamespacedName.String()) } - r.updateTriggerTotals(reqLogger, scaledObject, req.NamespacedName.String()) + r.updateMetrics(scaledObject, req.NamespacedName.String()) // ensure finalizer is set on this CR if err := r.ensureFinalizer(ctx, reqLogger, scaledObject); err != nil { @@ -480,44 +485,42 @@ func (r *ScaledObjectReconciler) scaledObjectGenerationChanged(logger logr.Logge return true, nil } -func (r *ScaledObjectReconciler) updateTriggerTotals(logger logr.Logger, scaledObject *kedav1alpha1.ScaledObject, namespacedName string) { - specChanged, err := r.scaledObjectGenerationChanged(logger, scaledObject) - if err != nil { - logger.Error(err, "failed to update trigger totals") - return - } +func (r *ScaledObjectReconciler) updateMetrics(scaledObject *kedav1alpha1.ScaledObject, namespacedName string) { + scaledObjectMetricsLock.Lock() + defer scaledObjectMetricsLock.Unlock() - if !specChanged { - return - } - - scaledObjectTriggersLock.Lock() - defer scaledObjectTriggersLock.Unlock() + metricsData, ok := scaledObjectMetricsMap[namespacedName] - if triggerTypes, ok := scaledObjectTriggers[namespacedName]; ok { - for _, triggerType := range triggerTypes { + if ok { + metrics.DecrementCRDTotal("scaled_object", metricsData.namespace) + for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } } + metrics.IncrementCRDTotal("scaled_object", scaledObject.Namespace) + metricsData.namespace = scaledObject.Namespace + triggerTypes := make([]string, len(scaledObject.Spec.Triggers)) for _, trigger := range scaledObject.Spec.Triggers { metrics.IncrementTriggerTotal(trigger.Type) triggerTypes = append(triggerTypes, trigger.Type) } + metricsData.triggerTypes = triggerTypes - scaledObjectTriggers[namespacedName] = triggerTypes + scaledObjectMetricsMap[namespacedName] = metricsData } func (r *ScaledObjectReconciler) updateTriggerTotalsOnDelete(namespacedName string) { - scaledObjectTriggersLock.Lock() - defer scaledObjectTriggersLock.Unlock() + scaledObjectMetricsLock.Lock() + defer scaledObjectMetricsLock.Unlock() - if triggerTypes, ok := scaledObjectTriggers[namespacedName]; ok { - for _, triggerType := range triggerTypes { + if metricsData, ok := scaledObjectMetricsMap[namespacedName]; ok { + metrics.DecrementCRDTotal("scaled_object", metricsData.namespace) + for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } } - delete(scaledObjectTriggers, namespacedName) + delete(scaledObjectMetricsMap, namespacedName) } From f670825eaec7e2bc9a100b6e5e282dd43aa3aeb0 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 7 Oct 2022 22:38:32 +0530 Subject: [PATCH 06/14] Add Prometheus metrics for ScaledJob totals Signed-off-by: Vighnesh Shenoy --- controllers/keda/scaledjob_controller.go | 73 ++++++++------------- controllers/keda/scaledjob_finalizer.go | 2 +- controllers/keda/scaledobject_controller.go | 2 +- controllers/keda/scaledobject_finalizer.go | 2 +- 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/controllers/keda/scaledjob_controller.go b/controllers/keda/scaledjob_controller.go index ecaacc7c8d8..3e7362320de 100644 --- a/controllers/keda/scaledjob_controller.go +++ b/controllers/keda/scaledjob_controller.go @@ -58,14 +58,19 @@ type ScaledJobReconciler struct { scaleHandler scaling.ScaleHandler } +type scaledJobMetricsData struct { + namespace string + triggerTypes []string +} + var ( - scaledJobTriggers map[string][]string - scaledJobTriggersLock *sync.Mutex + scaledJobMetricsMap map[string]scaledJobMetricsData + scaledJobMetricsLock *sync.Mutex ) func init() { - scaledJobTriggers = make(map[string][]string) - scaledJobTriggersLock = &sync.Mutex{} + scaledJobMetricsMap = make(map[string]scaledJobMetricsData) + scaledJobMetricsLock = &sync.Mutex{} } // SetupWithManager initializes the ScaledJobReconciler instance and starts a new controller managed by the passed Manager instance. @@ -107,7 +112,7 @@ func (r *ScaledJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if scaledJob.GetDeletionTimestamp() != nil { return ctrl.Result{}, r.finalizeScaledJob(ctx, reqLogger, scaledJob, req.NamespacedName.String()) } - r.updateTriggerTotals(reqLogger, scaledJob, req.NamespacedName.String()) + r.updateMetrics(scaledJob, req.NamespacedName.String()) // ensure finalizer is set on this CR if err := r.ensureFinalizer(ctx, reqLogger, scaledJob); err != nil { @@ -263,62 +268,42 @@ func (r *ScaledJobReconciler) stopScaleLoop(ctx context.Context, logger logr.Log return nil } -// scaledJobGenerationChanged returns true if ScaledJob's Generation was changed, ie. ScaledJob.Spec was changed -func (r *ScaledJobReconciler) scaledJobGenerationChanged(logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob) (bool, error) { - key, err := cache.MetaNamespaceKeyFunc(scaledJob) - if err != nil { - logger.Error(err, "Error getting key for scaledJob") - return true, err - } - - value, loaded := r.scaledJobGenerations.Load(key) - if loaded { - generation := value.(int64) - if generation == scaledJob.Generation { - return false, nil - } - } - return true, nil -} - -func (r *ScaledJobReconciler) updateTriggerTotals(logger logr.Logger, scaledJob *kedav1alpha1.ScaledJob, namespacedName string) { - specChanged, err := r.scaledJobGenerationChanged(logger, scaledJob) - if err != nil { - logger.Error(err, "failed to update trigger totals") - return - } - - if !specChanged { - return - } +func (r *ScaledJobReconciler) updateMetrics(scaledJob *kedav1alpha1.ScaledJob, namespacedName string) { + scaledJobMetricsLock.Lock() + defer scaledJobMetricsLock.Unlock() - scaledJobTriggersLock.Lock() - defer scaledJobTriggersLock.Unlock() + metricsData, ok := scaledJobMetricsMap[namespacedName] - if triggerTypes, ok := scaledJobTriggers[namespacedName]; ok { - for _, triggerType := range triggerTypes { + if ok { + metrics.DecrementCRDTotal("scaled_job", metricsData.namespace) + for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } } + metrics.IncrementCRDTotal("scaled_job", scaledJob.Namespace) + metricsData.namespace = scaledJob.Namespace + triggerTypes := make([]string, len(scaledJob.Spec.Triggers)) for _, trigger := range scaledJob.Spec.Triggers { metrics.IncrementTriggerTotal(trigger.Type) triggerTypes = append(triggerTypes, trigger.Type) } + metricsData.triggerTypes = triggerTypes - scaledJobTriggers[namespacedName] = triggerTypes + scaledJobMetricsMap[namespacedName] = metricsData } -func (r *ScaledJobReconciler) updateTriggerTotalsOnDelete(namespacedName string) { - scaledJobTriggersLock.Lock() - defer scaledJobTriggersLock.Unlock() +func (r *ScaledJobReconciler) updateMetricsOnDelete(namespacedName string) { + scaledJobMetricsLock.Lock() + defer scaledJobMetricsLock.Unlock() - if triggerTypes, ok := scaledJobTriggers[namespacedName]; ok { - for _, triggerType := range triggerTypes { + if metricsData, ok := scaledJobMetricsMap[namespacedName]; ok { + metrics.DecrementCRDTotal("scaled_job", metricsData.namespace) + for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } } - delete(scaledJobTriggers, namespacedName) + delete(scaledJobMetricsMap, namespacedName) } diff --git a/controllers/keda/scaledjob_finalizer.go b/controllers/keda/scaledjob_finalizer.go index faf63bb1f7c..21872f5ab98 100644 --- a/controllers/keda/scaledjob_finalizer.go +++ b/controllers/keda/scaledjob_finalizer.go @@ -50,7 +50,7 @@ func (r *ScaledJobReconciler) finalizeScaledJob(ctx context.Context, logger logr return err } - r.updateTriggerTotalsOnDelete(namespacedName) + r.updateMetricsOnDelete(namespacedName) } logger.Info("Successfully finalized ScaledJob") diff --git a/controllers/keda/scaledobject_controller.go b/controllers/keda/scaledobject_controller.go index 3a92a3cc49b..da5fea2dfe8 100644 --- a/controllers/keda/scaledobject_controller.go +++ b/controllers/keda/scaledobject_controller.go @@ -511,7 +511,7 @@ func (r *ScaledObjectReconciler) updateMetrics(scaledObject *kedav1alpha1.Scaled scaledObjectMetricsMap[namespacedName] = metricsData } -func (r *ScaledObjectReconciler) updateTriggerTotalsOnDelete(namespacedName string) { +func (r *ScaledObjectReconciler) updateMetricsOnDelete(namespacedName string) { scaledObjectMetricsLock.Lock() defer scaledObjectMetricsLock.Unlock() diff --git a/controllers/keda/scaledobject_finalizer.go b/controllers/keda/scaledobject_finalizer.go index 77a74fdb314..24ee3563dcc 100644 --- a/controllers/keda/scaledobject_finalizer.go +++ b/controllers/keda/scaledobject_finalizer.go @@ -79,7 +79,7 @@ func (r *ScaledObjectReconciler) finalizeScaledObject(ctx context.Context, logge return err } - r.updateTriggerTotalsOnDelete(namespacedName) + r.updateMetricsOnDelete(namespacedName) } logger.Info("Successfully finalized ScaledObject") From 309147cfceeab375855be9ec9989c777747b96f7 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Sat, 8 Oct 2022 00:26:46 +0530 Subject: [PATCH 07/14] Add e2e test Signed-off-by: Vighnesh Shenoy --- .../prometheus_metrics_test.go | 171 +++++++++++++++--- 1 file changed, 146 insertions(+), 25 deletions(-) diff --git a/tests/internals/prometheus_metrics/prometheus_metrics_test.go b/tests/internals/prometheus_metrics/prometheus_metrics_test.go index 3db46563a56..123815e2bdb 100644 --- a/tests/internals/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/internals/prometheus_metrics/prometheus_metrics_test.go @@ -33,6 +33,7 @@ var ( ) type templateData struct { + TestName string TestNamespace string DeploymentName string ScaledObjectName string @@ -176,6 +177,51 @@ spec: targetPort: 8080 selector: app: keda-operator +` + + authenticationTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.TestName}}-secret + namespace: {{.TestNamespace}} +type: Opaque +stringData: + key: value +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TestName}}-ta1 + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: param + name: {{.TestName}}-secret + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TestName}}-ta2 + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: param + name: {{.TestName}}-secret + key: key +--- +apiVersion: keda.sh/v1alpha1 +kind: ClusterTriggerAuthentication +metadata: + name: {{.TestName}}-cta + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: param + name: {{.TestName}}-secret + key: key +--- ` ) @@ -194,7 +240,7 @@ func TestScaler(t *testing.T) { "replica count should be 2 after 2 minute") testHPAScalerMetricValue(t) - testTriggerTotalMetric(t, kc, data) + testOperatorMetrics(t, kc, data) // cleanup DeleteKubernetesResources(t, kc, testNamespace, data, templates) @@ -202,6 +248,7 @@ func TestScaler(t *testing.T) { func getTemplateData() (templateData, []Template) { return templateData{ + TestName: testName, TestNamespace: testNamespace, DeploymentName: deploymentName, ScaledObjectName: scaledObjectName, @@ -215,6 +262,7 @@ func getTemplateData() (templateData, []Template) { {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, {Name: "clientTemplate", Config: clientTemplate}, {Name: "serviceTemplate", Config: serviceTemplate}, + {Name: "authenticatioNTemplate", Config: authenticationTemplate}, } } @@ -225,10 +273,10 @@ func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*promMo parser := expfmt.TextParser{} // Ensure EOL reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n")) - family, err := parser.TextToMetricFamilies(reader) + families, err := parser.TextToMetricFamilies(reader) assert.NoErrorf(t, err, "cannot parse metrics - %s", err) - return family + return families } func testHPAScalerMetricValue(t *testing.T) { @@ -254,29 +302,52 @@ func testHPAScalerMetricValue(t *testing.T) { } } -func testTriggerTotalMetric(t *testing.T, kc *kubernetes.Clientset, data templateData) { - t.Log("--- testing trigger total metric ---") - testTriggerTotalMetricValue(t, getTriggerTotalsManually(t, kc)) +func testOperatorMetrics(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing operator metrics ---") + testOperatorMetricValues(t, kc) KubectlApplyWithTemplate(t, data, "cronScaledJobTemplate", cronScaledJobTemplate) - testTriggerTotalMetricValue(t, getTriggerTotalsManually(t, kc)) + testOperatorMetricValues(t, kc) KubectlDeleteWithTemplate(t, data, "cronScaledJobTemplate", cronScaledJobTemplate) - testTriggerTotalMetricValue(t, getTriggerTotalsManually(t, kc)) + testOperatorMetricValues(t, kc) } -func getTriggerTotalsManually(t *testing.T, kc *kubernetes.Clientset) map[string]int { +func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[string]int, map[string]map[string]int) { kedaKc := GetKedaKubernetesClient(t) triggerTotals := make(map[string]int) + crdTotals := map[string]map[string]int{ + "scaled_object": {}, + "scaled_job": {}, + "trigger_authentication": {}, + "cluster_trigger_authentication": {}, + } namespaceList, err := kc.CoreV1().Namespaces().List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list namespaces - %s", err) + clusterTriggerAuthenticationList, err := kedaKc.ClusterTriggerAuthentications().List(context.Background(), v1.ListOptions{}) + assert.NoErrorf(t, err, "failed to list clusterTriggerAuthentications with err - %s") + + for _, clusterTriggerAuth := range clusterTriggerAuthenticationList.Items { + namespace := clusterTriggerAuth.Namespace + if namespace == "" { + namespace = "default" + } + crdTotals["cluster_trigger_authentication"][namespace]++ + } + for _, namespace := range namespaceList.Items { + namespaceName := namespace.Name + if namespace.Name == "" { + namespaceName = "default" + } + scaledObjectList, err := kedaKc.ScaledObjects(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list scaledObjects in namespace - %s with err - %s", namespace.Name, err) + crdTotals["scaled_object"][namespaceName] = len(scaledObjectList.Items) for _, scaledObject := range scaledObjectList.Items { for _, trigger := range scaledObject.Spec.Triggers { triggerTotals[trigger.Type]++ @@ -286,34 +357,84 @@ func getTriggerTotalsManually(t *testing.T, kc *kubernetes.Clientset) map[string scaledJobList, err := kedaKc.ScaledJobs(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list scaledJobs in namespace - %s with err - %s", namespace.Name, err) + crdTotals["scaled_job"][namespaceName] = len(scaledJobList.Items) for _, scaledJob := range scaledJobList.Items { for _, trigger := range scaledJob.Spec.Triggers { triggerTotals[trigger.Type]++ } } + + triggerAuthList, err := kedaKc.TriggerAuthentications(namespace.Name).List(context.Background(), v1.ListOptions{}) + assert.NoErrorf(t, err, "failed to list triggerAuthentications in namespace - %s with err - %s", namespace.Name, err) + + crdTotals["trigger_authentication"][namespaceName] = len(triggerAuthList.Items) } - return triggerTotals + return triggerTotals, crdTotals } -func testTriggerTotalMetricValue(t *testing.T, expected map[string]int) { - family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure http://%s.keda:8080/metrics", serviceName)) +func testOperatorMetricValues(t *testing.T, kc *kubernetes.Clientset) { + families := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure http://%s.keda:8080/metrics", serviceName)) + expectedTriggerTotals, expectedCrdTotals := getOperatorMetricsManually(t, kc) - if val, ok := family["keda_operator_trigger_totals"]; ok { - var found bool - metrics := val.GetMetric() - for _, metric := range metrics { - labels := metric.GetLabel() - for _, label := range labels { - if *label.Name == "type" { - assert.Equalf(t, float64(expected[*label.Value]), *metric.Gauge.Value, "expected %f got %f for type %s", - float64(expected[*label.Value]), *metric.Gauge.Value, *label.Value) - found = true - } + checkTriggerTotalValues(t, families, expectedTriggerTotals) + checkCRDTotalValues(t, families, expectedCrdTotals) +} + +func checkTriggerTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]int) { + t.Log("--- testing trigger total metrics ---") + + family, ok := families["keda_operator_trigger_totals"] + if !ok { + t.Errorf("metric not available") + return + } + + metrics := family.GetMetric() + for _, metric := range metrics { + labels := metric.GetLabel() + for _, label := range labels { + if *label.Name == "type" { + triggerType := *label.Value + metricValue := *metric.Gauge.Value + expectedMetricValue := float64(expected[triggerType]) + + assert.Equalf(t, expectedMetricValue, metricValue, "expected %f got %f for trigger type %s", + expectedMetricValue, metricValue, triggerType) + + delete(expected, triggerType) } } - assert.Equal(t, true, found) - } else { + } + + assert.Equal(t, 0, len(expected)) +} + +func checkCRDTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]map[string]int) { + t.Log("--- testing crd total metrics ---") + + family, ok := families["keda_operator_crd_totals"] + if !ok { t.Errorf("metric not available") + return + } + + metrics := family.GetMetric() + for _, metric := range metrics { + labels := metric.GetLabel() + var namespace, crdType string + for _, label := range labels { + if *label.Name == "type" { + crdType = *label.Value + } else if *label.Name == "namespace" { + namespace = *label.Value + } + } + + metricValue := *metric.Gauge.Value + expectedMetricValue := float64(expected[crdType][namespace]) + + assert.Equalf(t, expectedMetricValue, metricValue, "expected %f got %f for crd type %s & namespace %s", + expectedMetricValue, metricValue, crdType, namespace) } } From 5639253e47531d18376081b1985d02c95a32bf9e Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Sat, 8 Oct 2022 05:11:04 +0530 Subject: [PATCH 08/14] Update CHANGELOG Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8067cdf3af..5dcea0790b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,10 +38,16 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### New +<<<<<<< HEAD - **General**: Expand Prometheus metric with label "ScalerName" to distinguish different triggers. The scaleName is defined per Trigger.Name ([#3588](https://github.com/kedacore/keda/issues/3588) - **General:** Introduce new Loki Scaler ([#3699](https://github.com/kedacore/keda/issues/3699)) - **General**: Add ratelimitting parameters to keda manager to allow override of client defaults ([#3730](https://github.com/kedacore/keda/issues/2920)) - **General**: Provide Prometheus metric with indication of total number of custom resources per namespace for each custom resource type (CRD). ([#2637](https://github.com/kedacore/keda/issues/2637)|[#2638](https://github.com/kedacore/keda/issues/2638)|[#2639](https://github.com/kedacore/keda/issues/2639)) +||||||| parent of 35ee4c95 (Update CHANGELOG) +- **General**: Provide Prometheus metric with indication of total number of (Cluster)TriggerAuthentication objects. ([#2639](https://github.com/kedacore/keda/issues/2639)) +======= +- **General**: Provide Prometheus metric with indication of total number of custom resources per namespace. ([#2637](https://github.com/kedacore/keda/issues/2637)|[#2638](https://github.com/kedacore/keda/issues/2638)|[#2639](https://github.com/kedacore/keda/issues/2639)) +>>>>>>> 35ee4c95 (Update CHANGELOG) - **General**: Provide Prometheus metric with indication of total number of triggers per trigger type in `ScaledJob`/`ScaledObject`. ([#3663](https://github.com/kedacore/keda/issues/3663)) - **AWS Scalers**: Add setting AWS endpoint url. ([#3337](https://github.com/kedacore/keda/issues/3337)) - **Azure Service Bus Scaler**: Add support for Shared Access Signature (SAS) tokens for authentication. ([#2920](https://github.com/kedacore/keda/issues/2920)) From 6839abd9a6c88d8f4d52ade3454bb2bf892d5c86 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Sat, 8 Oct 2022 07:55:09 +0530 Subject: [PATCH 09/14] Move finalization logic into separate files. Signed-off-by: Vighnesh Shenoy --- ...clustertriggerauthentication_controller.go | 46 +----------- .../clustertriggerauthentication_finalizer.go | 19 +++++ .../keda/triggerauthentication_controller.go | 46 +----------- .../keda/triggerauthentication_finalizer.go | 19 +++++ controllers/keda/util/finalizer.go | 73 +++++++++++++++++++ main.go | 10 +-- 6 files changed, 121 insertions(+), 92 deletions(-) create mode 100644 controllers/keda/clustertriggerauthentication_finalizer.go create mode 100644 controllers/keda/triggerauthentication_finalizer.go create mode 100644 controllers/keda/util/finalizer.go diff --git a/controllers/keda/clustertriggerauthentication_controller.go b/controllers/keda/clustertriggerauthentication_controller.go index 3ddfacbecdf..6ba31500c2d 100644 --- a/controllers/keda/clustertriggerauthentication_controller.go +++ b/controllers/keda/clustertriggerauthentication_controller.go @@ -20,10 +20,8 @@ import ( "context" "sync" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -32,7 +30,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - "github.com/kedacore/keda/v2/controllers/keda/util" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metrics" ) @@ -40,18 +37,13 @@ import ( // ClusterTriggerAuthenticationReconciler reconciles a ClusterTriggerAuthentication object type ClusterTriggerAuthenticationReconciler struct { client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder + record.EventRecorder } type clusterTriggerAuthMetricsData struct { namespace string } -const ( - clusterTriggerAuthenticationFinalizer = "finalizer.keda.sh" -) - var ( clusterTriggerAuthMetricsMap map[string]clusterTriggerAuthMetricsData clusterTriggerAuthMetricsLock *sync.Mutex @@ -88,7 +80,7 @@ func (r *ClusterTriggerAuthenticationReconciler) Reconcile(ctx context.Context, r.updateMetrics(clusterTriggerAuthentication, req.NamespacedName.String()) if clusterTriggerAuthentication.ObjectMeta.Generation == 1 { - r.Recorder.Event(clusterTriggerAuthentication, corev1.EventTypeNormal, eventreason.ClusterTriggerAuthenticationAdded, "New ClusterTriggerAuthentication configured") + r.EventRecorder.Event(clusterTriggerAuthentication, corev1.EventTypeNormal, eventreason.ClusterTriggerAuthenticationAdded, "New ClusterTriggerAuthentication configured") } return ctrl.Result{}, nil } @@ -100,38 +92,6 @@ func (r *ClusterTriggerAuthenticationReconciler) SetupWithManager(mgr ctrl.Manag Complete(r) } -func (r *ClusterTriggerAuthenticationReconciler) ensureFinalizer(ctx context.Context, logger logr.Logger, clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication) error { - if !util.Contains(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer) { - logger.Info("Adding Finalizer for the ClusterTriggerAuthentication") - clusterTriggerAuth.SetFinalizers(append(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer)) - - // Update CR - err := r.Client.Update(ctx, clusterTriggerAuth) - if err != nil { - logger.Error(err, "Failed to update ClusterTriggerAuthentication with a finalizer", "finalizer", clusterTriggerAuthenticationFinalizer) - return err - } - } - return nil -} - -func (r *ClusterTriggerAuthenticationReconciler) finalizeClusterTriggerAuthentication(ctx context.Context, logger logr.Logger, - clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication, namespacedName string) error { - if util.Contains(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer) { - clusterTriggerAuth.SetFinalizers(util.Remove(clusterTriggerAuth.GetFinalizers(), clusterTriggerAuthenticationFinalizer)) - if err := r.Client.Update(ctx, clusterTriggerAuth); err != nil { - logger.Error(err, "Failed to update ClusterTriggerAuthentication after removing a finalizer", "finalizer", clusterTriggerAuthenticationFinalizer) - return err - } - - r.updateMetricsOnDelete(namespacedName) - } - - logger.Info("Successfully finalized ClusterTriggerAuthentication") - r.Recorder.Event(clusterTriggerAuth, corev1.EventTypeNormal, eventreason.TriggerAuthenticationDeleted, "ClusterTriggerAuthentication was deleted") - return nil -} - func (r *ClusterTriggerAuthenticationReconciler) updateMetrics(clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication, namespacedName string) { clusterTriggerAuthMetricsLock.Lock() defer clusterTriggerAuthMetricsLock.Unlock() @@ -145,7 +105,7 @@ func (r *ClusterTriggerAuthenticationReconciler) updateMetrics(clusterTriggerAut } // this method is idempotent, so it can be called multiple times without side-effects -func (r *ClusterTriggerAuthenticationReconciler) updateMetricsOnDelete(namespacedName string) { +func (r *ClusterTriggerAuthenticationReconciler) UpdateMetricsOnDelete(namespacedName string) { clusterTriggerAuthMetricsLock.Lock() defer clusterTriggerAuthMetricsLock.Unlock() diff --git a/controllers/keda/clustertriggerauthentication_finalizer.go b/controllers/keda/clustertriggerauthentication_finalizer.go new file mode 100644 index 00000000000..4fff965b325 --- /dev/null +++ b/controllers/keda/clustertriggerauthentication_finalizer.go @@ -0,0 +1,19 @@ +package keda + +import ( + "context" + + "github.com/go-logr/logr" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/controllers/keda/util" +) + +func (r *ClusterTriggerAuthenticationReconciler) ensureFinalizer(ctx context.Context, logger logr.Logger, clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication) error { + return util.EnsureAuthenticationResourceFinalizer(ctx, logger, r, clusterTriggerAuth) +} + +func (r *ClusterTriggerAuthenticationReconciler) finalizeClusterTriggerAuthentication(ctx context.Context, logger logr.Logger, + clusterTriggerAuth *kedav1alpha1.ClusterTriggerAuthentication, namespacedName string) error { + return util.FinalizeAuthenticationResource(ctx, logger, r, clusterTriggerAuth, namespacedName) +} diff --git a/controllers/keda/triggerauthentication_controller.go b/controllers/keda/triggerauthentication_controller.go index 52884664cc8..a5b23816a6b 100644 --- a/controllers/keda/triggerauthentication_controller.go +++ b/controllers/keda/triggerauthentication_controller.go @@ -20,10 +20,8 @@ import ( "context" "sync" - "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -32,7 +30,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - "github.com/kedacore/keda/v2/controllers/keda/util" "github.com/kedacore/keda/v2/pkg/eventreason" "github.com/kedacore/keda/v2/pkg/metrics" ) @@ -40,18 +37,13 @@ import ( // TriggerAuthenticationReconciler reconciles a TriggerAuthentication object type TriggerAuthenticationReconciler struct { client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder + record.EventRecorder } type triggerAuthMetricsData struct { namespace string } -const ( - triggerAuthenticationFinalizer = "finalizer.keda.sh" -) - var ( triggerAuthMetricsMap map[string]triggerAuthMetricsData triggerAuthMetricsLock *sync.Mutex @@ -88,7 +80,7 @@ func (r *TriggerAuthenticationReconciler) Reconcile(ctx context.Context, req ctr r.updateMetrics(triggerAuthentication, req.NamespacedName.String()) if triggerAuthentication.ObjectMeta.Generation == 1 { - r.Recorder.Event(triggerAuthentication, corev1.EventTypeNormal, eventreason.TriggerAuthenticationAdded, "New TriggerAuthentication configured") + r.EventRecorder.Event(triggerAuthentication, corev1.EventTypeNormal, eventreason.TriggerAuthenticationAdded, "New TriggerAuthentication configured") } return ctrl.Result{}, nil @@ -101,38 +93,6 @@ func (r *TriggerAuthenticationReconciler) SetupWithManager(mgr ctrl.Manager) err Complete(r) } -func (r *TriggerAuthenticationReconciler) ensureFinalizer(ctx context.Context, logger logr.Logger, triggerAuth *kedav1alpha1.TriggerAuthentication) error { - if !util.Contains(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer) { - logger.Info("Adding Finalizer for the TriggerAuthentication") - triggerAuth.SetFinalizers(append(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer)) - - // Update CR - err := r.Client.Update(ctx, triggerAuth) - if err != nil { - logger.Error(err, "Failed to update TriggerAuthentication with a finalizer", "finalizer", triggerAuthenticationFinalizer) - return err - } - } - return nil -} - -func (r *TriggerAuthenticationReconciler) finalizeTriggerAuthentication(ctx context.Context, logger logr.Logger, - triggerAuth *kedav1alpha1.TriggerAuthentication, namespacedName string) error { - if util.Contains(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer) { - triggerAuth.SetFinalizers(util.Remove(triggerAuth.GetFinalizers(), triggerAuthenticationFinalizer)) - if err := r.Client.Update(ctx, triggerAuth); err != nil { - logger.Error(err, "Failed to update TriggerAuthentication after removing a finalizer", "finalizer", triggerAuthenticationFinalizer) - return err - } - - r.updateMetricsOnDelete(namespacedName) - } - - logger.Info("Successfully finalized TriggerAuthentication") - r.Recorder.Event(triggerAuth, corev1.EventTypeNormal, eventreason.TriggerAuthenticationDeleted, "TriggerAuthentication was deleted") - return nil -} - func (r *TriggerAuthenticationReconciler) updateMetrics(triggerAuth *kedav1alpha1.TriggerAuthentication, namespacedName string) { triggerAuthMetricsLock.Lock() defer triggerAuthMetricsLock.Unlock() @@ -146,7 +106,7 @@ func (r *TriggerAuthenticationReconciler) updateMetrics(triggerAuth *kedav1alpha } // this method is idempotent, so it can be called multiple times without side-effects -func (r *TriggerAuthenticationReconciler) updateMetricsOnDelete(namespacedName string) { +func (r *TriggerAuthenticationReconciler) UpdateMetricsOnDelete(namespacedName string) { triggerAuthMetricsLock.Lock() defer triggerAuthMetricsLock.Unlock() diff --git a/controllers/keda/triggerauthentication_finalizer.go b/controllers/keda/triggerauthentication_finalizer.go new file mode 100644 index 00000000000..916f1e0d5d2 --- /dev/null +++ b/controllers/keda/triggerauthentication_finalizer.go @@ -0,0 +1,19 @@ +package keda + +import ( + "context" + + "github.com/go-logr/logr" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/controllers/keda/util" +) + +func (r *TriggerAuthenticationReconciler) ensureFinalizer(ctx context.Context, logger logr.Logger, triggerAuth *kedav1alpha1.TriggerAuthentication) error { + return util.EnsureAuthenticationResourceFinalizer(ctx, logger, r, triggerAuth) +} + +func (r *TriggerAuthenticationReconciler) finalizeTriggerAuthentication(ctx context.Context, logger logr.Logger, + triggerAuth *kedav1alpha1.TriggerAuthentication, namespacedName string) error { + return util.FinalizeAuthenticationResource(ctx, logger, r, triggerAuth, namespacedName) +} diff --git a/controllers/keda/util/finalizer.go b/controllers/keda/util/finalizer.go new file mode 100644 index 00000000000..3873db4c7bb --- /dev/null +++ b/controllers/keda/util/finalizer.go @@ -0,0 +1,73 @@ +package util + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/eventreason" +) + +const ( + authenticationFinalizer = "finalizer.keda.sh" +) + +type authenticationReconciler interface { + client.Client + record.EventRecorder + UpdateMetricsOnDelete(string) +} + +func EnsureAuthenticationResourceFinalizer(ctx context.Context, logger logr.Logger, reconciler authenticationReconciler, authResource client.Object) error { + var authResourceType string + switch authResource.(type) { + case *kedav1alpha1.TriggerAuthentication: + authResourceType = "TriggerAuthentication" + case *kedav1alpha1.ClusterTriggerAuthentication: + authResourceType = "ClusterTriggerAuthentication" + } + + if !Contains(authResource.GetFinalizers(), authenticationFinalizer) { + logger.Info(fmt.Sprintf("Adding Finalizer for the %s", authResourceType)) + authResource.SetFinalizers(append(authResource.GetFinalizers(), authenticationFinalizer)) + + // Update CR + err := reconciler.Update(ctx, authResource) + if err != nil { + logger.Error(err, fmt.Sprintf("Failed to update %s with a finalizer", authResourceType), "finalizer", authenticationFinalizer) + return err + } + } + return nil +} + +func FinalizeAuthenticationResource(ctx context.Context, logger logr.Logger, reconciler authenticationReconciler, authResource client.Object, namespacedName string) error { + var authResourceType, reason string + switch authResource.(type) { + case *kedav1alpha1.TriggerAuthentication: + authResourceType = "TriggerAuthentication" + reason = eventreason.TriggerAuthenticationDeleted + case *kedav1alpha1.ClusterTriggerAuthentication: + authResourceType = "ClusterTriggerAuthentication" + reason = eventreason.ClusterTriggerAuthenticationDeleted + } + + if Contains(authResource.GetFinalizers(), authenticationFinalizer) { + authResource.SetFinalizers(Remove(authResource.GetFinalizers(), authenticationFinalizer)) + if err := reconciler.Update(ctx, authResource); err != nil { + logger.Error(err, fmt.Sprintf("Failed to update %s after removing a finalizer", authResourceType), "finalizer", authenticationFinalizer) + return err + } + + reconciler.UpdateMetricsOnDelete(namespacedName) + } + + logger.Info(fmt.Sprintf("Successfully finalized %s", authResourceType)) + reconciler.Event(authResource, corev1.EventTypeNormal, reason, fmt.Sprintf("%s was deleted", authResourceType)) + return nil +} diff --git a/main.go b/main.go index 9e091e81324..2c547f1493c 100644 --- a/main.go +++ b/main.go @@ -169,17 +169,15 @@ func main() { os.Exit(1) } if err = (&kedacontrollers.TriggerAuthenticationReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: eventRecorder, + Client: mgr.GetClient(), + EventRecorder: eventRecorder, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TriggerAuthentication") os.Exit(1) } if err = (&kedacontrollers.ClusterTriggerAuthenticationReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: eventRecorder, + Client: mgr.GetClient(), + EventRecorder: eventRecorder, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterTriggerAuthentication") os.Exit(1) From 2c981d640165a4587c1710a90e9e7534fca02031 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Tue, 1 Nov 2022 11:52:32 +0530 Subject: [PATCH 10/14] Change metric name Signed-off-by: Vighnesh Shenoy --- pkg/metrics/operator_prom_metrics.go | 2 +- .../prometheus_metrics_test.go | 32 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/metrics/operator_prom_metrics.go b/pkg/metrics/operator_prom_metrics.go index b964dc9b756..28c99f9b569 100644 --- a/pkg/metrics/operator_prom_metrics.go +++ b/pkg/metrics/operator_prom_metrics.go @@ -34,7 +34,7 @@ var ( crdTotalsGaugeVec = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "keda_operator", - Subsystem: "crd", + Subsystem: "cr", Name: "totals", }, []string{"type", "namespace"}, diff --git a/tests/internals/prometheus_metrics/prometheus_metrics_test.go b/tests/internals/prometheus_metrics/prometheus_metrics_test.go index 123815e2bdb..dd117ed94fe 100644 --- a/tests/internals/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/internals/prometheus_metrics/prometheus_metrics_test.go @@ -317,7 +317,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str kedaKc := GetKedaKubernetesClient(t) triggerTotals := make(map[string]int) - crdTotals := map[string]map[string]int{ + crTotals := map[string]map[string]int{ "scaled_object": {}, "scaled_job": {}, "trigger_authentication": {}, @@ -335,7 +335,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str if namespace == "" { namespace = "default" } - crdTotals["cluster_trigger_authentication"][namespace]++ + crTotals["cluster_trigger_authentication"][namespace]++ } for _, namespace := range namespaceList.Items { @@ -347,7 +347,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str scaledObjectList, err := kedaKc.ScaledObjects(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list scaledObjects in namespace - %s with err - %s", namespace.Name, err) - crdTotals["scaled_object"][namespaceName] = len(scaledObjectList.Items) + crTotals["scaled_object"][namespaceName] = len(scaledObjectList.Items) for _, scaledObject := range scaledObjectList.Items { for _, trigger := range scaledObject.Spec.Triggers { triggerTotals[trigger.Type]++ @@ -357,7 +357,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str scaledJobList, err := kedaKc.ScaledJobs(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list scaledJobs in namespace - %s with err - %s", namespace.Name, err) - crdTotals["scaled_job"][namespaceName] = len(scaledJobList.Items) + crTotals["scaled_job"][namespaceName] = len(scaledJobList.Items) for _, scaledJob := range scaledJobList.Items { for _, trigger := range scaledJob.Spec.Triggers { triggerTotals[trigger.Type]++ @@ -367,18 +367,18 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str triggerAuthList, err := kedaKc.TriggerAuthentications(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list triggerAuthentications in namespace - %s with err - %s", namespace.Name, err) - crdTotals["trigger_authentication"][namespaceName] = len(triggerAuthList.Items) + crTotals["trigger_authentication"][namespaceName] = len(triggerAuthList.Items) } - return triggerTotals, crdTotals + return triggerTotals, crTotals } func testOperatorMetricValues(t *testing.T, kc *kubernetes.Clientset) { families := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure http://%s.keda:8080/metrics", serviceName)) - expectedTriggerTotals, expectedCrdTotals := getOperatorMetricsManually(t, kc) + expectedTriggerTotals, expectedCrTotals := getOperatorMetricsManually(t, kc) checkTriggerTotalValues(t, families, expectedTriggerTotals) - checkCRDTotalValues(t, families, expectedCrdTotals) + checkCRTotalValues(t, families, expectedCrTotals) } func checkTriggerTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]int) { @@ -410,10 +410,10 @@ func checkTriggerTotalValues(t *testing.T, families map[string]*promModel.Metric assert.Equal(t, 0, len(expected)) } -func checkCRDTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]map[string]int) { - t.Log("--- testing crd total metrics ---") +func checkCRTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]map[string]int) { + t.Log("--- testing cr total metrics ---") - family, ok := families["keda_operator_crd_totals"] + family, ok := families["keda_operator_cr_totals"] if !ok { t.Errorf("metric not available") return @@ -422,19 +422,19 @@ func checkCRDTotalValues(t *testing.T, families map[string]*promModel.MetricFami metrics := family.GetMetric() for _, metric := range metrics { labels := metric.GetLabel() - var namespace, crdType string + var namespace, crType string for _, label := range labels { if *label.Name == "type" { - crdType = *label.Value + crType = *label.Value } else if *label.Name == "namespace" { namespace = *label.Value } } metricValue := *metric.Gauge.Value - expectedMetricValue := float64(expected[crdType][namespace]) + expectedMetricValue := float64(expected[crType][namespace]) - assert.Equalf(t, expectedMetricValue, metricValue, "expected %f got %f for crd type %s & namespace %s", - expectedMetricValue, metricValue, crdType, namespace) + assert.Equalf(t, expectedMetricValue, metricValue, "expected %f got %f for cr type %s & namespace %s", + expectedMetricValue, metricValue, crType, namespace) } } From 96bbc4fb777b29edb00a369402b8b62dadba8eb8 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 4 Nov 2022 02:02:10 +0530 Subject: [PATCH 11/14] Change metric name Signed-off-by: Vighnesh Shenoy --- pkg/metrics/operator_prom_metrics.go | 2 +- tests/internals/prometheus_metrics/prometheus_metrics_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/operator_prom_metrics.go b/pkg/metrics/operator_prom_metrics.go index 28c99f9b569..f0988de7a59 100644 --- a/pkg/metrics/operator_prom_metrics.go +++ b/pkg/metrics/operator_prom_metrics.go @@ -34,7 +34,7 @@ var ( crdTotalsGaugeVec = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "keda_operator", - Subsystem: "cr", + Subsystem: "customresource", Name: "totals", }, []string{"type", "namespace"}, diff --git a/tests/internals/prometheus_metrics/prometheus_metrics_test.go b/tests/internals/prometheus_metrics/prometheus_metrics_test.go index dd117ed94fe..5618563bbbe 100644 --- a/tests/internals/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/internals/prometheus_metrics/prometheus_metrics_test.go @@ -411,9 +411,9 @@ func checkTriggerTotalValues(t *testing.T, families map[string]*promModel.Metric } func checkCRTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]map[string]int) { - t.Log("--- testing cr total metrics ---") + t.Log("--- testing customresource total metrics ---") - family, ok := families["keda_operator_cr_totals"] + family, ok := families["keda_operator_customresource_totals"] if !ok { t.Errorf("metric not available") return From 721dbd370bb1687e55addb233f07f35c53bed8d5 Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 4 Nov 2022 16:17:20 +0530 Subject: [PATCH 12/14] Change metric name Signed-off-by: Vighnesh Shenoy --- pkg/metrics/operator_prom_metrics.go | 2 +- tests/internals/prometheus_metrics/prometheus_metrics_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/operator_prom_metrics.go b/pkg/metrics/operator_prom_metrics.go index f0988de7a59..f93439344f5 100644 --- a/pkg/metrics/operator_prom_metrics.go +++ b/pkg/metrics/operator_prom_metrics.go @@ -34,7 +34,7 @@ var ( crdTotalsGaugeVec = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "keda_operator", - Subsystem: "customresource", + Subsystem: "resource", Name: "totals", }, []string{"type", "namespace"}, diff --git a/tests/internals/prometheus_metrics/prometheus_metrics_test.go b/tests/internals/prometheus_metrics/prometheus_metrics_test.go index 5618563bbbe..4d003094129 100644 --- a/tests/internals/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/internals/prometheus_metrics/prometheus_metrics_test.go @@ -411,9 +411,9 @@ func checkTriggerTotalValues(t *testing.T, families map[string]*promModel.Metric } func checkCRTotalValues(t *testing.T, families map[string]*promModel.MetricFamily, expected map[string]map[string]int) { - t.Log("--- testing customresource total metrics ---") + t.Log("--- testing resource total metrics ---") - family, ok := families["keda_operator_customresource_totals"] + family, ok := families["keda_operator_resource_totals"] if !ok { t.Errorf("metric not available") return From e4ff597db62d67d6575ce29e827d6bcf113878de Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 4 Nov 2022 18:00:36 +0530 Subject: [PATCH 13/14] Use constants Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 2 -- .../keda/clustertriggerauthentication_controller.go | 6 +++--- controllers/keda/scaledjob_controller.go | 6 +++--- controllers/keda/scaledobject_controller.go | 6 +++--- controllers/keda/triggerauthentication_controller.go | 6 +++--- pkg/metrics/operator_prom_metrics.go | 7 +++++++ .../prometheus_metrics/prometheus_metrics_test.go | 9 +++++---- 7 files changed, 24 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dcea0790b6..5d6fecd0b1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,6 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### New -<<<<<<< HEAD - **General**: Expand Prometheus metric with label "ScalerName" to distinguish different triggers. The scaleName is defined per Trigger.Name ([#3588](https://github.com/kedacore/keda/issues/3588) - **General:** Introduce new Loki Scaler ([#3699](https://github.com/kedacore/keda/issues/3699)) - **General**: Add ratelimitting parameters to keda manager to allow override of client defaults ([#3730](https://github.com/kedacore/keda/issues/2920)) @@ -47,7 +46,6 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio - **General**: Provide Prometheus metric with indication of total number of (Cluster)TriggerAuthentication objects. ([#2639](https://github.com/kedacore/keda/issues/2639)) ======= - **General**: Provide Prometheus metric with indication of total number of custom resources per namespace. ([#2637](https://github.com/kedacore/keda/issues/2637)|[#2638](https://github.com/kedacore/keda/issues/2638)|[#2639](https://github.com/kedacore/keda/issues/2639)) ->>>>>>> 35ee4c95 (Update CHANGELOG) - **General**: Provide Prometheus metric with indication of total number of triggers per trigger type in `ScaledJob`/`ScaledObject`. ([#3663](https://github.com/kedacore/keda/issues/3663)) - **AWS Scalers**: Add setting AWS endpoint url. ([#3337](https://github.com/kedacore/keda/issues/3337)) - **Azure Service Bus Scaler**: Add support for Shared Access Signature (SAS) tokens for authentication. ([#2920](https://github.com/kedacore/keda/issues/2920)) diff --git a/controllers/keda/clustertriggerauthentication_controller.go b/controllers/keda/clustertriggerauthentication_controller.go index 6ba31500c2d..8e9f6b392d2 100644 --- a/controllers/keda/clustertriggerauthentication_controller.go +++ b/controllers/keda/clustertriggerauthentication_controller.go @@ -97,10 +97,10 @@ func (r *ClusterTriggerAuthenticationReconciler) updateMetrics(clusterTriggerAut defer clusterTriggerAuthMetricsLock.Unlock() if metricsData, ok := clusterTriggerAuthMetricsMap[namespacedName]; ok { - metrics.DecrementCRDTotal("cluster_trigger_authentication", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.ClusterTriggerAuthenticationResource, metricsData.namespace) } - metrics.IncrementCRDTotal("cluster_trigger_authentication", clusterTriggerAuth.Namespace) + metrics.IncrementCRDTotal(metrics.ClusterTriggerAuthenticationResource, clusterTriggerAuth.Namespace) clusterTriggerAuthMetricsMap[namespacedName] = clusterTriggerAuthMetricsData{namespace: clusterTriggerAuth.Namespace} } @@ -110,7 +110,7 @@ func (r *ClusterTriggerAuthenticationReconciler) UpdateMetricsOnDelete(namespace defer clusterTriggerAuthMetricsLock.Unlock() if metricsData, ok := clusterTriggerAuthMetricsMap[namespacedName]; ok { - metrics.DecrementCRDTotal("cluster_trigger_authentication", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.ClusterTriggerAuthenticationResource, metricsData.namespace) } delete(clusterTriggerAuthMetricsMap, namespacedName) diff --git a/controllers/keda/scaledjob_controller.go b/controllers/keda/scaledjob_controller.go index 3e7362320de..2b940607b10 100644 --- a/controllers/keda/scaledjob_controller.go +++ b/controllers/keda/scaledjob_controller.go @@ -275,13 +275,13 @@ func (r *ScaledJobReconciler) updateMetrics(scaledJob *kedav1alpha1.ScaledJob, n metricsData, ok := scaledJobMetricsMap[namespacedName] if ok { - metrics.DecrementCRDTotal("scaled_job", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.ScaledJobResource, metricsData.namespace) for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } } - metrics.IncrementCRDTotal("scaled_job", scaledJob.Namespace) + metrics.IncrementCRDTotal(metrics.ScaledJobResource, scaledJob.Namespace) metricsData.namespace = scaledJob.Namespace triggerTypes := make([]string, len(scaledJob.Spec.Triggers)) @@ -299,7 +299,7 @@ func (r *ScaledJobReconciler) updateMetricsOnDelete(namespacedName string) { defer scaledJobMetricsLock.Unlock() if metricsData, ok := scaledJobMetricsMap[namespacedName]; ok { - metrics.DecrementCRDTotal("scaled_job", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.ScaledJobResource, metricsData.namespace) for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } diff --git a/controllers/keda/scaledobject_controller.go b/controllers/keda/scaledobject_controller.go index da5fea2dfe8..f853f94419f 100644 --- a/controllers/keda/scaledobject_controller.go +++ b/controllers/keda/scaledobject_controller.go @@ -492,13 +492,13 @@ func (r *ScaledObjectReconciler) updateMetrics(scaledObject *kedav1alpha1.Scaled metricsData, ok := scaledObjectMetricsMap[namespacedName] if ok { - metrics.DecrementCRDTotal("scaled_object", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.ScaledObjectResource, metricsData.namespace) for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } } - metrics.IncrementCRDTotal("scaled_object", scaledObject.Namespace) + metrics.IncrementCRDTotal(metrics.ScaledObjectResource, scaledObject.Namespace) metricsData.namespace = scaledObject.Namespace triggerTypes := make([]string, len(scaledObject.Spec.Triggers)) @@ -516,7 +516,7 @@ func (r *ScaledObjectReconciler) updateMetricsOnDelete(namespacedName string) { defer scaledObjectMetricsLock.Unlock() if metricsData, ok := scaledObjectMetricsMap[namespacedName]; ok { - metrics.DecrementCRDTotal("scaled_object", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.ScaledObjectResource, metricsData.namespace) for _, triggerType := range metricsData.triggerTypes { metrics.DecrementTriggerTotal(triggerType) } diff --git a/controllers/keda/triggerauthentication_controller.go b/controllers/keda/triggerauthentication_controller.go index a5b23816a6b..b4d4f3d6e52 100644 --- a/controllers/keda/triggerauthentication_controller.go +++ b/controllers/keda/triggerauthentication_controller.go @@ -98,10 +98,10 @@ func (r *TriggerAuthenticationReconciler) updateMetrics(triggerAuth *kedav1alpha defer triggerAuthMetricsLock.Unlock() if metricsData, ok := triggerAuthMetricsMap[namespacedName]; ok { - metrics.DecrementCRDTotal("trigger_authentication", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.TriggerAuthenticationResource, metricsData.namespace) } - metrics.IncrementCRDTotal("trigger_authentication", triggerAuth.Namespace) + metrics.IncrementCRDTotal(metrics.TriggerAuthenticationResource, triggerAuth.Namespace) triggerAuthMetricsMap[namespacedName] = triggerAuthMetricsData{namespace: triggerAuth.Namespace} } @@ -111,7 +111,7 @@ func (r *TriggerAuthenticationReconciler) UpdateMetricsOnDelete(namespacedName s defer triggerAuthMetricsLock.Unlock() if metricsData, ok := triggerAuthMetricsMap[namespacedName]; ok { - metrics.DecrementCRDTotal("trigger_authentication", metricsData.namespace) + metrics.DecrementCRDTotal(metrics.TriggerAuthenticationResource, metricsData.namespace) } delete(triggerAuthMetricsMap, namespacedName) diff --git a/pkg/metrics/operator_prom_metrics.go b/pkg/metrics/operator_prom_metrics.go index f93439344f5..f00fb5cc4fa 100644 --- a/pkg/metrics/operator_prom_metrics.go +++ b/pkg/metrics/operator_prom_metrics.go @@ -21,6 +21,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" ) +const ( + ClusterTriggerAuthenticationResource = "cluster_trigger_authentication" + TriggerAuthenticationResource = "trigger_authentication" + ScaledObjectResource = "scaled_object" + ScaledJobResource = "scaled_job" +) + var ( triggerTotalsGaugeVec = prometheus.NewGaugeVec( prometheus.GaugeOpts{ diff --git a/tests/internals/prometheus_metrics/prometheus_metrics_test.go b/tests/internals/prometheus_metrics/prometheus_metrics_test.go index 4d003094129..6168a9fe7df 100644 --- a/tests/internals/prometheus_metrics/prometheus_metrics_test.go +++ b/tests/internals/prometheus_metrics/prometheus_metrics_test.go @@ -15,6 +15,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "github.com/kedacore/keda/v2/pkg/metrics" . "github.com/kedacore/keda/v2/tests/helper" ) @@ -335,7 +336,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str if namespace == "" { namespace = "default" } - crTotals["cluster_trigger_authentication"][namespace]++ + crTotals[metrics.ClusterTriggerAuthenticationResource][namespace]++ } for _, namespace := range namespaceList.Items { @@ -347,7 +348,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str scaledObjectList, err := kedaKc.ScaledObjects(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list scaledObjects in namespace - %s with err - %s", namespace.Name, err) - crTotals["scaled_object"][namespaceName] = len(scaledObjectList.Items) + crTotals[metrics.ScaledObjectResource][namespaceName] = len(scaledObjectList.Items) for _, scaledObject := range scaledObjectList.Items { for _, trigger := range scaledObject.Spec.Triggers { triggerTotals[trigger.Type]++ @@ -357,7 +358,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str scaledJobList, err := kedaKc.ScaledJobs(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list scaledJobs in namespace - %s with err - %s", namespace.Name, err) - crTotals["scaled_job"][namespaceName] = len(scaledJobList.Items) + crTotals[metrics.ScaledJobResource][namespaceName] = len(scaledJobList.Items) for _, scaledJob := range scaledJobList.Items { for _, trigger := range scaledJob.Spec.Triggers { triggerTotals[trigger.Type]++ @@ -367,7 +368,7 @@ func getOperatorMetricsManually(t *testing.T, kc *kubernetes.Clientset) (map[str triggerAuthList, err := kedaKc.TriggerAuthentications(namespace.Name).List(context.Background(), v1.ListOptions{}) assert.NoErrorf(t, err, "failed to list triggerAuthentications in namespace - %s with err - %s", namespace.Name, err) - crTotals["trigger_authentication"][namespaceName] = len(triggerAuthList.Items) + crTotals[metrics.TriggerAuthenticationResource][namespaceName] = len(triggerAuthList.Items) } return triggerTotals, crTotals From f3881f8d96ba796bdaff5a4a10df29e322e12a0f Mon Sep 17 00:00:00 2001 From: Vighnesh Shenoy Date: Fri, 4 Nov 2022 21:12:44 +0530 Subject: [PATCH 14/14] Fix changelog :facepalm: Signed-off-by: Vighnesh Shenoy --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d6fecd0b1a..30c34894607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,14 +38,10 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio ### New -- **General**: Expand Prometheus metric with label "ScalerName" to distinguish different triggers. The scaleName is defined per Trigger.Name ([#3588](https://github.com/kedacore/keda/issues/3588) +- **General**: Expand Prometheus metric with label "ScalerName" to distinguish different triggers. The scaleName is defined per Trigger.Name ([#3588](https://github.com/kedacore/keda/issues/3588)) - **General:** Introduce new Loki Scaler ([#3699](https://github.com/kedacore/keda/issues/3699)) - **General**: Add ratelimitting parameters to keda manager to allow override of client defaults ([#3730](https://github.com/kedacore/keda/issues/2920)) - **General**: Provide Prometheus metric with indication of total number of custom resources per namespace for each custom resource type (CRD). ([#2637](https://github.com/kedacore/keda/issues/2637)|[#2638](https://github.com/kedacore/keda/issues/2638)|[#2639](https://github.com/kedacore/keda/issues/2639)) -||||||| parent of 35ee4c95 (Update CHANGELOG) -- **General**: Provide Prometheus metric with indication of total number of (Cluster)TriggerAuthentication objects. ([#2639](https://github.com/kedacore/keda/issues/2639)) -======= -- **General**: Provide Prometheus metric with indication of total number of custom resources per namespace. ([#2637](https://github.com/kedacore/keda/issues/2637)|[#2638](https://github.com/kedacore/keda/issues/2638)|[#2639](https://github.com/kedacore/keda/issues/2639)) - **General**: Provide Prometheus metric with indication of total number of triggers per trigger type in `ScaledJob`/`ScaledObject`. ([#3663](https://github.com/kedacore/keda/issues/3663)) - **AWS Scalers**: Add setting AWS endpoint url. ([#3337](https://github.com/kedacore/keda/issues/3337)) - **Azure Service Bus Scaler**: Add support for Shared Access Signature (SAS) tokens for authentication. ([#2920](https://github.com/kedacore/keda/issues/2920))