diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c83a8e186a..675644eda99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio Here is an overview of all new **experimental** features: - TODO ([#XXX](https://github.com/kedacore/keda/issues/XXX)) +- **Datadog Scaler**: Add support to use the Cluster Agent as source of metrics ([#5355](https://github.com/kedacore/keda/issues/5355)) ### Improvements diff --git a/pkg/scalers/datadog_scaler.go b/pkg/scalers/datadog_scaler.go index c06ede68aeb..aa8b499eaff 100644 --- a/pkg/scalers/datadog_scaler.go +++ b/pkg/scalers/datadog_scaler.go @@ -2,7 +2,10 @@ package scalers import ( "context" + "errors" "fmt" + "io" + "net/http" "regexp" "strconv" "strings" @@ -10,34 +13,61 @@ import ( datadog "github.com/DataDog/datadog-api-client-go/api/v1/datadog" "github.com/go-logr/logr" + "github.com/tidwall/gjson" v2 "k8s.io/api/autoscaling/v2" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/metrics/pkg/apis/external_metrics" + "github.com/kedacore/keda/v2/pkg/scalers/authentication" "github.com/kedacore/keda/v2/pkg/scalers/scalersconfig" kedautil "github.com/kedacore/keda/v2/pkg/util" ) type datadogScaler struct { - metadata *datadogMetadata - apiClient *datadog.APIClient - logger logr.Logger + metadata *datadogMetadata + apiClient *datadog.APIClient + httpClient *http.Client + logger logr.Logger + useClusterAgentProxy bool } type datadogMetadata struct { - apiKey string - appKey string - datadogSite string + + // AuthParams Cluster Agent Proxy + datadogNamespace string + datadogMetricsService string + datadogMetricsServicePort int + unsafeSsl bool + + // bearer auth Cluster Agent Proxy + enableBearerAuth bool + bearerToken string + + // TriggerMetadata Cluster Agent Proxy + datadogMetricServiceURL string + datadogMetricName string + datadogMetricNamespace string + activationTargetValue float64 + + // AuthParams Datadog API + apiKey string + appKey string + datadogSite string + + // TriggerMetadata Datadog API query string - queryValue float64 queryAggegrator string activationQueryValue float64 - vType v2.MetricTargetType - metricName string age int timeWindowOffset int lastAvailablePointOffset int - useFiller bool - fillValue float64 + + // TriggerMetadata Common + hpaMetricName string + fillValue float64 + targetValue float64 + useFiller bool + vType v2.MetricTargetType } const maxString = "max" @@ -53,19 +83,42 @@ func init() { func NewDatadogScaler(ctx context.Context, config *scalersconfig.ScalerConfig) (Scaler, error) { logger := InitializeLogger(config, "datadog_scaler") - meta, err := parseDatadogMetadata(config, logger) - if err != nil { - return nil, fmt.Errorf("error parsing Datadog metadata: %w", err) + var useClusterAgentProxy bool + var meta *datadogMetadata + var err error + var apiClient *datadog.APIClient + var httpClient *http.Client + + if val, ok := config.TriggerMetadata["useClusterAgentProxy"]; ok { + useClusterAgentProxy, err = strconv.ParseBool(val) + if err != nil { + return nil, fmt.Errorf("error parsing useClusterAgentProxy: %w", err) + } } - apiClient, err := newDatadogConnection(ctx, meta, config) - if err != nil { - return nil, fmt.Errorf("error establishing Datadog connection: %w", err) + if useClusterAgentProxy { + meta, err = parseDatadogClusterAgentMetadata(config, logger) + if err != nil { + return nil, fmt.Errorf("error parsing Datadog metadata: %w", err) + } + httpClient = kedautil.CreateHTTPClient(config.GlobalHTTPTimeout, meta.unsafeSsl) + } else { + meta, err = parseDatadogAPIMetadata(config, logger) + if err != nil { + return nil, fmt.Errorf("error parsing Datadog metadata: %w", err) + } + apiClient, err = newDatadogAPIConnection(ctx, meta, config) + if err != nil { + return nil, fmt.Errorf("error establishing Datadog connection: %w", err) + } } + return &datadogScaler{ - metadata: meta, - apiClient: apiClient, - logger: logger, + metadata: meta, + apiClient: apiClient, + httpClient: httpClient, + logger: logger, + useClusterAgentProxy: useClusterAgentProxy, }, nil } @@ -79,7 +132,17 @@ func parseDatadogQuery(q string) (bool, error) { return true, nil } -func parseDatadogMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger) (*datadogMetadata, error) { +// buildClusterAgentURL builds the URL for the Cluster Agent Metrics API service +func buildClusterAgentURL(datadogMetricsService, datadogNamespace string, datadogMetricsServicePort int) string { + return fmt.Sprintf("https://%s.%s:%d/apis/external.metrics.k8s.io/v1beta1", datadogMetricsService, datadogNamespace, datadogMetricsServicePort) +} + +// buildMetricURL builds the URL for the Datadog metric +func buildMetricURL(datadogClusterAgentURL, datadogMetricNamespace, datadogMetricName string) string { + return fmt.Sprintf("%s/namespaces/%s/%s", datadogClusterAgentURL, datadogMetricNamespace, datadogMetricName) +} + +func parseDatadogAPIMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger) (*datadogMetadata, error) { meta := datadogMetadata{} if val, ok := config.TriggerMetadata["age"]; ok { @@ -137,17 +200,23 @@ func parseDatadogMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger return nil, fmt.Errorf("no query given") } - if val, ok := config.TriggerMetadata["queryValue"]; ok { - queryValue, err := strconv.ParseFloat(val, 64) + if val, ok := config.TriggerMetadata["targetValue"]; ok { + targetValue, err := strconv.ParseFloat(val, 64) + if err != nil { + return nil, fmt.Errorf("targetValue parsing error %w", err) + } + meta.targetValue = targetValue + } else if val, ok := config.TriggerMetadata["queryValue"]; ok { + targetValue, err := strconv.ParseFloat(val, 64) if err != nil { return nil, fmt.Errorf("queryValue parsing error %w", err) } - meta.queryValue = queryValue + meta.targetValue = targetValue } else { if config.AsMetricSource { - meta.queryValue = 0 + meta.targetValue = 0 } else { - return nil, fmt.Errorf("no queryValue given") + return nil, fmt.Errorf("no targetValue or queryValue given") } } @@ -223,14 +292,140 @@ func parseDatadogMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger meta.datadogSite = siteVal - metricName := meta.query[0:strings.Index(meta.query, "{")] - meta.metricName = GenerateMetricNameWithIndex(config.TriggerIndex, kedautil.NormalizeString(fmt.Sprintf("datadog-%s", metricName))) + hpaMetricName := meta.query[0:strings.Index(meta.query, "{")] + meta.hpaMetricName = GenerateMetricNameWithIndex(config.TriggerIndex, kedautil.NormalizeString(fmt.Sprintf("datadog-%s", hpaMetricName))) + + return &meta, nil +} + +func parseDatadogClusterAgentMetadata(config *scalersconfig.ScalerConfig, logger logr.Logger) (*datadogMetadata, error) { + meta := datadogMetadata{} + + if val, ok := config.AuthParams["datadogNamespace"]; ok { + meta.datadogNamespace = val + } else { + return nil, fmt.Errorf("no datadogNamespace key given") + } + + if val, ok := config.AuthParams["datadogMetricsService"]; ok { + meta.datadogMetricsService = val + } else { + return nil, fmt.Errorf("no datadogMetricsService key given") + } + + if val, ok := config.AuthParams["datadogMetricsServicePort"]; ok { + port, err := strconv.Atoi(val) + if err != nil { + return nil, fmt.Errorf("datadogMetricServicePort parsing error %w", err) + } + meta.datadogMetricsServicePort = port + } else { + meta.datadogMetricsServicePort = 8443 + } + + meta.datadogMetricServiceURL = buildClusterAgentURL(meta.datadogMetricsService, meta.datadogNamespace, meta.datadogMetricsServicePort) + + meta.unsafeSsl = false + if val, ok := config.AuthParams["unsafeSsl"]; ok { + unsafeSsl, err := strconv.ParseBool(val) + if err != nil { + return nil, fmt.Errorf("error parsing unsafeSsl: %w", err) + } + meta.unsafeSsl = unsafeSsl + } + + if val, ok := config.TriggerMetadata["datadogMetricName"]; ok { + meta.datadogMetricName = val + } else { + return nil, fmt.Errorf("no datadogMetricName key given") + } + + if val, ok := config.TriggerMetadata["datadogMetricNamespace"]; ok { + meta.datadogMetricNamespace = val + } else { + return nil, fmt.Errorf("no datadogMetricNamespace key given") + } + + meta.hpaMetricName = "datadogmetric@" + meta.datadogMetricNamespace + ":" + meta.datadogMetricName + + if val, ok := config.TriggerMetadata["targetValue"]; ok { + targetValue, err := strconv.ParseFloat(val, 64) + if err != nil { + return nil, fmt.Errorf("targetValue parsing error %w", err) + } + meta.targetValue = targetValue + } else { + if config.AsMetricSource { + meta.targetValue = 0 + } else { + return nil, fmt.Errorf("no targetValue given") + } + } + + meta.activationTargetValue = 0 + if val, ok := config.TriggerMetadata["activationTargetValue"]; ok { + activationTargetValue, err := strconv.ParseFloat(val, 64) + if err != nil { + return nil, fmt.Errorf("activationTargetValue parsing error %w", err) + } + meta.activationTargetValue = activationTargetValue + } + + if val, ok := config.TriggerMetadata["metricUnavailableValue"]; ok { + fillValue, err := strconv.ParseFloat(val, 64) + if err != nil { + return nil, fmt.Errorf("metricUnavailableValue parsing error %w", err) + } + meta.fillValue = fillValue + meta.useFiller = true + } + + if val, ok := config.TriggerMetadata["type"]; ok { + logger.V(0).Info("trigger.metadata.type is deprecated in favor of trigger.metricType") + if config.MetricType != "" { + return nil, fmt.Errorf("only one of trigger.metadata.type or trigger.metricType should be defined") + } + val = strings.ToLower(val) + switch val { + case avgString: + meta.vType = v2.AverageValueMetricType + case "global": + meta.vType = v2.ValueMetricType + default: + return nil, fmt.Errorf("type has to be global or average") + } + } else { + metricType, err := GetMetricTargetType(config) + if err != nil { + return nil, fmt.Errorf("error getting scaler metric type: %w", err) + } + meta.vType = metricType + } + + authMode, ok := config.AuthParams["authMode"] + // no authMode specified + if !ok { + return &meta, nil + } + + authType := authentication.Type(strings.TrimSpace(authMode)) + switch authType { + case authentication.BearerAuthType: + if len(config.AuthParams["token"]) == 0 { + return nil, errors.New("no token provided") + } + + meta.bearerToken = config.AuthParams["token"] + meta.enableBearerAuth = true + default: + return nil, fmt.Errorf("err incorrect value for authMode is given: %s", authMode) + } return &meta, nil } -// newDatadogConnection tests a connection to the Datadog API -func newDatadogConnection(ctx context.Context, meta *datadogMetadata, config *scalersconfig.ScalerConfig) (*datadog.APIClient, error) { +// newDatadogAPIConnection tests a connection to the Datadog API +func newDatadogAPIConnection(ctx context.Context, meta *datadogMetadata, config *scalersconfig.ScalerConfig) (*datadog.APIClient, error) { ctx = context.WithValue( ctx, datadog.ContextAPIKeys, @@ -373,13 +568,73 @@ func (s *datadogScaler) getQueryResult(ctx context.Context) (float64, error) { } } +func (s *datadogScaler) getDatadogMetricValue(req *http.Request) (float64, error) { + resp, err := s.httpClient.Do(req) + + if err != nil { + return 0, fmt.Errorf("error getting metric value: %w", err) + } + + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + r := gjson.GetBytes(body, "message") + if r.Type == gjson.String { + return 0, fmt.Errorf("error getting metric value: %s", r.String()) + } + } + + valueLocation := "items.0.value" + r := gjson.GetBytes(body, valueLocation) + errorMsg := "the metric value must be of type number or a string representing a Quantity got: '%s'" + + if r.Type == gjson.String { + v, err := resource.ParseQuantity(r.String()) + if err != nil { + return 0, fmt.Errorf(errorMsg, r.String()) + } + return v.AsApproximateFloat64(), nil + } + if r.Type != gjson.Number { + return 0, fmt.Errorf(errorMsg, r.Type.String()) + } + return r.Num, nil +} + +func (s *datadogScaler) getDatadogClusterAgentHTTPRequest(ctx context.Context, url string) (*http.Request, error) { + var req *http.Request + var err error + + switch { + case s.metadata.enableBearerAuth: + req, err = http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", s.metadata.bearerToken)) + if err != nil { + return nil, err + } + return req, nil + + default: + req, err = http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return req, err + } + } + + return nil, nil +} + // GetMetricSpecForScaling returns the MetricSpec for the Horizontal Pod Autoscaler func (s *datadogScaler) GetMetricSpecForScaling(context.Context) []v2.MetricSpec { externalMetric := &v2.ExternalMetricSource{ Metric: v2.MetricIdentifier{ - Name: s.metadata.metricName, + Name: s.metadata.hpaMetricName, }, - Target: GetMetricTargetMili(s.metadata.vType, s.metadata.queryValue), + Target: GetMetricTargetMili(s.metadata.vType, s.metadata.targetValue), } metricSpec := v2.MetricSpec{ External: externalMetric, Type: externalMetricType, @@ -389,14 +644,33 @@ func (s *datadogScaler) GetMetricSpecForScaling(context.Context) []v2.MetricSpec // GetMetricsAndActivity returns value for a supported metric and an error if there is a problem getting the metric func (s *datadogScaler) GetMetricsAndActivity(ctx context.Context, metricName string) ([]external_metrics.ExternalMetricValue, bool, error) { - num, err := s.getQueryResult(ctx) + var metric external_metrics.ExternalMetricValue + var num float64 + var err error + + if s.useClusterAgentProxy { + url := buildMetricURL(s.metadata.datadogMetricServiceURL, s.metadata.datadogMetricNamespace, s.metadata.hpaMetricName) + + req, err := s.getDatadogClusterAgentHTTPRequest(ctx, url) + if (err != nil) || (req == nil) { + return []external_metrics.ExternalMetricValue{}, false, fmt.Errorf("error generating http request: %w", err) + } + + num, err = s.getDatadogMetricValue(req) + if err != nil { + return []external_metrics.ExternalMetricValue{}, false, fmt.Errorf("error getting metric value: %w", err) + } + + metric = GenerateMetricInMili(metricName, num) + return []external_metrics.ExternalMetricValue{metric}, num > s.metadata.activationTargetValue, nil + } + num, err = s.getQueryResult(ctx) if err != nil { s.logger.Error(err, "error getting metrics from Datadog") return []external_metrics.ExternalMetricValue{}, false, fmt.Errorf("error getting metrics from Datadog: %w", err) } - metric := GenerateMetricInMili(metricName, num) - + metric = GenerateMetricInMili(metricName, num) return []external_metrics.ExternalMetricValue{metric}, num > s.metadata.activationQueryValue, nil } diff --git a/pkg/scalers/datadog_scaler_test.go b/pkg/scalers/datadog_scaler_test.go index 41e32c7f0eb..bf12cadc60d 100644 --- a/pkg/scalers/datadog_scaler_test.go +++ b/pkg/scalers/datadog_scaler_test.go @@ -16,8 +16,16 @@ type datadogQueries struct { isError bool } +type datadogScalerType int64 + +const ( + apiType datadogScalerType = iota + clusterAgentType +) + type datadogMetricIdentifier struct { metadataTestData *datadogAuthMetadataTestData + typeOfScaler datadogScalerType triggerIndex int name string } @@ -90,7 +98,29 @@ func TestDatadogScalerParseQueries(t *testing.T) { } } -var testDatadogMetadata = []datadogAuthMetadataTestData{ +var testDatadogClusterAgentMetadata = []datadogAuthMetadataTestData{ + {"", map[string]string{}, map[string]string{}, true}, + + // all properly formed + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "datadogMetricNamespace": "default", "targetValue": "2", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "datadogMetricsServicePort": "8080", "unsafeSsl": "true", "authMode": "bearer"}, false}, + // Default Datadog service name and port + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "datadogMetricNamespace": "default", "targetValue": "2", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "unsafeSsl": "true", "authMode": "bearer"}, false}, + + // both metadata type and trigger type + {v2.AverageValueMetricType, map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "datadogMetricNamespace": "default", "targetValue": "2", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "unsafeSsl": "true", "authMode": "bearer"}, true}, + // missing DatadogMetric name + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricNamespace": "default", "targetValue": "2", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "unsafeSsl": "true", "authMode": "bearer"}, true}, + // missing DatadogMetric namespace + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "targetValue": "2", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "unsafeSsl": "true", "authMode": "bearer"}, true}, + // wrong port type + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "datadogMetricNamespace": "default", "targetValue": "2", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "datadogMetricsServicePort": "notanint", "unsafeSsl": "true", "authMode": "bearer"}, true}, + // wrong targetValue type + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "datadogMetricNamespace": "default", "targetValue": "notanint", "type": "global"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "datadogMetricsServicePort": "8080", "unsafeSsl": "true", "authMode": "bearer"}, true}, + // wrong type + {"", map[string]string{"useClusterAgentProxy": "true", "datadogMetricName": "nginx-hits", "datadogMetricNamespace": "default", "targetValue": "2", "type": "notatype"}, map[string]string{"token": "token", "datadogNamespace": "datadog", "datadogMetricsService": "datadog-cluster-agent-metrics-api", "datadogMetricsServicePort": "8080", "unsafeSsl": "true", "authMode": "bearer"}, true}, +} + +var testDatadogAPIMetadata = []datadogAuthMetadataTestData{ {"", map[string]string{}, map[string]string{}, true}, // all properly formed @@ -135,9 +165,22 @@ var testDatadogMetadata = []datadogAuthMetadataTestData{ {"", map[string]string{"query": "sum:trace.redis.command.hits.as_count()", "queryValue": "7"}, map[string]string{}, true}, } -func TestDatadogScalerAuthParams(t *testing.T) { - for _, testData := range testDatadogMetadata { - _, err := parseDatadogMetadata(&scalersconfig.ScalerConfig{TriggerMetadata: testData.metadata, AuthParams: testData.authParams, MetricType: testData.metricType}, logr.Discard()) +func TestDatadogScalerAPIAuthParams(t *testing.T) { + for _, testData := range testDatadogAPIMetadata { + _, err := parseDatadogAPIMetadata(&scalersconfig.ScalerConfig{TriggerMetadata: testData.metadata, AuthParams: testData.authParams, MetricType: testData.metricType}, logr.Discard()) + + if err != nil && !testData.isError { + t.Error("Expected success but got error", err) + } + if testData.isError && err == nil { + t.Error("Expected error but got success") + } + } +} + +func TestDatadogScalerClusterAgentAuthParams(t *testing.T) { + for _, testData := range testDatadogClusterAgentMetadata { + _, err := parseDatadogClusterAgentMetadata(&scalersconfig.ScalerConfig{TriggerMetadata: testData.metadata, AuthParams: testData.authParams, MetricType: testData.metricType}, logr.Discard()) if err != nil && !testData.isError { t.Error("Expected success but got error", err) @@ -149,19 +192,29 @@ func TestDatadogScalerAuthParams(t *testing.T) { } var datadogMetricIdentifiers = []datadogMetricIdentifier{ - {&testDatadogMetadata[1], 0, "s0-datadog-sum-trace-redis-command-hits"}, - {&testDatadogMetadata[1], 1, "s1-datadog-sum-trace-redis-command-hits"}, + {&testDatadogAPIMetadata[1], apiType, 0, "s0-datadog-sum-trace-redis-command-hits"}, + {&testDatadogAPIMetadata[1], apiType, 1, "s1-datadog-sum-trace-redis-command-hits"}, + {&testDatadogClusterAgentMetadata[1], clusterAgentType, 0, "datadogmetric@default:nginx-hits"}, } func TestDatadogGetMetricSpecForScaling(t *testing.T) { + var err error + var meta *datadogMetadata + for _, testData := range datadogMetricIdentifiers { - meta, err := parseDatadogMetadata(&scalersconfig.ScalerConfig{TriggerMetadata: testData.metadataTestData.metadata, AuthParams: testData.metadataTestData.authParams, TriggerIndex: testData.triggerIndex, MetricType: testData.metadataTestData.metricType}, logr.Discard()) + if testData.typeOfScaler == apiType { + meta, err = parseDatadogAPIMetadata(&scalersconfig.ScalerConfig{TriggerMetadata: testData.metadataTestData.metadata, AuthParams: testData.metadataTestData.authParams, TriggerIndex: testData.triggerIndex, MetricType: testData.metadataTestData.metricType}, logr.Discard()) + } else { + meta, err = parseDatadogClusterAgentMetadata(&scalersconfig.ScalerConfig{TriggerMetadata: testData.metadataTestData.metadata, AuthParams: testData.metadataTestData.authParams, TriggerIndex: testData.triggerIndex, MetricType: testData.metadataTestData.metricType}, logr.Discard()) + } if err != nil { t.Fatal("Could not parse metadata:", err) } + mockDatadogScaler := datadogScaler{ - metadata: meta, - apiClient: nil, + metadata: meta, + apiClient: nil, + httpClient: nil, } metricSpec := mockDatadogScaler.GetMetricSpecForScaling(context.Background()) @@ -171,3 +224,19 @@ func TestDatadogGetMetricSpecForScaling(t *testing.T) { } } } + +func TestBuildClusterAgentURL(t *testing.T) { + // Test valid inputs + url := buildClusterAgentURL("datadogMetricsService", "datadogNamespace", 8080) + if url != "https://datadogMetricsService.datadogNamespace:8080/apis/external.metrics.k8s.io/v1beta1" { + t.Error("Expected https://datadogMetricsService.datadogNamespace:8080/apis/external.metrics.k8s.io/v1beta1, got ", url) + } +} + +func TestBuildMetricURL(t *testing.T) { + // Test valid inputs + url := buildMetricURL("https://localhost:8080/apis/datadoghq.com/v1alpha1", "datadogMetricNamespace", "datadogMetricName") + if url != "https://localhost:8080/apis/datadoghq.com/v1alpha1/namespaces/datadogMetricNamespace/datadogMetricName" { + t.Error("Expected https://localhost:8080/apis/datadoghq.com/v1alpha1/namespaces/datadogMetricNamespace/datadogMetricName, got ", url) + } +} diff --git a/tests/scalers/datadog/datadog_test.go b/tests/scalers/datadog/datadog_api/datadog_api_test.go similarity index 94% rename from tests/scalers/datadog/datadog_test.go rename to tests/scalers/datadog/datadog_api/datadog_api_test.go index 99c46dc2366..f4125797291 100644 --- a/tests/scalers/datadog/datadog_test.go +++ b/tests/scalers/datadog/datadog_api/datadog_api_test.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package datadog_test +package datadog_api_test import ( "encoding/base64" @@ -28,7 +28,7 @@ var ( testNamespace = fmt.Sprintf("%s-ns", testName) deploymentName = fmt.Sprintf("%s-deployment", testName) monitoredDeploymentName = fmt.Sprintf("%s-monitored-deployment", testName) - servciceName = fmt.Sprintf("%s-service", testName) + serviceName = fmt.Sprintf("%s-service", testName) triggerAuthName = fmt.Sprintf("%s-ta", testName) scaledObjectName = fmt.Sprintf("%s-so", testName) secretName = fmt.Sprintf("%s-secret", testName) @@ -46,7 +46,7 @@ type templateData struct { TestNamespace string DeploymentName string MonitoredDeploymentName string - ServciceName string + ServiceName string ScaledObjectName string TriggerAuthName string SecretName string @@ -174,7 +174,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{.ServciceName}} + name: {{.ServiceName}} namespace: {{.TestNamespace}} spec: ports: @@ -213,7 +213,7 @@ spec: triggers: - type: datadog metadata: - query: "avg:nginx.net.request_per_s{cluster_name:{{.KuberneteClusterName}}}" + query: "avg:nginx.net.request_per_s{cluster_name:{{.KuberneteClusterName}}, kube_namespace:{{.TestNamespace}}}" queryValue: "2" activationQueryValue: "3" age: "120" @@ -231,7 +231,7 @@ spec: - image: busybox name: test command: ["/bin/sh"] - args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServciceName}}/; sleep 0.5; done"]` + args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServiceName}}/; sleep 5; done"]` heavyLoadTemplate = `apiVersion: v1 kind: Pod @@ -243,10 +243,10 @@ spec: - image: busybox name: test command: ["/bin/sh"] - args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServciceName}}/; sleep 0.1; done"]` + args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServiceName}}/; sleep 0.1; done"]` ) -func TestDatadogScaler(t *testing.T) { +func TestDatadogScalerAPI(t *testing.T) { // setup t.Log("--- setting up ---") require.NotEmpty(t, datadogAppKey, "DATADOG_APP_KEY env variable is required for datadog tests") @@ -261,11 +261,16 @@ func TestDatadogScaler(t *testing.T) { // install datadog CreateNamespace(t, kc, testNamespace) - installDatadog(t) // Create kubernetes resources KubectlApplyMultipleWithTemplate(t, data, templates) + // Deploy Datadog Agent + installDatadog(t) + + t.Log("--- creating ScaledObject ---") + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 180, 3), "replica count should be %d after 3 minutes", minReplicaCount) @@ -318,7 +323,7 @@ func getTemplateData() (templateData, []Template) { TestNamespace: testNamespace, DeploymentName: deploymentName, MonitoredDeploymentName: monitoredDeploymentName, - ServciceName: servciceName, + ServiceName: serviceName, TriggerAuthName: triggerAuthName, ScaledObjectName: scaledObjectName, SecretName: secretName, @@ -336,6 +341,5 @@ func getTemplateData() (templateData, []Template) { {Name: "serviceTemplate", Config: serviceTemplate}, {Name: "deploymentTemplate", Config: deploymentTemplate}, {Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate}, - {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, } } diff --git a/tests/scalers/datadog/datadog_dca/datadog_dca_test.go b/tests/scalers/datadog/datadog_dca/datadog_dca_test.go new file mode 100644 index 00000000000..66512515484 --- /dev/null +++ b/tests/scalers/datadog/datadog_dca/datadog_dca_test.go @@ -0,0 +1,441 @@ +//go:build e2e +// +build e2e + +package datadog_dca_test + +import ( + "encoding/base64" + "fmt" + "os" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "datadog-dca-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + dcaServiceAccount = fmt.Sprintf("%s-sa", testName) + dcaClusterRole = fmt.Sprintf("%s-cr", testName) + dcaClusterRoleBinding = fmt.Sprintf("%s-crb", testName) + dcaSAToken = fmt.Sprintf("%s-sa-token", testName) + datadogConfigName = fmt.Sprintf("%s-datadog-config", testName) + datadogMetricName = fmt.Sprintf("%s-datadog-metric", testName) + + deploymentName = fmt.Sprintf("%s-deployment", testName) + monitoredDeploymentName = fmt.Sprintf("%s-monitored-deployment", testName) + serviceName = fmt.Sprintf("%s-service", testName) + triggerAuthName = fmt.Sprintf("%s-ta", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + secretName = fmt.Sprintf("%s-secret", testName) + configName = fmt.Sprintf("%s-config", testName) + datadogAPIKey = os.Getenv("DATADOG_API_KEY") + datadogAppKey = os.Getenv("DATADOG_APP_KEY") + datadogSite = os.Getenv("DATADOG_SITE") + datadogHelmRepo = "https://helm.datadoghq.com" + kuberneteClusterName = "keda-datadog-cluster" + minReplicaCount = 0 + maxReplicaCount = 2 +) + +type templateData struct { + TestNamespace string + DcaServiceAccount string + DcaClusterRole string + DcaClusterRoleBinding string + DcaServiceAccountToken string + DatadogConfigName string + DatadogConfigNamespace string + DatadogConfigMetricsService string + DatadogConfigUnsafeSSL string + DatadogConfigAuthMode string + DatadogMetricName string + + DeploymentName string + MonitoredDeploymentName string + ServiceName string + ScaledObjectName string + TriggerAuthName string + SecretName string + ConfigName string + DatadogAPIKey string + DatadogAppKey string + DatadogSite string + KuberneteClusterName string + MinReplicaCount string + MaxReplicaCount string +} + +const ( + datadogMetricTemplate = `apiVersion: datadoghq.com/v1alpha1 +kind: DatadogMetric +metadata: + name: {{.DatadogMetricName}} + namespace: {{.TestNamespace}} + annotations: + external-metrics.datadoghq.com/always-active: "true" +spec: + query: "avg:nginx.net.request_per_s{cluster_name:{{.KuberneteClusterName}}, kube_namespace:{{.TestNamespace}}}" +` + + dcaServiceAccountTemplate = `apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{.DcaServiceAccount}} + namespace: {{.TestNamespace}} +` + dcaClusterRoleTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{.DcaClusterRole}} +rules: +- apiGroups: + - external.metrics.k8s.io + resources: + - '*' + verbs: ["get", "watch", "list"] +` + dcaClusterRoleBindingTemplate = `apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{.DcaClusterRoleBinding}} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{.DcaClusterRole}} +subjects: +- kind: ServiceAccount + name: {{.DcaServiceAccount}} + namespace: {{.TestNamespace}} +` + dcaServiceAccountTokenTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.DcaServiceAccountToken}} + namespace: {{.TestNamespace}} + annotations: + kubernetes.io/service-account.name: {{.DcaServiceAccount}} +type: kubernetes.io/service-account-token +` + secretTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + apiKey: {{.DatadogAPIKey}} + appKey: {{.DatadogAppKey}} + datadogSite: {{.DatadogSite}} +` + datadogConfigTemplate = `apiVersion: v1 +kind: Secret +metadata: + name: {{.DatadogConfigName}} + namespace: {{.TestNamespace}} +data: + datadogNamespace: {{.DatadogConfigNamespace}} + datadogMetricsService: {{.DatadogConfigMetricsService}} + datadogUnsafeSSL: {{.DatadogConfigUnsafeSSL}} + datadogAuthMode: {{.DatadogConfigAuthMode}} +` + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: {{.TriggerAuthName}} + namespace: {{.TestNamespace}} +spec: + secretTargetRef: + - parameter: token + name: {{.DcaServiceAccountToken}} + key: token + - parameter: datadogNamespace + name: {{.DatadogConfigName}} + key: datadogNamespace + - parameter: datadogMetricsService + name: {{.DatadogConfigName}} + key: datadogMetricsService + - parameter: unsafeSsl + name: {{.DatadogConfigName}} + key: datadogUnsafeSSL + - parameter: authMode + name: {{.DatadogConfigName}} + key: datadogAuthMode +` + configTemplate = `apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.ConfigName}} + namespace: {{.TestNamespace}} +data: + status.conf: | + server { + listen 81; + location /nginx_status { + stub_status on; + } + } +` + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: nginx + image: nginxinc/nginx-unprivileged + ports: + - containerPort: 80 +` + monitoredDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nginx + name: {{.MonitoredDeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + creationTimestamp: null + labels: + app: nginx + annotations: + ad.datadoghq.com/nginx.check_names: '["nginx"]' + ad.datadoghq.com/nginx.init_configs: '[{}]' + ad.datadoghq.com/nginx.instances: | + [ + { + "nginx_status_url":"http://%%host%%:81/nginx_status/" + } + ] + spec: + containers: + - image: nginx + name: nginx + ports: + - containerPort: 80 + - containerPort: 81 + volumeMounts: + - mountPath: /etc/nginx/conf.d/status.conf + subPath: status.conf + readOnly: true + name: "config" + volumes: + - name: "config" + configMap: + name: {{.ConfigName}} +` + serviceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: {{.ServiceName}} + namespace: {{.TestNamespace}} +spec: + ports: + - name: default + port: 80 + protocol: TCP + targetPort: 80 + - name: status + port: 81 + protocol: TCP + targetPort: 81 + selector: + app: nginx +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + minReplicaCount: {{.MinReplicaCount}} + maxReplicaCount: {{.MaxReplicaCount}} + pollingInterval: 1 + cooldownPeriod: 1 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 10 + triggers: + - type: datadog + metadata: + useClusterAgentProxy: "true" + datadogMetricName: {{.DatadogMetricName}} + datadogMetricNamespace: {{.TestNamespace}} + targetValue: "2" + activationTargetValue: "3" + metricType: "Value" + authenticationRef: + name: {{.TriggerAuthName}} +` + lightLoadTemplate = `apiVersion: v1 +kind: Pod +metadata: + name: fake-light-traffic + namespace: {{.TestNamespace}} +spec: + containers: + - image: busybox + name: test + command: ["/bin/sh"] + args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServiceName}}/; sleep 5; done"]` + + heavyLoadTemplate = `apiVersion: v1 +kind: Pod +metadata: + name: fake-heavy-traffic + namespace: {{.TestNamespace}} +spec: + containers: + - image: busybox + name: test + command: ["/bin/sh"] + args: ["-c", "while true; do wget -O /dev/null -o /dev/null http://{{.ServiceName}}/; sleep 0.1; done"]` +) + +func TestDatadogScalerDCA(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, datadogAppKey, "DATADOG_APP_KEY env variable is required for datadog tests") + require.NotEmpty(t, datadogAPIKey, "DATADOG_API_KEY env variable is required for datadog tests") + require.NotEmpty(t, datadogSite, "DATADOG_SITE env variable is required for datadog tests") + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + t.Cleanup(func() { + DeleteKubernetesResources(t, testNamespace, data, templates) + }) + + CreateKubernetesResources(t, kc, testNamespace, data, templates) + installDatadog(t) + + t.Log("--- creating DatadogMetric & ScaledObject ---") + KubectlApplyWithTemplate(t, data, "datadogMetricTemplate", datadogMetricTemplate) + KubectlApplyWithTemplate(t, data, "scaledObjectTemplate", scaledObjectTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 180, 3), + "replica count should be %d after 3 minutes", minReplicaCount) + + // test scaling + testActivation(t, kc, data) + testScaleOut(t, kc, data) + testScaleIn(t, kc, data) +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing activation ---") + KubectlApplyWithTemplate(t, data, "lightLoadTemplate", lightLoadTemplate) + + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, minReplicaCount, 60) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale out ---") + KubectlApplyWithTemplate(t, data, "heavyLoadTemplate", heavyLoadTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", maxReplicaCount) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale in ---") + KubectlDeleteWithTemplate(t, data, "lightLoadTemplate", lightLoadTemplate) + KubectlDeleteWithTemplate(t, data, "heavyLoadTemplate", heavyLoadTemplate) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) +} + +func installDatadog(t *testing.T) { + t.Log("--- installing datadog ---") + _, err := ExecuteCommand(fmt.Sprintf("helm repo add datadog %s", datadogHelmRepo)) + assert.NoErrorf(t, err, "cannot execute command - %s", err) + _, err = ExecuteCommand("helm repo update") + assert.NoErrorf(t, err, "cannot execute command - %s", err) + _, err = ExecuteCommand(fmt.Sprintf(`helm upgrade --install --set datadog.apiKey=%s --set datadog.appKey=%s --set datadog.site=%s --set datadog.clusterName=%s --set datadog.kubelet.tlsVerify=false --set clusterAgent.metricsProvider.enabled=true --set clusterAgent.metricsProvider.registerAPIService=false --set clusterAgent.metricsProvider.useDatadogMetrics=true --namespace %s --wait %s datadog/datadog`, + datadogAPIKey, + datadogAppKey, + datadogSite, + kuberneteClusterName, + testNamespace, + testName)) + assert.NoErrorf(t, err, "cannot execute command - %s", err) +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + DcaServiceAccount: dcaServiceAccount, + DcaClusterRole: dcaClusterRole, + DcaClusterRoleBinding: dcaClusterRoleBinding, + DcaServiceAccountToken: dcaSAToken, + DatadogConfigName: datadogConfigName, + DatadogConfigNamespace: base64.StdEncoding.EncodeToString([]byte(testNamespace)), + DatadogConfigMetricsService: base64.StdEncoding.EncodeToString([]byte(testName + "-cluster-agent-metrics-api")), + DatadogConfigUnsafeSSL: base64.StdEncoding.EncodeToString([]byte("true")), + DatadogConfigAuthMode: base64.StdEncoding.EncodeToString([]byte("bearer")), + DatadogMetricName: datadogMetricName, + DeploymentName: deploymentName, + MonitoredDeploymentName: monitoredDeploymentName, + ServiceName: serviceName, + TriggerAuthName: triggerAuthName, + ScaledObjectName: scaledObjectName, + SecretName: secretName, + ConfigName: configName, + DatadogAPIKey: base64.StdEncoding.EncodeToString([]byte(datadogAPIKey)), + DatadogAppKey: base64.StdEncoding.EncodeToString([]byte(datadogAppKey)), + DatadogSite: base64.StdEncoding.EncodeToString([]byte(datadogSite)), + KuberneteClusterName: kuberneteClusterName, + MinReplicaCount: fmt.Sprintf("%v", minReplicaCount), + MaxReplicaCount: fmt.Sprintf("%v", maxReplicaCount), + }, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "dcaServiceAccountTemplate", Config: dcaServiceAccountTemplate}, + {Name: "dcaClusterRoleTemplate", Config: dcaClusterRoleTemplate}, + {Name: "dcaClusterRoleBindingTemplate", Config: dcaClusterRoleBindingTemplate}, + {Name: "dcaServiceAccountTokenTemplate", Config: dcaServiceAccountTokenTemplate}, + {Name: "configTemplate", Config: configTemplate}, + {Name: "datadogConfigTemplate", Config: datadogConfigTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "serviceTemplate", Config: serviceTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "monitoredDeploymentTemplate", Config: monitoredDeploymentTemplate}, + } +}