Use go templates to render the builtin promql queries

pkg/controller/scheduler.go

@@ -513,7 +513,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 		}
 
 		if metric.Name == "istio_requests_total" {
-			val, err := c.observer.GetDeploymentCounter(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
+			val, err := c.observer.GetIstioSuccessRate(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
 			if err != nil {
 				if strings.Contains(err.Error(), "no values found") {
 					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
@@ -531,7 +531,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 		}
 
 		if metric.Name == "istio_request_duration_seconds_bucket" {
-			val, err := c.observer.GetDeploymentHistogram(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
+			val, err := c.observer.GetIstioRequestDuration(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
 			if err != nil {
 				c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err)
 				return false

pkg/metrics/envoy.go

@@ -0,0 +1,65 @@
+package metrics
+
+import (
+	"fmt"
+	"net/url"
+	"strconv"
+)
+
+const envoySuccessRateQuery = `
+sum(rate(
+envoy_cluster_upstream_rq{kubernetes_namespace="{{ .Namespace }}",
+kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+envoy_response_code!~"5.*"}
+[{{ .Interval }}])) 
+/ 
+sum(rate(
+envoy_cluster_upstream_rq{kubernetes_namespace="{{ .Namespace }}",
+kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"}
+[{{ .Interval }}])) 
+* 100
+`
+
+func (c *Observer) GetEnvoySuccessRate(name string, namespace string, metric string, interval string) (float64, error) {
+	if c.metricsServer == "fake" {
+		return 100, nil
+	}
+
+	meta := struct {
+		Name      string
+		Namespace string
+		Interval  string
+	}{
+		name,
+		namespace,
+		interval,
+	}
+
+	query, err := render(meta, envoySuccessRateQuery)
+	if err != nil {
+		return 0, err
+	}
+
+	var rate *float64
+	querySt := url.QueryEscape(query)
+	result, err := c.queryMetric(querySt)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, v := range result.Data.Result {
+		metricValue := v.Value[1]
+		switch metricValue.(type) {
+		case string:
+			f, err := strconv.ParseFloat(metricValue.(string), 64)
+			if err != nil {
+				return 0, err
+			}
+			rate = &f
+		}
+	}
+	if rate == nil {
+		return 0, fmt.Errorf("no values found for metric %s", metric)
+	}
+	return *rate, nil
+}

pkg/metrics/envoy_test.go

@@ -0,0 +1,28 @@
+package metrics
+
+import (
+	"testing"
+)
+
+func Test_EnvoySuccessRateQueryRender(t *testing.T) {
+	meta := struct {
+		Name      string
+		Namespace string
+		Interval  string
+	}{
+		"podinfo",
+		"default",
+		"1m",
+	}
+
+	query, err := render(meta, envoySuccessRateQuery)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := `sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",envoy_response_code!~"5.*"}[1m])) / sum(rate(envoy_cluster_upstream_rq{kubernetes_namespace="default",kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"}[1m])) * 100`
+
+	if query != expected {
+		t.Errorf("\nGot %s \nWanted %s", query, expected)
+	}
+}

pkg/metrics/istio.go

@@ -0,0 +1,123 @@
+package metrics
+
+import (
+	"fmt"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+const istioSuccessRateQuery = `
+sum(rate(
+istio_requests_total{reporter="destination",
+destination_workload_namespace="{{ .Namespace }}",
+destination_workload=~"{{ .Name }}",
+response_code!~"5.*"}
+[{{ .Interval }}])) 
+/ 
+sum(rate(
+istio_requests_total{reporter="destination",
+destination_workload_namespace="{{ .Namespace }}",
+destination_workload=~"{{ .Name }}"}
+[{{ .Interval }}])) 
+* 100
+`
+
+// GetIstioSuccessRate returns the requests success rate (non 5xx) using istio_requests_total metric
+func (c *Observer) GetIstioSuccessRate(name string, namespace string, metric string, interval string) (float64, error) {
+	if c.metricsServer == "fake" {
+		return 100, nil
+	}
+
+	meta := struct {
+		Name      string
+		Namespace string
+		Interval  string
+	}{
+		name,
+		namespace,
+		interval,
+	}
+
+	query, err := render(meta, istioSuccessRateQuery)
+	if err != nil {
+		return 0, err
+	}
+
+	var rate *float64
+	querySt := url.QueryEscape(query)
+	result, err := c.queryMetric(querySt)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, v := range result.Data.Result {
+		metricValue := v.Value[1]
+		switch metricValue.(type) {
+		case string:
+			f, err := strconv.ParseFloat(metricValue.(string), 64)
+			if err != nil {
+				return 0, err
+			}
+			rate = &f
+		}
+	}
+	if rate == nil {
+		return 0, fmt.Errorf("no values found for metric %s", metric)
+	}
+	return *rate, nil
+}
+
+const istioRequestDurationQuery = `
+histogram_quantile(0.99, sum(rate(
+istio_request_duration_seconds_bucket{reporter="destination",
+destination_workload_namespace="{{ .Namespace }}",
+destination_workload=~"{{ .Name }}"}
+[{{ .Interval }}])) by (le))
+`
+
+// GetIstioRequestDuration returns the 99P requests delay using istio_request_duration_seconds_bucket metrics
+func (c *Observer) GetIstioRequestDuration(name string, namespace string, metric string, interval string) (time.Duration, error) {
+	if c.metricsServer == "fake" {
+		return 1, nil
+	}
+
+	meta := struct {
+		Name      string
+		Namespace string
+		Interval  string
+	}{
+		name,
+		namespace,
+		interval,
+	}
+
+	query, err := render(meta, istioRequestDurationQuery)
+	if err != nil {
+		return 0, err
+	}
+
+	var rate *float64
+	querySt := url.QueryEscape(query)
+	result, err := c.queryMetric(querySt)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, v := range result.Data.Result {
+		metricValue := v.Value[1]
+		switch metricValue.(type) {
+		case string:
+			f, err := strconv.ParseFloat(metricValue.(string), 64)
+			if err != nil {
+				return 0, err
+			}
+			rate = &f
+		}
+	}
+	if rate == nil {
+		return 0, fmt.Errorf("no values found for metric %s", metric)
+	}
+	ms := time.Duration(int64(*rate*1000)) * time.Millisecond
+	return ms, nil
+}

pkg/metrics/istio_test.go

@@ -0,0 +1,51 @@
+package metrics
+
+import (
+	"testing"
+)
+
+func Test_IstioSuccessRateQueryRender(t *testing.T) {
+	meta := struct {
+		Name      string
+		Namespace string
+		Interval  string
+	}{
+		"podinfo",
+		"default",
+		"1m",
+	}
+
+	query, err := render(meta, istioSuccessRateQuery)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := `sum(rate(istio_requests_total{reporter="destination",destination_workload_namespace="default",destination_workload=~"podinfo",response_code!~"5.*"}[1m])) / sum(rate(istio_requests_total{reporter="destination",destination_workload_namespace="default",destination_workload=~"podinfo"}[1m])) * 100`
+
+	if query != expected {
+		t.Errorf("\nGot %s \nWanted %s", query, expected)
+	}
+}
+
+func Test_IstioRequestDurationQueryRender(t *testing.T) {
+	meta := struct {
+		Name      string
+		Namespace string
+		Interval  string
+	}{
+		"podinfo",
+		"default",
+		"1m",
+	}
+
+	query, err := render(meta, istioRequestDurationQuery)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := `histogram_quantile(0.99, sum(rate(istio_request_duration_seconds_bucket{reporter="destination",destination_workload_namespace="default",destination_workload=~"podinfo"}[1m])) by (le))`
+
+	if query != expected {
+		t.Errorf("\nGot %s \nWanted %s", query, expected)
+	}
+}