refactor: add resiliency & concurrency and optimize code (grafana#3349)

Fixes grafana#3062 Signed-off-by: Anıl Mısırlıoğlu <[email protected]> Co-authered-by: Furkan Türkal <[email protected]> Co-authered-by: Ayşe Sert <[email protected]> Signed-off-by: Anıl Mısırlıoğlu <[email protected]>
udmire · Dec 16, 2022 · bcf746f · bcf746f
1 parent cee1cbb
commit bcf746f
Show file tree

Hide file tree

Showing 8 changed files with 209 additions and 151 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -96,6 +96,8 @@
 ### Mimirtool
 
 * [ENHANCEMENT] Added `mimirtool rules delete-namespace` command to delete all of the rule groups in a namespace including the namespace itself. #3136
+* [ENHANCEMENT] Refactor `mimirtool analyze prometheus`: add concurrency and resiliency #3062
+  * Add `--concurrency` flag. Default: number of logical CPUs
 * [BUGFIX] `--log.level=debug` now correctly prints the response from the remote endpoint when a request fails. #3180
 
 ### Documentation

diff --git a/pkg/mimirtool/analyze/grafana.go b/pkg/mimirtool/analyze/grafana.go
@@ -13,14 +13,15 @@ import (
 
 	"github.com/grafana/regexp"
 	"github.com/pkg/errors"
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/promql/parser"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/grafana/mimir/pkg/mimirtool/minisdk"
 )
 
 type MetricsInGrafana struct {
-	MetricsUsed    []string            `json:"metricsUsed"`
+	MetricsUsed    model.LabelValues   `json:"metricsUsed"`
 	OverallMetrics map[string]struct{} `json:"-"`
 	Dashboards     []DashboardMetrics  `json:"dashboards"`
 }
@@ -80,7 +81,6 @@ func ParseMetricsInBoard(mig *MetricsInGrafana, board minisdk.Board) {
 		Metrics:     metricsInBoard,
 		ParseErrors: parseErrs,
 	})
-
 }
 
 func metricsFromTemplating(templating minisdk.Templating, metrics map[string]struct{}) []error {

diff --git a/pkg/mimirtool/analyze/prometheus.go b/pkg/mimirtool/analyze/prometheus.go
@@ -6,9 +6,9 @@
 package analyze
 
 type MetricsInPrometheus struct {
-	TotalActiveSeries      int `json:"total_active_series"`
-	InUseActiveSeries      int `json:"in_use_active_series"`
-	AdditionalActiveSeries int `json:"additional_active_series"`
+	TotalActiveSeries      uint64 `json:"total_active_series"`
+	InUseActiveSeries      uint64 `json:"in_use_active_series"`
+	AdditionalActiveSeries uint64 `json:"additional_active_series"`
 
 	InUseMetricCounts      []MetricCount `json:"in_use_metric_counts"`
 	AdditionalMetricCounts []MetricCount `json:"additional_metric_counts"`

diff --git a/pkg/mimirtool/analyze/ruler.go b/pkg/mimirtool/analyze/ruler.go
@@ -9,14 +9,15 @@ import (
 	"sort"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/promql/parser"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/grafana/mimir/pkg/mimirtool/rules/rwrulefmt"
 )
 
 type MetricsInRuler struct {
-	MetricsUsed    []string            `json:"metricsUsed"`
+	MetricsUsed    model.LabelValues   `json:"metricsUsed"`
 	OverallMetrics map[string]struct{} `json:"-"`
 	RuleGroups     []RuleGroupMetrics  `json:"ruleGroups"`
 }

diff --git a/pkg/mimirtool/commands/analyse.go b/pkg/mimirtool/commands/analyse.go
@@ -6,11 +6,13 @@
 package commands
 
 import (
+	"runtime"
+	"strconv"
+
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-type AnalyzeCommand struct {
-}
+type AnalyzeCommand struct{}
 
 func (cmd *AnalyzeCommand) Register(app *kingpin.Application, envVars EnvVarNames) {
 	analyzeCmd := app.Command("analyze", "Run analysis against your Prometheus, Grafana, and Grafana Mimir to see which metrics are being used and exported.")
@@ -38,6 +40,9 @@ func (cmd *AnalyzeCommand) Register(app *kingpin.Application, envVars EnvVarName
 	prometheusAnalyzeCmd.Flag("ruler-metrics-file", "The path for the input file containing the metrics from ruler-analyze command").
 		Default("metrics-in-ruler.json").
 		StringVar(&paCmd.rulerMetricsFile)
+	prometheusAnalyzeCmd.Flag("concurrency", "Concurrency (Default: runtime.NumCPU())").
+		Default(strconv.Itoa(runtime.NumCPU())).
+		IntVar(&paCmd.concurrency)
 	prometheusAnalyzeCmd.Flag("output", "The path for the output file").
 		Default("prometheus-metrics.json").
 		StringVar(&paCmd.outputFile)

diff --git a/pkg/mimirtool/commands/analyse_grafana.go b/pkg/mimirtool/commands/analyse_grafana.go
@@ -13,6 +13,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/grafana-tools/sdk"
 	"gopkg.in/alecthomas/kingpin.v2"
 
@@ -77,19 +79,19 @@ func unmarshalDashboard(data []byte, link sdk.FoundBoard) (minisdk.Board, error)
 }
 
 func writeOut(mig *analyze.MetricsInGrafana, outputFile string) error {
-	var metricsUsed []string
+	var metricsUsed model.LabelValues
 	for metric := range mig.OverallMetrics {
-		metricsUsed = append(metricsUsed, metric)
+		metricsUsed = append(metricsUsed, model.LabelValue(metric))
 	}
-	sort.Strings(metricsUsed)
+	sort.Sort(metricsUsed)
 
 	mig.MetricsUsed = metricsUsed
 	out, err := json.MarshalIndent(mig, "", "  ")
 	if err != nil {
 		return err
 	}
 
-	if err := os.WriteFile(outputFile, out, os.FileMode(int(0666))); err != nil {
+	if err := os.WriteFile(outputFile, out, os.FileMode(int(0o666))); err != nil {
 		return err
 	}