Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: mimirtool analyze prometheus: add concurrency and resiliency #3349

Merged
merged 1 commit into from
Nov 9, 2022
+209 −151
Merged

refactor: mimirtool analyze prometheus: add concurrency and resiliency #3349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@
### Mimirtool

* [ENHANCEMENT] Added `mimirtool rules delete-namespace` command to delete all of the rule groups in a namespace including the namespace itself. #3136
* [ENHANCEMENT] Refactor `mimirtool analyze prometheus`: add concurrency and resiliency #3062
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're used to reference the PR number instead of the issue number (so #3349 instead of #3062). This will make our release tooling matching the PR when checking the CHANGELOG.

@anilmisirlioglu would you mind opening a follow up PR to change it, please?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure

* Add `--concurrency` flag. Default: number of logical CPUs
* [BUGFIX] `--log.level=debug` now correctly prints the response from the remote endpoint when a request fails. #3180

### Documentation
Expand Down
4 changes: 2 additions & 2 deletions pkg/mimirtool/analyze/grafana.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,15 @@ import (

"github.com/grafana/regexp"
"github.com/pkg/errors"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/promql/parser"
log "github.com/sirupsen/logrus"

"github.com/grafana/mimir/pkg/mimirtool/minisdk"
)

type MetricsInGrafana struct {
MetricsUsed []string `json:"metricsUsed"`
MetricsUsed model.LabelValues `json:"metricsUsed"`
OverallMetrics map[string]struct{} `json:"-"`
Dashboards []DashboardMetrics `json:"dashboards"`
}
Expand Down Expand Up @@ -80,7 +81,6 @@ func ParseMetricsInBoard(mig *MetricsInGrafana, board minisdk.Board) {
Metrics: metricsInBoard,
ParseErrors: parseErrs,
})

}

func metricsFromTemplating(templating minisdk.Templating, metrics map[string]struct{}) []error {
Expand Down
6 changes: 3 additions & 3 deletions pkg/mimirtool/analyze/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
package analyze

type MetricsInPrometheus struct {
TotalActiveSeries int `json:"total_active_series"`
InUseActiveSeries int `json:"in_use_active_series"`
AdditionalActiveSeries int `json:"additional_active_series"`
TotalActiveSeries uint64 `json:"total_active_series"`
InUseActiveSeries uint64 `json:"in_use_active_series"`
AdditionalActiveSeries uint64 `json:"additional_active_series"`

InUseMetricCounts []MetricCount `json:"in_use_metric_counts"`
AdditionalMetricCounts []MetricCount `json:"additional_metric_counts"`
Expand Down
3 changes: 2 additions & 1 deletion pkg/mimirtool/analyze/ruler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ import (
"sort"

"github.com/pkg/errors"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/promql/parser"
log "github.com/sirupsen/logrus"

"github.com/grafana/mimir/pkg/mimirtool/rules/rwrulefmt"
)

type MetricsInRuler struct {
MetricsUsed []string `json:"metricsUsed"`
MetricsUsed model.LabelValues `json:"metricsUsed"`
OverallMetrics map[string]struct{} `json:"-"`
RuleGroups []RuleGroupMetrics `json:"ruleGroups"`
}
Expand Down
9 changes: 7 additions & 2 deletions pkg/mimirtool/commands/analyse.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
package commands

import (
"runtime"
"strconv"

"gopkg.in/alecthomas/kingpin.v2"
)

type AnalyzeCommand struct {
}
type AnalyzeCommand struct{}

func (cmd *AnalyzeCommand) Register(app *kingpin.Application, envVars EnvVarNames) {
analyzeCmd := app.Command("analyze", "Run analysis against your Prometheus, Grafana, and Grafana Mimir to see which metrics are being used and exported.")
Expand Down Expand Up @@ -38,6 +40,9 @@ func (cmd *AnalyzeCommand) Register(app *kingpin.Application, envVars EnvVarName
prometheusAnalyzeCmd.Flag("ruler-metrics-file", "The path for the input file containing the metrics from ruler-analyze command").
Default("metrics-in-ruler.json").
StringVar(&paCmd.rulerMetricsFile)
prometheusAnalyzeCmd.Flag("concurrency", "Concurrency (Default: runtime.NumCPU())").
Default(strconv.Itoa(runtime.NumCPU())).
IntVar(&paCmd.concurrency)
prometheusAnalyzeCmd.Flag("output", "The path for the output file").
Default("prometheus-metrics.json").
StringVar(&paCmd.outputFile)
Expand Down
10 changes: 6 additions & 4 deletions pkg/mimirtool/commands/analyse_grafana.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import (
"sort"
"time"

"github.com/prometheus/common/model"

"github.com/grafana-tools/sdk"
"gopkg.in/alecthomas/kingpin.v2"

Expand Down Expand Up @@ -77,19 +79,19 @@ func unmarshalDashboard(data []byte, link sdk.FoundBoard) (minisdk.Board, error)
}

func writeOut(mig *analyze.MetricsInGrafana, outputFile string) error {
var metricsUsed []string
var metricsUsed model.LabelValues
for metric := range mig.OverallMetrics {
metricsUsed = append(metricsUsed, metric)
metricsUsed = append(metricsUsed, model.LabelValue(metric))
}
sort.Strings(metricsUsed)
sort.Sort(metricsUsed)

mig.MetricsUsed = metricsUsed
out, err := json.MarshalIndent(mig, "", " ")
if err != nil {
return err
}

if err := os.WriteFile(outputFile, out, os.FileMode(int(0666))); err != nil {
if err := os.WriteFile(outputFile, out, os.FileMode(int(0o666))); err != nil {
return err
}

Expand Down
Loading