From 501d860170901580850f53347e54f5d97463d3a6 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 25 Jan 2018 09:18:07 -0200 Subject: [PATCH 1/6] improved error when victorops fails --- notify/notify.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/notify/notify.go b/notify/notify.go index 930c7cd0e1..147a63f54a 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -16,6 +16,7 @@ package notify import ( "fmt" "sort" + "strings" "sync" "time" @@ -602,7 +603,11 @@ func (r RetryStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Ale numFailedNotifications.WithLabelValues(r.integration.name).Inc() level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "err", err) if !retry { - return ctx, alerts, fmt.Errorf("cancelling notify retry for %q due to unrecoverable error: %s", r.integration.name, err) + var alertnames []string + for _, al := range alerts { + alertnames = append(alertnames, al.Name()) + } + return ctx, alerts, fmt.Errorf("cancelling notify retry for %q (alerts: %s) due to unrecoverable error: %s", r.integration.name, strings.Join(alertnames, ", "), err) } // Save this error to be able to return the last seen error by an From bdf3499805834e595adef527f4a5ec6f9281e4c3 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 25 Jan 2018 09:28:33 -0200 Subject: [PATCH 2/6] moved to debug --- notify/notify.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/notify/notify.go b/notify/notify.go index 147a63f54a..58d6e12a4d 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -16,7 +16,6 @@ package notify import ( "fmt" "sort" - "strings" "sync" "time" @@ -601,13 +600,13 @@ func (r RetryStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Ale case <-tick.C: if retry, err := r.integration.Notify(ctx, alerts...); err != nil { numFailedNotifications.WithLabelValues(r.integration.name).Inc() - level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "err", err) + var alertnames []string + for _, al := range alerts { + alertnames = append(alertnames, al.Name()) + } + level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "alerts", alertnames, "err", err) if !retry { - var alertnames []string - for _, al := range alerts { - alertnames = append(alertnames, al.Name()) - } - return ctx, alerts, fmt.Errorf("cancelling notify retry for %q (alerts: %s) due to unrecoverable error: %s", r.integration.name, strings.Join(alertnames, ", "), err) + return ctx, alerts, fmt.Errorf("cancelling notify retry for %q due to unrecoverable error: %s", r.integration.name, err) } // Save this error to be able to return the last seen error by an From 3f3899c343bce4a72ec103a3ad58524da5decadc Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 25 Jan 2018 09:34:32 -0200 Subject: [PATCH 3/6] allocate mem only once --- notify/notify.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notify/notify.go b/notify/notify.go index 58d6e12a4d..53032e5118 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -600,7 +600,7 @@ func (r RetryStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Ale case <-tick.C: if retry, err := r.integration.Notify(ctx, alerts...); err != nil { numFailedNotifications.WithLabelValues(r.integration.name).Inc() - var alertnames []string + alertnames := make([]string, 0, len(alerts)) for _, al := range alerts { alertnames = append(alertnames, al.Name()) } From 6a896f6ef1dd780adce53c9a7875371cfc19bac7 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 25 Jan 2018 09:52:53 -0200 Subject: [PATCH 4/6] joining strings --- notify/notify.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/notify/notify.go b/notify/notify.go index 53032e5118..67d11f221f 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -16,6 +16,7 @@ package notify import ( "fmt" "sort" + "strings" "sync" "time" @@ -604,7 +605,7 @@ func (r RetryStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Ale for _, al := range alerts { alertnames = append(alertnames, al.Name()) } - level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "alerts", alertnames, "err", err) + level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "alerts", strings.Join(alertnames, ", "), "err", err) if !retry { return ctx, alerts, fmt.Errorf("cancelling notify retry for %q due to unrecoverable error: %s", r.integration.name, err) } From 2d2224edc75a4269c3f798654554e99cb4418b74 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 25 Jan 2018 14:25:22 -0200 Subject: [PATCH 5/6] logging receiver name --- notify/notify.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/notify/notify.go b/notify/notify.go index 67d11f221f..6bfa165b62 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -16,7 +16,6 @@ package notify import ( "fmt" "sort" - "strings" "sync" "time" @@ -239,7 +238,7 @@ func createStage(rc *config.Receiver, tmpl *template.Template, wait func() time. var s MultiStage s = append(s, NewWaitStage(wait)) s = append(s, NewDedupStage(notificationLog, recv)) - s = append(s, NewRetryStage(i)) + s = append(s, NewRetryStage(i, recv)) s = append(s, NewSetNotifiesStage(notificationLog, recv)) fs = append(fs, s) @@ -552,12 +551,14 @@ func (n *DedupStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Al // succeeds. It aborts if the context is canceled or timed out. type RetryStage struct { integration Integration + recv *nflogpb.Receiver } // NewRetryStage returns a new instance of a RetryStage. -func NewRetryStage(i Integration) *RetryStage { +func NewRetryStage(i Integration, recv *nflogpb.Receiver) *RetryStage { return &RetryStage{ integration: i, + recv: recv, } } @@ -601,11 +602,7 @@ func (r RetryStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Ale case <-tick.C: if retry, err := r.integration.Notify(ctx, alerts...); err != nil { numFailedNotifications.WithLabelValues(r.integration.name).Inc() - alertnames := make([]string, 0, len(alerts)) - for _, al := range alerts { - alertnames = append(alertnames, al.Name()) - } - level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "alerts", strings.Join(alertnames, ", "), "err", err) + level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "receiver", r.recv.GroupName, "err", err) if !retry { return ctx, alerts, fmt.Errorf("cancelling notify retry for %q due to unrecoverable error: %s", r.integration.name, err) } From 71a1e582d2b19469079c18c327cc7b3bf6b77f79 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Fri, 26 Jan 2018 11:20:05 -0200 Subject: [PATCH 6/6] passing only group name --- notify/notify.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notify/notify.go b/notify/notify.go index 6bfa165b62..dc16c760cc 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -238,7 +238,7 @@ func createStage(rc *config.Receiver, tmpl *template.Template, wait func() time. var s MultiStage s = append(s, NewWaitStage(wait)) s = append(s, NewDedupStage(notificationLog, recv)) - s = append(s, NewRetryStage(i, recv)) + s = append(s, NewRetryStage(i, rc.Name)) s = append(s, NewSetNotifiesStage(notificationLog, recv)) fs = append(fs, s) @@ -551,14 +551,14 @@ func (n *DedupStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Al // succeeds. It aborts if the context is canceled or timed out. type RetryStage struct { integration Integration - recv *nflogpb.Receiver + groupName string } // NewRetryStage returns a new instance of a RetryStage. -func NewRetryStage(i Integration, recv *nflogpb.Receiver) *RetryStage { +func NewRetryStage(i Integration, groupName string) *RetryStage { return &RetryStage{ integration: i, - recv: recv, + groupName: groupName, } } @@ -602,7 +602,7 @@ func (r RetryStage) Exec(ctx context.Context, l log.Logger, alerts ...*types.Ale case <-tick.C: if retry, err := r.integration.Notify(ctx, alerts...); err != nil { numFailedNotifications.WithLabelValues(r.integration.name).Inc() - level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "receiver", r.recv.GroupName, "err", err) + level.Debug(l).Log("msg", "Notify attempt failed", "attempt", i, "integration", r.integration.name, "receiver", r.groupName, "err", err) if !retry { return ctx, alerts, fmt.Errorf("cancelling notify retry for %q due to unrecoverable error: %s", r.integration.name, err) }