fix: critical slashing alert (#159)

* fix: critical slashing alert * fix: remove useless condition
lidofinance · Apr 14, 2023 · 006ec38 · 006ec38
1 parent c6d458a
commit 006ec38
Show file tree

Hide file tree

Showing 8 changed files with 17 additions and 106 deletions.
diff --git a/docker/prometheus/alerts_rules.test b/docker/prometheus/alerts_rules.test
@@ -2,7 +2,6 @@
 evaluation_interval: 1m
 rule_files:
   - alerts_rules.yml
-  - fires.yml
 tests:
   - interval: 10s
     input_series:
@@ -143,9 +142,7 @@ tests:
               field_name: "Operator 1"
               field_value: "[1](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var=Operator+1&from=-660000.000000&to=540000.000000)"
               url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-              footer_text: |-
-                Operator 1 • 1 fires in last 1h
-                Epoch • 1
+              footer_text: "Epoch • 1"
               footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
       - alertname: NumValidatorsWithNegativeDelta
         eval_time: 17m
@@ -162,9 +159,7 @@ tests:
               field_name: "Operator 1"
               field_value: "[2](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var=Operator+1&from=-180000.000000&to=1020000.000000)"
               url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-              footer_text: |-
-                Operator 1 • 2 fires in last 1h
-                Epoch • 2
+              footer_text: "Epoch • 2"
               footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
       - alertname: NumValidatorsWithNegativeDelta
         eval_time: 25m

diff --git a/docker/prometheus/alerts_rules.yml b/docker/prometheus/alerts_rules.yml
@@ -3,7 +3,7 @@ groups:
     rules:
 
       - alert: UserSlashedValidators
-        expr: delta(ethereum_validators_monitoring_user_validators{status="slashed"}[1m]) > 0 AND ON(nos_name) ethereum_validators_monitoring_user_validators{status="slashed"} > 0
+        expr: (ethereum_validators_monitoring_user_validators{status="slashed"} - ethereum_validators_monitoring_user_validators{status="slashed"} offset 1m) > 0
         labels:
           severity: critical
         annotations:
@@ -44,13 +44,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_validator_count_with_negative_balances_delta_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_validator_count_with_negative_balances_delta" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: NumValidatorsWithMissAttestationLastNEpoch
@@ -66,13 +60,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_validator_count_miss_attestation_last_n_epoch_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_validator_count_miss_attestation_last_n_epoch" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: NumValidatorsWithHighIncDelayAttestationLastNEpoch
@@ -86,13 +74,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_validator_count_high_inc_delay_last_n_epoch_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_validator_count_high_inc_delay_last_n_epoch" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: NumValidatorsWithInvalidPropertyAttestationLastNEpoch
@@ -106,13 +88,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_validator_count_invalid_attestation_property_last_n_epoch_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_validator_count_invalid_attestation_property_last_n_epoch" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: NumValidatorsWithMissPropose
@@ -128,13 +104,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_validator_count_miss_propose_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_validator_count_miss_propose" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: NumValidatorsWithSyncParticipationLessAvgLastNEpoch
@@ -150,13 +120,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_validator_count_with_sync_participation_less_avg_last_n_epoch_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_validator_count_with_sync_participation_less_avg_last_n_epoch" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: HighRewardNumValidatorsWithSyncParticipationLessAvgLastNEpoch
@@ -172,13 +136,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_high_reward_validator_count_with_sync_participation_less_avg_last_n_epoch_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_high_reward_validator_count_with_sync_participation_less_avg_last_n_epoch" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: HighRewardNumValidatorsWithMissAttestationLastNEpoch
@@ -194,13 +152,7 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_high_reward_validator_count_miss_attestation_last_n_epoch_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_high_reward_validator_count_miss_attestation_last_n_epoch" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: HighRewardNumValidatorsWithMissPropose
@@ -216,11 +168,5 @@ groups:
           field_name: '{{ $labels.nos_name }}'
           field_value: '[{{ $value | printf "%.0f" }}](http://127.0.0.1:8082/d/3wimU2H7h/nodeoperators/?var-nos_name_var={{ urlquery $labels.nos_name }}&from={{ with query "(time() - 1200) * 1000" }}{{ . | first | value | printf "%f" }}{{ end }}&to={{ with query "time() * 1000" }}{{ . | first | value | printf "%f" }}{{ end }})'
           url: "http://127.0.0.1:8082/d/HRgPmpNnz/validators"
-          footer_text: 'Fires (last 1h):
-          {{- "\n" -}}
-          {{- range query "sum_over_time(sgn(ethereum_validators_monitoring_high_reward_validator_count_miss_propose_alert)[1h:] offset -25s) AND ethereum_validators_monitoring_high_reward_validator_count_miss_propose" -}}
-            {{- .Labels.nos_name }} - {{ .Value }}
-            {{- "\n" -}}
-          {{- end -}}
-          Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
+          footer_text: 'Epoch • {{ with query "ethereum_validators_monitoring_epoch_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
           footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
diff --git a/docker/prometheus/fires.yml b/docker/prometheus/fires.yml
diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml
@@ -11,7 +11,6 @@ alerting:
 
 rule_files:
   - "alerts_rules.yml"
-  - "fires.yml"
 
 scrape_configs:
   - job_name: 'cadvisor'

diff --git a/src/common/alertmanager/alerts/CriticalMissedAttestations.ts b/src/common/alertmanager/alerts/CriticalMissedAttestations.ts
@@ -52,7 +52,7 @@ export class CriticalMissedAttestations extends Alert {
   alertBody(ruleResult: AlertRuleResult): AlertRequestBody {
     return {
       startsAt: new Date(this.sendTimestamp).toISOString(),
-      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 1)).toISOString(),
+      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 2)).toISOString(),
       labels: {
         alertname: this.alertname,
         severity: 'critical',

diff --git a/src/common/alertmanager/alerts/CriticalMissedProposes.ts b/src/common/alertmanager/alerts/CriticalMissedProposes.ts
@@ -49,7 +49,7 @@ export class CriticalMissedProposes extends Alert {
   alertBody(ruleResult: AlertRuleResult): AlertRequestBody {
     return {
       startsAt: new Date(this.sendTimestamp).toISOString(),
-      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 1)).toISOString(),
+      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 2)).toISOString(),
       labels: { alertname: this.alertname, severity: 'critical', ...this.config.get('CRITICAL_ALERTS_ALERTMANAGER_LABELS') },
       annotations: {
         summary: `${Object.values(ruleResult).length} Node Operators with CRITICAL count of missed proposes in the last 12 hours`,

diff --git a/src/common/alertmanager/alerts/CriticalNegativeDelta.ts b/src/common/alertmanager/alerts/CriticalNegativeDelta.ts
@@ -52,7 +52,7 @@ export class CriticalNegativeDelta extends Alert {
   alertBody(ruleResult: AlertRuleResult): AlertRequestBody {
     return {
       startsAt: new Date(this.sendTimestamp).toISOString(),
-      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 1)).toISOString(),
+      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 2)).toISOString(),
       labels: { alertname: this.alertname, severity: 'critical', ...this.config.get('CRITICAL_ALERTS_ALERTMANAGER_LABELS') },
       annotations: {
         summary: `${Object.values(ruleResult).length} Node Operators with CRITICAL count of validators with negative delta`,

diff --git a/src/common/alertmanager/alerts/CriticalSlashing.ts b/src/common/alertmanager/alerts/CriticalSlashing.ts
@@ -29,13 +29,14 @@ export class CriticalSlashing extends Alert {
   }
 
   sendRule(ruleResult: AlertRuleResult): boolean {
+    this.sendTimestamp = Date.now();
     return !!Object.values(ruleResult).length;
   }
 
   alertBody(ruleResult: AlertRuleResult): AlertRequestBody {
     return {
       startsAt: new Date(this.sendTimestamp).toISOString(),
-      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 1)).toISOString(),
+      endsAt: new Date(new Date(this.sendTimestamp).setMinutes(new Date(this.sendTimestamp).getMinutes() + 2)).toISOString(),
       labels: { alertname: this.alertname, severity: 'critical', ...this.config.get('CRITICAL_ALERTS_ALERTMANAGER_LABELS') },
       annotations: {
         summary: `${Object.values(ruleResult).length} Node Operators with SLASHED validators`,