From 6a2751b42956d16988c00ea6db8d822611a8da58 Mon Sep 17 00:00:00 2001 From: Matthias Loibl Date: Tue, 30 Jun 2020 14:32:14 +0200 Subject: [PATCH] Update ThanosReceiveNoUpload to select sum == 0 Right now the alert actually fires in wrong conditions. We select all sum(instances == 0) which is always 0... Instead we want to sum(instances) == 0 Signed-off-by: Matthias Loibl --- examples/alerts/alerts.md | 2 +- examples/alerts/alerts.yaml | 2 +- mixin/alerts/receive.libsonnet | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index 8f025dd5d6..3e3ffa0ddd 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -462,7 +462,7 @@ rules: expr: | (up{job=~"thanos-receive.*"} - 1) + on (instance) # filters to only alert on current instance last 2h - sum by (instance) (increase(thanos_shipper_uploads_total{job=~"thanos-receive.*"}[2h]) == 0) + (sum by (instance) (increase(thanos_shipper_uploads_total{job=~"thanos-receive.*"}[2h])) == 0) for: 2h labels: severity: critical diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index 990677ac8c..94f1af004c 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -224,7 +224,7 @@ groups: expr: | (up{job=~"thanos-receive.*"} - 1) + on (instance) # filters to only alert on current instance last 2h - sum by (instance) (increase(thanos_shipper_uploads_total{job=~"thanos-receive.*"}[2h]) == 0) + (sum by (instance) (increase(thanos_shipper_uploads_total{job=~"thanos-receive.*"}[2h])) == 0) for: 2h labels: severity: critical diff --git a/mixin/alerts/receive.libsonnet b/mixin/alerts/receive.libsonnet index de5d6cce33..115a3fb930 100644 --- a/mixin/alerts/receive.libsonnet +++ b/mixin/alerts/receive.libsonnet @@ -107,7 +107,7 @@ expr: ||| (up{%(selector)s} - 1) + on (instance) # filters to only alert on current instance last 2h - sum by (instance) (increase(thanos_shipper_uploads_total{%(selector)s}[2h]) == 0) + (sum by (instance) (increase(thanos_shipper_uploads_total{%(selector)s}[2h])) == 0) ||| % thanos.receive, 'for': '2h', labels: {