mixin: Exclude cache "add" operations from alerting

Exclude alerts from firing about cache "add" operations failing since this is expected during normal operation. Related #9386 Signed-off-by: Nick Pillitteri <[email protected]>
grafana · Oct 17, 2024 · b5e02a6 · b5e02a6
1 parent 9d8252f
commit b5e02a6
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 12 deletions.
diff --git a/operations/mimir-mixin-compiled-baremetal/alerts.yaml b/operations/mimir-mixin-compiled-baremetal/alerts.yaml
@@ -107,15 +107,15 @@ groups:
           expr: |
             (
               sum by(cluster, namespace, name, operation) (
-                rate(thanos_memcached_operation_failures_total[1m])
+                rate(thanos_memcached_operation_failures_total{operation!="add"}[1m])
                 or
-                rate(thanos_cache_operation_failures_total[1m])
+                rate(thanos_cache_operation_failures_total{operation!="add"}[1m])
               )
               /
               sum by(cluster, namespace, name, operation) (
-                rate(thanos_memcached_operations_total[1m])
+                rate(thanos_memcached_operations_total{operation!="add"}[1m])
                 or
-                rate(thanos_cache_operations_total[1m])
+                rate(thanos_cache_operations_total{operation!="add"}[1m])
               )
             ) * 100 > 5
           for: 5m

diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml
@@ -107,15 +107,15 @@ groups:
           expr: |
             (
               sum by(cluster, namespace, name, operation) (
-                rate(thanos_memcached_operation_failures_total[1m])
+                rate(thanos_memcached_operation_failures_total{operation!="add"}[1m])
                 or
-                rate(thanos_cache_operation_failures_total[1m])
+                rate(thanos_cache_operation_failures_total{operation!="add"}[1m])
               )
               /
               sum by(cluster, namespace, name, operation) (
-                rate(thanos_memcached_operations_total[1m])
+                rate(thanos_memcached_operations_total{operation!="add"}[1m])
                 or
-                rate(thanos_cache_operations_total[1m])
+                rate(thanos_cache_operations_total{operation!="add"}[1m])
               )
             ) * 100 > 5
           for: 5m

diff --git a/operations/mimir-mixin/alerts/alerts.libsonnet b/operations/mimir-mixin/alerts/alerts.libsonnet
@@ -202,18 +202,20 @@ local utils = import 'mixin-utils/utils.libsonnet';
         },
         {
           alert: $.alertName('CacheRequestErrors'),
+          // Specifically exclude "add" operations which are used for cache invalidation and "locking" since
+          // they are expected to sometimes fail in normal operation (such as when a "lock" alread exists).
           expr: |||
             (
               sum by(%(group_by)s, name, operation) (
-                rate(thanos_memcached_operation_failures_total[%(range_interval)s])
+                rate(thanos_memcached_operation_failures_total{operation!="add"}[%(range_interval)s])
                 or
-                rate(thanos_cache_operation_failures_total[%(range_interval)s])
+                rate(thanos_cache_operation_failures_total{operation!="add"}[%(range_interval)s])
               )
               /
               sum by(%(group_by)s, name, operation) (
-                rate(thanos_memcached_operations_total[%(range_interval)s])
+                rate(thanos_memcached_operations_total{operation!="add"}[%(range_interval)s])
                 or
-                rate(thanos_cache_operations_total[%(range_interval)s])
+                rate(thanos_cache_operations_total{operation!="add"}[%(range_interval)s])
               )
             ) * 100 > 5
           ||| % {