feat: set up the NarrowSelectorsOnIntegers alert for a smooth Prom3 u…

…pgrade
openshift · Jan 10, 2025 · 883058b · 883058b
1 parent 0264620
commit 883058b
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 0 deletions.
diff --git a/assets/cluster-monitoring-operator/prometheus-rule.yaml b/assets/cluster-monitoring-operator/prometheus-rule.yaml
@@ -30,6 +30,16 @@ spec:
       for: 15m
       labels:
         severity: warning
+    - alert: NarrowSelectorsOnIntegers
+      annotations:
+        description: Queries to Prometheus {{$labels.namespace}}/{{$labels.pod}} could be too restrictive.
+        summary: |
+          Some queries with selectors on the values of the "le" label of classic histograms or/and the "quantile" label of summaries
+          may not take into account that values could also be floats.
+      expr: increase(prometheus_parser_narrow_selectors_on_integers_count{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
   - name: openshift-kubernetes.rules
     rules:
     - expr: sum(rate(container_cpu_usage_seconds_total{container="",pod!=""}[5m])) BY (pod, namespace)

diff --git a/jsonnet/rules.libsonnet b/jsonnet/rules.libsonnet
@@ -27,6 +27,21 @@ function(params) {
             severity: 'warning',
           },
         },
+        {
+          expr: 'increase(prometheus_parser_narrow_selectors_on_integers_count{job=~"prometheus-k8s|prometheus-user-workload"}[5m]) > 0',
+          alert: 'NarrowSelectorsOnIntegers',
+          'for': '15m',
+          annotations: {
+            description: 'Queries to Prometheus {{$labels.namespace}}/{{$labels.pod}} could be too restrictive.',
+            summary: |||
+              Some queries with selectors on the values of the "le" label of classic histograms or/and the "quantile" label of summaries
+              may not take into account that values could also be floats.
+            |||,
+          },
+          labels: {
+            severity: 'warning',
+          },
+        },
       ],
     },
     {

diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go
@@ -1332,6 +1332,8 @@ func (f *Factory) PrometheusK8s(grpcTLS *v1.Secret, telemetrySecret *v1.Secret)
 	if f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.LogLevel != "" {
 		p.Spec.LogLevel = f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.LogLevel
 	}
+	// Temp, debugging only
+	p.Spec.LogLevel = "debug"
 
 	if f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.Retention != "" {
 		p.Spec.Retention = monv1.Duration(f.config.ClusterMonitoringConfiguration.PrometheusK8sConfig.Retention)