fix(mixins): disk space utilization panels (#8212)

* fix(mixins): disk space utilization panels * add changelog entry Signed-off-by: QuentinBisson <[email protected]> --------- Signed-off-by: QuentinBisson <[email protected]>
grafana · Jun 5, 2024 · fc7ef9f · fc7ef9f
1 parent 1b66a4c
commit fc7ef9f
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 25 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -108,6 +108,7 @@
 * [BUGFIX] Dashboards: fix regular expression for matching read-path gRPC ingester methods to include querying of exemplars, label-related queries, or active series queries. #7676
 * [BUGFIX] Dashboards: fix user id abbreviations and column heads for Top Tenants dashboard. #7724
 * [BUGFIX] Dashboards: fix incorrect query used for "queue length" panel on "Ruler" dashboard. #8006
+* [BUGFIX] Dashboards: fix disk space utilization panels when running with a recent version of kube-state-metrics. #8212
 
 ### Jsonnet
 

diff --git a/...oring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml b/...oring-values-generated/mimir-distributed/templates/metamonitoring/grafana-dashboards.yaml
@@ -590,7 +590,7 @@ data:
                       "span": 12,
                       "targets": [
                          {
-                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(alertmanager).*\"\n  }\n)\n",
+                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(alertmanager).*\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(alertmanager).*\"}\n)\n",
                             "format": "time_series",
                             "legendFormat": "{{persistentvolumeclaim}}",
                             "legendLink": null
@@ -3874,7 +3874,7 @@ data:
                       "span": 4,
                       "targets": [
                          {
-                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(compactor).*\"\n  }\n)\n",
+                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(compactor).*\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(compactor).*\"}\n)\n",
                             "format": "time_series",
                             "legendFormat": "{{persistentvolumeclaim}}",
                             "legendLink": null
@@ -8753,7 +8753,7 @@ data:
                       "span": 4,
                       "targets": [
                          {
-                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(distributor|ingester|mimir-write).*\"\n  }\n)\n",
+                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(distributor|ingester|mimir-write).*\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(distributor|ingester|mimir-write).*\"}\n)\n",
                             "format": "time_series",
                             "legendFormat": "{{persistentvolumeclaim}}",
                             "legendLink": null
@@ -9224,7 +9224,7 @@ data:
                       "span": 4,
                       "targets": [
                          {
-                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"\n  }\n)\n",
+                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}\n)\n",
                             "format": "time_series",
                             "legendFormat": "{{persistentvolumeclaim}}",
                             "legendLink": null
@@ -17431,7 +17431,7 @@ data:
                       "span": 4,
                       "targets": [
                          {
-                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(store-gateway).*\"\n  }\n)\n",
+                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(store-gateway).*\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(store-gateway).*\"}\n)\n",
                             "format": "time_series",
                             "legendFormat": "{{persistentvolumeclaim}}",
                             "legendLink": null
@@ -38373,7 +38373,7 @@ data:
                       "span": 4,
                       "targets": [
                          {
-                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(ingester).*\"\n  }\n)\n",
+                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(ingester).*\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", persistentvolumeclaim=~\".*(ingester).*\"}\n)\n",
                             "format": "time_series",
                             "legendFormat": "{{persistentvolumeclaim}}",
                             "legendLink": null

diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager-resources.json
diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-compactor-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-compactor-resources.json
diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-overview-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-overview-resources.json
diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-reads-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-reads-resources.json
diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-writes-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-writes-resources.json
diff --git a/operations/mimir-mixin/config.libsonnet b/operations/mimir-mixin/config.libsonnet
@@ -534,15 +534,8 @@
         disk_utilization:
           |||
             max by(persistentvolumeclaim) (
-              kubelet_volume_stats_used_bytes{%(namespaceMatcher)s} /
-              kubelet_volume_stats_capacity_bytes{%(namespaceMatcher)s}
-            )
-            and
-            count by(persistentvolumeclaim) (
-              kube_persistentvolumeclaim_labels{
-                %(namespaceMatcher)s,
-                %(containerMatcher)s
-              }
+              kubelet_volume_stats_used_bytes{%(namespaceMatcher)s, %(persistentVolumeClaimMatcher)s} /
+              kubelet_volume_stats_capacity_bytes{%(namespaceMatcher)s, %(persistentVolumeClaimMatcher)s}
             )
           |||,
       },

diff --git a/operations/mimir-mixin/dashboards/dashboard-utils.libsonnet b/operations/mimir-mixin/dashboards/dashboard-utils.libsonnet
@@ -493,7 +493,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     $.queryPanel(
       $._config.resources_panel_queries[$._config.deployment_type].disk_utilization % {
         namespaceMatcher: $.namespaceMatcher(),
-        containerMatcher: $.containerLabelNameMatcher(containerName),
+        persistentVolumeClaimMatcher: $.containerPersistentVolumeClaimMatcher(containerName),
         instanceLabel: $._config.per_instance_label,
         instanceName: instanceName,
         instanceDataDir: $._config.instance_data_mountpoint,
@@ -513,9 +513,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
     $.containerDiskSpaceUtilizationPanel($._config.instance_names[componentName], $._config.container_names[componentName]),
 
   // The provided containerName should be a regexp from $._config.container_names.
-  containerLabelNameMatcher(containerName)::
-    // Check only the prefix so that a multi-zone deployment matches too.
-    'label_name=~"(%s).*"' % containerName,
+  containerPersistentVolumeClaimMatcher(containerName)::
+    'persistentvolumeclaim=~".*(%s).*"' % containerName,
 
   // The provided componentName should be the name of a component among the ones defined in $._config.instance_names.
   containerNetworkingRowByComponent(title, componentName)::