Merge branch 'bugfix/3474-lower-disk-alert-thresholds' into q/2.10

eve/main.yml

@@ -148,7 +148,7 @@ models:
         elif [ "$DEBUG_STEPS" = all ]; then
           RUN_STEP=1
           REASON="\"debug\" property set to \"all\""
-        elif [[ "$DEBUG_STEPS" =~ ^[a-z\-]+(~[a-z\-]+)*$ ]]; then
+        elif [[ "$DEBUG_STEPS" =~ ^[a-z0-9\-]+(~[a-z0-9\-]+)*$ ]]; then
           IFS="~" read -ra SELECTED <<< "$DEBUG_STEPS"
           for selected in "${SELECTED[@]}"; do
             if [ "$selected" = "$STEP_NAME" ]; then

salt/metalk8s/addons/prometheus-operator/config/prometheus.yaml

@@ -22,9 +22,9 @@ spec:
           threshold: 20
       node_filesystem_almost_out_of_space:
         warning:
-          available: 5  # Percentage of free space left
+          available: 20  # Percentage of free space left
         critical:
-          available: 3
+          available: 12
       node_filesystem_files_filling_up:
         warning:
           hours: 24  # Hours before there is no inode left
@@ -34,9 +34,9 @@ spec:
           threshold: 20
       node_filesystem_almost_out_of_files:
         warning:
-          available: 5  # Percentage of free inodes left
+          available: 15  # Percentage of free inodes left
         critical:
-          available: 3
+          available: 8
       node_network_receive_errors:
         warning:
           error_rate: 0.01  # Rate of receive errors for the last 2m

tools/rule_extractor/alerting_rules.csv

@@ -80,12 +80,12 @@ KubeVersionMismatch,warning,Different semantic versions of Kubernetes components
 KubeClientErrors,warning,Kubernetes API server client is experiencing errors.
 NodeFilesystemSpaceFillingUp,warning,Filesystem is predicted to run out of space within the next 24 hours.
 NodeFilesystemSpaceFillingUp,critical,Filesystem is predicted to run out of space within the next 4 hours.
-NodeFilesystemAlmostOutOfSpace,warning,Filesystem has less than 5% space left.
-NodeFilesystemAlmostOutOfSpace,critical,Filesystem has less than 3% space left.
+NodeFilesystemAlmostOutOfSpace,warning,Filesystem has less than 20% space left.
+NodeFilesystemAlmostOutOfSpace,critical,Filesystem has less than 12% space left.
 NodeFilesystemFilesFillingUp,warning,Filesystem is predicted to run out of inodes within the next 24 hours.
 NodeFilesystemFilesFillingUp,critical,Filesystem is predicted to run out of inodes within the next 4 hours.
-NodeFilesystemAlmostOutOfFiles,warning,Filesystem has less than 5% inodes left.
-NodeFilesystemAlmostOutOfFiles,critical,Filesystem has less than 3% inodes left.
+NodeFilesystemAlmostOutOfFiles,warning,Filesystem has less than 15% inodes left.
+NodeFilesystemAlmostOutOfFiles,critical,Filesystem has less than 8% inodes left.
 NodeNetworkReceiveErrs,warning,Network interface is reporting many receive errors.
 NodeNetworkTransmitErrs,warning,Network interface is reporting many transmit errors.
 NodeHighNumberConntrackEntriesUsed,warning,Number of conntrack are getting close to the limit

tools/rule_extractor/alerting_rules.json

@@ -492,15 +492,15 @@
         "severity": "critical"
     },
     {
-        "message": "Filesystem has less than 5% space left.",
+        "message": "Filesystem has less than 20% space left.",
         "name": "NodeFilesystemAlmostOutOfSpace",
-        "query": "(node_filesystem_avail_bytes{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"node-exporter\"} * 100 < 5 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
+        "query": "(node_filesystem_avail_bytes{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"node-exporter\"} * 100 < 20 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
         "severity": "warning"
     },
     {
-        "message": "Filesystem has less than 3% space left.",
+        "message": "Filesystem has less than 12% space left.",
         "name": "NodeFilesystemAlmostOutOfSpace",
-        "query": "(node_filesystem_avail_bytes{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"node-exporter\"} * 100 < 3 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
+        "query": "(node_filesystem_avail_bytes{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_size_bytes{fstype!=\"\",job=\"node-exporter\"} * 100 < 12 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
         "severity": "critical"
     },
     {
@@ -516,15 +516,15 @@
         "severity": "critical"
     },
     {
-        "message": "Filesystem has less than 5% inodes left.",
+        "message": "Filesystem has less than 15% inodes left.",
         "name": "NodeFilesystemAlmostOutOfFiles",
-        "query": "(node_filesystem_files_free{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_files{fstype!=\"\",job=\"node-exporter\"} * 100 < 5 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
+        "query": "(node_filesystem_files_free{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_files{fstype!=\"\",job=\"node-exporter\"} * 100 < 15 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
         "severity": "warning"
     },
     {
-        "message": "Filesystem has less than 3% inodes left.",
+        "message": "Filesystem has less than 8% inodes left.",
         "name": "NodeFilesystemAlmostOutOfFiles",
-        "query": "(node_filesystem_files_free{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_files{fstype!=\"\",job=\"node-exporter\"} * 100 < 3 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
+        "query": "(node_filesystem_files_free{fstype!=\"\",job=\"node-exporter\"} / node_filesystem_files{fstype!=\"\",job=\"node-exporter\"} * 100 < 8 and node_filesystem_readonly{fstype!=\"\",job=\"node-exporter\"} == 0)",
         "severity": "critical"
     },
     {