From b46d756c0ecd03289eb1bdc5edeb4bbfb13a836c Mon Sep 17 00:00:00 2001
From: Christoph Kleineweber <c.kleineweber@sap.com>
Date: Thu, 8 Dec 2022 14:51:38 +0100
Subject: [PATCH 1/2] Add instructions to set up alerting channels to
 Prometheus example

---
 prometheus/README.md                |  7 ++++
 prometheus/alertmanager-values.yaml | 52 +++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 prometheus/alertmanager-values.yaml

diff --git a/prometheus/README.md b/prometheus/README.md
index 97ebb003..5e90c93e 100644
--- a/prometheus/README.md
+++ b/prometheus/README.md
@@ -60,6 +60,13 @@ The provided `values.yaml` covers the following adjustments:
 - Client certificate injection to support scraping of workload secured with Istio strict mTLS
 - Active scraping of workload annotated with prometheus.io/scrape
 
+3. You can connect the Alertmanager to your notification channel (for instance, Slack or VictorOps) by providing an [Alertmanager configuration](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) to the `alertmanager.config` value.
+
+   The [alertmanager-values.yaml](./alertmanager-values.yaml) example provides a configuration that sends notifications for alerts with high severity to a Slack channel. To deploy it, download the file, adapt `<channel-name>`, `<api-url>` and `<cluster-domain>` to your environment, and run the Helm upgrade command to deploy the configuration:
+   ```bash
+    helm upgrade --install -n ${KYMA_NS} ${HELM_RELEASE} prometheus-community/kube-prometheus-stack -f https://raw.githubusercontent.com/kyma-project/examples/main/prometheus/values.yaml -f ./alertmanager-values.yaml --set grafana.adminPassword=myPwd
+   ```
+
 ### Activate scraping of Istio metrics & Grafana dashboards
 
 1. To configure Prometheus for scraping of the Istio-specific metrics from any istio-proxy running in the cluster, deploy a PodMonitor, which scrapes any Pod that has a port with name `.*-envoy-prom` exposed.
diff --git a/prometheus/alertmanager-values.yaml b/prometheus/alertmanager-values.yaml
new file mode 100644
index 00000000..3e303637
--- /dev/null
+++ b/prometheus/alertmanager-values.yaml
@@ -0,0 +1,52 @@
+alertmanager:
+  config:
+   global:
+     resolve_timeout: 5m
+   inhibit_rules:
+     - source_matchers:
+         - 'severity = critical'
+       target_matchers:
+         - 'severity =~ warning|info'
+       equal:
+         - 'namespace'
+         - 'alertname'
+     - source_matchers:
+         - 'severity = warning'
+       target_matchers:
+         - 'severity = info'
+       equal:
+         - 'namespace'
+         - 'alertname'
+     - source_matchers:
+         - 'alertname = InfoInhibitor'
+       target_matchers:
+         - 'severity = info'
+       equal:
+         - 'namespace'
+   route:
+     group_by: ['namespace']
+     group_wait: 30s
+     group_interval: 5m
+     repeat_interval: 12h
+     receiver: 'null'
+     routes:
+     - receiver: 'null'
+       matchers:
+         - alertname =~ "InfoInhibitor|Watchdog"
+     - continue: true
+       match_re:
+         severity: critical
+       receiver: "slack"
+   receivers:
+   - name: 'null'
+   - name: "slack"
+     slack_configs:
+     - channel: "<channel-name>"
+       send_resolved: true
+       api_url: "<api-url>"
+       icon_emoji: ":ghost:"
+       title: '{{ template "__subject" . }}'
+       title_link: 'https://grafana.<cluster-domain>'
+       text: '{{ range .Alerts }}`}}<!channel>{{`{{- "\n" -}}`}}{{`{{ .Annotations.description }}`}}{{`{{- "\n" -}}`}}{{`{{ end }}'
+   templates:
+   - '/etc/alertmanager/config/*.tmpl'

From 9253b135bae43cd6572e825ff0b11d1956beb560 Mon Sep 17 00:00:00 2001
From: Christoph Kleineweber <c.kleineweber@sap.com>
Date: Fri, 9 Dec 2022 11:35:22 +0100
Subject: [PATCH 2/2] Add Alertmanager datasource to Grafana

---
 prometheus/values.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/prometheus/values.yaml b/prometheus/values.yaml
index 7a5083f7..d43935e8 100644
--- a/prometheus/values.yaml
+++ b/prometheus/values.yaml
@@ -75,7 +75,7 @@ prometheus:
     ruleSelectorNilUsesHelmValues: false
 
 ####### This block enabled scrape discovery based on typical prometheus annotations
-  
+
     additionalScrapeConfigs:
       # Scrape config for service endpoints.
       #
@@ -174,3 +174,12 @@ prometheus:
           - source_labels: [__meta_kubernetes_pod_name]
             action: replace
             target_label: kubernetes_pod_name
+
+grafana:
+  additionalDataSources:
+    - name: Alertmanager
+      type: alertmanager
+      url: http://{{ printf "%s-kube-prometh-alertmanager.%s" .Release.Name .Release.Namespace }}:9093
+      access: proxy
+      jsonData:
+        implementation: prometheus