Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add alerts for Rules and Alerting SLOs #300

Merged
merged 4 commits into from
Sep 20, 2022
Merged

Add alerts for Rules and Alerting SLOs #300

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2e8eaae
Add alerts for Rules and Alerting SLOs
jessicalins Aug 24, 2022
b84b14f
Add container selector to APIRulesSyncAvailabilityErrorBudgetBurning …
jessicalins Aug 25, 2022
15078dd
Fix telemeter namespace, update sync SLO query in alerts
jessicalins Sep 19, 2022
716e0d4
Refactor instanceNamespace function
jessicalins Sep 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions observability/dashboards/slo.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
local utils = import '../utils.jsonnet';

function(instanceName, environment, dashboardName) {
// Validate our inputs.
assert std.member(['telemeter', 'mst'], instanceName),
Expand All @@ -23,16 +25,17 @@ function(instanceName, environment, dashboardName) {
datasource: 'telemeter-prod-01-prometheus',
upNamespace: 'observatorium-mst-production',
apiJob: 'observatorium-observatorium-mst-api',
metricsNamespace: 'observatorium-metrics-production',
},
stage: {
datasource: 'app-sre-stage-01-prometheus',
upNamespace: 'observatorium-mst-stage',
apiJob: 'observatorium-observatorium-mst-api',
metricsNamespace: 'observatorium-metrics-stage',
},
},
},
local instance = instanceConfig[instanceName][environment],
local instanceNamespace(name) = if name == 'telemeter' then instance.metricsNamespace else instance.upNamespace,
// This is part of a dirty hack because I can't figure out how to do an auto-incrementing counter in Jsonnet.
// Each grafana dashboard that requests data needs a unique ID, we use the panels per row + a unqiue index per panel
// to generate a continuous stream of integers from 0...
Expand Down Expand Up @@ -482,8 +485,8 @@ function(instanceName, environment, dashboardName) {
availabilityRow(
'95% of rules are successfully synced to Thanos Ruler',
0.95,
'sum(rate(client_api_requests_total{client="reload",container="thanos-rule-syncer",namespace="%s",code=~"5.+"}[28d]))' % instanceNamespace(instanceName),
'sum(rate(client_api_requests_total{client="reload",container="thanos-rule-syncer",namespace="%s",code!~"4.+"}[28d]))' % instanceNamespace(instanceName),
'sum(rate(client_api_requests_total{client="reload",container="thanos-rule-syncer",namespace="%s",code=~"5.+"}[28d]))' % utils.instanceNamespace(instanceName, instance.metricsNamespace, instance.upNamespace),
'sum(rate(client_api_requests_total{client="reload",container="thanos-rule-syncer",namespace="%s",code!~"4.+"}[28d]))' % utils.instanceNamespace(instanceName, instance.metricsNamespace, instance.upNamespace),
10
) +
titleRow('API > Rules Read (/rules) > Availability') +
Expand All @@ -506,15 +509,15 @@ function(instanceName, environment, dashboardName) {
availabilityRow(
'95% of alerts are successfully delivered to Alertmanager',
0.95,
'sum(rate(thanos_alert_sender_alerts_dropped_total{container="thanos-rule",namespace="%s"}[28d]))' % instanceNamespace(instanceName),
'sum(rate(thanos_alert_sender_alerts_sent_total{container="thanos-rule",namespace="%s"}[28d]))' % instanceNamespace(instanceName),
'sum(rate(thanos_alert_sender_alerts_dropped_total{container="thanos-rule",namespace="%s"}[28d]))' % utils.instanceNamespace(instanceName, instance.metricsNamespace, instance.upNamespace),
'sum(rate(thanos_alert_sender_alerts_sent_total{container="thanos-rule",namespace="%s"}[28d]))' % utils.instanceNamespace(instanceName, instance.metricsNamespace, instance.upNamespace),
13
) +
availabilityRow(
'95% of alerts are successfully delivered to upstream targets',
0.95,
'sum(rate(alertmanager_notifications_failed_total{service="observatorium-alertmanager",namespace="%s"}[28d]))' % instanceNamespace(instanceName),
'sum(rate(alertmanager_notifications_total{service="observatorium-alertmanager",namespace="%s"}[28d]))' % instanceNamespace(instanceName),
'sum(rate(alertmanager_notifications_failed_total{service="observatorium-alertmanager",namespace="%s"}[28d]))' % utils.instanceNamespace(instanceName, instance.metricsNamespace, instance.upNamespace),
'sum(rate(alertmanager_notifications_total{service="observatorium-alertmanager",namespace="%s"}[28d]))' % utils.instanceNamespace(instanceName, instance.metricsNamespace, instance.upNamespace),
14
),
local apiLogsPanels =
Expand Down
84 changes: 79 additions & 5 deletions observability/prometheusrules.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
local loki = (import 'github.com/grafana/loki/production/loki-mixin/mixin.libsonnet');
local slo = import 'github.com/metalmatze/slo-libsonnet/slo-libsonnet/slo.libsonnet';
local lokiTenants = import './observatorium-logs/loki-tenant-alerts.libsonnet';
local utils = import './utils.jsonnet';

local config = (import 'config.libsonnet') {
thanos+: {
Expand Down Expand Up @@ -288,9 +289,10 @@ local renderAlerts(name, environment, mixin) = {
},
],

local apiSLOs = function(instance, upNamespace, apiJob) {
local apiSLOs = function(instance, upNamespace, metricsNamespace, apiJob) {
local apiJobSelector = 'job="' + apiJob + '"',
local upNamespaceSelector = 'namespace="' + upNamespace + '"',

slos: [
{
name: 'rhobs-' + instance + '-api-metrics-write-availability.slo',
Expand Down Expand Up @@ -370,17 +372,89 @@ local renderAlerts(name, environment, mixin) = {
}),
],
},
{
name: 'rhobs-' + instance + '-api-rules-raw-write-availability.slo',
slos: [
slo.errorburn({
alertName: 'APIRulesRawWriteAvailabilityErrorBudgetBurning',
alertMessage: 'API /rules/raw endpoint is burning too much error budget to guarantee availability SLOs',
metric: 'http_requests_total',
selectors: [apiJobSelector, 'group="metricsv1"', 'handler="rules-raw"', 'code=~"^(2..|3..|5..)$"', 'method=~"PUT"'],
errorSelectors: ['code=~"5.+"'],
target: 0.95,
}),
],
},
{
name: 'rhobs-' + instance + '-api-rules-sync-availability.slo',
slos: [
slo.errorburn({
alertName: 'APIRulesSyncAvailabilityErrorBudgetBurning',
alertMessage: 'API /reload endpoint is burning too much error budget to guarantee availability SLOs',
metric: 'client_api_requests_total',
selectors: ['client="reload"', 'container="thanos-rule-syncer"', 'namespace="' + utils.instanceNamespace(instance, metricsNamespace, upNamespace) + '"', 'code=~"^(2..|3..|5..)$"'],
errorSelectors: ['code=~"5.+"'],
target: 0.95,
}),
],
},
{
name: 'rhobs-' + instance + '-api-rules-read-availability.slo',
slos: [
slo.errorburn({
alertName: 'APIRulesReadAvailabilityErrorBudgetBurning',
alertMessage: 'API /rules endpoint is burning too much error budget to guarantee availability SLOs',
metric: 'http_requests_total',
selectors: [apiJobSelector, 'group="metricsv1"', 'handler=~"rules"', 'code=~"^(2..|3..|5..)$"'],
errorSelectors: ['code=~"5.+"'],
target: 0.90,
}),
],
},
{

name: 'rhobs-' + instance + '-api-rules-raw-read-availability.slo',
slos: [
slo.errorburn({
alertName: 'APIRulesRawReadAvailabilityErrorBudgetBurning',
alertMessage: 'API /rules/raw endpoint is burning too much error budget to guarantee availability SLOs',
metric: 'http_requests_total',
selectors: [apiJobSelector, 'group="metricsv1"', 'handler=~"rules-raw"', 'code=~"^(2..|3..|5..)$"'],
errorSelectors: ['code=~"5.+"'],
target: 0.90,
}),
],
},
{
name: 'rhobs-' + instance + '-api-alerting-availability.slo',
slos: [
slo.errorburn({
alertName: 'APIAlertmanagerAvailabilityErrorBudgetBurning',
alertMessage: 'API Thanos Rule failing to send alerts to Alertmanager and is burning too much error budget to guarantee availability SLOs',
metric: 'thanos_alert_sender_alerts_dropped_total',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we need this alert - we seem to use already the thanos mixin and it contains the ThanosRuleSenderIsFailingAlerts alert, though I think it is not written specifically for our SLOs.

selectors: ['container="thanos-rule"', 'namespace="' + utils.instanceNamespace(instance, metricsNamespace, upNamespace) + '"'],
target: 0.95,
}),
slo.errorburn({
alertName: 'APIAlertmanagerNotificationsAvailabilityErrorBudgetBurning',
alertMessage: 'API Alertmanager failing to deliver alerts to upstream targets and is burning too much error budget to guarantee availability SLOs',
metric: 'alertmanager_notifications_failed_total',
selectors: ['service="observatorium-alertmanager"', 'namespace="' + utils.instanceNamespace(instance, metricsNamespace, upNamespace) + '"'],
target: 0.95,
}),
],
},
],
},

local mstStageSLOs = apiSLOs('mst', 'observatorium-mst-stage', 'observatorium-observatorium-mst-api').slos,
local mstProductionSLOs = apiSLOs('mst', 'observatorium-mst-production', 'observatorium-observatorium-mst-api').slos,
local mstStageSLOs = apiSLOs('mst', 'observatorium-mst-stage', 'observatorium-mst-stage', 'observatorium-observatorium-mst-api').slos,
local mstProductionSLOs = apiSLOs('mst', 'observatorium-mst-production', 'observatorium-mst-production', 'observatorium-observatorium-mst-api').slos,

'rhobs-slos-mst-stage.prometheusrules': renderAlerts('rhobs-slos-mst-stage', 'stage', flatten(mstStageSLOs)),
'rhobs-slos-mst-production.prometheusrules': renderAlerts('rhobs-slos-mst-production', 'production', flatten(mstProductionSLOs)),

local telemeterStageSLOs = telemeterServerSLOs + apiSLOs('telemeter', 'observatorium-stage', 'observatorium-observatorium-api').slos,
local telemeterProductionSLOs = telemeterServerSLOs + apiSLOs('telemeter', 'observatorium-production', 'observatorium-observatorium-api').slos,
local telemeterStageSLOs = telemeterServerSLOs + apiSLOs('telemeter', 'observatorium-stage', 'observatorium-metrics-stage', 'observatorium-observatorium-api').slos,
local telemeterProductionSLOs = telemeterServerSLOs + apiSLOs('telemeter', 'observatorium-production', 'observatorium-metrics-production', 'observatorium-observatorium-api').slos,

'rhobs-slos-telemeter-stage.prometheusrules': renderAlerts('rhobs-slos-telemeter-stage', 'stage', flatten(telemeterStageSLOs)),
'rhobs-slos-telemeter-production.prometheusrules': renderAlerts('rhobs-slos-telemeter-production', 'production', flatten(telemeterProductionSLOs)),
Expand Down
3 changes: 3 additions & 0 deletions observability/utils.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
instanceNamespace(name, metricsNamespace, upNamespace): if name == 'telemeter' then metricsNamespace else upNamespace,
}
Loading