Add AlertmanagerNotificationsFailing alert

See #226 and #189
ooni · Oct 31, 2018 · e7a5b2e · e7a5b2e
1 parent 1d1a22f
commit e7a5b2e
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/ansible/roles/prometheus/files/alert_rules.yml b/ansible/roles/prometheus/files/alert_rules.yml
@@ -139,4 +139,11 @@ groups:
     annotations:
       summary: '`certbot` failed at {{ $labels.instance }}'
       description: 'SSL cert for  expires in {{ printf "%.0f" $value }} days.'
+
+- name: Prometheus self-care
+  rules:
+  - alert: AlertmanagerNotificationsFailing
+    expr: rate(alertmanager_notifications_failed_total[1m]) > 0
+    annotations:
+      summary: 'Alertmanager {{ $labels.instance }} fails {{ $labels.integration }} notifications.'
 ...