feat(alertmanager): add dead man's switch healthcheck

willianpaixao · Jan 30, 2025 · 645178e · 645178e
1 parent 75796a4
commit 645178e
Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/kubernetes/raspberry/observability/kube-prometheus-stack/app/alertmanager-config.yaml b/kubernetes/raspberry/observability/kube-prometheus-stack/app/alertmanager-config.yaml
@@ -17,15 +17,26 @@ spec:
       - receiver: 'null'
         matchers:
           - name: 'alertname'
-            value: 'InfoInhibitor|NodeCPUHighUsage|NodeSystemSaturation|Watchdog'
+            value: 'InfoInhibitor|NodeCPUHighUsage|NodeSystemSaturation'
             matchType: '=~'
+      - receiver: 'dead-mans-switch'
+        groupWait: 0s
+        groupInterval: 5m
+        repeatInterval: 5m
+        matchers:
+          - name: 'alertname'
+            value: 'Watchdog'
+            matchType: '='
       - receiver: 'telegram'
         matchers:
           - name: 'severity'
             value: 'critical|page'
             matchType: '=~'
   receivers:
     - name: 'null'
+    - name: 'dead-mans-switch'
+      webhookConfigs:
+        - url: "${HEALTHCHECKS_IO_URL}"
     - name: 'telegram'
       telegramConfigs:
         - botToken:

diff --git a/kubernetes/raspberry/observability/kube-prometheus-stack/app/telegram-config.sops.yaml b/kubernetes/raspberry/observability/kube-prometheus-stack/app/telegram-config.sops.yaml
@@ -6,6 +6,7 @@ metadata:
 data:
     BOT_TOKEN: ENC[AES256_GCM,data:nv+mcZ8PsDtJrBlw9TWqENjyhHxWGN6RqiXrkZGEaDzPC8ducn9m+2vqGVtc8VUz4gViCNi1c+c8JI3XphyUdw==,iv:XlMkKzN+QjFep6p2ub8bebwanwokY8zsI1dyGCUh2hA=,tag:KO00y4tQVkST+jXY/p8oNw==,type:str]
     CHAT_ID: ENC[AES256_GCM,data:Iy8A7A4PvyShJM1TXpNONMzc+ro=,iv:qLdTA/uS6rr7eoCVARl/c/nzBABvGT1x6l8eyQdU260=,tag:DK91+mMXIXW5Za2F0D+AKQ==,type:str]
+    HEALTHCHECKS_IO_URL: ENC[AES256_GCM,data:XXy2j7B2rz/LXBelVlG4mNPPGufpc3a6eXhbXGRlAhM7rVfnVfK1Vlk0txORUqa1SJzN9XiV/ik=,iv:3UAGdPFLp7CjRjNb6pJZ5Ebb3VTSY2Vomq8pp3OLIak=,tag:Kd4UCvbQlCC4LQrDE0pGkQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -30,8 +31,8 @@ sops:
             UVc3WTRVSFFVSGNrNCt0MXFYUWZLbDQKpIJimfVIW3uR4TBL/emtALvJmKdYkh4t
             M1UOLPI4cDPK8vC6mgg6fZlJyzp0A3Q8kPoTV2STjjh9Ezpk2blVkQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-11-21T18:40:04Z"
-    mac: ENC[AES256_GCM,data:rCzaMX48ZDNAY1pG4JrmmncbSpyMfp7xxDntz0kt17iaIVzkrB94UiVd41xbdviJ79R2SOapFDbs8aOrH1uMa7PgGvSoW//xDqi0vKeR5Pae461WM8tu+7cZNVumUxwv2doS85a46UDZam9/aNeQB9qUwM2MtmSm9X1r/tdz0qs=,iv:AF4TW8fLQxvE/NTObL/JwxXpjMv1yeho2rf6cze2Umw=,tag:Wk46e9cNF2JFzHKF9rQ48w==,type:str]
+    lastmodified: "2025-01-30T17:51:17Z"
+    mac: ENC[AES256_GCM,data:7mgzaHv7n911mYBAyvO5ltbuhDk5NftQGZMYVoohDX5Zfpz3dnzm5dr5QrTtHqtYXmOZ7ciSg0WTesbCoP7F8yeUQVO1KzfxzBENoNZycIp8EiUvPtTWJU3+zKbIxqNVfeM+4UvnJ0m/P2un7V0hWGKGdZTMD+LdGacNQ5SEaMM=,iv:NEW2y6ae1YG5Kgzb7+ww3CtGbiJ2c6IiDd8gF8OzXf4=,tag:ywa1S3ihN6MsSTiUC38vgQ==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1