add pushgateway feature

containeroo · Sep 10, 2020 · 2d56967 · 2d56967
1 parent 1ca6bfc
commit 2d56967
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -4,6 +4,8 @@
 
 - Mirror two MinIO buckets
 - Delete files older than n days
+- Send status to prometheus pushgateway
+- Prometheus example rule
 
 Useful for GitLab backup mirroring between different MinIO instances.
 
@@ -15,13 +17,14 @@ Please refer to the example deployment in the `deploy/` directory of this reposi
 
 This image takes the following variables (all vars are required):
 
-| Variable                      | Description                                             | Example                                  |
-|-------------------------------|---------------------------------------------------------|------------------------------------------|
-| `${GITLAB_MINIO_URL}`         | URL for GitLab MinIO                                    | `https://minio.local.example.com`        |
-| `${GITLAB_BACKUP_RETENTION}`  | Delete backups older than ... days (default: `30d0h0m`) | `7d0h0m`                                 |
-| `${GITLAB_MINIO_ACCESSKEY}`   | GitLab MinIO accesskey (from k8s secret)                | `abc`                                    |
-| `${GITLAB_MINIO_SECRETKEY}`   | GitLab MinIO secretkey (from k8s secret)                | `def`                                    |
-| `${BACKUP_MINIO_URL}`         | URL for backup MinIO                                    | `https://minio-backup.local.example.com` |
-| `${BACKUP_MINIO_ACCESSKEY}`   | Backup MinIO accesskey (from k8s secret)                | `ghi`                                    |
-| `${BACKUP_MINIO_SECRETKEY}`   | Backup MinIO secretkey (from k8s secret)                | `jkl`                                    |
-| `${BACKUP_MINIO_BUCKET_NAME}` | Bucket name for backups (default: `gitlab`)             | `gitlab`                                 |
+| Variable                      | Description                                             | Example                                                           |
+| ----------------------------- | ------------------------------------------------------- | ----------------------------------------------------------------- |
+| `${GITLAB_MINIO_URL}`         | URL for GitLab MinIO                                    | `https://minio.local.example.com`                                 |
+| `${GITLAB_BACKUP_RETENTION}`  | Delete backups older than ... days (default: `30d0h0m`) | `7d0h0m`                                                          |
+| `${GITLAB_MINIO_ACCESSKEY}`   | GitLab MinIO accesskey (from k8s secret)                | `abc`                                                             |
+| `${GITLAB_MINIO_SECRETKEY}`   | GitLab MinIO secretkey (from k8s secret)                | `def`                                                             |
+| `${BACKUP_MINIO_URL}`         | URL for backup MinIO                                    | `https://minio-backup.local.example.com`                          |
+| `${BACKUP_MINIO_ACCESSKEY}`   | Backup MinIO accesskey (from k8s secret)                | `ghi`                                                             |
+| `${BACKUP_MINIO_SECRETKEY}`   | Backup MinIO secretkey (from k8s secret)                | `jkl`                                                             |
+| `${BACKUP_MINIO_BUCKET_NAME}` | Bucket name for backups (default: `gitlab`)             | `gitlab`                                                          |
+| `${PUSHGATEWAY_URL}`          | Prometheus pushgateway URL (optional)                   | `http://prometheus-pushgateway.monitoring.svc.cluster.local:9091` |
diff --git a/deploy/prometheusrule.yaml b/deploy/prometheusrule.yaml
@@ -0,0 +1,19 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    prometheus: default
+  name: gitlab-backup-mirror-rule
+  namespace: monitoring
+spec:
+  groups:
+  - name: gitlab-backup-mirror
+    rules:
+    - alert: GitlabBackupMirrorSyncError
+      annotations:
+        description: This alert is fired when sync fails
+        summary: Whether the snyc failed
+      expr: gbm_sync_failed > 0
+      labels:
+        severity: warning
diff --git a/gitlab-mirror.sh b/gitlab-mirror.sh
@@ -5,6 +5,25 @@ set -o errexit
 bucketname=${BACKUP_MINIO_BUCKET_NAME:-gitlab}
 retention=${GITLAB_BACKUP_RETENTION:-30d0h0m}
 
+function pushgateway {
+if [ -n "${PUSHGATEWAY_URL}" ]; then
+
+cat <<EOF | curl -L --data-binary @- ${PUSHGATEWAY_URL}/metrics/job/gitlab-backup-mirror/instance/sync
+# TYPE gbm_sync_failed gauge
+# HELP gbm_sync_failed Whether the snyc failed
+gbm_sync_failed $1
+EOF
+
+fi
+}
+
+function notify {
+pushgateway 1
+echo "ERROR: cannot sync"
+}
+
+trap notify ERR
+
 mc config host add gitlab ${GITLAB_MINIO_URL} ${GITLAB_MINIO_ACCESSKEY} ${GITLAB_MINIO_SECRETKEY} --api S3v4
 
 mc config host add backup ${BACKUP_MINIO_URL} ${BACKUP_MINIO_ACCESSKEY} ${BACKUP_MINIO_SECRETKEY} --api S3v4
@@ -14,3 +33,5 @@ mc mb --ignore-existing backup/gitlab
 mc rm --force --recursive --dangerous --older-than ${retention} gitlab/gitlab-backups
 
 mc mirror gitlab/gitlab-backups backup/${bucketname} --remove
+
+pushgateway 0