feat(prometheus): Support StatefulSet with Prometheus sidecar

.github/workflows/ci.yml

@@ -29,6 +29,7 @@ jobs:
           - "all"
           - "image"
           - "ingress"
+          - "prometheus_sidecar"
         k8s_version:
           - v1.25.0
           - v1.27.0

charts/bindplane/Chart.yaml

@@ -3,7 +3,7 @@ name: bindplane
 description: BindPlane OP is an open source observability pipeline.
 type: application
 # The chart's version
-version: 1.0.12
+version: 1.1.0
 # The BindPlane OP tagged release. If the user does not
 # set the `image.tag` values option, this version is used.
 appVersion: 1.39.0

charts/bindplane/README.md

@@ -1,6 +1,6 @@
 # bindplane
 
-![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.31.0](https://img.shields.io/badge/AppVersion-1.31.0-informational?style=flat-square)
+![Version: 1.1.0](https://img.shields.io/badge/Version-1.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.39.0](https://img.shields.io/badge/AppVersion-1.39.0-informational?style=flat-square)
 
 BindPlane OP is an open source observability pipeline.
 
@@ -63,6 +63,9 @@ BindPlane OP is an open source observability pipeline.
 | dev.collector.labels | string | `"configuration=test"` |  |
 | dev.namespace.create | bool | `false` |  |
 | dev.namespace.name | string | `""` |  |
+| dev.prometheus.create | bool | `false` |  |
+| dev.prometheus.image.name | string | `"prom/prometheus"` |  |
+| dev.prometheus.image.tag | string | `"v2.47.2"` |  |
 | email.sendgrid.token | string | `""` | The sendgrid API token to use when authenticating to Sendgrid. |
 | email.type | string | `""` | The optional email backend type to use. Valid options include `sendgrid`. Requires an auth type other than `system`. |
 | eventbus.kafka.auth.plain.password | string | `""` | Password to use for plain authentication. |
@@ -96,6 +99,15 @@ BindPlane OP is an open source observability pipeline.
 | ingress.tls.secret | string | `""` | Name of the Kubernetes secret which contains the TLS certificate. This secret must be created and managed outside of the Helm chart. See the [ingress TLS documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/#tls) for more details. |
 | multiAccount | bool | `false` | Whether or not to enable multi account (tenant). |
 | podSecurityContext | object | `{"fsGroup":65534}` | The Pod spec's securityContext: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod. |
+| prometheus.enable | bool | `false` | when enabled, Prometheus will be used as the measurements backend. Prometheus is the recommended backend for production deployments. |
+| prometheus.enableSideCar | bool | `false` | When enabled, the Prometheus measurements backend will be deployed as a sidecar container. This option is only valid when BindPlane is running as a single node statefulset. |
+| prometheus.host | string | `"127.0.0.1"` | The Prometheus hostname or IP address. |
+| prometheus.port | int | `9090` | The Prometheus TCP port. |
+| prometheus.sidecar.resources.limits.memory | string | `"500Mi"` | Memory limit. |
+| prometheus.sidecar.resources.requests.cpu | string | `"250m"` | CPU request. |
+| prometheus.sidecar.resources.requests.memory | string | `"250Mi"` | Memory request. |
+| prometheus.sidecar.storageClass | string | `""` | The Kubernetes storage class to use for the volumeClaimTemplate. If unset, the volume claim will use the cluster's default storage class. |
+| prometheus.sidecar.volumeSize | string | `"10Gi"` | Persistent volume size. |
 | resources.limits.memory | string | `"500Mi"` | Memory limit. |
 | resources.requests.cpu | string | `"250m"` | CPU request. |
 | resources.requests.memory | string | `"250Mi"` | Memory request. |

charts/bindplane/templates/bindplane.yaml

@@ -310,6 +310,40 @@ spec:
             preStop:
               exec:
                 command: ["sh", "-c", "sleep 5",]
+        {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
+        {{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
+        - name: prometheus
+          image: prom/prometheus:v2.47.2
+          ports:
+            - containerPort: 9090
+              name: http
+          args:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --web.config.file=/etc/prometheus/web.yml
+            - --storage.tsdb.retention.time=2d
+            - --web.enable-remote-write-receiver
+            - --web.listen-address={{ .Values.prometheus.host }}:{{ .Values.prometheus.port }}
+            - --storage.tsdb.path=/var/lib/prometheus/tsdb
+            - --web.console.templates=/etc/prometheus/consoles
+            - --web.console.libraries=/etc/prometheus/console_libraries
+          {{- with .Values.prometheus.sidecar.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: {{ include "bindplane.fullname" . }}-prometheus-config
+              mountPath: /etc/prometheus/prometheus.yml
+              subPath: prometheus.yml
+            - name: {{ include "bindplane.fullname" . }}-prometheus-config
+              mountPath: /etc/prometheus/rules.yml
+              subPath: rules.yml
+            - name: {{ include "bindplane.fullname" . }}-prometheus-config
+              mountPath: /etc/prometheus/web.yml
+              subPath: web.yml
+            - name: {{ include "bindplane.fullname" . }}-prometheus-data
+              mountPath: /var/lib/prometheus/tsdb
+        {{- end }}
+        {{- end }}
       terminationGracePeriodSeconds: 60
       volumes:
         {{- if eq .Values.eventbus.type "pubsub" }}
@@ -336,6 +370,13 @@ spec:
             secretName: {{ .Values.eventbus.kafka.tls.secret.name }}
         {{- end }}
         {{- end }}
+        {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
+        {{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
+        - name: {{ include "bindplane.fullname" . }}-prometheus-config
+          configMap:
+            name: {{ include "bindplane.fullname" . }}-prometheus-config
+        {{- end }}
+        {{- end }}
   {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
   volumeClaimTemplates:
     {{- if eq .Values.backend.type "bbolt" }}
@@ -357,4 +398,25 @@ spec:
         storageClassName: {{ .Values.backend.bbolt.storageClass }}
         {{- end }}
     {{- end }}
+    {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
+    {{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
+    - metadata:
+        name: {{ include "bindplane.fullname" . }}-prometheus-data
+        labels:
+          app.kubernetes.io/name: {{ include "bindplane.name" . }}
+          app.kubernetes.io/stack: bindplane
+          app.kubernetes.io/component: server
+          app.kubernetes.io/instance: {{ .Release.Name }}
+          app.kubernetes.io/managed-by: {{ .Release.Service }}
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: {{ .Values.prometheus.sidecar.volumeSize }}
+        {{- if .Values.prometheus.sidecar.storageClass }}
+        storageClassName: {{ .Values.prometheus.sidecar.storageClass }}
+        {{- end }}
+      {{- end }}
+      {{- end }}
   {{- end }}

charts/bindplane/templates/prometheus_config.yaml

@@ -0,0 +1,27 @@
+{{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
+{{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bindplane.fullname" . }}-prometheus-config
+  labels:
+    app.kubernetes.io/name: {{ include "bindplane.name" . }}
+    app.kubernetes.io/stack: bindplane
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  prometheus.yml: |
+    scrape_configs: []
+    rule_files: [/etc/prometheus/rules.yml]
+  rules.yml: |
+    groups:
+    - name: configuration-rollups
+      interval: 1m
+      rules:
+      - record: bindplane_agent_measurements:rollup:rate:1m
+        expr: sum without (agent) (rate(bindplane_agent_measurements{}[1m] offset 10s))
+  web.yml: |
+    # web.yml is empty but must be present for Prometheus to start
+{{- end }}
+{{- end }}
+

charts/bindplane/values.yaml

@@ -30,9 +30,31 @@ backend:
 # TODO(jsirianni): Support authentication and TLS.
 # This is undocumented for now, as Prometheus support has not been released.
 prometheus:
+  # -- when enabled, Prometheus will be used as the measurements backend. Prometheus is the recommended backend for production deployments.
   enable: false
-  host: ""
+  # -- The Prometheus hostname or IP address.
+  host: "127.0.0.1"
+  # -- The Prometheus TCP port.
   port: 9090
+  # -- When enabled, the Prometheus measurements backend will be deployed as a sidecar container. This option is only valid when BindPlane is running as a single node statefulset.
+  enableSideCar: false
+  sidecar:
+    resources:
+      requests:
+        # -- Memory request.
+        memory: 250Mi
+        # -- CPU request.
+        cpu: 250m
+      limits:
+        # -- Memory limit.
+        memory: 500Mi
+        # Disable cpu limit by default, for burstable qos class
+        # cpu: 500m
+    # -- Persistent volume size.
+    volumeSize: 10Gi
+    # -- The Kubernetes storage class to use for the volumeClaimTemplate. If unset, the volume claim will use the cluster's default storage class.
+    storageClass: ""
+
 
 eventbus:
   # The eventbus type to use when BindPlane is deployed with multiple pods (Deployment). Available options include `pubsub`. By default, this option is not required as BindPlane OP operates as a StatefulSet with one pod.

test/cases/prometheus_sidecar/values.yaml

@@ -0,0 +1,9 @@
+# Required options
+config:
+  username: bpuser
+  password: bppass
+  secret_key: 12D8FB6E-1532-4A4C-97AF-95A430BE5E6E
+  sessions_secret: 4484766F-5016-4077-B8E0-0DE1D637854B
+prometheus:
+  enable: true
+  enableSideCar: true