epic(prometheus): Always use Prometheus

.github/workflows/ci.yml

@@ -1,8 +1,6 @@
 name: CI
 on:
   pull_request:
-    branches:
-      - main
 
 jobs:
   helm:
@@ -26,10 +24,8 @@ jobs:
       matrix:
         case:
           - "default"
-          - "all"
           - "image"
           - "ingress"
-          - "prometheus_sidecar"
         k8s_version:
           - v1.25.0
           - v1.27.0

charts/bindplane/Chart.yaml

@@ -3,7 +3,7 @@ name: bindplane
 description: BindPlane OP is an open source observability pipeline.
 type: application
 # The chart's version
-version: 1.1.10
+version: 1.2.0
 # The BindPlane OP tagged release. If the user does not
 # set the `image.tag` values option, this version is used.
 appVersion: 1.45.0

charts/bindplane/README.md

@@ -1,6 +1,6 @@
 # bindplane
 
-![Version: 1.1.10](https://img.shields.io/badge/Version-1.1.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.45.0](https://img.shields.io/badge/AppVersion-1.45.0-informational?style=flat-square)
+![Version: 1.2.0](https://img.shields.io/badge/Version-1.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.45.0](https://img.shields.io/badge/AppVersion-1.45.0-informational?style=flat-square)
 
 BindPlane OP is an open source observability pipeline.
 
@@ -105,19 +105,19 @@ BindPlane OP is an open source observability pipeline.
 | prometheus.auth.password | string | `""` | Prometheus basic authentication password. |
 | prometheus.auth.type | string | `"none"` | Prometheus authentication. Supported options include `none` and `basic`. |
 | prometheus.auth.username | string | `""` | Prometheus basic authentication username. |
-| prometheus.enable | bool | `false` | when enabled, Prometheus will be used as the measurements backend. Prometheus is the recommended backend for production deployments. |
-| prometheus.enableSideCar | bool | `false` | When enabled, the Prometheus measurements backend will be deployed as a sidecar container. This option is only valid when BindPlane is running as a single node statefulset. When using this option, leave all other Prometheus options unset and at their default values. |
-| prometheus.host | string | `"127.0.0.1"` | The Prometheus hostname or IP address used for querying and writing metrics. |
+| prometheus.enableSideCar | bool | `false` | When enabled, the Prometheus measurements backend will be deployed as a sidecar container. This option is only valid when BindPlane is running as a single node statefulset. |
+| prometheus.host | string | `""` | The Prometheus hostname or IP address used for querying and writing metrics. Defaults to the service name of the Prometheus StatefulSet deployed by this chart. |
 | prometheus.port | int | `9090` | The Prometheus TCP port used for querying and writing metrics. |
 | prometheus.queryPathPrefix | string | `""` | Optional Prometheus query path prefix. Useful when overriding the query endpoints when using systems such as Mimir. |
+| prometheus.remote | bool | `false` | When true, the chart will not deploy Prometheus. Instead, the user should provide a Prometheus instance to use. |
 | prometheus.remoteWrite.host | string | `""` | Optional hostname or IP address of the remote write endpoint. This value overrides the `prometheus.host` for remote write. |
 | prometheus.remoteWrite.path | string | `"/api/v1/write"` | Path of the remote write endpoint. This value should default to `/api/v1/write`. |
 | prometheus.remoteWrite.port | int | `9090` | Optional TCP port of the remote write endpoint. This value overrides the `prometheus.port` for remote write. |
-| prometheus.sidecar.resources.limits.memory | string | `"500Mi"` | Memory limit. |
-| prometheus.sidecar.resources.requests.cpu | string | `"250m"` | CPU request. |
-| prometheus.sidecar.resources.requests.memory | string | `"250Mi"` | Memory request. |
-| prometheus.sidecar.storageClass | string | `""` | The Kubernetes storage class to use for the volumeClaimTemplate. If unset, the volume claim will use the cluster's default storage class. |
-| prometheus.sidecar.volumeSize | string | `"10Gi"` | Persistent volume size. |
+| prometheus.resources.limits.memory | string | `"500Mi"` | Memory limit. |
+| prometheus.resources.requests.cpu | string | `"250m"` | CPU request. |
+| prometheus.resources.requests.memory | string | `"500Mi"` | Memory request. |
+| prometheus.storage.storageClass | string | `""` | The Kubernetes storage class to use for the volumeClaimTemplate. If unset, the volume claim will use the cluster's default storage class. |
+| prometheus.storage.volumeSize | string | `"10Gi"` | Persistent volume size. |
 | prometheus.tls.enable | bool | `false` | Whether or not to use TLS when connecting to Prometheus. |
 | prometheus.tls.insecure | bool | `false` | Whether or not to skip verification of the Prometheus server's certificate. |
 | prometheus.tls.secret.caSubPath | string | `""` | The secret's subPath which contains the certificate authority. |

charts/bindplane/templates/bindplane.yaml

@@ -240,13 +240,18 @@ spec:
             {{- end }}
             - name: BINDPLANE_PORT
               value: "3001"
-            {{- if .Values.prometheus.enable }}
             - name: BINDPLANE_PROMETHEUS_ENABLE
               value: "true"
             - name: BINDPLANE_PROMETHEUS_ENABLE_REMOTE
               value: "true"
             - name: BINDPLANE_PROMETHEUS_HOST
+              {{- if .Values.prometheus.enableSideCar }}
+              value: localhost
+              {{- else if .Values.prometheus.remote }}
               value: {{ .Values.prometheus.host }}
+              {{- else }}
+              value: {{ include "bindplane.fullname" . }}-prometheus
+              {{- end }}
             - name: BINDPLANE_PROMETHEUS_PORT
               value: "{{ .Values.prometheus.port }}"
             {{- if .Values.prometheus.queryPathPrefix }}
@@ -287,7 +292,6 @@ spec:
               value: /prometheus-client.key
             {{- end }}
             {{- end }}
-            {{- end }}
           {{- with .Values.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -344,7 +348,7 @@ spec:
               subPath: {{ .Values.eventbus.kafka.tls.secret.keySubPath }}
             {{- end }}
             {{- end }}
-            {{- if and (.Values.prometheus.enable) (.Values.prometheus.tls.enable) }}
+            {{- if .Values.prometheus.tls.enable }}
             {{- if .Values.prometheus.tls.secret.caSubPath }}
             - mountPath: /prometheus-ca.crt
               name: {{ .Values.prometheus.tls.secret.name }}
@@ -366,43 +370,27 @@ spec:
               exec:
                 command: ["sh", "-c", "sleep 5",]
         {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
-        {{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
+        {{- if and (.Values.prometheus.enableSideCar) (eq .Values.prometheus.remote false)}}
         - name: prometheus
-          image: prom/prometheus:v2.47.2
-          ports:
-            - containerPort: 9090
-              name: http
-          args:
-            - --config.file=/etc/prometheus/prometheus.yml
-            - --web.config.file=/etc/prometheus/web.yml
-            - --storage.tsdb.retention.time=2d
-            - --web.enable-remote-write-receiver
-            - --web.listen-address={{ .Values.prometheus.host }}:{{ .Values.prometheus.port }}
-            - --storage.tsdb.path=/var/lib/prometheus/tsdb
-            - --web.console.templates=/etc/prometheus/consoles
-            - --web.console.libraries=/etc/prometheus/console_libraries
-          {{- with .Values.prometheus.sidecar.resources }}
-          resources:
-            {{- toYaml . | nindent 12 }}
-          {{- end }}
+          image: ghcr.io/observiq/bindplane-prometheus:{{ include "bindplane.tag" . }}
+          imagePullPolicy: IfNotPresent
           securityContext:
             runAsNonRoot: true
             readOnlyRootFilesystem: true
             runAsUser: 65534
             capabilities:
               drop: ["ALL"]
+          ports:
+            - name: http
+              containerPort: 9090
+              protocol: TCP
+          {{- with .Values.prometheus.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           volumeMounts:
-            - name: {{ include "bindplane.fullname" . }}-prometheus-config
-              mountPath: /etc/prometheus/prometheus.yml
-              subPath: prometheus.yml
-            - name: {{ include "bindplane.fullname" . }}-prometheus-config
-              mountPath: /etc/prometheus/rules.yml
-              subPath: rules.yml
-            - name: {{ include "bindplane.fullname" . }}-prometheus-config
-              mountPath: /etc/prometheus/web.yml
-              subPath: web.yml
-            - name: {{ include "bindplane.fullname" . }}-prometheus-data
-              mountPath: /var/lib/prometheus/tsdb
+            - mountPath: /prometheus
+              name: {{ include "bindplane.fullname" . }}-prometheus-data
         {{- end }}
         {{- end }}
       terminationGracePeriodSeconds: 60
@@ -431,21 +419,14 @@ spec:
             secretName: {{ .Values.eventbus.kafka.tls.secret.name }}
         {{- end }}
         {{- end }}
-        {{- if and (.Values.prometheus.enable) (.Values.prometheus.tls.enable) }}
+        {{- if .Values.prometheus.tls.enable }}
         {{- if .Values.prometheus.tls.secret.name }}
         - name: {{ .Values.prometheus.tls.secret.name }}
           secret:
             defaultMode: 0400
             secretName: {{ .Values.prometheus.tls.secret.name }}
         {{- end }}
         {{- end }}
-        {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
-        {{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
-        - name: {{ include "bindplane.fullname" . }}-prometheus-config
-          configMap:
-            name: {{ include "bindplane.fullname" . }}-prometheus-config
-        {{- end }}
-        {{- end }}
   {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
   volumeClaimTemplates:
     {{- if eq .Values.backend.type "bbolt" }}
@@ -467,8 +448,7 @@ spec:
         storageClassName: {{ .Values.backend.bbolt.storageClass }}
         {{- end }}
     {{- end }}
-    {{- if eq (include "bindplane.deployment_type" .) "StatefulSet" }}
-    {{- if and (.Values.prometheus.enable) (.Values.prometheus.enableSideCar) }}
+    {{- if and (.Values.prometheus.enableSideCar) (eq .Values.prometheus.remote false)}}
     - metadata:
         name: {{ include "bindplane.fullname" . }}-prometheus-data
         labels:
@@ -482,10 +462,9 @@ spec:
           - ReadWriteOnce
         resources:
           requests:
-            storage: {{ .Values.prometheus.sidecar.volumeSize }}
-        {{- if .Values.prometheus.sidecar.storageClass }}
-        storageClassName: {{ .Values.prometheus.sidecar.storageClass }}
+            storage: {{ .Values.prometheus.storage.volumeSize }}
+        {{- if .Values.prometheus.storage.storageClass }}
+        storageClassName: {{ .Values.prometheus.storage.storageClass }}
         {{- end }}
-      {{- end }}
-      {{- end }}
+    {{- end }}
   {{- end }}

charts/bindplane/templates/prometheus.deprecated.yaml

@@ -0,0 +1,129 @@
+{{- if .Values.dev.prometheus.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "bindplane.fullname" . }}-prometheus
+data:
+  # Both of these should be empty, but might contain
+  # configuration in the future.
+  prometheus.yml: |
+    scrape_configs: []
+    rule_files: [/etc/prometheus/rules.yml]
+  web.yml: |
+    # This is an empty config.
+  rules.yml: |
+    groups:
+    - name: configuration-rollups
+      interval: 1m
+      rules:
+      - record: bindplane_agent_measurements:rollup:rate:1m
+        expr: sum without (agent) (rate(bindplane_agent_measurements{}[1m9s999ms] offset 10s))
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "bindplane.fullname" . }}-prometheus
+  labels:
+    app.kubernetes.io/name: test-prometheus
+spec:
+  replicas: 1
+  serviceName: bindplane-test-prometheus
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: test-prometheus
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: test-prometheus
+    spec:
+      initContainers:
+        # Set permissions on /prometheus volume.
+        - name: setup-volumes
+          image: {{ .Values.dev.prometheus.image.name }}:{{ .Values.dev.prometheus.image.tag }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - "chown"
+            - "65534:"
+            - "/prometheus"
+          volumeMounts:
+            - mountPath: /prometheus
+              name: tsdb
+      containers:
+        - name: opentelemetry-container
+          image: {{ .Values.dev.prometheus.image.name }}:{{ .Values.dev.prometheus.image.tag }}
+          imagePullPolicy: IfNotPresent
+          command:
+            - /bin/prometheus
+            - --web.listen-address=:9090
+            - --config.file=/etc/prometheus/prometheus.yml 
+            - --web.config.file=/etc/prometheus/web.yml
+            - --web.console.libraries=/usr/share/prometheus/console_libraries 
+            - --web.console.templates=/usr/share/prometheus/consoles
+            - --web.enable-remote-write-receiver
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention.time=2d
+          securityContext:
+            readOnlyRootFilesystem: true
+          ports:
+            - name: http
+              containerPort: 9090
+              protocol: TCP
+          resources:
+            requests:
+              memory: 300Mi
+              cpu: 100m
+            limits:
+              memory: 300Mi
+          volumeMounts:
+            # prometheus and web configuration
+            # from configmap.
+            - mountPath: /etc/prometheus/prometheus.yml
+              subPath: prometheus.yml
+              name: config
+            - mountPath: /etc/prometheus/web.yml
+              subPath: web.yml
+              name: config
+            - mountPath: /etc/prometheus/rules.yml
+              subPath: rules.yml
+              name: config
+            # time series database persistent volume.
+            - mountPath: /prometheus
+              name: tsdb
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "bindplane.fullname" . }}-prometheus
+  # Delete persistent volumes when the statefulset is deleted.
+  persistentVolumeClaimRetentionPolicy:
+    whenDeleted: Retain
+    whenScaled: Retain
+  volumeClaimTemplates:
+    - metadata:
+        name: tsdb
+        labels:
+          app.kubernetes.io/name: test-prometheus
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 60Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "bindplane.fullname" . }}-prometheus
+  labels:
+    app.kubernetes.io/name: test-prometheus
+spec:
+  ports:
+    - port: 9090
+      protocol: TCP
+      targetPort: http
+      name: http
+  selector:
+    app.kubernetes.io/name: test-prometheus
+  sessionAffinity: None
+  type: ClusterIP
+{{- end }}