Add CPU based autoscaling, add signoz collector config

exceptionless · Oct 13, 2024 · 6f6cee7 · 6f6cee7
1 parent 08c97ea
commit 6f6cee7
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 12 deletions.
diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml
@@ -6,15 +6,15 @@ metadata:
 spec:
   version: 8.15.2
   image: exceptionless/elasticsearch:8.15.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
-  monitoring:
-    metrics:
-      elasticsearchRefs:
-        - name: elastic-monitor
-          namespace: elastic-system
-    logs:
-      elasticsearchRefs:
-        - name: elastic-monitor
-          namespace: elastic-system
+  # monitoring:
+  #   metrics:
+  #     elasticsearchRefs:
+  #       - name: elastic-monitor
+  #         namespace: elastic-system
+  #   logs:
+  #     elasticsearchRefs:
+  #       - name: elastic-monitor
+  #         namespace: elastic-system
   secureSettings:
     - secretName: ex-prod-snapshots
   http:

diff --git a/k8s/ex-prod-values.yaml b/k8s/ex-prod-values.yaml
@@ -2,13 +2,17 @@ appMode: Production
 appScope: prod
 app:
   replicaCount: 3
+  minReplicaCount: 2
+  maxReplicaCount: 5
   defaultDomain: be.exceptionless.io
   domains:
     - be.exceptionless.io
     - app.exceptionless.io
     - app.exceptionless.com
 api:
   replicaCount: 5
+  minReplicaCount: 2
+  maxReplicaCount: 5
   defaultDomain: collector.exceptionless.io
   domains:
     - collector.exceptionless.io
@@ -21,7 +25,9 @@ jobs:
   eventNotifications:
     replicaCount: 2
   eventPosts:
-    replicaCount: 10
+    replicaCount: 2
+    minReplicaCount: 2
+    maxReplicaCount: 10
   eventUserDescriptions:
     replicaCount: 2
   mailMessage:
@@ -30,6 +36,8 @@ jobs:
     replicaCount: 2
   workitem:
     replicaCount: 4
+    minReplicaCount: 2
+    maxReplicaCount: 5
 
 config:
   EX_EnableSnapshotJobs: "true"

diff --git a/k8s/ex-setup.ps1 b/k8s/ex-setup.ps1
@@ -147,6 +147,10 @@ helm install ex-$ENV-redis bitnami/redis --values ex-$ENV-redis-values.yaml --na
 # upgrade redis server
 helm upgrade ex-$ENV-redis bitnami/redis --reset-values --values ex-$ENV-redis-values.yaml --namespace ex-$ENV
 
+# install signoz otel collector
+helm repo add signoz https://charts.signoz.io
+helm install signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY"
+
 # install exceptionless app
 $VERSION="8.0.0"
 helm install ex-$ENV .\exceptionless --namespace ex-$ENV --values ex-$ENV-values.yaml `

diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml
@@ -10,7 +10,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  replicas: {{ .Values.api.replicaCount }}
+  replicas: {{ if not (kindIs "invalid" ( .Values.api | default dict ).replicaCount) }}{{ .Values.api.replicaCount }}{{ else }}{{ 2 }}{{ end }}
   selector:
     matchLabels:
       component: {{ template "exceptionless.fullname" . }}-api
@@ -90,6 +90,28 @@ spec:
             claimName: {{ template "exceptionless.fullname" . }}-storage
 {{- end }}
 
+{{- if not (kindIs "invalid" ( .Values.api | default dict ).replicaCount) }}
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ template "exceptionless.fullname" . }}-api-scaler
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ template "exceptionless.fullname" . }}-api
+  minReplicas: {{ if not (kindIs "invalid" ( .Values.api | default dict ).minReplicaCount) }}{{ .Values.api.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
+  maxReplicas: {{ .Values.api.maxReplicaCount }}
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 50
+{{- end }}
+
 ---
 apiVersion: v1
 kind: Service

diff --git a/k8s/exceptionless/templates/app.yaml b/k8s/exceptionless/templates/app.yaml
@@ -10,7 +10,7 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  replicas: {{ .Values.app.replicaCount }}
+  replicas: {{ if not (kindIs "invalid" ( .Values.app | default dict ).replicaCount) }}{{ .Values.app.replicaCount }}{{ else }}{{ 2 }}{{ end }}
   selector:
     matchLabels:
       component: {{ template "exceptionless.fullname" . }}-app
@@ -88,6 +88,28 @@ spec:
             claimName: {{ template "exceptionless.fullname" . }}-storage
 {{- end }}
 
+{{- if not (kindIs "invalid" ( .Values.app | default dict ).replicaCount) }}
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ template "exceptionless.fullname" . }}-app-scaler
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ template "exceptionless.fullname" . }}-app
+  minReplicas: {{ if not (kindIs "invalid" ( .Values.app | default dict ).minReplicaCount) }}{{ .Values.app.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
+  maxReplicas: {{ .Values.app.maxReplicaCount }}
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 50
+{{- end }}
+
 ---
 apiVersion: v1
 kind: Service

diff --git a/k8s/exceptionless/templates/jobs.yaml b/k8s/exceptionless/templates/jobs.yaml
@@ -343,6 +343,28 @@ spec:
             - name: EX_OTEL_RESOURCE_ATTRIBUTES
               value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID)
 
+{{- if not (kindIs "invalid" ( .Values.jobs.eventPosts | default dict ).maxReplicaCount) }}
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ template "exceptionless.fullname" . }}-jobs-event-posts-scaler
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ template "exceptionless.fullname" . }}-jobs-event-posts
+  minReplicas: {{ if not (kindIs "invalid" ( .Values.jobs.eventPosts | default dict ).minReplicaCount) }}{{ .Values.jobs.eventPosts.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
+  maxReplicas: {{ .Values.jobs.eventPosts.maxReplicaCount }}
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 50
+{{- end }}
+
 ---
 apiVersion: batch/v1
 kind: CronJob
@@ -945,6 +967,28 @@ spec:
             claimName: {{ template "exceptionless.fullname" . }}-storage
 {{- end }}
 
+{{- if not (kindIs "invalid" ( .Values.jobs.workitem | default dict ).maxReplicaCount) }}
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ template "exceptionless.fullname" . }}-jobs-work-item-scaler
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ template "exceptionless.fullname" . }}-jobs-work-item
+  minReplicas: {{ if not (kindIs "invalid" ( .Values.jobs.workitem | default dict ).minReplicaCount) }}{{ .Values.jobs.workitem.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
+  maxReplicas: {{ .Values.jobs.workitem.maxReplicaCount }}
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 50
+{{- end }}
+
 ---
 apiVersion: apps/v1
 kind: Deployment

diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml
@@ -0,0 +1,17 @@
+global:
+  cloud: aks
+  clusterName: ex-k8s-v6
+  deploymentEnvironment: prod
+otelCollectorEndpoint: ingest.us.signoz.cloud:443
+otelInsecure: false
+presets:
+  otlpExporter:
+    enabled: true
+  loggingExporter:
+    enabled: false
+  logsCollection:
+    enabled: false
+  resourceDetection:
+    detectors:
+      - azure
+      - system