From 6f6cee7ab9822bd422205b23d9e9eba98199e4f6 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 13 Oct 2024 18:33:56 -0500 Subject: [PATCH] Add CPU based autoscaling, add signoz collector config --- k8s/ex-prod-elasticsearch.yaml | 18 +++++------ k8s/ex-prod-values.yaml | 10 +++++- k8s/ex-setup.ps1 | 4 +++ k8s/exceptionless/templates/api.yaml | 24 ++++++++++++++- k8s/exceptionless/templates/app.yaml | 24 ++++++++++++++- k8s/exceptionless/templates/jobs.yaml | 44 +++++++++++++++++++++++++++ k8s/signoz.yaml | 17 +++++++++++ 7 files changed, 129 insertions(+), 12 deletions(-) create mode 100644 k8s/signoz.yaml diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index 9dc3fa976c..6aa7237eb1 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -6,15 +6,15 @@ metadata: spec: version: 8.15.2 image: exceptionless/elasticsearch:8.15.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch - monitoring: - metrics: - elasticsearchRefs: - - name: elastic-monitor - namespace: elastic-system - logs: - elasticsearchRefs: - - name: elastic-monitor - namespace: elastic-system + # monitoring: + # metrics: + # elasticsearchRefs: + # - name: elastic-monitor + # namespace: elastic-system + # logs: + # elasticsearchRefs: + # - name: elastic-monitor + # namespace: elastic-system secureSettings: - secretName: ex-prod-snapshots http: diff --git a/k8s/ex-prod-values.yaml b/k8s/ex-prod-values.yaml index d16c497b07..2b30acdbbf 100644 --- a/k8s/ex-prod-values.yaml +++ b/k8s/ex-prod-values.yaml @@ -2,6 +2,8 @@ appMode: Production appScope: prod app: replicaCount: 3 + minReplicaCount: 2 + maxReplicaCount: 5 defaultDomain: be.exceptionless.io domains: - be.exceptionless.io @@ -9,6 +11,8 @@ app: - app.exceptionless.com api: replicaCount: 5 + minReplicaCount: 2 + maxReplicaCount: 5 defaultDomain: collector.exceptionless.io domains: - collector.exceptionless.io @@ -21,7 +25,9 @@ jobs: eventNotifications: replicaCount: 2 eventPosts: - replicaCount: 10 + replicaCount: 2 + minReplicaCount: 2 + maxReplicaCount: 10 eventUserDescriptions: replicaCount: 2 mailMessage: @@ -30,6 +36,8 @@ jobs: replicaCount: 2 workitem: replicaCount: 4 + minReplicaCount: 2 + maxReplicaCount: 5 config: EX_EnableSnapshotJobs: "true" diff --git a/k8s/ex-setup.ps1 b/k8s/ex-setup.ps1 index 834a3dc0b7..fc284bfa67 100644 --- a/k8s/ex-setup.ps1 +++ b/k8s/ex-setup.ps1 @@ -147,6 +147,10 @@ helm install ex-$ENV-redis bitnami/redis --values ex-$ENV-redis-values.yaml --na # upgrade redis server helm upgrade ex-$ENV-redis bitnami/redis --reset-values --values ex-$ENV-redis-values.yaml --namespace ex-$ENV +# install signoz otel collector +helm repo add signoz https://charts.signoz.io +helm install signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY" + # install exceptionless app $VERSION="8.0.0" helm install ex-$ENV .\exceptionless --namespace ex-$ENV --values ex-$ENV-values.yaml ` diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml index 25aef0c952..9e81de4c15 100644 --- a/k8s/exceptionless/templates/api.yaml +++ b/k8s/exceptionless/templates/api.yaml @@ -10,7 +10,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: - replicas: {{ .Values.api.replicaCount }} + replicas: {{ if not (kindIs "invalid" ( .Values.api | default dict ).replicaCount) }}{{ .Values.api.replicaCount }}{{ else }}{{ 2 }}{{ end }} selector: matchLabels: component: {{ template "exceptionless.fullname" . }}-api @@ -90,6 +90,28 @@ spec: claimName: {{ template "exceptionless.fullname" . }}-storage {{- end }} +{{- if not (kindIs "invalid" ( .Values.api | default dict ).replicaCount) }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "exceptionless.fullname" . }}-api-scaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "exceptionless.fullname" . }}-api + minReplicas: {{ if not (kindIs "invalid" ( .Values.api | default dict ).minReplicaCount) }}{{ .Values.api.minReplicaCount }}{{ else }}{{ 2 }}{{ end }} + maxReplicas: {{ .Values.api.maxReplicaCount }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 +{{- end }} + --- apiVersion: v1 kind: Service diff --git a/k8s/exceptionless/templates/app.yaml b/k8s/exceptionless/templates/app.yaml index bdfff12bff..3738f053bb 100644 --- a/k8s/exceptionless/templates/app.yaml +++ b/k8s/exceptionless/templates/app.yaml @@ -10,7 +10,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: - replicas: {{ .Values.app.replicaCount }} + replicas: {{ if not (kindIs "invalid" ( .Values.app | default dict ).replicaCount) }}{{ .Values.app.replicaCount }}{{ else }}{{ 2 }}{{ end }} selector: matchLabels: component: {{ template "exceptionless.fullname" . }}-app @@ -88,6 +88,28 @@ spec: claimName: {{ template "exceptionless.fullname" . }}-storage {{- end }} +{{- if not (kindIs "invalid" ( .Values.app | default dict ).replicaCount) }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "exceptionless.fullname" . }}-app-scaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "exceptionless.fullname" . }}-app + minReplicas: {{ if not (kindIs "invalid" ( .Values.app | default dict ).minReplicaCount) }}{{ .Values.app.minReplicaCount }}{{ else }}{{ 2 }}{{ end }} + maxReplicas: {{ .Values.app.maxReplicaCount }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 +{{- end }} + --- apiVersion: v1 kind: Service diff --git a/k8s/exceptionless/templates/jobs.yaml b/k8s/exceptionless/templates/jobs.yaml index f73b03c22e..12cfe09a0d 100644 --- a/k8s/exceptionless/templates/jobs.yaml +++ b/k8s/exceptionless/templates/jobs.yaml @@ -343,6 +343,28 @@ spec: - name: EX_OTEL_RESOURCE_ATTRIBUTES value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) +{{- if not (kindIs "invalid" ( .Values.jobs.eventPosts | default dict ).maxReplicaCount) }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "exceptionless.fullname" . }}-jobs-event-posts-scaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "exceptionless.fullname" . }}-jobs-event-posts + minReplicas: {{ if not (kindIs "invalid" ( .Values.jobs.eventPosts | default dict ).minReplicaCount) }}{{ .Values.jobs.eventPosts.minReplicaCount }}{{ else }}{{ 2 }}{{ end }} + maxReplicas: {{ .Values.jobs.eventPosts.maxReplicaCount }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 +{{- end }} + --- apiVersion: batch/v1 kind: CronJob @@ -945,6 +967,28 @@ spec: claimName: {{ template "exceptionless.fullname" . }}-storage {{- end }} +{{- if not (kindIs "invalid" ( .Values.jobs.workitem | default dict ).maxReplicaCount) }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "exceptionless.fullname" . }}-jobs-work-item-scaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "exceptionless.fullname" . }}-jobs-work-item + minReplicas: {{ if not (kindIs "invalid" ( .Values.jobs.workitem | default dict ).minReplicaCount) }}{{ .Values.jobs.workitem.minReplicaCount }}{{ else }}{{ 2 }}{{ end }} + maxReplicas: {{ .Values.jobs.workitem.maxReplicaCount }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 +{{- end }} + --- apiVersion: apps/v1 kind: Deployment diff --git a/k8s/signoz.yaml b/k8s/signoz.yaml new file mode 100644 index 0000000000..a9fa1e2512 --- /dev/null +++ b/k8s/signoz.yaml @@ -0,0 +1,17 @@ +global: + cloud: aks + clusterName: ex-k8s-v6 + deploymentEnvironment: prod +otelCollectorEndpoint: ingest.us.signoz.cloud:443 +otelInsecure: false +presets: + otlpExporter: + enabled: true + loggingExporter: + enabled: false + logsCollection: + enabled: false + resourceDetection: + detectors: + - azure + - system