From 9633bb9a99bf7f26d45e7bffa188c8bd042464f3 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 24 Apr 2024 20:16:52 -0700 Subject: [PATCH 1/3] Update deployment for 1.17 Also turn off gossip and mingle as workers were missing a lot of heartbeats. Recommended from this blog post: https://www.cloudamqp.com/docs/celery.html#commandline-arguments --- helm-chart/banzai/templates/workers-large.yaml | 3 +++ helm-chart/banzai/templates/workers.yaml | 3 +++ helm-chart/banzai/values-prod.yaml | 10 +++++----- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/helm-chart/banzai/templates/workers-large.yaml b/helm-chart/banzai/templates/workers-large.yaml index 2e5f47dc..cd5419b2 100644 --- a/helm-chart/banzai/templates/workers-large.yaml +++ b/helm-chart/banzai/templates/workers-large.yaml @@ -35,6 +35,9 @@ spec: - "-A" - "banzai" - "worker" + - "--without-heartbeat" + - "--without-gossip" + - "--without-mingle" - "--concurrency" - "1" - "-l" diff --git a/helm-chart/banzai/templates/workers.yaml b/helm-chart/banzai/templates/workers.yaml index cec78ab7..6d4fb60f 100644 --- a/helm-chart/banzai/templates/workers.yaml +++ b/helm-chart/banzai/templates/workers.yaml @@ -35,6 +35,9 @@ spec: - "-A" - "banzai" - "worker" + - "--without-heartbeat" + - "--without-gossip" + - "--without-mingle" - "--concurrency" - "1" - "-l" diff --git a/helm-chart/banzai/values-prod.yaml b/helm-chart/banzai/values-prod.yaml index ef292db1..a9a53711 100644 --- a/helm-chart/banzai/values-prod.yaml +++ b/helm-chart/banzai/values-prod.yaml @@ -6,15 +6,15 @@ # will be started when the CPU usage rises above the configured threshold. horizontalPodAutoscaler: enabled: true - minReplicas: 12 - maxReplicas: 12 + minReplicas: 15 + maxReplicas: 15 targetCPUUtilizationPercentage: 50 - minLargeReplicas: 3 - maxLargeReplicas: 3 + minLargeReplicas: 7 + maxLargeReplicas: 7 image: repository: ghcr.io/lcogt/banzai - tag: "1.11.0" + tag: "1.17.0" pullPolicy: IfNotPresent # Values for the OCS Ingester library, used by BANZAI. From d54bb9d9a44e8544a5b742f1dc67bf01228a34f8 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 28 Aug 2024 08:58:47 -0500 Subject: [PATCH 2/3] Update changelog --- CHANGES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 68106182..e2d7709f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,9 @@ +1.18.1 (2024-08-28) +------------------- +- Update celery config to turn off gossip and mingle as workers were missing lots of heartbeats and were throwing errors. See https://www.cloudamqp.com/docs/celery.html#commandline-arguments for details + 1.18.0 (2024-08-09) +------------------- - Added support for frames that are composed of sub-exposures that are stacked at site From 51061d2b851fdc3552b3a541b0ae831edfe49b97 Mon Sep 17 00:00:00 2001 From: Matt Daily Date: Wed, 28 Aug 2024 09:03:00 -0500 Subject: [PATCH 3/3] Uprevv versions of helm chart. --- helm-chart/banzai/Chart.yaml | 4 ++-- helm-chart/banzai/values-prod.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm-chart/banzai/Chart.yaml b/helm-chart/banzai/Chart.yaml index 77555cac..f24eece2 100644 --- a/helm-chart/banzai/Chart.yaml +++ b/helm-chart/banzai/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "1.14.1" +appVersion: "1.18.1" description: A Helm chart to deploy the BANZAI pipeline name: banzai -version: 1.14.1 +version: 1.18.1 diff --git a/helm-chart/banzai/values-prod.yaml b/helm-chart/banzai/values-prod.yaml index a9a53711..150c629b 100644 --- a/helm-chart/banzai/values-prod.yaml +++ b/helm-chart/banzai/values-prod.yaml @@ -14,7 +14,7 @@ horizontalPodAutoscaler: image: repository: ghcr.io/lcogt/banzai - tag: "1.17.0" + tag: "1.18.1" pullPolicy: IfNotPresent # Values for the OCS Ingester library, used by BANZAI.