feat: add extraContainers and extraInitContainers values for all …

…components (#856) * feat: add `scheduler.extraContainers` value Signed-off-by: Mathew Wicks <[email protected]> * feat: add `airflow.extraInitContainers` value Signed-off-by: Mathew Wicks <[email protected]> * feat: add `web.extra*Containers` values Signed-off-by: Mathew Wicks <[email protected]> * feat: add `workers.extra*Containers` values Signed-off-by: Mathew Wicks <[email protected]> * feat: add `triggerer.extra*Containers` values Signed-off-by: Mathew Wicks <[email protected]> * feat: add `flower.extra*Containers` values Signed-off-by: Mathew Wicks <[email protected]> --------- Signed-off-by: Mathew Wicks <[email protected]>
airflow-helm · May 1, 2024 · c4fa882 · c4fa882
1 parent a13412a
commit c4fa882
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 10 deletions.
diff --git a/charts/airflow/README.md b/charts/airflow/README.md
@@ -224,6 +224,7 @@ Parameter | Description | Default
 `airflow.protectedPipPackages` | pip packages that are protected from upgrade/downgrade by `extraPipPackages` | `["apache-airflow"]`
 `airflow.extraEnv` | extra environment variables for the airflow Pods | `[]`
 `airflow.extraContainers` | extra containers for the airflow Pods | `[]`
+`airflow.extraInitContainers` | extra init-containers for the airflow Pods | `[]`
 `airflow.extraVolumeMounts` | extra VolumeMounts for the airflow Pods | `[]`
 `airflow.extraVolumes` | extra Volumes for the airflow Pods | `[]`
 `airflow.clusterDomain` | kubernetes cluster domain name | `cluster.local`
@@ -256,11 +257,12 @@ Parameter | Description | Default
 `scheduler.podDisruptionBudget.*` | configs for the PodDisruptionBudget of the scheduler | `<see values.yaml>`
 `scheduler.logCleanup.*` | configs for the log-cleanup sidecar of the scheduler | `<see values.yaml>`
 `scheduler.numRuns` | the value of the `airflow --num_runs` parameter used to run the airflow scheduler | `-1`
+`scheduler.livenessProbe.*` | configs for the scheduler Pods' liveness probe | `<see values.yaml>`
 `scheduler.extraPipPackages` | extra pip packages to install in the scheduler Pods | `[]`
+`scheduler.extraContainers` | extra containers for the scheduler Pods | `[]`
+`scheduler.extraInitContainers` | extra init-containers for the scheduler Pods | `[]`
 `scheduler.extraVolumeMounts` | extra VolumeMounts for the scheduler Pods | `[]`
 `scheduler.extraVolumes` | extra Volumes for the scheduler Pods | `[]`
-`scheduler.livenessProbe.*` | configs for the scheduler Pods' liveness probe | `<see values.yaml>`
-`scheduler.extraInitContainers` | extra init containers to run in the scheduler Pods | `[]`
 
 </details>
 
@@ -287,6 +289,8 @@ Parameter | Description | Default
 `web.readinessProbe.*` | configs for the web Pods' readiness probe | `<see values.yaml>`
 `web.livenessProbe.*` | configs for the web Pods' liveness probe | `<see values.yaml>`
 `web.extraPipPackages` | extra pip packages to install in the web Pods | `[]`
+`web.extraContainers` | extra containers for the web Pods | `[]`
+`web.extraInitContainers` | extra init-containers for the web Pods | `[]`
 `web.extraVolumeMounts` | extra VolumeMounts for the web Pods | `[]`
 `web.extraVolumes` | extra Volumes for the web Pods | `[]`
 
@@ -317,6 +321,8 @@ Parameter | Description | Default
 `workers.logCleanup.*` | configs for the log-cleanup sidecar of the worker Pods | `<see values.yaml>`
 `workers.livenessProbe.*` | configs for the worker Pods' liveness probe | `<see values.yaml>`
 `workers.extraPipPackages` | extra pip packages to install in the worker Pods | `[]`
+`workers.extraContainers` | extra containers for the worker Pods | `[]`
+`workers.extraInitContainers` | extra init-containers for the worker Pods | `[]`
 `workers.extraVolumeMounts` | extra VolumeMounts for the worker Pods | `[]`
 `workers.extraVolumes` | extra Volumes for the worker Pods | `[]`
 
@@ -344,6 +350,8 @@ Parameter | Description | Default
 `triggerer.capacity` | maximum number of triggers each triggerer will run at once (sets `AIRFLOW__TRIGGERER__DEFAULT_CAPACITY`) | `1000`
 `triggerer.livenessProbe.*` | configs for the triggerer Pods' liveness probe | `<see values.yaml>`
 `triggerer.extraPipPackages` | extra pip packages to install in the triggerer Pods | `[]`
+`triggerer.extraContainers` | extra containers for the triggerer Pods | `[]`
+`triggerer.extraInitContainers` | extra init-containers for the triggerer Pods | `[]`
 `triggerer.extraVolumeMounts` | extra VolumeMounts for the triggerer Pods | `[]`
 `triggerer.extraVolumes` | extra Volumes for the triggerer Pods | `[]`
 
@@ -370,7 +378,9 @@ Parameter | Description | Default
 `flower.basicAuthSecret` | the name of a pre-created secret containing the basic authentication value for flower | `""`
 `flower.basicAuthSecretKey` | the key within `flower.basicAuthSecret` containing the basic authentication string | `""`
 `flower.service.*` | configs for the Service of the flower Pods | `<see values.yaml>`
-`flower.extraPipPackages` | extra pip packages to install in the flower Pod | `[]`
+`flower.extraPipPackages` | extra pip packages to install in the flower Pods | `[]`
+`flower.extraContainers` | extra containers for the flower Pods | `[]`
+`flower.extraInitContainers` | extra init-containers for the flower Pods | `[]`
 `flower.extraVolumeMounts` | extra VolumeMounts for the flower Pods | `[]`
 `flower.extraVolumes` | extra Volumes for the flower Pods | `[]`
 

diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml
@@ -97,6 +97,12 @@ spec:
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
+        {{- if .Values.airflow.extraInitContainers }}
+        {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }}
+        {{- end }}
+        {{- if .Values.flower.extraInitContainers }}
+        {{- toYaml .Values.flower.extraInitContainers | nindent 8 }}
+        {{- end }}
       containers:
         - name: airflow-flower
           {{- include "airflow.image" . | indent 10 }}
@@ -163,6 +169,9 @@ spec:
         {{- if .Values.airflow.extraContainers }}
         {{- toYaml .Values.airflow.extraContainers | nindent 8 }}
         {{- end }}
+        {{- if .Values.flower.extraContainers }}
+        {{- toYaml .Values.flower.extraContainers | nindent 8 }}
+        {{- end }}
       {{- if $volumes }}
       volumes:
         {{- $volumes | indent 8 }}

diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml
@@ -104,6 +104,9 @@ spec:
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
+        {{- if .Values.airflow.extraInitContainers }}
+        {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }}
+        {{- end }}
         {{- if .Values.scheduler.extraInitContainers }}
         {{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }}
         {{- end }}
@@ -245,6 +248,9 @@ spec:
         {{- if .Values.airflow.extraContainers }}
         {{- toYaml .Values.airflow.extraContainers | nindent 8 }}
         {{- end }}
+        {{- if .Values.scheduler.extraContainers }}
+        {{- toYaml .Values.scheduler.extraContainers | nindent 8 }}
+        {{- end }}
       {{- if or ($volumes) (include "airflow.executor.kubernetes_like" .) }}
       volumes:
         {{- $volumes | indent 8 }}

diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml
@@ -96,6 +96,12 @@ spec:
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
+        {{- if .Values.airflow.extraInitContainers }}
+        {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }}
+        {{- end }}
+        {{- if .Values.triggerer.extraInitContainers }}
+        {{- toYaml .Values.triggerer.extraInitContainers | nindent 8 }}
+        {{- end }}
       containers:
         - name: airflow-triggerer
           {{- include "airflow.image" . | indent 10 }}
@@ -171,6 +177,9 @@ spec:
         {{- if .Values.airflow.extraContainers }}
         {{- toYaml .Values.airflow.extraContainers | nindent 8 }}
         {{- end }}
+        {{- if .Values.triggerer.extraContainers }}
+        {{- toYaml .Values.triggerer.extraContainers | nindent 8 }}
+        {{- end }}
       {{- if $volumes }}
       volumes:
         {{- $volumes | indent 8 }}

diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml
@@ -98,6 +98,12 @@ spec:
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
+        {{- if .Values.airflow.extraInitContainers }}
+        {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }}
+        {{- end }}
+        {{- if .Values.web.extraInitContainers }}
+        {{- toYaml .Values.web.extraInitContainers | nindent 8 }}
+        {{- end }}
       containers:
         - name: airflow-web
           {{- include "airflow.image" . | indent 10 }}
@@ -155,6 +161,9 @@ spec:
         {{- if .Values.airflow.extraContainers }}
         {{- toYaml .Values.airflow.extraContainers | nindent 8 }}
         {{- end }}
+        {{- if .Values.web.extraContainers }}
+        {{- toYaml .Values.web.extraContainers | nindent 8 }}
+        {{- end }}
       volumes:
         {{- $volumes | indent 8 }}
         {{- if .Values.web.webserverConfig.enabled }}

diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml
@@ -101,6 +101,12 @@ spec:
         {{- end }}
         {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
         {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }}
+        {{- if .Values.airflow.extraInitContainers }}
+        {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }}
+        {{- end }}
+        {{- if .Values.workers.extraInitContainers }}
+        {{- toYaml .Values.workers.extraInitContainers | nindent 8 }}
+        {{- end }}
       containers:
         - name: airflow-worker
           {{- include "airflow.image" . | indent 10 }}
@@ -223,6 +229,9 @@ spec:
         {{- if .Values.airflow.extraContainers }}
         {{- toYaml .Values.airflow.extraContainers | nindent 8 }}
         {{- end }}
+        {{- if .Values.workers.extraContainers }}
+        {{- toYaml .Values.workers.extraContainers | nindent 8 }}
+        {{- end }}
       {{- if $volumes }}
       volumes:
         {{- $volumes | indent 8 }}

diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml
@@ -251,6 +251,12 @@ airflow:
   ##
   extraContainers: []
 
+  ## extra init-containers for the airflow Pods
+  ## - spec for Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraInitContainers: []
+
   ## extra VolumeMounts for the airflow Pods
   ## - spec for VolumeMount:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core
@@ -723,6 +729,18 @@ scheduler:
   ##
   extraPipPackages: []
 
+  ## extra containers for the scheduler Pods
+  ## - spec for Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraContainers: []
+
+  ## extra init-containers for the scheduler Pods
+  ## - spec of Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraInitContainers: []
+
   ## extra VolumeMounts for the scheduler Pods
   ## - spec of VolumeMount:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core
@@ -735,12 +753,6 @@ scheduler:
   ##
   extraVolumes: []
 
-  ## extra init containers to run in the scheduler Pods
-  ## - spec of Container:
-  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
-  ##
-  extraInitContainers: []
-
 ###################################
 ## COMPONENT | Airflow Webserver
 ###################################
@@ -897,6 +909,18 @@ web:
   ##
   extraPipPackages: []
 
+  ## extra containers for the web Pods
+  ## - spec for Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraContainers: []
+
+  ## extra init-containers for the web Pods
+  ## - spec of Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraInitContainers: []
+
   ## extra VolumeMounts for the web Pods
   ## - spec for VolumeMount:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core
@@ -1093,6 +1117,18 @@ workers:
   ##
   extraPipPackages: []
 
+  ## extra containers for the worker Pods
+  ## - spec for Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraContainers: []
+
+  ## extra init-containers for the worker Pods
+  ## - spec of Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraInitContainers: []
+
   ## extra VolumeMounts for the worker Pods
   ## - spec for VolumeMount:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core
@@ -1218,6 +1254,18 @@ triggerer:
   ##
   extraPipPackages: []
 
+  ## extra containers for the triggerer Pods
+  ## - spec for Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraContainers: []
+
+  ## extra init-containers for the triggerer Pods
+  ## - spec of Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraInitContainers: []
+
   ## extra VolumeMounts for the triggerer Pods
   ## - spec for VolumeMount:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core
@@ -1358,14 +1406,26 @@ flower:
     timeoutSeconds: 5
     failureThreshold: 6
 
-  ## extra pip packages to install in the flower Pod
+  ## extra pip packages to install in the flower Pods
   ##
   ## ____ EXAMPLE _______________
   ##   extraPipPackages:
   ##     - "SomeProject==1.0.0"
   ##
   extraPipPackages: []
 
+  ## extra containers for the flower Pods
+  ## - spec for Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraContainers: []
+
+  ## extra init-containers for the flower Pods
+  ## - spec of Container:
+  ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core
+  ##
+  extraInitContainers: []
+
   ## extra VolumeMounts for the flower Pods
   ## - spec for VolumeMount:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core