diff --git a/charts/airflow/README.md b/charts/airflow/README.md index 3c8a6b15..243a8894 100644 --- a/charts/airflow/README.md +++ b/charts/airflow/README.md @@ -224,6 +224,7 @@ Parameter | Description | Default `airflow.protectedPipPackages` | pip packages that are protected from upgrade/downgrade by `extraPipPackages` | `["apache-airflow"]` `airflow.extraEnv` | extra environment variables for the airflow Pods | `[]` `airflow.extraContainers` | extra containers for the airflow Pods | `[]` +`airflow.extraInitContainers` | extra init-containers for the airflow Pods | `[]` `airflow.extraVolumeMounts` | extra VolumeMounts for the airflow Pods | `[]` `airflow.extraVolumes` | extra Volumes for the airflow Pods | `[]` `airflow.clusterDomain` | kubernetes cluster domain name | `cluster.local` @@ -256,11 +257,12 @@ Parameter | Description | Default `scheduler.podDisruptionBudget.*` | configs for the PodDisruptionBudget of the scheduler | `` `scheduler.logCleanup.*` | configs for the log-cleanup sidecar of the scheduler | `` `scheduler.numRuns` | the value of the `airflow --num_runs` parameter used to run the airflow scheduler | `-1` +`scheduler.livenessProbe.*` | configs for the scheduler Pods' liveness probe | `` `scheduler.extraPipPackages` | extra pip packages to install in the scheduler Pods | `[]` +`scheduler.extraContainers` | extra containers for the scheduler Pods | `[]` +`scheduler.extraInitContainers` | extra init-containers for the scheduler Pods | `[]` `scheduler.extraVolumeMounts` | extra VolumeMounts for the scheduler Pods | `[]` `scheduler.extraVolumes` | extra Volumes for the scheduler Pods | `[]` -`scheduler.livenessProbe.*` | configs for the scheduler Pods' liveness probe | `` -`scheduler.extraInitContainers` | extra init containers to run in the scheduler Pods | `[]` @@ -287,6 +289,8 @@ Parameter | Description | Default `web.readinessProbe.*` | configs for the web Pods' readiness probe | `` `web.livenessProbe.*` | configs for the web Pods' liveness probe | `` `web.extraPipPackages` | extra pip packages to install in the web Pods | `[]` +`web.extraContainers` | extra containers for the web Pods | `[]` +`web.extraInitContainers` | extra init-containers for the web Pods | `[]` `web.extraVolumeMounts` | extra VolumeMounts for the web Pods | `[]` `web.extraVolumes` | extra Volumes for the web Pods | `[]` @@ -317,6 +321,8 @@ Parameter | Description | Default `workers.logCleanup.*` | configs for the log-cleanup sidecar of the worker Pods | `` `workers.livenessProbe.*` | configs for the worker Pods' liveness probe | `` `workers.extraPipPackages` | extra pip packages to install in the worker Pods | `[]` +`workers.extraContainers` | extra containers for the worker Pods | `[]` +`workers.extraInitContainers` | extra init-containers for the worker Pods | `[]` `workers.extraVolumeMounts` | extra VolumeMounts for the worker Pods | `[]` `workers.extraVolumes` | extra Volumes for the worker Pods | `[]` @@ -344,6 +350,8 @@ Parameter | Description | Default `triggerer.capacity` | maximum number of triggers each triggerer will run at once (sets `AIRFLOW__TRIGGERER__DEFAULT_CAPACITY`) | `1000` `triggerer.livenessProbe.*` | configs for the triggerer Pods' liveness probe | `` `triggerer.extraPipPackages` | extra pip packages to install in the triggerer Pods | `[]` +`triggerer.extraContainers` | extra containers for the triggerer Pods | `[]` +`triggerer.extraInitContainers` | extra init-containers for the triggerer Pods | `[]` `triggerer.extraVolumeMounts` | extra VolumeMounts for the triggerer Pods | `[]` `triggerer.extraVolumes` | extra Volumes for the triggerer Pods | `[]` @@ -370,7 +378,9 @@ Parameter | Description | Default `flower.basicAuthSecret` | the name of a pre-created secret containing the basic authentication value for flower | `""` `flower.basicAuthSecretKey` | the key within `flower.basicAuthSecret` containing the basic authentication string | `""` `flower.service.*` | configs for the Service of the flower Pods | `` -`flower.extraPipPackages` | extra pip packages to install in the flower Pod | `[]` +`flower.extraPipPackages` | extra pip packages to install in the flower Pods | `[]` +`flower.extraContainers` | extra containers for the flower Pods | `[]` +`flower.extraInitContainers` | extra init-containers for the flower Pods | `[]` `flower.extraVolumeMounts` | extra VolumeMounts for the flower Pods | `[]` `flower.extraVolumes` | extra Volumes for the flower Pods | `[]` diff --git a/charts/airflow/templates/flower/flower-deployment.yaml b/charts/airflow/templates/flower/flower-deployment.yaml index dca41531..bfd71953 100644 --- a/charts/airflow/templates/flower/flower-deployment.yaml +++ b/charts/airflow/templates/flower/flower-deployment.yaml @@ -97,6 +97,12 @@ spec: {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} + {{- if .Values.airflow.extraInitContainers }} + {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.flower.extraInitContainers }} + {{- toYaml .Values.flower.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: airflow-flower {{- include "airflow.image" . | indent 10 }} @@ -163,6 +169,9 @@ spec: {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.flower.extraContainers }} + {{- toYaml .Values.flower.extraContainers | nindent 8 }} + {{- end }} {{- if $volumes }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/scheduler/scheduler-deployment.yaml b/charts/airflow/templates/scheduler/scheduler-deployment.yaml index cf8e1438..30ed6f3f 100644 --- a/charts/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/charts/airflow/templates/scheduler/scheduler-deployment.yaml @@ -104,6 +104,9 @@ spec: {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} + {{- if .Values.airflow.extraInitContainers }} + {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }} + {{- end }} {{- if .Values.scheduler.extraInitContainers }} {{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }} {{- end }} @@ -245,6 +248,9 @@ spec: {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.scheduler.extraContainers }} + {{- toYaml .Values.scheduler.extraContainers | nindent 8 }} + {{- end }} {{- if or ($volumes) (include "airflow.executor.kubernetes_like" .) }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/triggerer/triggerer-deployment.yaml b/charts/airflow/templates/triggerer/triggerer-deployment.yaml index 480a281a..bac60bb1 100644 --- a/charts/airflow/templates/triggerer/triggerer-deployment.yaml +++ b/charts/airflow/templates/triggerer/triggerer-deployment.yaml @@ -96,6 +96,12 @@ spec: {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} + {{- if .Values.airflow.extraInitContainers }} + {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.triggerer.extraInitContainers }} + {{- toYaml .Values.triggerer.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: airflow-triggerer {{- include "airflow.image" . | indent 10 }} @@ -171,6 +177,9 @@ spec: {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.triggerer.extraContainers }} + {{- toYaml .Values.triggerer.extraContainers | nindent 8 }} + {{- end }} {{- if $volumes }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/templates/webserver/webserver-deployment.yaml b/charts/airflow/templates/webserver/webserver-deployment.yaml index 4aa33bdf..74d24a0f 100644 --- a/charts/airflow/templates/webserver/webserver-deployment.yaml +++ b/charts/airflow/templates/webserver/webserver-deployment.yaml @@ -98,6 +98,12 @@ spec: {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} + {{- if .Values.airflow.extraInitContainers }} + {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.web.extraInitContainers }} + {{- toYaml .Values.web.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: airflow-web {{- include "airflow.image" . | indent 10 }} @@ -155,6 +161,9 @@ spec: {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.web.extraContainers }} + {{- toYaml .Values.web.extraContainers | nindent 8 }} + {{- end }} volumes: {{- $volumes | indent 8 }} {{- if .Values.web.webserverConfig.enabled }} diff --git a/charts/airflow/templates/worker/worker-statefulset.yaml b/charts/airflow/templates/worker/worker-statefulset.yaml index e19b647c..02743115 100644 --- a/charts/airflow/templates/worker/worker-statefulset.yaml +++ b/charts/airflow/templates/worker/worker-statefulset.yaml @@ -101,6 +101,12 @@ spec: {{- end }} {{- include "airflow.init_container.check_db" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} {{- include "airflow.init_container.wait_for_db_migrations" (dict "Release" .Release "Values" .Values "volumeMounts" $volumeMounts) | indent 8 }} + {{- if .Values.airflow.extraInitContainers }} + {{- toYaml .Values.airflow.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.workers.extraInitContainers }} + {{- toYaml .Values.workers.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: airflow-worker {{- include "airflow.image" . | indent 10 }} @@ -223,6 +229,9 @@ spec: {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} + {{- if .Values.workers.extraContainers }} + {{- toYaml .Values.workers.extraContainers | nindent 8 }} + {{- end }} {{- if $volumes }} volumes: {{- $volumes | indent 8 }} diff --git a/charts/airflow/values.yaml b/charts/airflow/values.yaml index cb06e796..e589e8b6 100644 --- a/charts/airflow/values.yaml +++ b/charts/airflow/values.yaml @@ -251,6 +251,12 @@ airflow: ## extraContainers: [] + ## extra init-containers for the airflow Pods + ## - spec for Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraInitContainers: [] + ## extra VolumeMounts for the airflow Pods ## - spec for VolumeMount: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core @@ -723,6 +729,18 @@ scheduler: ## extraPipPackages: [] + ## extra containers for the scheduler Pods + ## - spec for Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraContainers: [] + + ## extra init-containers for the scheduler Pods + ## - spec of Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraInitContainers: [] + ## extra VolumeMounts for the scheduler Pods ## - spec of VolumeMount: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core @@ -735,12 +753,6 @@ scheduler: ## extraVolumes: [] - ## extra init containers to run in the scheduler Pods - ## - spec of Container: - ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core - ## - extraInitContainers: [] - ################################### ## COMPONENT | Airflow Webserver ################################### @@ -897,6 +909,18 @@ web: ## extraPipPackages: [] + ## extra containers for the web Pods + ## - spec for Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraContainers: [] + + ## extra init-containers for the web Pods + ## - spec of Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraInitContainers: [] + ## extra VolumeMounts for the web Pods ## - spec for VolumeMount: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core @@ -1093,6 +1117,18 @@ workers: ## extraPipPackages: [] + ## extra containers for the worker Pods + ## - spec for Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraContainers: [] + + ## extra init-containers for the worker Pods + ## - spec of Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraInitContainers: [] + ## extra VolumeMounts for the worker Pods ## - spec for VolumeMount: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core @@ -1218,6 +1254,18 @@ triggerer: ## extraPipPackages: [] + ## extra containers for the triggerer Pods + ## - spec for Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraContainers: [] + + ## extra init-containers for the triggerer Pods + ## - spec of Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraInitContainers: [] + ## extra VolumeMounts for the triggerer Pods ## - spec for VolumeMount: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core @@ -1358,7 +1406,7 @@ flower: timeoutSeconds: 5 failureThreshold: 6 - ## extra pip packages to install in the flower Pod + ## extra pip packages to install in the flower Pods ## ## ____ EXAMPLE _______________ ## extraPipPackages: @@ -1366,6 +1414,18 @@ flower: ## extraPipPackages: [] + ## extra containers for the flower Pods + ## - spec for Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraContainers: [] + + ## extra init-containers for the flower Pods + ## - spec of Container: + ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#container-v1-core + ## + extraInitContainers: [] + ## extra VolumeMounts for the flower Pods ## - spec for VolumeMount: ## https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#volumemount-v1-core