(BSR) feat(airflow): deploy CI/CD to GKE Airflow (dev) (#3782)

pass-culture · Feb 10, 2025 · 6665219 · 6665219
1 parent dd1d61b
commit 6665219
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 7 deletions.
diff --git a/.github/workflows/on_cd__workflow.yml b/.github/workflows/on_cd__workflow.yml
@@ -70,12 +70,29 @@ jobs:
       GITHUB_ENV_NAME: "dev"
       ENV_SHORT_NAME: "dev"
       COMPOSER_DAGS_BUCKET:  ${{ vars.COMPOSER_DAGS_BUCKET_DEV }}
-      COMPOSER_NAME: "data-composer-dev"
+      AIRFLOW_NAME: "data-composer-dev"
       DATA_GCP_PROJECT: "passculture-data-ehp"
       APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_DEV }}
     secrets:
       GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }}
       GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }}
+  airflow-deploy-on-dev:
+    # deploying new custom gke airflow on dev
+    if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/production' }}
+    uses: ./.github/workflows/reusable_deploy_composer.yml
+    needs: [test-orchestration, test-jobs, linter]
+    with:
+      GITHUB_ENV_NAME: "dev"
+      ENV_SHORT_NAME: "dev"
+      COMPOSER_DAGS_BUCKET:  ${{ vars.GKE_AIRFLOW_DAGS_BUCKET_DEV }}
+      AIRFLOW_NAME: "gke-data-airflow-dev"
+      DATA_GCP_PROJECT: "passculture-data-ehp"
+      APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_DEV }}
+      IS_COMPOSER: false
+    secrets:
+      GCP_EHP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_EHP_WORKLOAD_IDENTITY_PROVIDER }}
+      GCP_EHP_SERVICE_ACCOUNT: ${{ secrets.GCP_EHP_SERVICE_ACCOUNT }}
+
   composer-deploy-on-stg:
     if: github.ref == 'refs/heads/master'
     uses: ./.github/workflows/reusable_deploy_composer.yml
@@ -84,7 +101,7 @@ jobs:
       GITHUB_ENV_NAME: "staging"
       ENV_SHORT_NAME: "stg"
       COMPOSER_DAGS_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_STG }}
-      COMPOSER_NAME: "data-composer-stg"
+      AIRFLOW_NAME: "data-composer-stg"
       DATA_GCP_PROJECT: "passculture-data-ehp"
       APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_STG }}
     secrets:
@@ -99,7 +116,7 @@ jobs:
       GITHUB_ENV_NAME: "production"
       ENV_SHORT_NAME: "prod"
       COMPOSER_DAGS_BUCKET: ${{ vars.COMPOSER_DAGS_BUCKET_PROD }}
-      COMPOSER_NAME: "data-composer-prod"
+      AIRFLOW_NAME: "data-composer-prod"
       DATA_GCP_PROJECT: "passculture-data-prod"
       APPLICATIVE_EXTERNAL_CONNECTION_ID: ${{ vars.APPLICATIVE_EXTERNAL_CONNECTION_ID_PROD }}
     secrets:

diff --git a/.github/workflows/reusable_deploy_composer.yml b/.github/workflows/reusable_deploy_composer.yml
@@ -11,7 +11,7 @@ on:
       COMPOSER_DAGS_BUCKET:
         type: string
         required: true
-      COMPOSER_NAME:
+      AIRFLOW_NAME:
         type: string
         required: true
       GITHUB_ENV_NAME:
@@ -23,6 +23,11 @@ on:
       APPLICATIVE_EXTERNAL_CONNECTION_ID:
         type: string
         required: false
+      IS_COMPOSER:
+        type: boolean
+        required: true
+        default: true
+        description: "Temporary; If false, the workflow will not wait for the composer to be deployed as we are not deploying a composer but custom airflow"
     secrets:
       GCP_EHP_WORKLOAD_IDENTITY_PROVIDER:
         required: true
@@ -125,8 +130,9 @@ jobs:
           gsutil cp target/static_index.html gs://$COMPOSER_DAGS_BUCKET/plugins/static/dbt_docs/index.html
 
       - name: "Wait for composer to be deployed (10s x 6 times)"
+        if: ${{ inputs.IS_COMPOSER }}
         run: |
-          ./wait_for_dag_deployed.sh ${{ inputs.COMPOSER_NAME }} ${{ env.GCP_REGION }} airflow_monitoring 6 10 ${{ inputs.DATA_GCP_PROJECT }}
+          ./wait_for_dag_deployed.sh ${{ inputs.AIRFLOW_NAME }} ${{ env.GCP_REGION }} airflow_monitoring 6 10 ${{ inputs.DATA_GCP_PROJECT }}
       - name: "Post to a Slack channel"
         if: always()
         uses: slackapi/[email protected]
@@ -145,7 +151,7 @@ jobs:
                   "author_icon": "https://github.com/${{github.actor}}.png",
                   "title": "Composer déployé",
                   "title_link": "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}",
-                  "text": "Composer a été déployé sur l'environnement ${{ inputs.COMPOSER_NAME }}"
+                  "text": "Airflow a été déployé sur l'environnement ${{ inputs.AIRFLOW_NAME }}"
                 }
               ],
               "unfurl_links": false,

diff --git a/orchestration/plugins/dbt_docs.py b/orchestration/plugins/dbt_docs.py
@@ -5,14 +5,21 @@
 from flask_appbuilder import BaseView as AppBuilderBaseView
 from flask_appbuilder import expose
 
+from airflow.exceptions import AirflowConfigException
 from airflow.security import permissions
 from airflow.www.auth import has_access
 
 
 def get_airflow_home() -> str:
     if os.environ.get("LOCAL_ENV", None) == "1":
         return "/opt/airflow/"
-    return "/home/airflow/gcs"
+    if os.environ.get("DAG_FOLDER", None) == "/opt/airflow/dags":
+        return "/opt/airflow/"
+    if os.environ.get("DAG_FOLDER", None) == "/home/airflow/gcs/dags":
+        return "/home/airflow/gcs"
+    raise AirflowConfigException(
+        "Airflow home not found, failed to determine environment"
+    )
 
 
 AIRFLOW_HOME = get_airflow_home()