build-deploy-pudl.yml

name: build-deploy-pudl
on:
  workflow_dispatch:
  push:
    tags:
      - "v20*"
    branches:
      - try-google-batch
  schedule:
    - cron: "0 6 * * 1-5" # Weekdays at midnight on MST

env:
  GCP_BILLING_PROJECT: ${{ secrets.GCP_BILLING_PROJECT }}
  BUILD_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule
  GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule
  GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }}
  GCS_OUTPUT_BUCKET: gs://nightly-build-outputs.catalyst.coop
  BATCH_JOB_JSON: batch_job.json

jobs:
  build_and_deploy_pudl:
    name: Build Docker image, push to Docker Hub and deploy to a GCE VM
    runs-on: ubuntu-latest
    permissions:
      contents: write
      id-token: write
    steps:
      - name: Use pudl-deployment-dev vm and dev branch if running on a schedule
        if: ${{ (github.event_name == 'schedule') }}
        run: |
          echo "This action was triggered by a schedule."
          echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV
          echo "BUILD_REF=dev" >> $GITHUB_ENV

      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          ref: ${{ env.BUILD_REF }}

      - name: Set action environment variables
        run: |
          echo "NIGHTLY_TAG=nightly-$(date +%Y-%m-%d)" >> $GITHUB_ENV
          echo "BUILD_ID=$(date +%Y-%m-%d-%H%M)-$(git rev-parse --short HEAD)-${BUILD_REF}" >> $GITHUB_ENV

      - name: Show freshly set envvars
        run: |
          echo "GCE_INSTANCE: $GCE_INSTANCE"
          echo "BUILD_REF: $BUILD_REF"
          echo "NIGHTLY_TAG: $NIGHTLY_TAG"
          echo "BUILD_ID: $BUILD_ID"

      - name: Tag nightly build
        if: ${{ (github.event_name == 'schedule') }}
        run: |
          git config user.email "pudl@catalyst.coop"
          git config user.name "pudlbot"
          git tag -a -m "$NIGHTLY_TAG" $NIGHTLY_TAG $BUILD_REF
          git push origin $NIGHTLY_TAG

      - name: Docker Metadata
        id: docker_metadata
        uses: docker/metadata-action@v5
        with:
          images: catalystcoop/pudl-etl
          flavor: |
            latest=auto
          tags: |
            type=raw,value=${{ env.BUILD_REF}}
            type=ref,event=tag

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to DockerHub
        if: github.event_name != 'pull_request'
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build image and push to Docker Hub
        uses: docker/build-push-action@v5
        with:
          context: .
          file: docker/Dockerfile
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.docker_metadata.outputs.tags }}
          labels: ${{ steps.docker_metadata.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: inspect docker manifest right after pushing
        run: docker inspect docker.io/catalystcoop/pudl-etl:${{ env.BUILD_REF}}

      - id: "auth"
        uses: "google-github-actions/auth@v2"
        with:
          workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
          service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"

      # Setup gcloud CLI
      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v2

      - name: inspect docker manifest right before submitting Batch job
        run: docker inspect docker.io/catalystcoop/pudl-etl:${{ env.BUILD_REF}}

      # Deploy PUDL image to GCE
      - name: Make GCP Batch config file
        env:
          DAGSTER_PG_PASSWORD: ${{ secrets.DAGSTER_PG_PASSWORD }}
          PUDL_OUTPUT_PATH: ${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_ID }}
        run: |-
          ./devtools/generate_batch_config.py \
            --container-image "docker.io/catalystcoop/pudl-etl:${{ env.BUILD_REF}}" \
            --container-command "micromamba" \
            --container-arg="run" \
            --container-arg="--prefix" \
            --container-arg="/home/mambauser/env" \
            --container-arg="--attach" \
            --container-arg='' \
            --container-arg="bash" \
            --container-arg="./docker/gcp_pudl_etl.sh" \
            --container-env-file="./docker/.env" \
            --container-env BUILD_REF=${{ env.BUILD_REF}} \
            --container-env BUILD_ID=${{ env.BUILD_ID }} \
            --container-env NIGHTLY_TAG=${{ env.NIGHTLY_TAG }} \
            --container-env GITHUB_ACTION_TRIGGER=${{ github.event_name }} \
            --container-env SLACK_TOKEN=${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }} \
            --container-env GCE_INSTANCE=${{ env.GCE_INSTANCE }} \
            --container-env GCE_INSTANCE_ZONE=${{ env.GCE_INSTANCE_ZONE }} \
            --container-env GCP_BILLING_PROJECT=${{ secrets.GCP_BILLING_PROJECT }} \
            --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \
            --container-env DAGSTER_PG_USERNAME="postgres" \
            --container-env DAGSTER_PG_PASSWORD="$DAGSTER_PG_PASSWORD" \
            --container-env DAGSTER_PG_HOST="104.154.182.24" \
            --container-env DAGSTER_PG_DB="dagster-storage" \
            --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \
            --container-env PUDL_BOT_PAT=${{ secrets.PUDL_BOT_PAT }} \
            --container-env ZENODO_SANDBOX_TOKEN_PUBLISH=${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} \
            --container-env PUDL_SETTINGS_YML="/home/mambauser/pudl/src/pudl/package_data/settings/etl_full.yml" \
            --container-env PUDL_GCS_OUTPUT=${{ env.PUDL_OUTPUT_PATH }} \
            --output ${{ env.BATCH_JOB_JSON }}

      # Start the VM
      - name: Kick off batch job
        run: gcloud batch jobs submit --job-prefix run-etl-test --config ${{ env.BATCH_JOB_JSON }} --location us-west1

      - name: Post to a pudl-deployments channel
        id: slack
        uses: slackapi/slack-github-action@v1
        with:
          channel-id: "C03FHB9N0PQ"
          slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.GCS_OUTPUT_BUCKET }}/${{ env.BUILD_REF }}"
        env:
          channel-id: "C03FHB9N0PQ"
          SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }}