python.yaml

name: Python Connectors

on:
  push:
    branches: [main]
    paths:
      - "estuary-cdk/**"
      - "source-airtable/**"
      - "source-asana/**"
      - "source-facebook-marketing/**"
      - "source-gladly/**"
      - "source-google-ads/**"
      - "source-google-sheets-native/**"
      - "source-hubspot-native/**"
      - "source-hubspot/**"
      - "source-notion/**"
      - "source-linkedin-pages/**"
      - "source-linkedin-ads-v2/**"
      - "source-oracle-flashback/**"
      - "source-klaviyo/**"
      - "source-recharge/**"
      - "source-stripe-native/**"
      - "source-zendesk-support/**"
      - "source-iterable/**"
      - "source-pendo/**"
      - "source-twilio/**"
      - "source-jira-native/**"
      - "source-mixpanel-native/**"
      - "source-brevo/**"
      - "source-impact-native/**"

  pull_request:
    branches: [main]
    paths:
      - "estuary-cdk/**"
      - "source-airtable/**"
      - "source-asana/**"
      - "source-facebook-marketing/**"
      - "source-gladly/**"
      - "source-google-ads/**"
      - "source-google-sheets-native/**"
      - "source-hubspot-native/**"
      - "source-hubspot/**"
      - "source-notion/**"
      - "source-linkedin-pages/**"
      - "source-linkedin-ads-v2/**"
      - "source-oracle-flashback/**"
      - "source-recharge/**"
      - "source-klaviyo/**"
      - "source-recharge/**"
      - "source-stripe-native/**"
      - "source-zendesk-support/**"
      - "source-iterable/**"
      - "source-jira-native/**"
      - "source-pendo/**"
      - "source-twilio/**"
      - "source-jira-native/**"
      - "source-mixpanel-native/**"
      - "source-brevo/**"
      - "source-impact-native/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  py_connector:
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        connector:
          # Note that every entry here must specify usage_rate. We're unable to
          # set a default and selectively override it with `0.0`, because GH actions
          # considers zero values to be "unset" (which is bs and I'm salty about it).
          - name: source-airtable
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-asana
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-facebook-marketing
            type: capture
            version: v4
            usage_rate: "1.0"
          - name: source-gladly
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-google-ads
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-google-sheets-native
            type: capture
            version: v1
            usage_rate: "0.0"
          - name: source-hubspot-native
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-hubspot
            type: capture
            version: v5
            usage_rate: "1.0"
          - name: source-notion
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-oracle-flashback
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-linkedin-pages
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-linkedin-ads-v2
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-klaviyo
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-recharge
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-stripe-native
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-zendesk-support
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-iterable
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-pendo
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-twilio
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-jira-native
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-mixpanel-native
            type: capture
            version: v2
            usage_rate: "1.0"
          - name: source-brevo
            type: capture
            version: v1
            usage_rate: "1.0"
          - name: source-impact-native
            type: capture
            version: v2
            usage_rate: "1.0"

    steps:
      - uses: actions/checkout@v4

      - name: Download flow-network-tunnel and add it to $PATH
        run: |
          mkdir ./network-tunnel
          docker pull ghcr.io/estuary/network-tunnel:dev
          docker cp $(docker create --name temp ghcr.io/estuary/network-tunnel:dev sh):/flow-network-tunnel ./network-tunnel/flow-network-tunnel
          docker rm temp
          echo "${PWD}/network-tunnel" >> $GITHUB_PATH

      - name: Common Setup
        id: setup
        uses: ./.github/actions/setup
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          gcp_project_id: ${{ secrets.GCP_PROJECT_ID }}
          gcp_service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install Poetry
        uses: snok/install-poetry@v1

      - name: ${{ matrix.connector.name }} tests
        run: |
          cd ${{ matrix.connector.name }}
          poetry install
          source $(poetry env info --path)/bin/activate
          cd ..
          pytest ${{ matrix.connector.name }}

      # Uncomment to allow each connector to have its own Dockerfile
      #- name: Find Dockerfile for ${{ matrix.connector.name }}
      #  id: dockerfile
      #  run: |
      #    DOCKERFILE_PATH="${PWD}/estuary-cdk/common.Dockerfile"
      #    if [ -f "${{ matrix.connector.name }}/Dockerfile" ]; then
      #      DOCKERFILE_PATH="${PWD}/${{ matrix.connector.name }}/Dockerfile"
      #    fi
      #    echo "path=${DOCKERFILE_PATH}" >> $GITHUB_OUTPUT

      - name: Build ${{ matrix.connector.name }} Docker Image
        uses: docker/build-push-action@v6
        with:
          context: .
          file: estuary-cdk/common.Dockerfile
          load: true
          build-args: |
            CONNECTOR_NAME=${{ matrix.connector.name }}
            CONNECTOR_TYPE=${{ matrix.connector.type }}
            USAGE_RATE=${{ matrix.connector.usage_rate }}
          tags: ghcr.io/estuary/${{ matrix.connector.name }}:local

      - name: Deployment
        uses: ./.github/actions/deploy
        with:
          connector: ${{ matrix.connector.name }}
          pg_database: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }}
          pg_host: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }}
          pg_password: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }}
          pg_user: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }}
          tag_sha: ${{ steps.setup.outputs.tag_sha }}
          tag_version: ${{ matrix.connector.version }}
          variants: ${{ matrix.connector.variants }}