source-twilio: check for updated messages based on the lookback window #1005
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Connectors | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "estuary-cdk/**" | |
| - "source-airtable/**" | |
| - "source-asana/**" | |
| - "source-facebook-marketing/**" | |
| - "source-gladly/**" | |
| - "source-google-ads/**" | |
| - "source-google-sheets-native/**" | |
| - "source-hubspot-native/**" | |
| - "source-hubspot/**" | |
| - "source-notion/**" | |
| - "source-linkedin-pages/**" | |
| - "source-linkedin-ads-v2/**" | |
| - "source-oracle-flashback/**" | |
| - "source-klaviyo/**" | |
| - "source-recharge/**" | |
| - "source-stripe-native/**" | |
| - "source-zendesk-support/**" | |
| - "source-iterable/**" | |
| - "source-pendo/**" | |
| - "source-twilio/**" | |
| - "source-jira-native/**" | |
| - "source-mixpanel-native/**" | |
| - "source-brevo/**" | |
| - "source-impact-native/**" | |
| - "source-front/**" | |
| - "source-genesys/**" | |
| - "source-braintree-native/**" | |
| - "source-intercom-native/**" | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - "estuary-cdk/**" | |
| - "source-airtable/**" | |
| - "source-asana/**" | |
| - "source-facebook-marketing/**" | |
| - "source-gladly/**" | |
| - "source-google-ads/**" | |
| - "source-google-sheets-native/**" | |
| - "source-hubspot-native/**" | |
| - "source-hubspot/**" | |
| - "source-notion/**" | |
| - "source-linkedin-pages/**" | |
| - "source-linkedin-ads-v2/**" | |
| - "source-oracle-flashback/**" | |
| - "source-recharge/**" | |
| - "source-klaviyo/**" | |
| - "source-recharge/**" | |
| - "source-stripe-native/**" | |
| - "source-zendesk-support/**" | |
| - "source-iterable/**" | |
| - "source-jira-native/**" | |
| - "source-pendo/**" | |
| - "source-twilio/**" | |
| - "source-jira-native/**" | |
| - "source-mixpanel-native/**" | |
| - "source-brevo/**" | |
| - "source-impact-native/**" | |
| - "source-front/**" | |
| - "source-genesys/**" | |
| - "source-braintree-native/**" | |
| - "source-intercom-native/**" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| py_connector: | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| connector: | |
| # Note that every entry here must specify usage_rate. We're unable to | |
| # set a default and selectively override it with `0.0`, because GH actions | |
| # considers zero values to be "unset" (which is bs and I'm salty about it). | |
| - name: source-airtable | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-asana | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-facebook-marketing | |
| type: capture | |
| version: v4 | |
| usage_rate: "1.0" | |
| - name: source-gladly | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-google-ads | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-google-sheets-native | |
| type: capture | |
| version: v1 | |
| usage_rate: "0.0" | |
| - name: source-hubspot-native | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-hubspot | |
| type: capture | |
| version: v5 | |
| usage_rate: "1.0" | |
| - name: source-notion | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-oracle-flashback | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-linkedin-pages | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-linkedin-ads-v2 | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-klaviyo | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-recharge | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-stripe-native | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-zendesk-support | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-iterable | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-pendo | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-twilio | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-jira-native | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-mixpanel-native | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-brevo | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-impact-native | |
| type: capture | |
| version: v2 | |
| usage_rate: "1.0" | |
| - name: source-front | |
| type: capture | |
| version: v3 | |
| usage_rate: "1.0" | |
| - name: source-genesys | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-braintree-native | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| - name: source-intercom-native | |
| type: capture | |
| version: v1 | |
| usage_rate: "1.0" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download flow-network-tunnel and add it to $PATH | |
| run: | | |
| mkdir ./network-tunnel | |
| docker pull ghcr.io/estuary/network-tunnel:dev | |
| docker cp $(docker create --name temp ghcr.io/estuary/network-tunnel:dev sh):/flow-network-tunnel ./network-tunnel/flow-network-tunnel | |
| docker rm temp | |
| echo "${PWD}/network-tunnel" >> $GITHUB_PATH | |
| - name: Common Setup | |
| id: setup | |
| uses: ./.github/actions/setup | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| gcp_project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| gcp_service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| - name: ${{ matrix.connector.name }} tests | |
| run: | | |
| cd ${{ matrix.connector.name }} | |
| poetry install | |
| source $(poetry env info --path)/bin/activate | |
| cd .. | |
| pytest ${{ matrix.connector.name }} | |
| # Uncomment to allow each connector to have its own Dockerfile | |
| #- name: Find Dockerfile for ${{ matrix.connector.name }} | |
| # id: dockerfile | |
| # run: | | |
| # DOCKERFILE_PATH="${PWD}/estuary-cdk/common.Dockerfile" | |
| # if [ -f "${{ matrix.connector.name }}/Dockerfile" ]; then | |
| # DOCKERFILE_PATH="${PWD}/${{ matrix.connector.name }}/Dockerfile" | |
| # fi | |
| # echo "path=${DOCKERFILE_PATH}" >> $GITHUB_OUTPUT | |
| - name: Build ${{ matrix.connector.name }} Docker Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: estuary-cdk/common.Dockerfile | |
| load: true | |
| build-args: | | |
| CONNECTOR_NAME=${{ matrix.connector.name }} | |
| CONNECTOR_TYPE=${{ matrix.connector.type }} | |
| USAGE_RATE=${{ matrix.connector.usage_rate }} | |
| tags: ghcr.io/estuary/${{ matrix.connector.name }}:local | |
| - name: Deployment | |
| uses: ./.github/actions/deploy | |
| with: | |
| connector: ${{ matrix.connector.name }} | |
| pg_database: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_DATABASE }} | |
| pg_host: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_HOST }} | |
| pg_password: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_PASSWORD }} | |
| pg_user: ${{ secrets.POSTGRES_CONNECTOR_REFRESH_USER }} | |
| tag_sha: ${{ steps.setup.outputs.tag_sha }} | |
| tag_version: ${{ matrix.connector.version }} | |
| variants: ${{ matrix.connector.variants }} |