Metadata Extract - Loaders #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Metadata Extract - Loaders | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'The environment to run the workflow in' | |
| required: true | |
| type: choice | |
| default: 'preview' | |
| options: | |
| - 'preview' | |
| - 'production' | |
| schedule: | |
| - cron: '0 9 * * 1' # Run at midnight UTC on Mondays | |
| jobs: | |
| get_variants_list: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| sdk_loaders_matrix: ${{ steps.setmatrix_sdk_loaders.outputs.sdk_loaders_matrix }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Install hub-utils | |
| run: pipx install git+https://github.com/meltano/hub-utils.git | |
| - name: Get Variants List - SDK Loaders | |
| id: get-variants-list-sdk-loaders | |
| run: echo "VARIANTS_SDK_LOADERS=$(hub-utils get-variant-names $(pwd) --plugin-type=loaders --metadata-type=sdk)" >> $GITHUB_OUTPUT | |
| - name: Set Dynamic Matrix - SDK Loaders | |
| id: setmatrix_sdk_loaders | |
| run: | | |
| matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" | |
| echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" | |
| metadata_extract_sdk_loaders: | |
| name: SDK Loaders - Metadata Extract | |
| needs: get_variants_list | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment || github.event_name == 'schedule' && 'production' || 'preview' }} | |
| env: | |
| AWS_S3_BUCKET: "${{secrets.HUB_METADATA_S3_BUCKET }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.get_variants_list.outputs.sdk_loaders_matrix) }} | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Configure AWS credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| aws-region: us-west-2 | |
| role-session-name: "GitHubActions" | |
| - name: Install hub-utils | |
| run: pipx install git+https://github.com/meltano/hub-utils.git | |
| # This starts the attempts to install and extract metadata. Since we don't | |
| # know what python versions are accepted we start with the latest and continue | |
| # decreasing until we have a success or run out of python versions to attempt. | |
| - name: Install Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Extract Metadata (3.12) | |
| id: extract_312 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.11 (Only if 3.12 failed) | |
| if: always() && (steps.extract_312.outcome == 'failure') | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Extract Metadata (3.11) | |
| if: always() && (steps.extract_312.outcome == 'failure') | |
| id: extract_311 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.10 (Only if 3.11 failed) | |
| if: always() && (steps.extract_311.outcome == 'failure') | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Extract Metadata (3.10) | |
| if: always() && (steps.extract_311.outcome == 'failure') | |
| id: extract_310 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.9 (Only if 3.10 failed) | |
| if: always() && (steps.extract_310.outcome == 'failure') | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Extract Metadata (3.9) | |
| if: always() && (steps.extract_310.outcome == 'failure') | |
| id: extract_39 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data | |
| continue-on-error: true | |
| - name: Install Python 3.8 (Only if 3.9 failed) | |
| if: always() && (steps.extract_39.outcome == 'failure') | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.8' | |
| - name: Extract Metadata (3.8) | |
| if: always() && (steps.extract_39.outcome == 'failure') | |
| id: extract_38 | |
| run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data |