From 4dfb80aa2ac1cc3662d95038bb666285fec80111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 5 Nov 2024 13:55:57 -0600 Subject: [PATCH 1/2] ci: Use uv in more places in CI --- .github/workflows/check-yaml-format.yml | 10 ++++++---- .github/workflows/metadata-extract-airbyte.yml | 10 ++++++++-- .../workflows/metadata-extract-extractors.yml | 17 ++++++++++++++--- .github/workflows/metadata-extract-loaders.yml | 11 +++++++++-- .github/workflows/metadata-refresh.yml | 8 ++++++-- .github/workflows/test-plugin-command.yml | 13 +++++++++---- .github/workflows/test_dispatcher.yml | 2 ++ .github/workflows/test_meltano_add_install.yml | 3 ++- 8 files changed, 56 insertions(+), 18 deletions(-) diff --git a/.github/workflows/check-yaml-format.yml b/.github/workflows/check-yaml-format.yml index 6ca9a4439..76955e389 100644 --- a/.github/workflows/check-yaml-format.yml +++ b/.github/workflows/check-yaml-format.yml @@ -1,3 +1,5 @@ +name: Check YAML format + on: push: branches: @@ -20,10 +22,10 @@ jobs: files: | _data/**/*.yml - - name: Install yamllint - if: steps.changed-files.outputs.any_changed == 'true' - run: pipx install yamllint + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" - name: Lint YAML files if: steps.changed-files.outputs.any_changed == 'true' - run: yamllint ${{ steps.changed-files.outputs.all_changed_files }} + run: uvx yamllint ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml index 50dd8e4e7..9eaa74638 100644 --- a/.github/workflows/metadata-extract-airbyte.yml +++ b/.github/workflows/metadata-extract-airbyte.yml @@ -22,9 +22,12 @@ jobs: steps: - uses: actions/checkout@v4.2.2 + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main - name: Get Variants List - Airbyte id: get-variants-list-airbyte @@ -81,9 +84,12 @@ jobs: - uses: actions/download-artifact@v3 with: name: extractor--${{ matrix.source-name }}--airbyte.json + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main - name: Upload Metadata run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml index 3d23e7a09..cdd05d2aa 100644 --- a/.github/workflows/metadata-extract-extractors.yml +++ b/.github/workflows/metadata-extract-extractors.yml @@ -23,9 +23,12 @@ jobs: steps: - uses: actions/checkout@v4.2.2 + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main - name: Get Variants List - SDK Extractors (p1) id: get-variants-list-sdk-ex-p1 @@ -69,8 +72,12 @@ jobs: aws-region: us-west-2 role-session-name: "GitHubActions" + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" + - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main # This starts the attempts to install and extract metadata. Since we don't # know what python versions are accepted we start with the latest and continue @@ -142,8 +149,12 @@ jobs: aws-region: us-west-2 role-session-name: "GitHubActions" + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" + - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main - name: Install Python uses: actions/setup-python@v5 diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml index 07f5d4b96..8ebee156e 100644 --- a/.github/workflows/metadata-extract-loaders.yml +++ b/.github/workflows/metadata-extract-loaders.yml @@ -22,9 +22,12 @@ jobs: steps: - uses: actions/checkout@v4.2.2 + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main - name: Get Variants List - SDK Loaders id: get-variants-list-sdk-loaders @@ -58,8 +61,12 @@ jobs: aws-region: us-west-2 role-session-name: "GitHubActions" + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" + - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable + run: uv tool install git+https://github.com/meltano/hub-utils.git@main # This starts the attempts to install and extract metadata. Since we don't # know what python versions are accepted we start with the latest and continue diff --git a/.github/workflows/metadata-refresh.yml b/.github/workflows/metadata-refresh.yml index 727bbe768..dc35eaaf2 100644 --- a/.github/workflows/metadata-refresh.yml +++ b/.github/workflows/metadata-refresh.yml @@ -34,8 +34,12 @@ jobs: aws-region: us-west-2 role-session-name: "GitHubActions" + - uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" + - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git@main + run: uv tool install git+https://github.com/meltano/hub-utils.git@main - name: Download Raw JSON Output run: hub-utils download-metadata "/home/runner/work/downloaded_data" --ignore-list-str="extractors/tap-ga4/connorflyn,extractors/tap-ga4/meltanolabs,extractors/tap-ga4/sehnem,extractors/tap-jira/meltanolabs,extractors/tap-salesforce-commerce/hotgluexyz,loaders/target-s3/crowemi,loaders/target-s3-parquet/jkausti,extractors/tap-rest-api-msdk/widen" @@ -45,7 +49,7 @@ jobs: - name: Get current date id: date - run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - name: Create Pull Request uses: peter-evans/create-pull-request@v7 diff --git a/.github/workflows/test-plugin-command.yml b/.github/workflows/test-plugin-command.yml index 266d848ca..f61e27a9f 100644 --- a/.github/workflows/test-plugin-command.yml +++ b/.github/workflows/test-plugin-command.yml @@ -79,20 +79,25 @@ jobs: uses: actions/setup-python@v5 with: # https://github.com/actions/setup-python - python-version: 3.9 + python-version: 3.10 cache: 'pip' + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: ">=0.4.30" + - name: Install plugin id: plugin-install run: | - pipx install ${{ env.PIP_URL }} 2>&1 | tee install-output.txt - echo "::set-output name=failed::${PIPESTATUS[0]}" + uv tool install ${{ env.PIP_URL }} 2>&1 | tee install-output.txt + echo "failed=${PIPESTATUS[0]}" >> $GITHUB_ENV - name: Create installation failure log if: steps.plugin-install.outputs.failed == '1' run: | echo "JOB_OUTPUT<> $GITHUB_ENV - echo "melty-bot % pipx install ${{ env.PIP_URL }}" >> $GITHUB_ENV + echo "melty-bot % uv tool install ${{ env.PIP_URL }}" >> $GITHUB_ENV cat install-output.txt >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV diff --git a/.github/workflows/test_dispatcher.yml b/.github/workflows/test_dispatcher.yml index 1ca0d7cbe..8ab5f2138 100644 --- a/.github/workflows/test_dispatcher.yml +++ b/.github/workflows/test_dispatcher.yml @@ -12,6 +12,7 @@ on: jobs: get_changed_files: + name: Get changed files runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -38,6 +39,7 @@ jobs: run: echo "matrix={\"changed_file\":${{ steps.filter_files.outputs.filtered_files }}}" >> "$GITHUB_OUTPUT" dispatch_tests: + name: Dispatch tests for ${{ matrix.name }} (${{ matrix.variant }}) runs-on: ubuntu-latest needs: - get_changed_files diff --git a/.github/workflows/test_meltano_add_install.yml b/.github/workflows/test_meltano_add_install.yml index 6a20ee9cf..ab59319e3 100644 --- a/.github/workflows/test_meltano_add_install.yml +++ b/.github/workflows/test_meltano_add_install.yml @@ -27,6 +27,7 @@ jobs: matrix: true test: + name: Install plugin with Meltano runs-on: ubuntu-latest needs: get_changed_plugins strategy: @@ -45,7 +46,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v3 with: - version: "0.4.26" + version: ">=0.4.30" - name: Install Meltano run: uv tool install meltano From 9afabf4a1e2db2d317fecf5662df43a40b5cbd5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 5 Nov 2024 14:03:11 -0600 Subject: [PATCH 2/2] Touch a plugin --- _data/meltano/extractors/tap-fedidb/edgarrmondragon.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/meltano/extractors/tap-fedidb/edgarrmondragon.yml b/_data/meltano/extractors/tap-fedidb/edgarrmondragon.yml index e00222c72..3157fc65a 100644 --- a/_data/meltano/extractors/tap-fedidb/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-fedidb/edgarrmondragon.yml @@ -28,7 +28,7 @@ settings: label: Batch Config Encoding Compression name: batch_config.encoding.compression options: - - label: Gzip + - label: GZIP value: gzip - label: None value: none @@ -37,7 +37,7 @@ settings: label: Batch Config Encoding Format name: batch_config.encoding.format options: - - label: Jsonl + - label: JSONL value: jsonl - label: Parquet value: parquet