forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
airbyte-ci: add format commands (airbytehq#31831)
Co-authored-by: Ben Church <[email protected]> Co-authored-by: bnchrch <[email protected]> Co-authored-by: alafanechere <[email protected]> Co-authored-by: Augustin <[email protected]> Co-authored-by: Marius Posta <[email protected]> Co-authored-by: alafanechere <[email protected]>
- Loading branch information
7 people
authored
Nov 14, 2023
1 parent
c75325b
commit ac3eb28
Showing
46 changed files
with
1,104 additions
and
631 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,9 @@ | ||
| name: Format Code (Python + Java) | ||
| name: Check for formatting errors | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| # Cancel any previous runs on the same branch if they are still in progress | ||
| cancel-in-progress: true | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
|
|
@@ -10,8 +12,8 @@ on: | |
| - master | ||
| pull_request: | ||
| jobs: | ||
| format-and-commit: | ||
| runs-on: ubuntu-latest | ||
| format-fix: | ||
| runs-on: "conn-prod-xlarge-runner" | ||
| name: "Apply All Formatting Rules" | ||
| timeout-minutes: 40 | ||
| steps: | ||
|
|
@@ -20,16 +22,16 @@ jobs: | |
| with: | ||
| ref: ${{ github.head_ref }} | ||
| # Important that this is set so that CI checks are triggered again | ||
| # Without this we would be be forever waiting on required checks to pass | ||
| # Without this we would be forever waiting on required checks to pass | ||
| token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} | ||
|
|
||
| # IMPORTANT! This is nessesary to make sure that a status is reported on the PR | ||
| # even if the workflow is skipped. If we used github actions filters, the workflow | ||
| # IMPORTANT! This is necessary to make sure that a status is reported on the PR | ||
| # even if the workflow is skipped. If we used GitHub Actions filters, the workflow | ||
| # would not be reported as skipped, but instead would be forever pending. | ||
| # | ||
| # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. | ||
| # | ||
| # Also it gets worse | ||
| # Also, it gets worse | ||
| # | ||
| # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. | ||
| # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. | ||
|
|
@@ -42,59 +44,35 @@ jobs: | |
| - '**/*' | ||
| - '!**/*.md' | ||
| - uses: actions/setup-java@v3 | ||
| with: | ||
| distribution: "zulu" | ||
| java-version: "17" | ||
|
|
||
| - uses: actions/setup-python@v4 | ||
| - name: Run airbyte-ci format fix | ||
| uses: ./.github/actions/run-dagger-pipeline | ||
| with: | ||
| python-version: "3.10" | ||
|
|
||
| - name: Set up CI Gradle Properties | ||
| run: | | ||
| mkdir -p ~/.gradle/ | ||
| cat > ~/.gradle/gradle.properties <<EOF | ||
| org.gradle.jvmargs=-Xmx8g -Xss4m \ | ||
| --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ | ||
| --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ | ||
| --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ | ||
| --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ | ||
| --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED | ||
| org.gradle.workers.max=8 | ||
| org.gradle.vfs.watch=false | ||
| EOF | ||
| - name: Format | ||
| if: steps.changes.outputs.format_any_changed == 'true' | ||
| uses: Wandalen/[email protected] | ||
| with: | ||
| command: ./gradlew format --scan --info --stacktrace | ||
| attempt_limit: 3 | ||
| attempt_delay: 5000 # in ms | ||
| context: "pull_request" | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | ||
| subcommand: "format fix all" | ||
|
|
||
| # This is helpful in the case that we change a previously committed generated file to be ignored by git. | ||
| - name: Remove any files that have been gitignored | ||
| run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached | ||
|
|
||
| - name: Commit Formatting Changes (PR) | ||
| uses: stefanzweifel/git-auto-commit-action@v4 | ||
| uses: stefanzweifel/git-auto-commit-action@v5 | ||
| # do not commit if master branch | ||
| if: github.ref != 'refs/heads/master' | ||
| with: | ||
| commit_message: Automated Commit - Formatting Changes | ||
| commit_user_name: Octavia Squidington III | ||
| commit_user_email: [email protected] | ||
|
|
||
| - name: "Fail on Formatting Changes (Master)" | ||
| if: github.ref == 'refs/heads/master' | ||
| run: git --no-pager diff && test -z "$(git --no-pager diff)" | ||
|
|
||
| notify-failure-slack-channel: | ||
| name: "Notify Slack Channel on Build Failures" | ||
| runs-on: ubuntu-latest | ||
| needs: | ||
| - format-and-commit | ||
| - format-fix | ||
| if: ${{ failure() && github.ref == 'refs/heads/master' }} | ||
| steps: | ||
| - name: Checkout Airbyte | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,5 @@ | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
|
|
||
| import time | ||
| from typing import Any, Optional | ||
|
|
||
|
|
||
4 changes: 4 additions & 0 deletions
4
airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,7 @@ | ||
| # | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
| from typing import Union | ||
|
|
||
| import dpath.util | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
54 changes: 54 additions & 0 deletions
54
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| # | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
| from typing import List | ||
|
|
||
| import dagger | ||
| from pipelines.helpers.utils import sh_dash_c | ||
|
|
||
|
|
||
| def run_check( | ||
| container: dagger.Container, | ||
| check_commands: List[str], | ||
| ) -> dagger.Container: | ||
| """Checks whether the repository is formatted correctly. | ||
| Args: | ||
| container: (dagger.Container): The container to run the formatting check in | ||
| check_commands (List[str]): The list of commands to run to check the formatting | ||
| """ | ||
| return container.with_exec(sh_dash_c(check_commands), skip_entrypoint=True) | ||
|
|
||
|
|
||
| async def run_format( | ||
| container: dagger.Container, | ||
| format_commands: List[str], | ||
| ) -> dagger.Container: | ||
| """Formats the repository. | ||
| Args: | ||
| container: (dagger.Container): The container to run the formatter in | ||
| format_commands (List[str]): The list of commands to run to format the repository | ||
| """ | ||
| format_container = container.with_exec(sh_dash_c(format_commands), skip_entrypoint=True) | ||
| return await format_container.directory("/src").export(".") | ||
|
|
||
|
|
||
| def mount_repo_for_formatting( | ||
| dagger_client: dagger.Client, | ||
| container: dagger.Container, | ||
| include: List[str], | ||
| ) -> dagger.Container: | ||
| """Mounts the relevant parts of the repository: the code to format and the formatting config | ||
| Args: | ||
| container: (dagger.Container): The container to mount the repository in | ||
| include (List[str]): The list of files to include in the container | ||
| """ | ||
| container = container.with_mounted_directory( | ||
| "/src", | ||
| dagger_client.host().directory( | ||
| ".", | ||
| include=include, | ||
| ), | ||
| ).with_workdir("/src") | ||
|
|
||
| return container |
Oops, something went wrong.