github workflows: repo-wide auto-format (airbytehq#29798)

Co-authored-by: postamar <[email protected]>

.github/workflows/gradle.yml

@@ -89,9 +89,70 @@ jobs:
   #      - run: |
   #          echo '${{ toJSON(needs) }}'
 
+  format:
+    needs: changes
+    runs-on: ubuntu-latest
+    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+    if: needs.changes.outputs.any_change == 'true' || (always() && github.ref == 'refs/heads/master')
+    name: "Apply All Formatting Rules"
+    timeout-minutes: 20
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Cache Build Artifacts
+        uses: ./.github/actions/cache-build-artifacts
+        with:
+          cache-key: ${{ secrets.CACHE_VERSION }}-format
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m \
+            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Format
+        uses: Wandalen/[email protected]
+        with:
+          command: ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+
+      # This is helpful in the case that we change a previously committed generated file to be ignored by git.
+      - name: Remove any files that have been gitignored
+        run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached
+
+      - name: Commit Formatting Changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Automated Commit - Formatting Changes
+          commit_user_name: Octavia Squidington III
+          commit_user_email: [email protected]
+
   ## BUILDS
   octavia-cli-build:
-    needs: changes
+    needs:
+      - changes
+      - format
     runs-on: ubuntu-latest
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -133,16 +194,6 @@ jobs:
           org.gradle.vfs.watch=false
           EOF
 
-      - name: Format
-        uses: Wandalen/[email protected]
-        with:
-          command: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
-          attempt_limit: 3
-          attempt_delay: 5000 # in ms
-
-      - name: Ensure no file change
-        run: ./tools/bin/check_for_file_changes
-
       - name: Build
         uses: Wandalen/[email protected]
         with:
@@ -158,7 +209,9 @@ jobs:
           attempt_delay: 5000 # in ms
 
   python-cdk-build:
-    needs: changes
+    needs:
+      - changes
+      - format
     runs-on: ubuntu-latest
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.python_cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -203,20 +256,6 @@ jobs:
           org.gradle.vfs.watch=false
           EOF
 
-      - name: Format
-        uses: Wandalen/[email protected]
-        with:
-          command: SUB_BUILD=CDK ./gradlew format --scan --info --stacktrace
-          attempt_limit: 3
-          attempt_delay: 5000 # in ms
-
-      - name: Commit Formatting Changes
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          commit_message: Automated Commit - Formatting Changes
-          commit_user_name: Octavia Squidington III
-          commit_user_email: [email protected]
-
       - name: Build
         uses: Wandalen/[email protected]
         with:
@@ -225,7 +264,9 @@ jobs:
           attempt_delay: 5000 # in ms
 
   java-cdk-build:
-    needs: changes
+    needs:
+      - changes
+      - format
     runs-on: ubuntu-latest
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.java_cdk == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
@@ -272,6 +313,7 @@ jobs:
     name: "Connectors Base: Start Build EC2 Runner"
     needs:
       - changes
+      - format
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: |
       needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors_base == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master')

airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1,7 +1,3 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
 # generated by datamodel-codegen:
 #   filename:  declarative_component_schema.yaml
 

airbyte-ci/connectors/ci_credentials/ci_credentials/main.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
 import json
 import sys
 from json.decoder import JSONDecodeError

airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
 import base64
 import json
 import os

airbyte-ci/connectors/ci_credentials/tests/test_models.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
 import pytest
 from ci_credentials.models import Secret
 

airbyte-ci/connectors/ci_credentials/tests/test_secrets_manager.py

@@ -1,6 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
 import base64
 import json
 import re

airbyte-ci/connectors/common_utils/common_utils/google_api.py

@@ -1,6 +1,10 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import time
 from dataclasses import dataclass
-from typing import Mapping, Any, List, ClassVar
+from typing import Any, ClassVar, List, Mapping
 
 import jwt
 import requests

airbyte-ci/connectors/common_utils/common_utils/logger.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import datetime as dt
 import inspect
 import logging

airbyte-ci/connectors/common_utils/tests/test_logger.py

@@ -1,8 +1,11 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import re
 from datetime import datetime, timedelta
 
 import pytest
-
 from common_utils import Logger
 
 LOG_RE = re.compile(r"^\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\.\d{6})\] -" r"\s+(\w+)\s+- \[.*tests/test_logger.py:(\d+)\] # (.+)")

airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py

@@ -4,12 +4,11 @@
 
 import logging
 import sys
-from typing import List, Dict, Union, Set
-import yaml
+from typing import Dict, List, Set, Union
 
+import yaml
 from connector_ops import utils
 
-
 BACKWARD_COMPATIBILITY_REVIEWERS = {"connector-operations", "connector-extensibility"}
 TEST_STRICTNESS_LEVEL_REVIEWERS = {"connector-operations"}
 GA_BYPASS_REASON_REVIEWERS = {"connector-operations"}

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -11,12 +11,12 @@
 from glob import glob
 from pathlib import Path
 from typing import List, Optional, Set, Tuple, Union
-from pydash.objects import get
 
 import git
 import requests
 import yaml
 from ci_credentials import SecretsManager
+from pydash.objects import get
 from rich.console import Console
 
 console = Console()

airbyte-ci/connectors/connector_ops/tests/conftest.py

@@ -4,6 +4,7 @@
 
 
 from datetime import datetime
+
 import pandas as pd
 import pytest
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py

@@ -1,12 +1,13 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
 import pathlib
 
 import click
-from metadata_service.gcs_upload import upload_metadata_to_gcs, MetadataUploadInfo
-from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, validate_and_load, ValidatorOptions
 from metadata_service.constants import METADATA_FILE_NAME
+from metadata_service.gcs_upload import MetadataUploadInfo, upload_metadata_to_gcs
+from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load
 from pydantic import ValidationError
 
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/constants.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 METADATA_FILE_NAME = "metadata.yaml"
 ICON_FILE_NAME = "icon.svg"
 METADATA_FOLDER = "metadata"

airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py

@@ -1,7 +1,12 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import os
-import requests
 from typing import List
 
+import requests
+
 
 def get_docker_hub_auth_token() -> str:
     docker_username = os.environ.get("DOCKER_HUB_USERNAME")

airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py

@@ -1,25 +1,23 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-from pathlib import Path
-from typing import Tuple, Optional
-from pydash.objects import get
 
 import base64
 import hashlib
 import json
 import os
-import yaml
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, Tuple
 
+import yaml
 from google.cloud import storage
 from google.oauth2 import service_account
-
-from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER, ICON_FILE_NAME
-from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, validate_and_load, ValidatorOptions
-from metadata_service.models.transform import to_json_sanitized_dict
+from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME, METADATA_FOLDER
 from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
-
-from dataclasses import dataclass
+from metadata_service.models.transform import to_json_sanitized_dict
+from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load
+from pydash.objects import get
 
 
 @dataclass(frozen=True)

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  ActorDefinitionResourceRequirements.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  AirbyteInternal.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  AllowedHosts.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  ConnectorMetadataDefinitionV0.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  ConnectorRegistryDestinationDefinition.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  ConnectorRegistrySourceDefinition.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  ConnectorRegistryV0.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  ConnectorReleases.yaml
 

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 # generated by datamodel-codegen:
 #   filename:  JobType.yaml