Python formatting and linting #1311
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| pull_request: | |
| schedule: | |
| # Run daily at 01:34 so we get notified if CI is broken before a pull request | |
| # is submitted. | |
| - cron: "34 1 * * *" | |
| permissions: | |
| contents: read | |
| jobs: | |
| go-lint: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.21" | |
| - name: Read build tags | |
| id: tags | |
| run: echo tags=$(cat .go-build-tags) >> $GITHUB_OUTPUT | |
| - name: Download formatters | |
| run: | | |
| go install golang.org/x/tools/cmd/[email protected] | |
| go install mvdan.cc/[email protected] | |
| - name: Check formatting | |
| run: | | |
| unformatted=$( | |
| gofumpt -l . | |
| goimports -l --local github.com/G-Research/fasttrackml . | |
| ) | |
| if [ -n "$unformatted" ]; then | |
| for file in $unformatted; do | |
| echo "::error file=$file::$file is not formatted properly (hint: run \"make go-format\" to fix this)" | |
| done | |
| exit 1 | |
| fi | |
| - name: Download mockery | |
| run: go install github.com/vektra/mockery/[email protected] | |
| - name: Generate mocks | |
| run: make mocks-generate | |
| - name: Check with go vet | |
| run: go vet --tags "${{ steps.tags.outputs.tags }}" ./... | |
| - uses: dominikh/[email protected] | |
| with: | |
| version: "2022.1.3" | |
| build-tags: ${{ steps.tags.outputs.tags }} | |
| python-lint: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Python Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install pipenv | |
| run: pip install pipenv==2022.12.19 | |
| - name: Run Python linters | |
| run: make python-lint | |
| go-unit-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Unit Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.21" | |
| - name: Download mockery | |
| run: go install github.com/vektra/mockery/[email protected] | |
| - name: Generate mocks | |
| run: make mocks-generate | |
| - name: Run Go Unit Tests | |
| run: make test-go-unit | |
| go-integration-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Go Integration Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.21" | |
| - name: Run Integration Tests | |
| run: make service-test | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| python-integration-tests: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Python Integration Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| api: [aim, mlflow] | |
| fail-fast: false | |
| services: | |
| postgres: | |
| image: postgres | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: test | |
| LC_COLLATE: POSIX | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.21" | |
| - name: Run ${{ matrix.api }} integration tests | |
| run: ./tests/integration/python/${{ matrix.api }}/test.sh | |
| build: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Build software distribution for ${{ matrix.os }}/${{ matrix.arch }} | |
| strategy: | |
| matrix: | |
| os: [darwin, linux, windows] | |
| arch: [amd64, arm64] | |
| exclude: | |
| - os: windows | |
| arch: arm64 | |
| include: | |
| - os: darwin | |
| runner: macos-latest | |
| - os: linux | |
| runner: ubuntu-latest | |
| - os: windows | |
| runner: ubuntu-latest | |
| fail-fast: true | |
| runs-on: ${{ matrix.runner }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.21" | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install pipenv | |
| run: pip install pipenv==2022.12.19 | |
| - name: Install arm64 cross-compilation toolchain on Linux | |
| if: matrix.os == 'linux' && matrix.arch == 'arm64' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu libc6-dev-arm64-cross | |
| echo CC=aarch64-linux-gnu-gcc >> $GITHUB_ENV | |
| - name: Install Windows cross-compilation toolchain on Linux | |
| if: matrix.os == 'windows' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64-win32 | |
| echo CC=x86_64-w64-mingw32-gcc >> $GITHUB_ENV | |
| - name: Build software distribution | |
| run: make dist | |
| env: | |
| GOOS: ${{ matrix.os }} | |
| GOARCH: ${{ matrix.arch }} | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fasttrackml-archives | |
| path: dist/* | |
| - name: Upload wheels artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fasttrackml-wheels | |
| path: wheelhouse/*.whl | |
| build-image: | |
| if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id | |
| name: Build container image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Compute build args | |
| id: build-args | |
| run: | | |
| echo version=$(git describe --tags --always --match='v*' | sed 's/^v//') >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: fasttrackml | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=ref,event=pr | |
| type=edge | |
| - name: Build container image | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| build-args: | | |
| version=${{ steps.build-args.outputs.version }} | |
| platforms: linux/amd64,linux/arm64 | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| provenance: false | |
| sbom: false | |
| outputs: type=oci,dest=fasttrackml-oci.tar | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: fasttrackml-oci-image | |
| path: fasttrackml-oci.tar | |
| # Virtual job that can be configured as a required check before a PR can be merged. | |
| # As GitHub considers a check as successful if it is skipped, we need to check its status in | |
| # another workflow (check-required.yml) and create a check there. | |
| all-required-checks-done: | |
| name: All required checks done | |
| needs: | |
| - go-lint | |
| - python-lint | |
| - go-unit-tests | |
| - go-integration-tests | |
| - python-integration-tests | |
| - build | |
| - build-image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "All required checks done" | |
| # Publish any push to a branch or tag to ghcr.io as a convenience | |
| # Actual release to Docker Hub happens in a different workflow | |
| push-ghcr: | |
| name: Push to GitHub Container Registry | |
| if: github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| needs: all-required-checks-done | |
| permissions: | |
| packages: write | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: fasttrackml-oci-image | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Compute repo name | |
| id: repo | |
| run: echo name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_OUTPUT | |
| - name: Push to GitHub Container Registry | |
| run: | | |
| tags=$(tar -xOf fasttrackml-oci.tar index.json | jq -r '.manifests[].annotations."org.opencontainers.image.ref.name"') | |
| for tag in $tags | |
| do | |
| echo "::group::Pushing image to ghcr.io/${{ steps.repo.outputs.name }}:$tag" | |
| skopeo copy --all oci-archive:fasttrackml-oci.tar:$tag docker://ghcr.io/${{ steps.repo.outputs.name }}:$tag | |
| echo "::endgroup::" | |
| done |