ci.yml

name: CI

on:
  push:
  pull_request:
  schedule:
    # Run daily at 01:34 so we get notified if CI is broken before a pull request
    # is submitted.
    - cron: "34 1 * * *"

permissions:
  contents: read

jobs:
  go-lint:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Go Lint
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Read build tags
        id: tags
        run: echo tags=$(cat .go-build-tags) >> $GITHUB_OUTPUT

      - name: Install tools
        run: make install-tools

      - name: Check formatting
        run: |
          make go-format
          modified=$(git ls-files --modified -- '*.go')
          if [ -n "$modified" ]; then
            for file in $modified; do
              echo "::error file=$file::$file is not formatted properly (hint: run \"make go-format\" to fix this)"
            done
            exit 1
          fi

      - name: Check mocks
        run: |
          make mocks-generate
          modified=$(git ls-files --modified -- '*/mock_*.go')
          if [ -n "$modified" ]; then
            for file in $modified; do
              echo "::error file=$file::$file is not up to date (hint: run \"make mocks-generate\" to fix this)"
            done
            exit 1
          fi

      - name: Check with go vet
        run: go vet --tags "${{ steps.tags.outputs.tags }}" ./...

      - uses: dominikh/staticcheck-action@v1.3.0
        with:
          version: "2023.1.5"
          install-go: false
          build-tags: ${{ steps.tags.outputs.tags }}

  python-lint:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Python Lint
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Install pipenv
        run: pip install pipenv==2022.12.19

      - name: Run Python linters
        run: make python-lint

  go-unit-tests:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Go Unit Tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Run Go Unit Tests
        run: make test-go-unit

  go-integration-tests:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Go Integration Tests (${{ matrix.database-backend }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        database-backend: [sqlite, sqlcipher, postgres]
        include:
          - database-backend: sqlite
            database-uri: sqlite:///tmp/fasttrackml.db
          - database-backend: sqlcipher
            database-uri: sqlite:///tmp/fasttrackml.db?_key=passphrase
          - database-backend: postgres
            database-uri: postgres://postgres:postgres@postgres/postgres
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Run Integration Tests
        run: make service-test
        env:
          DOCKER_BUILDKIT: 1
          FML_DATABASE_URI: ${{ matrix.database-uri }}

  python-integration-tests:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Python Integration Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        api: [aim, mlflow]
      fail-fast: false
    services:
      postgres:
        image: postgres
        env:
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: test
          LC_COLLATE: POSIX
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Run ${{ matrix.api }} integration tests
        run: ./tests/integration/python/${{ matrix.api }}/test.sh

  build:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Build software distribution for ${{ matrix.os }}/${{ matrix.arch }}
    strategy:
      matrix:
        os: [darwin, linux, windows]
        arch: [amd64, arm64]
        exclude:
          - os: windows
            arch: arm64
        include:
          - os: darwin
            runner: macos-latest
          - os: linux
            runner: ubuntu-latest
          - os: windows
            runner: ubuntu-latest
      fail-fast: true
    runs-on: ${{ matrix.runner }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup Go
        uses: actions/setup-go@v4
        with:
          go-version: "1.21"

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Install pipenv
        run: pip install pipenv==2022.12.19

      - name: Install arm64 cross-compilation toolchain on Linux
        if: matrix.os == 'linux' && matrix.arch == 'arm64'
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu libc6-dev-arm64-cross
          echo CC=aarch64-linux-gnu-gcc >> $GITHUB_ENV

      - name: Install Windows cross-compilation toolchain on Linux
        if: matrix.os == 'windows'
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64-win32
          echo CC=x86_64-w64-mingw32-gcc >> $GITHUB_ENV

      - name: Build software distribution
        run: make dist
        env:
          GOOS: ${{ matrix.os }}
          GOARCH: ${{ matrix.arch }}

      - name: Upload binary artifact
        uses: actions/upload-artifact@v3
        with:
          name: fasttrackml-archives
          path: dist/*

      - name: Upload wheels artifact
        uses: actions/upload-artifact@v3
        with:
          name: fasttrackml-wheels
          path: wheelhouse/*.whl

  build-image:
    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
    name: Build container image
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Compute build args
        id: build-args
        run: |
          echo version=$(git describe --tags --always --match='v*' | sed 's/^v//') >> $GITHUB_OUTPUT

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: fasttrackml
          tags: |
            type=ref,event=branch
            type=ref,event=tag
            type=ref,event=pr
            type=edge

      - name: Build container image
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            version=${{ steps.build-args.outputs.version }}
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          provenance: false
          sbom: false
          outputs: type=oci,dest=fasttrackml-oci.tar

      - name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: fasttrackml-oci-image
          path: fasttrackml-oci.tar

  # Virtual job that can be configured as a required check before a PR can be merged.
  # As GitHub considers a check as successful if it is skipped, we need to check its status in
  # another workflow (check-required.yml) and create a check there.
  all-required-checks-done:
    name: All required checks done
    needs:
      - go-lint
      - python-lint
      - go-unit-tests
      - go-integration-tests
      - python-integration-tests
      - build
      - build-image
    runs-on: ubuntu-latest
    steps:
      - run: echo "All required checks done"

  # Publish any push to a branch or tag to ghcr.io as a convenience
  # Actual release to Docker Hub happens in a different workflow
  push-ghcr:
    name: Push to GitHub Container Registry
    if: github.event_name == 'push'
    runs-on: ubuntu-latest
    needs: all-required-checks-done
    permissions:
      packages: write
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          name: fasttrackml-oci-image

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Compute repo name
        id: repo
        run: echo name=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_OUTPUT

      - name: Push to GitHub Container Registry
        run: |
          tags=$(tar -xOf fasttrackml-oci.tar index.json | jq -r '.manifests[].annotations."org.opencontainers.image.ref.name"')
          for tag in $tags
          do
            echo "::group::Pushing image to ghcr.io/${{ steps.repo.outputs.name }}:$tag"
            skopeo copy --all oci-archive:fasttrackml-oci.tar:$tag docker://ghcr.io/${{ steps.repo.outputs.name }}:$tag
            echo "::endgroup::"
          done