ci.yml

name: CI

on:
  push:
    branches:
      - master
    tags:
      - "**"
  pull_request: {}

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      webknossos: ${{ steps.filter.outputs.webknossos || github.ref == 'refs/heads/master' }}
      cluster_tools: ${{ steps.filter.outputs.cluster_tools || github.ref == 'refs/heads/master' }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            webknossos:
              - 'webknossos/**'
            cluster_tools:
              - 'cluster_tools/**'

  cluster_tools:
    needs: changes
    if: ${{ needs.changes.outputs.cluster_tools == 'true' }}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      max-parallel: 4
      matrix:
        executors: [multiprocessing, slurm, kubernetes, dask]
        python-version: ["3.12", "3.11", "3.10", "3.9"]
    defaults:
      run:
        working-directory: cluster_tools
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          architecture: "x64"
      - name: Build/pull dockered-slurm image
        if: ${{ matrix.executors == 'slurm' }}
        run: |
          cd ./dockered-slurm

          echo docker compose up
          docker compose up -d

          # Register cluster (with retry)
          for i in {1..5}; do
            echo register_cluster
            ./register_cluster.sh && s=0 && break || s=$?
            sleep 10
          done

          # Show log output for debugging
          docker logs slurmctld
          docker logs c1
          docker logs c2

          # Run setup.py on all three nodes
          docker exec -w /cluster_tools slurmctld bash -c "poetry install" &
          docker exec -w /cluster_tools c1 bash -c "poetry install" &
          docker exec -w /cluster_tools c2 bash -c "poetry install" &
          wait

      - name: Setup Kubernetes-in-Docker
        if: ${{ matrix.executors == 'kubernetes' }}
        run: |
          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
          chmod +x ./kind
          sed -i "s#__PATH__#$(pwd)#g" tests/cluster-config.yaml
          ./kind create cluster --config=tests/cluster-config.yaml
          ./kind export kubeconfig

          cp ../requirements.txt .
          docker build -f tests/Dockerfile -t scalableminds/cluster-tools:latest .
          ./kind load docker-image scalableminds/cluster-tools:latest

      - name: Install dependencies (without docker)
        if: ${{ matrix.executors == 'multiprocessing' }}
        run: |
          pip install -r ../requirements.txt
          poetry install

      - name: Install dependencies (without docker)
        if: ${{ matrix.executors == 'kubernetes' || matrix.executors == 'dask' }}
        run: |
          pip install -r ../requirements.txt
          poetry install --all-extras

      - name: Check typing
        if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }}
        run: ./typecheck.sh

      - name: Check formatting
        if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }}
        run: ./format.sh check

      - name: Lint code
        if: ${{ matrix.executors == 'multiprocessing' && matrix.python-version == '3.11' }}
        run: ./lint.sh

      - name: Run multiprocessing tests
        if: ${{ matrix.executors == 'multiprocessing' }}
        run: |
          cd tests
          PYTEST_EXECUTORS=multiprocessing,sequential,test_pickling,debug_sequential \
            poetry run python -m pytest -sv test_all.py test_multiprocessing.py

      - name: Run slurm tests
        if: ${{ matrix.executors == 'slurm' }}
        run: |
          cd ./dockered-slurm
          docker exec \
            -w /cluster_tools/tests \
            -e PYTEST_EXECUTORS=slurm \
            slurmctld bash -c "poetry run python -m pytest -sv test_all.py test_slurm.py"
          docker exec \
            -w /cluster_tools/tests \
            slurmctld bash -c "poetry run python test_deref_main.py"

      - name: Run kubernetes tests
        if: ${{ matrix.executors == 'kubernetes' }}
        run: |
          cd tests
          PYTEST_EXECUTORS=kubernetes poetry run python -m pytest -sv test_all.py test_kubernetes.py

      - name: Run dask tests
        if: ${{ matrix.executors == 'dask' }}
        run: |
          cd tests
          PYTEST_EXECUTORS=dask poetry run python -m pytest -sv test_all.py test_dask.py

  webknossos_linux:
    needs: changes
    if: |
      ${{ needs.changes.outputs.cluster_tools == 'true' }} ||
      ${{ needs.changes.outputs.webknossos == 'true' }}
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: ["3.12", "3.11", "3.10", "3.9"]
        group: [1, 2, 3]
      fail-fast: false
    defaults:
      run:
        working-directory: webknossos

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          architecture: "x64"

      - name: Install dependencies
        run: |
          pip install -r ../requirements.txt
          poetry install --extras all --with examples --with dev

      - name: Check formatting
        if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }}
        run: ./format.sh check

      - name: Lint code
        if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }}
        run: ./lint.sh

      - name: Check typing
        if: ${{ matrix.group == 1 && matrix.python-version == '3.11' }}
        run: ./typecheck.sh

      - name: Python tests
        timeout-minutes: 30
        env:
          WK_TOKEN: ${{ secrets.WK_TOKEN }}
        run: ./test.sh -vv -p no:faulthandler --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration

      - name: Check if git is dirty
        run: |
          git diff --no-ext-diff --quiet --exit-code
          [[ -z $(git status -s) ]]

  webknossos_cli_docker:
    needs: [cluster_tools, webknossos_linux]
    if: |
      always() &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled') &&
      !github.event.pull_request.head.repo.fork
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Write version file
        run: |
          pip install -r requirements.txt
          pushd webknossos

          PKG_VERSION="$(dunamai from git)"
          echo "__version__ = '$PKG_VERSION'" > ./webknossos/version.py
          poetry version "$PKG_VERSION"

            popd

      - name: Build docker image
        run: docker build -t scalableminds/webknossos-cli:$GITHUB_SHA -f webknossos/Dockerfile .

      - name: Smoke test docker
        run: |
          docker run --rm \
            -v$(pwd)/webknossos/testdata:/webknossos/testdata \
            scalableminds/webknossos-cli:$GITHUB_SHA \
            webknossos convert \
              --jobs 2 \
              --voxel-size 1,1,1 \
              testdata/tiff testoutput/tiff

      - name: Login to docker
        env:
          DOCKER_USER: ${{ secrets.DOCKER_USER }}
          DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
        run: |
          echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin

      - name: Push docker images
        run: |
          docker push scalableminds/webknossos-cli:$GITHUB_SHA

      - name: Push docker images (for tag)
        if: github.ref_type == 'tag'
        run: |
          CI_TAG=$(git describe --tags)
          docker tag \
            scalableminds/webknossos-cli:$GITHUB_SHA \
            scalableminds/webknossos-cli:$CI_TAG
          docker push scalableminds/webknossos-cli:$CI_TAG
          docker tag \
            scalableminds/webknossos-cli:$GITHUB_SHA \
            scalableminds/webknossos-cli:latest
          docker push scalableminds/webknossos-cli:latest

      - name: Push docker images (for branch)
        if: github.ref_type == 'branch'
        run: |
          CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}
          NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_}
          docker tag \
            scalableminds/webknossos-cli:$GITHUB_SHA \
            scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH
          docker push scalableminds/webknossos-cli:$NORMALIZED_CI_BRANCH

  docs:
    needs: [cluster_tools, webknossos_linux]
    runs-on: ubuntu-latest
    if: |
      always() &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled') &&
      !github.event.pull_request.head.repo.fork

    steps:
      - uses: actions/checkout@v3
      - uses: actions/checkout@v3
        with:
          repository: scalableminds/webknossos
          path: docs/wk-repo
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          architecture: "x64"

      - name: Install dependencies
        run: |
          pip3 install -r requirements.txt

      - name: Build Docs
        run: |
          cd docs
          ./generate.sh --persist

      - name: Push docs (for branch)
        if: github.ref_type == 'branch'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: "eu-west-1"
        run: |
          CI_BRANCH=${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}
          NORMALIZED_CI_BRANCH=${CI_BRANCH//[\/-]/_}
          aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${NORMALIZED_CI_BRANCH}

      - name: Push docs (for tag)
        if: github.ref_type == 'tag'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: "eu-west-1"
        run: |
          CI_TAG=$(git describe --tags)
          aws s3 sync --acl public-read docs/out s3://static.webknossos.org/docs/${CI_TAG}

      - name: Check links (on master)
        if: github.ref == 'refs/heads/master'
        env: # Or as an environment variable
          SLACK_HOOK: ${{ secrets.LINK_CHECKER_SLACK_HOOK }}
        run: |
          cd docs
          poetry run linkchecker --config linkcheckerrc https://docs.webknossos.org > link_status || \
            curl -X POST --data-urlencode "payload={\"text\": \":warning: Broken Links on doc.webknossos.org :warning:\n"'```'"\n$(cat link_status)\n"'```"}' \
              "$SLACK_HOOK"

  pypi_and_gh_release:
    needs: [cluster_tools, webknossos_linux]
    if: |
      always() &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled') &&
      github.ref_type == 'tag' &&
      !github.event.pull_request.head.repo.fork
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.8
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          architecture: "x64"
      - name: Install dependencies
        run: pip3 install -r requirements.txt
      - name: Publish python packages
        env:
          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PASSWORD }}
        run: _tooling/publish.sh
      - name: Prepare github release
        run: |
          VERSION="$(dunamai from git)"
          _tooling/changelog_for_version.sh $VERSION > Changelog.md
      - name: Publish github release
        id: create_release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ github.ref }}
          release_name: ${{ github.ref }}
          body_path: Changelog.md
          draft: false
          prerelease: false

  complete:
    needs:
      [
        cluster_tools,
        webknossos_linux,
        webknossos_cli_docker,
        docs,
        pypi_and_gh_release,
      ]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check failure
        if: |
          contains(needs.*.result, 'failure') ||
          contains(needs.*.result, 'cancelled')
        run: exit 1
      - name: Success
        run: echo Success!