cicd.yaml

# CI/CD pipeline builds the Docker image and pushes it to the GitHub Container Registry
# Then it runs the Sphinx documentation build and deploys it to GitHub Pages
# Finally it tests the pymimic3 package
# -- DEBUG --
# Trigger: >git commit --allow-empty -m "Trigger pipeline" && git push

name: CI/CD Pipeline

on: [push, pull_request, workflow_dispatch]

permissions:
  contents: write

env:
  CONTAINER_PYTEST_RESULTS: /workdir/tests/data/pytest-results/${{ github.sha }}
  CONTAINER_BASH_RESULTS: /workdir/tests/data/bash-results/${{ github.sha }}

jobs:
  build-and-push:
    needs: deploy
    # |-> 1. pull code
    # |-> 2. set up docker buildx 
    # |-> 3. login to docker hub
    # |-> 4. setup cache dir
    # |      - ?=self-hosted
    # |-> 5. extract branch name
    # |-> 6. setup cache on runner machine
    # |      - ?=self-hosted|github-hosted
    # |-> 7. build and export docker image
    # |      - ?=self-hosted|github-hosted
    # |-> 8. push the image
    # |-> 9. move cache
    # '-> 10. scan image for vulnerabilities
    runs-on: self-hosted # -> ubuntu-latest to run in github cloud
    outputs:
      branch_name: ${{ steps.get_branch.outputs.branch_name }}
      docker_image_tar: ${{ steps.get_branch.outputs.docker_image_tar }}
      docker_volumes: ${{ steps.get_volumes.outputs.docker_volumes }}
      bash_results: ${{ steps.get_volumes.outputs.bash_results }}
      pytest_results: ${{ steps.get_volumes.outputs.pytest_results }}
      local_registry: ${{ steps.check_registry.outputs.local_registry }}
    steps:
      # => 1. Pull code into runner env
      - name: Checkout code
        uses: actions/checkout@v4

      # => 2. Set up Docker Buildx step
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver-opts: |
            network=host
          buildkitd-flags: --allow-insecure-entitlement network.host

      # => 3. Login to docker hub to push images
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          # Secrets stored in repository
          username: tensorpod
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Setup volume mounts
        shell: bash
        id: get_volumes
        run: |
          if [ -z "$DOCKER_CACHE_DIR" ]; then
            echo "Error: DOCKER_CACHE_DIR environment variable is not set."
            exit 1
          fi
          TERM=xterm-color
          DOCKER_VOLUME_MOUNTS="\
           -v $DOCKER_CACHE_DIR/.github-action-cache/control-dataset:/workdir/tests/data/control-dataset\
           -v $DOCKER_CACHE_DIR/.github-action-cache/mimiciii-demo:/workdir/tests/data/mimiciii-demo\
           -v $DOCKER_CACHE_DIR/.github-action-cache/${{ runner.name }}/semitemp:/workdir/tests/data/semitemp\
           -v $DOCKER_CACHE_DIR/.github-action-cache/bash-results:/workdir/tests/data/bash-results\
           -v $DOCKER_CACHE_DIR/.github-action-cache/pytest-results:/workdir/tests/data/pytest-results"
         
          echo "DOCKER_VOLUME_MOUNTS=$DOCKER_VOLUME_MOUNTS" >> $GITHUB_ENV
          echo "docker_volumes=$DOCKER_VOLUME_MOUNTS" >> $GITHUB_OUTPUT
 
          echo "::group::DOCKER_VOLUME_MOUNTS"
          echo "$DOCKER_VOLUME_MOUNTS" | sed 's/ -v /\n  -v /g'
          echo "::endgroup::"
 
          BASH_RESULTS=$DOCKER_CACHE_DIR/.github-action-cache/bash-results/${{ github.sha }}
          PYTEST_RESULTS=$DOCKER_CACHE_DIR/.github-action-cache/pytest-results/${{ github.sha }}

          echo "::group::BASH/PYTEST RESULTS"
          echo $BASH_RESULTS
          echo $PYTEST_RESULTS
          echo "::endgroup::"
 
          echo "bash_results=$BASH_RESULTS" >> $GITHUB_OUTPUT
          echo "pytest_results=$PYTEST_RESULTS" >> $GITHUB_OUTPUT

      - name: Check if local registry is running
        id: check_registry
        run: |
          echo "Checking if local registry is running at localhost:5000..."
          check_registry() {
              local url=$1
              if curl -s -f -m 5 "http://${url}/v2/_catalog" > /dev/null 2>&1; then
                  return 0
              else
                  return 1
              fi
          }

          if check_registry "localhost:5000"; then
              echo "local_registry=true" >> $GITHUB_OUTPUT
              echo "Local registry is running on localhost:5000."
              echo "registry_host=localhost:5000" >> $GITHUB_OUTPUT
          elif check_registry "registry:5000"; then
              echo "local_registry=true" >> $GITHUB_OUTPUT
              echo "registry_host=registry:5000" >> $GITHUB_OUTPUT
              echo "Local registry is running on registry:5000 (Docker network)."
          else
              echo "local_registry=false" >> $GITHUB_OUTPUT
              echo "Error: No local registry is running or accessible." >&2
          fi

      # => 4. Create cache directory if runing locally
      - name: Set up Docker cache/output directory on self-hosted
        if: runner.environment == 'self-hosted'
        run: |
          echo cache dir is: $DOCKER_CACHE_DIR
          
          if [ -n "$DOCKER_CACHE_DIR" ]; then
            echo "Using runner's DOCKER_CACHE_DIR: $DOCKER_CACHE_DIR"
          else
            DOCKER_CACHE_DIR="/tmp"
            echo "DOCKER_CACHE_DIR not set on runner. Using default: $DOCKER_CACHE_DIR"
          fi

          echo "DOCKER_CACHE_DIR=$DOCKER_CACHE_DIR" >> $GITHUB_ENV
          
          mkdir -p $DOCKER_CACHE_DIR
          mkdir -p $DOCKER_CACHE_DIR/.buildx-cache
          mkdir -p $DOCKER_CACHE_DIR/.buildx-image-cache/tensorpod/pymimic3/

          if [[ steps.check_registry.outputs.local_registry == 'false' ]]; then
            echo "Copying docker cache from $DOCKER_CACHE_DIR/.buildx-cache to $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache"
            mkdir -p "$DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache"
            cp -r "$DOCKER_CACHE_DIR/.buildx-cache" "$DOCKER_CACHE_DIR/${{ runner.name }}/"
          fi

          echo "::group::DOCKER_CACHE_DIR"
          echo $DOCKER_CACHE_DIR
          echo $DOCKER_CACHE_DIR/.buildx-image-cache/tensorpod/pymimic3/
          echo "::endgroup::"

      # => 5. Extract branch name (may fail on pull requests, untested)
      - name: Extract branch name
        shell: bash
        id: get_branch
        run: |
          BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
          echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT

          echo "::group::BRANCH_NAME"
          echo $BRANCH_NAME
          echo "::endgroup::"

          DOCKER_IMAGE_TAR=$DOCKER_CACHE_DIR/.buildx-image-cache/tensorpod/pymimic3/$BRANCH_NAME.tar
          echo "DOCKER_IMAGE_TAR=$DOCKER_IMAGE_TAR" >> $GITHUB_ENV
          echo "docker_image_tar=$DOCKER_IMAGE_TAR" >> $GITHUB_OUTPUT
          echo "::group::DOCKER_IMAGE_TAR"
          echo $DOCKER_IMAGE_TAR
          echo "::endgroup::"
      
      # => 6. Setup cache on runner machine: self-hosted
      - name: Cache Docker layers on self-hosted
        if: |
          runner.environment == 'self-hosted' &&
          steps.check_registry.outputs.local_registry == 'false'
        uses: maxnowack/local-cache@v2
        with: 
          path: ${{ env.DOCKER_CACHE_DIR }}/${{ runner.name }}/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-   
      
      # => 6. Setup cache on github cache: github hosted
      - name: Cache Docker layers on github-hosted
        if: runner.environment == 'github-hosted'
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-

      # => 7. Build and push Docker image: self-hosted (part 1) 
      #       - local registry reachable:
      #           - build and push to local registry
      #           - tag and push to Docker Hub
      #           - don't export image locally 
      #           - don't load the image into docker
      #       - local registry not reachable:
      #           - build and export image locally
      #           - don't push to local registry
      #           - push to docker hub in subsequent step
      - name: Build and push to local registry
        if: |
          steps.check_registry.outputs.local_registry == 'true' && 
          runner.environment == 'self-hosted'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/Dockerfile.ci
          push: true
          tags: |
            tensorpod/pymimic3:latest
            tensorpod/pymimic3:${{ env.BRANCH_NAME }}
            localhost:5000/pymimic3:latest
            localhost:5000/pymimic3:${{ env.BRANCH_NAME }}
          cache-from: type=registry,ref=localhost:5000/pymimic3:buildcache
          cache-to: type=registry,ref=localhost:5000/pymimic3:buildcache,mode=max

      - name: Build and export to tar
        if: |
          steps.check_registry.outputs.local_registry != 'true' && 
          runner.environment == 'self-hosted'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/Dockerfile.ci
          load: true
          tags: |
            tensorpod/pymimic3:latest
            tensorpod/pymimic3:${{ env.BRANCH_NAME }}
          cache-from: type=local,src=${{ env.DOCKER_CACHE_DIR }}/.buildx-cache
          cache-to: type=local,dest=${{ env.DOCKER_CACHE_DIR }}/.buildx-cache-new,mode=max
          outputs: type=docker,dest=${{ env.DOCKER_IMAGE_TAR }}

      # => 7. Build and push Docker image: github hosted
      - name: Build and export Docker image on github-hosted
        if: runner.environment == 'github-hosted'
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/Dockerfile.ci
          push: ${{ github.event_name != 'pull_request' }}
          tags: |
            tensorpod/pymimic3:latest
            tensorpod/pymimic3:${{ env.BRANCH_NAME }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Upload artifact
        if: runner.environment == 'github-hosted'
        uses: actions/upload-artifact@v4
        with:
          name: pymimic3.$BRANCH_NAME
          path: /tmp/.buildx-image-cache/tensorpod/pymimic3/${{ env.BRANCH_NAME }}.tar

      # => 8. Push Docker image
      # It seems the docker image is not automatically loaded after build
      - name: Push Docker image
        if: github.event_name != 'pull_request' && runner.environment == 'self-hosted'
        run: |
          # Login to Docker Hub
          echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u tensorpod --password-stdin

          if [[ "${{ steps.check_registry.outputs.local_registry }}" == "true" ]]; then
            echo "Local registry is available. Pulling from local registry for subsequent jobs."
          else
            echo "No local registry available. Loading from tar and pushing to Docker Hub."
                        
            # Push to Docker Hub
            docker push tensorpod/pymimic3:latest
            docker push tensorpod/pymimic3:$BRANCH_NAME
          fi

      - name: Docker cleanup
        run: |
          docker image prune -f
          docker volume prune -f
          docker builder prune

      # => 9. Move the cache with -new suffix to original cache
      - name: Move cache
        if: ${{ steps.check_registry.outputs.local_registry == 'false' }}
        run: |
          if [ "${{ runner.environment }}" = "self-hosted" ]; then
            rm -rf $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache
            mv $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache-new $DOCKER_CACHE_DIR/.buildx-cache
          elif [ "${{ runner.environment }}" = "github-hosted" ]; then
            rm -rf tmp/.buildx-cache
            mv tmp/.buildx-cache-new tmp/.buildx-cache
          fi

      # => 10. Scan image for vulnerabilities
      # - name: Scan image for vulnerabilities
     #   uses: anchore/scan-action@v3
      #   with:
      #     image: "tensorpod/pymimic3:$BRANCH_NAME"
      #     fail-build: false
      #     severity-cutoff: high

  deploy:
    # needs: build-and-push
    # env:
    #   BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
    #   DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
    #   DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
    #   BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
    #   PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
    #   LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: self-hosted
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.x'


      - name: Build documentation
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run command in docker
          echo "Pulling image from local registry"
          docker pull localhost:5000/pymimic3:latest
          echo "Tagging image"
          docker tag localhost:5000/pymimic3:latest tensorpod/pymimic3:latest
          echo "Running sphinx build"
          docker run --rm tensorpod/pymimic3:latest bash -ic "pip install sphinx sphinx-rtd-theme && cd docs && make html"
        working-directory: .

      - name: Setup Pages
        uses: actions/configure-pages@v4

      - name: Upload artifact
        uses: actions/upload-pages-artifact@v3
        with:
          path: './build/html'

      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4

      - name: Fail
        shell: bash
        run: exit 1
 # # Documentation build and deploy
 # docs:
 #   runs-on: self-hosted
 #   needs: build-and-push
 #   steps:
 #     - uses: actions/checkout@v4
 #     - uses: actions/setup-python@v5
 #     - name: Install dependencies
 #       run: |
 #         pip install sphinx sphinx_rtd_theme myst_parser
 #     - name: Sphinx build
 #       run: |
 #         sphinx-build doc _build
 #     - name: Deploy to GitHub Pages
 #       uses: peaceiris/actions-gh-pages@v3
 #       if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
 #       with:
 #         publish_branch: gh-pages
 #         github_token: ${{ secrets.GITHUB_TOKEN }}
 #         publish_dir: _build/
 #         force_orphan: true
#
# Pytest run test

  setup-control-dataset:
    concurrency:
      group: setup-control-dataset
    runs-on: self-hosted
    needs: build-and-push
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup persistent test mounts
        shell: bash
        run: |
          mkdir -p $DOCKER_CACHE_DIR/.github-action-cache/{pytest-results/${{ github.sha }},bash-results/${{ github.sha }}}

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Create output dirs
        run: |
          mkdir -p BASH_RESULTS
          mkdir -p PYTEST_RESULTS

      - name: Setup control dataset
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run command in docker
          .github/scripts/run_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "make testing=true" \
          "$BASH_RESULTS" \
          "setup-control-dataset"

  setup-fixtures:
    needs: [build-and-push, setup-control-dataset]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run fixture setup tests
        shell: bash
        run: |
          # Create the semitemp directory
          mkdir -p $DOCKER_CACHE_DIR/.github-action-cache/${{ runner.name }}/semitemp
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_zsetup_all_fixtures.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "setup-control-dataset"

      - name: Run pytest suite tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_pytest_utils.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-pytest-utils"

  test-readers:
    needs: [setup-fixtures, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run reader tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_readers" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-readers"

  test-trackers:
    needs: [setup-fixtures, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run tracker tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_trackers" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-trackers"

  test-data-split:
    needs: [test-readers, test-trackers, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run data split tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_data_split.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-data-split"

  test-extraction:
    needs: [test-readers, test-trackers, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run extraction tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_extraction.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-extraction"

  test-preprocessors:
    needs: [test-extraction, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run preprocessor tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_preprocessor.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-preprocessors"

  test-feature-engineering:
    needs: [test-extraction, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run feature engineering tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_feature_engine.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-feature-engineering"

  test-discretizer:
    needs: [test-extraction, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run discretizer tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_discretizer.py" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-discretizer"

  test-parameters:
    needs: [test-preprocessors, test-feature-engineering, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run parameter tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_datasets/test_parameters" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-parameters"


  test-preprocessing:
    needs: [setup-fixtures, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run preprocessing tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_preprocessing" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-preprocessing"

  test-metrics:
    needs: [setup-fixtures, build-and-push]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run metrics tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_metrics" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-metrics"

  test-models:
    needs: [build-and-push, test-metrics, test-parameters, test-preprocessing]
    runs-on: self-hosted
    env: 
      BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
      DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
      DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
      BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
      PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
      LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Load image
        shell: bash
        run: |
          if [[ "$LOCAL_REGISTRY" == "true" ]]; then
            echo "Pulling image from local registry"
            docker pull localhost:5000/pymimic3:$BRANCH_NAME
            docker tag  localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
          else
            echo "Loading image from tar file"
            docker load -i $DOCKER_IMAGE_TAR
          fi
          echo "Docker images:"
          docker images

      - name: Run model tests
        shell: bash
        run: |
          # Enable color output
          export TERM=xterm-color
          export TERM=xterm-256color

          # Run pytest in docker
          .github/scripts/run_pytest_container.sh \
          "$DOCKER_VOLUME_MOUNTS" \
          "$BRANCH_NAME" \
          "$CONTAINER_PYTEST_RESULTS" \
          "tests/test_models" \
          "$BASH_RESULTS" \
          "$PYTEST_RESULTS" \
          "test-models"
  Cleanup:
    if: always()
    runs-on: self-hosted
    needs: [test-models]
    steps:
      - name: Cleanup semitemp directory
        if: always()
        shell: bash
        run: |
          rm -rf $DOCKER_CACHE_DIR/.github-action-cache/${{ runner.name }}/semitemp && \
          rm -rf $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache 2>&1 \
          | tee $BASH_RESULTS/cleanup-semitemp.txt

          echo "Exit status: $?"
          if [ $? -ne 0 ]; then
            echo "The command failed."
            exit 1
          fi
          # TODO!!!!! add cleanup old mongodb data