Attempted fix #381

Workflow file for this run

	# CI/CD pipeline builds the Docker image and pushes it to the GitHub Container Registry
	# Then it runs the Sphinx documentation build and deploys it to GitHub Pages
	# Finally it tests the pymimic3 package
	# -- DEBUG --
	# Trigger: >git commit --allow-empty -m "Trigger pipeline" && git push

	name: CI/CD Pipeline

	on: [push, pull_request, workflow_dispatch]

	permissions:
	contents: write

	env:
	CONTAINER_PYTEST_RESULTS: /workdir/tests/data/pytest-results/${{ github.sha }}
	CONTAINER_BASH_RESULTS: /workdir/tests/data/bash-results/${{ github.sha }}

	jobs:
	build-and-push:
	# \|-> 1. pull code
	# \|-> 2. set up docker buildx
	# \|-> 3. login to docker hub
	# \|-> 4. setup cache dir
	# \| - ?=self-hosted
	# \|-> 5. extract branch name
	# \|-> 6. setup cache on runner machine
	# \| - ?=self-hosted\|github-hosted
	# \|-> 7. build and export docker image
	# \| - ?=self-hosted\|github-hosted
	# \|-> 8. push the image
	# \|-> 9. move cache
	# '-> 10. scan image for vulnerabilities
	runs-on: self-hosted # -> ubuntu-latest to run in github cloud
	outputs:
	branch_name: ${{ steps.get_branch.outputs.branch_name }}
	docker_image_tar: ${{ steps.get_branch.outputs.docker_image_tar }}
	docker_volumes: ${{ steps.get_volumes.outputs.docker_volumes }}
	bash_results: ${{ steps.get_volumes.outputs.bash_results }}
	pytest_results: ${{ steps.get_volumes.outputs.pytest_results }}
	local_registry: ${{ steps.check_registry.outputs.local_registry }}
	steps:
	# => 1. Pull code into runner env
	- name: Checkout code
	uses: actions/checkout@v4

	# => 2. Set up Docker Buildx step
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	with:
	driver-opts: \|
	network=host
	buildkitd-flags: --allow-insecure-entitlement network.host

	# => 3. Login to docker hub to push images
	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	# Secrets stored in repository
	username: tensorpod
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Setup volume mounts
	shell: bash
	id: get_volumes
	run: \|
	if [ -z "$DOCKER_CACHE_DIR" ]; then
	echo "Error: DOCKER_CACHE_DIR environment variable is not set."
	exit 1
	fi
	TERM=xterm-color
	DOCKER_VOLUME_MOUNTS="\
	-v $DOCKER_CACHE_DIR/.github-action-cache/control-dataset:/workdir/tests/data/control-dataset\
	-v $DOCKER_CACHE_DIR/.github-action-cache/mimiciii-demo:/workdir/tests/data/mimiciii-demo\
	-v $DOCKER_CACHE_DIR/.github-action-cache/${{ runner.name }}/semitemp:/workdir/tests/data/semitemp\
	-v $DOCKER_CACHE_DIR/.github-action-cache/bash-results:/workdir/tests/data/bash-results\
	-v $DOCKER_CACHE_DIR/.github-action-cache/pytest-results:/workdir/tests/data/pytest-results"

	echo "DOCKER_VOLUME_MOUNTS=$DOCKER_VOLUME_MOUNTS" >> $GITHUB_ENV
	echo "docker_volumes=$DOCKER_VOLUME_MOUNTS" >> $GITHUB_OUTPUT

	echo "::group::DOCKER_VOLUME_MOUNTS"
	echo "$DOCKER_VOLUME_MOUNTS" \| sed 's/ -v /\n -v /g'
	echo "::endgroup::"

	BASH_RESULTS=$DOCKER_CACHE_DIR/.github-action-cache/bash-results/${{ github.sha }}
	PYTEST_RESULTS=$DOCKER_CACHE_DIR/.github-action-cache/pytest-results/${{ github.sha }}

	echo "::group::BASH/PYTEST RESULTS"
	echo $BASH_RESULTS
	echo $PYTEST_RESULTS
	echo "::endgroup::"

	echo "bash_results=$BASH_RESULTS" >> $GITHUB_OUTPUT
	echo "pytest_results=$PYTEST_RESULTS" >> $GITHUB_OUTPUT

	- name: Check if local registry is running
	id: check_registry
	run: \|
	echo "Checking if local registry is running at localhost:5000..."
	check_registry() {
	local url=$1
	if curl -s -f -m 5 "http://${url}/v2/_catalog" > /dev/null 2>&1; then
	return 0
	else
	return 1
	fi
	}

	if check_registry "localhost:5000"; then
	echo "local_registry=true" >> $GITHUB_OUTPUT
	echo "Local registry is running on localhost:5000."
	echo "registry_host=localhost:5000" >> $GITHUB_OUTPUT
	elif check_registry "registry:5000"; then
	echo "local_registry=true" >> $GITHUB_OUTPUT
	echo "registry_host=registry:5000" >> $GITHUB_OUTPUT
	echo "Local registry is running on registry:5000 (Docker network)."
	else
	echo "local_registry=false" >> $GITHUB_OUTPUT
	echo "Error: No local registry is running or accessible." >&2
	fi

	# => 4. Create cache directory if runing locally
	- name: Set up Docker cache/output directory on self-hosted
	if: runner.environment == 'self-hosted'
	run: \|
	echo cache dir is: $DOCKER_CACHE_DIR

	if [ -n "$DOCKER_CACHE_DIR" ]; then
	echo "Using runner's DOCKER_CACHE_DIR: $DOCKER_CACHE_DIR"
	else
	DOCKER_CACHE_DIR="/tmp"
	echo "DOCKER_CACHE_DIR not set on runner. Using default: $DOCKER_CACHE_DIR"
	fi

	echo "DOCKER_CACHE_DIR=$DOCKER_CACHE_DIR" >> $GITHUB_ENV

	mkdir -p $DOCKER_CACHE_DIR
	mkdir -p $DOCKER_CACHE_DIR/.buildx-cache
	mkdir -p $DOCKER_CACHE_DIR/.buildx-image-cache/tensorpod/pymimic3/

	if [[ steps.check_registry.outputs.local_registry == 'false' ]]; then
	echo "Copying docker cache from $DOCKER_CACHE_DIR/.buildx-cache to $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache"
	mkdir -p "$DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache"
	cp -r "$DOCKER_CACHE_DIR/.buildx-cache" "$DOCKER_CACHE_DIR/${{ runner.name }}/"
	fi

	echo "::group::DOCKER_CACHE_DIR"
	echo $DOCKER_CACHE_DIR
	echo $DOCKER_CACHE_DIR/.buildx-image-cache/tensorpod/pymimic3/
	echo "::endgroup::"

	# => 5. Extract branch name (may fail on pull requests, untested)
	- name: Extract branch name
	shell: bash
	id: get_branch
	run: \|
	BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
	echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
	echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT

	echo "::group::BRANCH_NAME"
	echo $BRANCH_NAME
	echo "::endgroup::"

	DOCKER_IMAGE_TAR=$DOCKER_CACHE_DIR/.buildx-image-cache/tensorpod/pymimic3/$BRANCH_NAME.tar
	echo "DOCKER_IMAGE_TAR=$DOCKER_IMAGE_TAR" >> $GITHUB_ENV
	echo "docker_image_tar=$DOCKER_IMAGE_TAR" >> $GITHUB_OUTPUT
	echo "::group::DOCKER_IMAGE_TAR"
	echo $DOCKER_IMAGE_TAR
	echo "::endgroup::"

	# => 6. Setup cache on runner machine: self-hosted
	- name: Cache Docker layers on self-hosted
	if: \|
	runner.environment == 'self-hosted' &&
	steps.check_registry.outputs.local_registry == 'false'
	uses: maxnowack/local-cache@v2
	with:
	path: ${{ env.DOCKER_CACHE_DIR }}/${{ runner.name }}/.buildx-cache
	key: ${{ runner.os }}-buildx-${{ github.sha }}
	restore-keys: \|
	${{ runner.os }}-buildx-

	# => 6. Setup cache on github cache: github hosted
	- name: Cache Docker layers on github-hosted
	if: runner.environment == 'github-hosted'
	uses: actions/cache@v3
	with:
	path: /tmp/.buildx-cache
	key: ${{ runner.os }}-buildx-${{ github.sha }}
	restore-keys: \|
	${{ runner.os }}-buildx-

	# => 7. Build and push Docker image: self-hosted (part 1)
	# - local registry reachable:
	# - build and push to local registry
	# - tag and push to Docker Hub
	# - don't export image locally
	# - don't load the image into docker
	# - local registry not reachable:
	# - build and export image locally
	# - don't push to local registry
	# - push to docker hub in subsequent step
	- name: Build and push to local registry
	if: \|
	steps.check_registry.outputs.local_registry == 'true' &&
	runner.environment == 'self-hosted' && false
	uses: docker/build-push-action@v6
	with:
	context: .
	file: ./docker/Dockerfile.ci
	push: true
	tags: \|
	tensorpod/pymimic3:latest
	tensorpod/pymimic3:${{ env.BRANCH_NAME }}
	localhost:5000/pymimic3:latest
	localhost:5000/pymimic3:${{ env.BRANCH_NAME }}
	cache-from: type=registry,ref=localhost:5000/pymimic3:buildcache
	cache-to: type=registry,ref=localhost:5000/pymimic3:buildcache,mode=max

	- name: Build and export to tar
	if: \|
	steps.check_registry.outputs.local_registry != 'true' &&
	runner.environment == 'self-hosted'
	uses: docker/build-push-action@v6
	with:
	context: .
	file: ./docker/Dockerfile.ci
	load: true
	tags: \|
	tensorpod/pymimic3:latest
	tensorpod/pymimic3:${{ env.BRANCH_NAME }}
	cache-from: type=local,src=${{ env.DOCKER_CACHE_DIR }}/.buildx-cache
	cache-to: type=local,dest=${{ env.DOCKER_CACHE_DIR }}/.buildx-cache-new,mode=max
	outputs: type=docker,dest=${{ env.DOCKER_IMAGE_TAR }}

	# => 7. Build and push Docker image: github hosted
	- name: Build and export Docker image on github-hosted
	if: runner.environment == 'github-hosted'
	uses: docker/build-push-action@v6
	with:
	context: .
	file: ./docker/Dockerfile.ci
	push: ${{ github.event_name != 'pull_request' }}
	tags: \|
	tensorpod/pymimic3:latest
	tensorpod/pymimic3:${{ env.BRANCH_NAME }}
	cache-from: type=gha
	cache-to: type=gha,mode=max

	- name: Upload artifact
	if: runner.environment == 'github-hosted'
	uses: actions/upload-artifact@v4
	with:
	name: pymimic3.$BRANCH_NAME
	path: /tmp/.buildx-image-cache/tensorpod/pymimic3/${{ env.BRANCH_NAME }}.tar

	# => 8. Push Docker image
	# It seems the docker image is not automatically loaded after build
	- name: Push Docker image
	if: github.event_name != 'pull_request' && runner.environment == 'self-hosted'
	run: \|
	# Login to Docker Hub
	echo "${{ secrets.DOCKER_PASSWORD }}" \| docker login -u tensorpod --password-stdin

	if [[ "${{ steps.check_registry.outputs.local_registry }}" == "true" ]]; then
	echo "Local registry is available. Pulling from local registry for subsequent jobs."
	else
	echo "No local registry available. Loading from tar and pushing to Docker Hub."

	# Push to Docker Hub
	docker push tensorpod/pymimic3:latest
	docker push tensorpod/pymimic3:$BRANCH_NAME
	fi

	- name: Docker cleanup
	run: \|
	docker image prune -f
	docker volume prune -f
	docker builder prune

	# => 9. Move the cache with -new suffix to original cache
	- name: Move cache
	if: ${{ steps.check_registry.outputs.local_registry == 'false' }}
	run: \|
	if [ "${{ runner.environment }}" = "self-hosted" ]; then
	rm -rf $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache
	mv $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache-new $DOCKER_CACHE_DIR/.buildx-cache
	elif [ "${{ runner.environment }}" = "github-hosted" ]; then
	rm -rf tmp/.buildx-cache
	mv tmp/.buildx-cache-new tmp/.buildx-cache
	fi

	# => 10. Scan image for vulnerabilities
	# - name: Scan image for vulnerabilities
	# uses: anchore/scan-action@v3
	# with:
	# image: "tensorpod/pymimic3:$BRANCH_NAME"
	# fail-build: false
	# severity-cutoff: high

	deploy:
	needs: build-and-push
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	environment:
	name: github-pages
	url: ${{ steps.deployment.outputs.page_url }}
	runs-on: self-hosted
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Build documentation
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color
	VOLUME_DIR="${{ runner.workspace }}/docs/build"
	VOLUMES="-v ${VOLUME_DIR}:/workdir/docs/build"

	# Run command in docker
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:latest
	echo "Tagging image"
	docker tag localhost:5000/pymimic3:latest tensorpod/pymimic3:latest
	echo "Running sphinx build"
	docker run $VOLUMES --rm tensorpod/pymimic3:latest bash -ic "pip install sphinx sphinx-rtd-theme && cd docs && make html"
	working-directory: .

	- name: Setup Pages
	uses: actions/configure-pages@v4

	- name: Upload artifact
	uses: actions/upload-pages-artifact@v3
	with:
	path: '${{ runner.workspace }}/docs/build/html'

	- name: Deploy to GitHub Pages
	id: deployment
	uses: actions/deploy-pages@v4
	# # Documentation build and deploy
	# docs:
	# runs-on: self-hosted
	# needs: build-and-push
	# steps:
	# - uses: actions/checkout@v4
	# - uses: actions/setup-python@v5
	# - name: Install dependencies
	# run: \|
	# pip install sphinx sphinx_rtd_theme myst_parser
	# - name: Sphinx build
	# run: \|
	# sphinx-build doc _build
	# - name: Deploy to GitHub Pages
	# uses: peaceiris/actions-gh-pages@v3
	# if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
	# with:
	# publish_branch: gh-pages
	# github_token: ${{ secrets.GITHUB_TOKEN }}
	# publish_dir: _build/
	# force_orphan: true
	#
	# Pytest run test

	setup-control-dataset:
	concurrency:
	group: setup-control-dataset
	runs-on: self-hosted
	needs: build-and-push
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup persistent test mounts
	shell: bash
	run: \|
	mkdir -p $DOCKER_CACHE_DIR/.github-action-cache/{pytest-results/${{ github.sha }},bash-results/${{ github.sha }}}

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Create output dirs
	run: \|
	mkdir -p BASH_RESULTS
	mkdir -p PYTEST_RESULTS

	- name: Setup control dataset
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run command in docker
	.github/scripts/run_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"make testing=true" \
	"$BASH_RESULTS" \
	"setup-control-dataset"

	setup-fixtures:
	needs: [build-and-push, setup-control-dataset]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run fixture setup tests
	shell: bash
	run: \|
	# Create the semitemp directory
	mkdir -p $DOCKER_CACHE_DIR/.github-action-cache/${{ runner.name }}/semitemp
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_zsetup_all_fixtures.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"setup-control-dataset"

	- name: Run pytest suite tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_pytest_utils.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-pytest-utils"

	test-readers:
	needs: [setup-fixtures, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run reader tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_readers" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-readers"

	test-trackers:
	needs: [setup-fixtures, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run tracker tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_trackers" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-trackers"

	test-data-split:
	needs: [test-readers, test-trackers, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run data split tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_data_split.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-data-split"

	test-extraction:
	needs: [test-readers, test-trackers, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run extraction tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_extraction.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-extraction"

	test-preprocessors:
	needs: [test-extraction, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run preprocessor tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_preprocessor.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-preprocessors"

	test-feature-engineering:
	needs: [test-extraction, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run feature engineering tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_feature_engine.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-feature-engineering"

	test-discretizer:
	needs: [test-extraction, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run discretizer tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_discretizer.py" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-discretizer"

	test-parameters:
	needs: [test-preprocessors, test-feature-engineering, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run parameter tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_datasets/test_parameters" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-parameters"


	test-preprocessing:
	needs: [setup-fixtures, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run preprocessing tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_preprocessing" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-preprocessing"

	test-metrics:
	needs: [setup-fixtures, build-and-push]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run metrics tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_metrics" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-metrics"

	test-models:
	needs: [build-and-push, test-metrics, test-parameters, test-preprocessing]
	runs-on: self-hosted
	env:
	BRANCH_NAME: ${{ needs.build-and-push.outputs.branch_name }}
	DOCKER_IMAGE_TAR: ${{ needs.build-and-push.outputs.docker_image_tar }}
	DOCKER_VOLUME_MOUNTS: ${{ needs.build-and-push.outputs.docker_volumes }}
	BASH_RESULTS: ${{ needs.build-and-push.outputs.bash_results }}
	PYTEST_RESULTS: ${{ needs.build-and-push.outputs.pytest_results }}
	LOCAL_REGISTRY: ${{ needs.build-and-push.outputs.local_registry }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Load image
	shell: bash
	run: \|
	if [[ "$LOCAL_REGISTRY" == "true" ]]; then
	echo "Pulling image from local registry"
	docker pull localhost:5000/pymimic3:$BRANCH_NAME
	docker tag localhost:5000/pymimic3:$BRANCH_NAME tensorpod/pymimic3:$BRANCH_NAME
	else
	echo "Loading image from tar file"
	docker load -i $DOCKER_IMAGE_TAR
	fi
	echo "Docker images:"
	docker images

	- name: Run model tests
	shell: bash
	run: \|
	# Enable color output
	export TERM=xterm-color
	export TERM=xterm-256color

	# Run pytest in docker
	.github/scripts/run_pytest_container.sh \
	"$DOCKER_VOLUME_MOUNTS" \
	"$BRANCH_NAME" \
	"$CONTAINER_PYTEST_RESULTS" \
	"tests/test_models" \
	"$BASH_RESULTS" \
	"$PYTEST_RESULTS" \
	"test-models"
	Cleanup:
	if: always()
	runs-on: self-hosted
	needs: [test-models]
	steps:
	- name: Cleanup semitemp directory
	if: always()
	shell: bash
	run: \|
	rm -rf $DOCKER_CACHE_DIR/.github-action-cache/${{ runner.name }}/semitemp && \
	rm -rf $DOCKER_CACHE_DIR/${{ runner.name }}/.buildx-cache 2>&1 \
	\| tee $BASH_RESULTS/cleanup-semitemp.txt

	echo "Exit status: $?"
	if [ $? -ne 0 ]; then
	echo "The command failed."
	exit 1
	fi
	# TODO!!!!! add cleanup old mongodb data

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Attempted fix #381

Workflow file

Attempted fix #381

Jobs

Run details

Workflow file for this run