From bd64a99fcf88799aacaa6988fdc77e24b742fe13 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 29 Oct 2024 15:08:32 -0700 Subject: [PATCH 1/8] Add pytest and coverage as additional dependencies for the automated unit test and test coverage --- viz_scripts/docker/environment36.dashboard.additions.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/viz_scripts/docker/environment36.dashboard.additions.yml b/viz_scripts/docker/environment36.dashboard.additions.yml index 59d26eb..49d927f 100644 --- a/viz_scripts/docker/environment36.dashboard.additions.yml +++ b/viz_scripts/docker/environment36.dashboard.additions.yml @@ -4,6 +4,8 @@ channels: - defaults dependencies: - seaborn=0.11.1 +- pytest +- coverage - pip: - nbparameterise==0.6 - devcron==0.4 From de3d1268fabc6acd5039e51a2e92038a17b23796 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 29 Oct 2024 15:17:35 -0700 Subject: [PATCH 2/8] Enable running of plots.py from pytest, aside from Notebook --- viz_scripts/plots.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/viz_scripts/plots.py b/viz_scripts/plots.py index 34e26bc..4c35465 100644 --- a/viz_scripts/plots.py +++ b/viz_scripts/plots.py @@ -9,7 +9,17 @@ sns.set_style("whitegrid") sns.set() -get_ipython().run_line_magic('matplotlib', 'inline') + +try: + # Import the function + from IPython import get_ipython + # Check if running in an IPython environment (like Jupyter Notebook) + if get_ipython() is not None: + get_ipython().run_line_magic('matplotlib', 'inline') +except ImportError: + # Handle the case where IPython is not installed + # We are running in regular Python (likely pytest), not Jupyter/IPython + pass # Module for pretty-printing outputs (e.g. head) to help users # understand what is going on From 0f3099ebacff89b269b50c7c932086da901d025b Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 29 Oct 2024 15:40:16 -0700 Subject: [PATCH 3/8] Add Git Action workflow --- .github/workflows/test_with_docker.yml | 34 ++++++++++++++++++++++++++ viz_scripts/docker/Dockerfile.test | 19 ++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/test_with_docker.yml create mode 100644 viz_scripts/docker/Dockerfile.test diff --git a/.github/workflows/test_with_docker.yml b/.github/workflows/test_with_docker.yml new file mode 100644 index 0000000..6a58265 --- /dev/null +++ b/.github/workflows/test_with_docker.yml @@ -0,0 +1,34 @@ +# This is a basic workflow to help you get started with Actions + +name: test-with-docker + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '5 4 * * 0' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout + uses: actions/checkout@v2 + + - name: Make sure that the workflow works + run: echo Smoke test + + - name: Run the tests using docker-compose + run: docker compose -f setup/docker-compose.tests.yml build + run: docker compose -f setup/docker-compose.tests.yml up --exit-code-from web-server \ No newline at end of file diff --git a/viz_scripts/docker/Dockerfile.test b/viz_scripts/docker/Dockerfile.test new file mode 100644 index 0000000..d9a84ea --- /dev/null +++ b/viz_scripts/docker/Dockerfile.test @@ -0,0 +1,19 @@ +# python 3 +ARG SERVER_IMAGE_TAG +FROM shankari/e-mission-server:master_${SERVER_IMAGE_TAG} + +VOLUME /plots + +ADD docker/environment36.dashboard.additions.yml / + +WORKDIR /usr/src/app + +RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission --file setup/environment36.notebook.additions.yml" +RUN /bin/bash -c "source setup/activate.sh && conda env update --name emission --file /environment36.dashboard.additions.yml" + +ADD docker/start_tests.sh /usr/src/app/.docker/start_tests.sh +RUN chmod u+x /usr/src/app/.docker/start_tests.sh + +EXPOSE 8888 + +CMD ["/bin/bash", "/usr/src/app/.docker/start_tests.sh"] From 5d8548a6cba4c30fba142574e29728dca97a505e Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 29 Oct 2024 15:40:49 -0700 Subject: [PATCH 4/8] Add docker-compose file for automated unit testing --- docker-compose.tests.yml | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docker-compose.tests.yml diff --git a/docker-compose.tests.yml b/docker-compose.tests.yml new file mode 100644 index 0000000..6ed827e --- /dev/null +++ b/docker-compose.tests.yml @@ -0,0 +1,51 @@ +version: "3" +services: + dashboard: + image: em-pub-dash-dev/frontend + build: + context: frontend + dockerfile: docker/Dockerfile.dev + depends_on: + - db + ports: + # DASH in numbers + - "3274:6060" + volumes: + - ./frontend:/public + - ./plots:/public/plots + networks: + - emission + notebook-server: + image: em-pub-dash-dev/viz-scripts + build: + context: viz_scripts + dockerfile: docker/Dockerfile.test + args: + SERVER_IMAGE_TAG: ${SERVER_IMAGE_TAG} + depends_on: + - db + environment: + - DB_HOST=db + - WEB_SERVER_HOST=0.0.0.0 + - CRON_MODE= + - STUDY_CONFIG=stage-program + ports: + # ipynb in numbers + - "47962:47962" + networks: + - emission + volumes: + - ./viz_scripts:/usr/src/app/saved-notebooks + - ./plots:/plots + db: + image: mongo:4.4.0 + volumes: + - mongo-data:/data/db + networks: + - emission + +networks: + emission: + +volumes: + mongo-data: From 599558951042a71674ea86defac5932b80cb72d4 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 29 Oct 2024 15:44:54 -0700 Subject: [PATCH 5/8] Initial unit test cases for plots.py --- viz_scripts/docker/start_tests.sh | 13 +++++++ viz_scripts/tests/test_plots.py | 60 +++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 viz_scripts/docker/start_tests.sh create mode 100644 viz_scripts/tests/test_plots.py diff --git a/viz_scripts/docker/start_tests.sh b/viz_scripts/docker/start_tests.sh new file mode 100644 index 0000000..06dbf8d --- /dev/null +++ b/viz_scripts/docker/start_tests.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e # Exit on error + +# change python environment +pwd +source setup/activate.sh || exit 1 +conda env list +cd saved-notebooks/tests || exit 1 + +echo "Starting unit tests..." +PYTHONPATH=. coverage run -m pytest test_plots.py + +coverage report diff --git a/viz_scripts/tests/test_plots.py b/viz_scripts/tests/test_plots.py new file mode 100644 index 0000000..23d5fef --- /dev/null +++ b/viz_scripts/tests/test_plots.py @@ -0,0 +1,60 @@ +import pytest as pytest +import pandas as pd +import numpy as np +import os +import pathlib +import sys +sys.path.append(str(pathlib.Path(__file__).parent.parent)) +import plots as plots + +# Test Data Fixtures +@pytest.fixture +def sample_labels(): + return ['Car', 'Bus', 'Train', 'Walk'] + +@pytest.fixture +def sample_values(): + return [100, 50, 3, 1] + +@pytest.fixture +def sample_labels_no_small(): + return ['Car', 'Bus'] + + +@pytest.fixture +def sample_values_no_small(): + return [100, 100] + +class TestCalculatePct: + def test_calculate_pct_basic(self, sample_labels, sample_values): + labels, values, pcts = plots.calculate_pct(sample_labels, sample_values) + assert len(labels) == len(sample_labels) + assert len(values) == len(sample_values) + assert sum(pcts) == pytest.approx(100.0, abs=0.1) + + def test_calculate_pct_empty(self): + labels, values, pcts = plots.calculate_pct([],[]) + assert len(labels) == 0 + assert len(values) == 0 + assert len(pcts) == 0 + + def test_calculate_pct_single(self): + labels, values, pcts = plots.calculate_pct(['Car'], [100]) + assert pcts == [100.0] + +class TestMergeSmallEntries: + def test_merge_small_entries_basic(self, sample_labels, sample_values): + labels, values, pcts = plots.merge_small_entries(sample_labels, sample_values) + assert all(pct > 2.0 for pct in pcts) + + def test_merge_small_entries_no_small(self, sample_labels_no_small, sample_values_no_small): + result_labels, result_values, result_pcts = plots.merge_small_entries(sample_labels_no_small, sample_values_no_small) + assert len(result_labels) == 2 + assert 'other' not in result_labels + assert 'OTHER' not in result_labels + + def test_merge_small_entries_some_small(self, sample_labels, sample_values): + result_labels, result_values, result_pcts = plots.merge_small_entries(sample_labels, sample_values) + print(result_labels) + assert len(result_labels) == 3 + assert result_labels[0] in ['Car', 'Bus','other', 'OTHER'] From 63867f470455e3157c9da28af93ba4e9d37ca10e Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 29 Oct 2024 15:52:27 -0700 Subject: [PATCH 6/8] Add extra line at the end --- .github/workflows/test_with_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_docker.yml b/.github/workflows/test_with_docker.yml index 6a58265..73e2947 100644 --- a/.github/workflows/test_with_docker.yml +++ b/.github/workflows/test_with_docker.yml @@ -31,4 +31,4 @@ jobs: - name: Run the tests using docker-compose run: docker compose -f setup/docker-compose.tests.yml build - run: docker compose -f setup/docker-compose.tests.yml up --exit-code-from web-server \ No newline at end of file + run: docker compose -f setup/docker-compose.tests.yml up --exit-code-from web-server From bba61b66424593296a1c71d34f5d606d42ac5941 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 5 Nov 2024 12:46:24 -0700 Subject: [PATCH 7/8] Simplify import of parent directory with the use of PYTHONPATH=../.. since emission is two directory above the current test_*.py files directory. Used importlib as the parent directory has - in saved-notebooks. --- viz_scripts/docker/start_tests.sh | 2 +- viz_scripts/tests/test_plots.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/viz_scripts/docker/start_tests.sh b/viz_scripts/docker/start_tests.sh index 06dbf8d..f979f23 100644 --- a/viz_scripts/docker/start_tests.sh +++ b/viz_scripts/docker/start_tests.sh @@ -8,6 +8,6 @@ conda env list cd saved-notebooks/tests || exit 1 echo "Starting unit tests..." -PYTHONPATH=. coverage run -m pytest test_plots.py +PYTHONPATH=../.. coverage run -m pytest test_plots.py -v coverage report diff --git a/viz_scripts/tests/test_plots.py b/viz_scripts/tests/test_plots.py index 23d5fef..26d26e3 100644 --- a/viz_scripts/tests/test_plots.py +++ b/viz_scripts/tests/test_plots.py @@ -1,11 +1,9 @@ import pytest as pytest import pandas as pd import numpy as np -import os -import pathlib -import sys -sys.path.append(str(pathlib.Path(__file__).parent.parent)) -import plots as plots +# Using import_module, as we have saved-notebooks as the directory +import importlib +plots = importlib.import_module('saved-notebooks.plots') # Test Data Fixtures @pytest.fixture From 7a1ed375e88c08a1187c27a5574419462ed7ec37 Mon Sep 17 00:00:00 2001 From: Ananta Shrestha Date: Tue, 19 Nov 2024 22:39:10 -0700 Subject: [PATCH 8/8] Fix run for tests with docker-compose in test_with_docker.yml --- .github/workflows/test_with_docker.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_with_docker.yml b/.github/workflows/test_with_docker.yml index 73e2947..0339e3c 100644 --- a/.github/workflows/test_with_docker.yml +++ b/.github/workflows/test_with_docker.yml @@ -30,5 +30,7 @@ jobs: run: echo Smoke test - name: Run the tests using docker-compose - run: docker compose -f setup/docker-compose.tests.yml build - run: docker compose -f setup/docker-compose.tests.yml up --exit-code-from web-server + working-directory: .github/workflows + run: | + docker compose -f ../../docker-compose.tests.yml build + docker compose -f ../../docker-compose.tests.yml up --exit-code-from notebook-server