From 5a6e73169266908f4fc4e2b1b4ac7dd10eabc912 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 6 Aug 2024 10:58:52 +0300 Subject: [PATCH 1/6] Fix group by error when annotating stations --- .../reports/administrative_areas_reports.py | 19 +++++++------ .../reports/turnout_reports_by_admin_areas.py | 27 ++++++++++++------- tally_ho/celeryapp.py | 2 +- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/tally_ho/apps/tally/views/reports/administrative_areas_reports.py b/tally_ho/apps/tally/views/reports/administrative_areas_reports.py index e5c56ba38..08a1db473 100644 --- a/tally_ho/apps/tally/views/reports/administrative_areas_reports.py +++ b/tally_ho/apps/tally/views/reports/administrative_areas_reports.py @@ -1990,7 +1990,12 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): 'sub_race_type': electrol_race.ballot_name}\ for electrol_race in filtered_electrol_races\ if electrol_race_has_results(qs, electrol_race)} - + tally_stations_qs = Station.objects.filter(tally_id=tally_id) + stations_by_id =\ + { + station.id:\ + station for station in tally_stations_qs + } # Calculate voters in counted stations and turnout percentage for race_type_obj in race_data_by_election_level_names.values(): # race_type_obj =\ @@ -1999,8 +2004,7 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): election_level_name = race_type_obj.get('election_level') # Calculate voters in counted stations qs =\ - Station.objects.filter( - tally_id=tally_id, + tally_stations_qs.filter( center__resultform__ballot__electrol_race__election_level=\ election_level_name, center__resultform__ballot__electrol_race__ballot_name=\ @@ -2015,13 +2019,12 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): race=F('center__resultform__ballot__electrol_race__election_level') ).values('id').annotate( races=ArrayAgg('race', distinct=True), - number=F('station_number'), - num_registrants=F('registrants') ) voters = 0 stations_processed = 0 registrants_in_processed_stations = 0 for station in station_ids_by_races: + station_obj = stations_by_id.get(station.get('id')) # Calculate stations processed and total registrants form_states =\ ResultForm.objects.filter( @@ -2031,7 +2034,7 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): center__resultform__ballot__electrol_race__ballot_name=\ sub_race_type, center__stations__id=station.get('id'), - station_number=station.get('number'), + station_number=station_obj.station_number, ).values_list('form_state', flat=True).distinct() station_is_processed =\ @@ -2048,7 +2051,7 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): stations_processed += 1 registrants_in_processed_stations +=\ - station.get('num_registrants') + station_obj.registrants # Calculate voters voted in processed stations votes =\ @@ -2059,7 +2062,7 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): result_form__ballot__electrol_race__ballot_name=\ sub_race_type, result_form__center__stations__id=station.get('id'), - result_form__station_number=station.get('number'), + result_form__station_number=station_obj.station_number, entry_version=EntryVersion.FINAL, active=True, ).annotate( diff --git a/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py b/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py index 93746c34c..4fe5eebf2 100644 --- a/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py +++ b/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py @@ -65,7 +65,7 @@ def get_stations_in_admin_area(tally_id, admin_level, admin_area): def get_result_forms_for_station_in_admin_area( - tally_id, admin_level, admin_area, station + tally_id, admin_level, admin_area, station, station_obj ): """ get distinct result forms for a given station @@ -84,12 +84,12 @@ def get_result_forms_for_station_in_admin_area( tally__id=tally_id, **area_filter_map, center__stations__id=station.get('id'), - station_number=station.get('number'), + station_number=station_obj.station_number, ).values_list('form_state', flat=True).distinct() def get_station_votes_in_admin_area( - tally_id, admin_level, admin_area, station + tally_id, admin_level, admin_area, station, station_obj ): """ Gets the station votes grouped by races for the given station. @@ -108,7 +108,7 @@ def get_station_votes_in_admin_area( result_form__tally__id=tally_id, **area_filter_map, result_form__center__stations__id=station.get('id'), - result_form__station_number=station.get('number'), + result_form__station_number=station_obj.station_number, result_form__ballot__electrol_race_id__in=station.get('races'), entry_version=EntryVersion.FINAL, active=True, @@ -148,6 +148,13 @@ def get_initial_queryset(self): ret_value = [] + tally_stations_qs = Station.objects.filter(tally_id=tally_id) + stations_by_id =\ + { + station.id:\ + station for station in tally_stations_qs + } + # Calculate voters in counted stations and turnout percentage for area in admin_areas_qs: response = {} @@ -168,28 +175,30 @@ def get_initial_queryset(self): race=F('center__resultform__ballot__electrol_race_id') ).values('id').annotate( races=ArrayAgg('race', distinct=True), - number=F('station_number'), - num_registrants=F('registrants') ) voters = 0 stations_processed = 0 registrants_in_processed_stations = 0 for station in station_ids_by_race: + station_obj = stations_by_id.get(station.get('id')) # Calculate stations processed and total registrants form_states = get_result_forms_for_station_in_admin_area( tally_id, - admin_level, area_code, station + admin_level, + area_code, + station, + station_obj ) if form_states.count() == 1 and \ form_states[0] == FormState.ARCHIVED: stations_processed += 1 registrants_in_processed_stations += \ - station.get('num_registrants') + station_obj.registrants # Calculate voters voted in processed stations votes = get_station_votes_in_admin_area( - tally_id, admin_level, area_code, station + tally_id, admin_level, area_code, station, station_obj ) if votes.count() != 0: diff --git a/tally_ho/celeryapp.py b/tally_ho/celeryapp.py index ccaed373e..73b1d4d27 100644 --- a/tally_ho/celeryapp.py +++ b/tally_ho/celeryapp.py @@ -6,7 +6,7 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", - "tally_ho.settings.default") + "tally_ho.settings.local_settings") app = Celery(__name__) # Using a string here means the worker will not have to From 6c42942a935a2ea2a04a179444b71ee4d152e8d4 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 6 Aug 2024 11:11:49 +0300 Subject: [PATCH 2/6] Point to correct settings file --- tally_ho/celeryapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tally_ho/celeryapp.py b/tally_ho/celeryapp.py index 73b1d4d27..ccaed373e 100644 --- a/tally_ho/celeryapp.py +++ b/tally_ho/celeryapp.py @@ -6,7 +6,7 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", - "tally_ho.settings.local_settings") + "tally_ho.settings.default") app = Celery(__name__) # Using a string here means the worker will not have to From 085884a5964bb6f9e34e1fb80a0fc36284091afa Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 6 Aug 2024 11:12:17 +0300 Subject: [PATCH 3/6] Fix ruff check command --- .github/workflows/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 47e516035..23c40669f 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -58,7 +58,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements/dev.pip pip install pytest-cov - ruff . --exit-non-zero-on-fix + ruff check . --exit-non-zero-on-fix pytest tally_ho --doctest-modules --junitxml=coverage.xml --cov=tally_ho --cov-report=xml --cov-report=html coverage xml ls -al From 9b3af9be522d6847283602e2c78f0f60590ec50a Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 6 Aug 2024 19:29:44 +0300 Subject: [PATCH 4/6] Guard logic with if condition to prevent attribute error --- .../apps/tally/views/reports/administrative_areas_reports.py | 2 ++ .../apps/tally/views/reports/turnout_reports_by_admin_areas.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tally_ho/apps/tally/views/reports/administrative_areas_reports.py b/tally_ho/apps/tally/views/reports/administrative_areas_reports.py index 08a1db473..f9760babd 100644 --- a/tally_ho/apps/tally/views/reports/administrative_areas_reports.py +++ b/tally_ho/apps/tally/views/reports/administrative_areas_reports.py @@ -2025,6 +2025,8 @@ def create_results_power_point_headers(tally_id, filtered_electrol_races, qs): registrants_in_processed_stations = 0 for station in station_ids_by_races: station_obj = stations_by_id.get(station.get('id')) + if station_obj is None: + continue # Calculate stations processed and total registrants form_states =\ ResultForm.objects.filter( diff --git a/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py b/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py index 4fe5eebf2..89c4884b6 100644 --- a/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py +++ b/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py @@ -181,6 +181,8 @@ def get_initial_queryset(self): registrants_in_processed_stations = 0 for station in station_ids_by_race: station_obj = stations_by_id.get(station.get('id')) + if station_obj is None: + continue # Calculate stations processed and total registrants form_states = get_result_forms_for_station_in_admin_area( tally_id, From e67671cdce679c2b7566d1b70dff14980c3b1743 Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 6 Aug 2024 20:47:43 +0300 Subject: [PATCH 5/6] Code refactor to reuse tally stations queryset --- .../reports/turnout_reports_by_admin_areas.py | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py b/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py index 89c4884b6..4f8ea1fbd 100644 --- a/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py +++ b/tally_ho/apps/tally/views/reports/turnout_reports_by_admin_areas.py @@ -44,7 +44,7 @@ def get_sub_constituencies(tally_id): ).values("area_name", "id") -def get_stations_in_admin_area(tally_id, admin_level, admin_area): +def get_stations_in_admin_area(tally_stations_qs, admin_level, admin_area): """ get stations in a given administrative area across any of the 4 supported admin levels. @@ -59,13 +59,11 @@ def get_stations_in_admin_area(tally_id, admin_level, admin_area): area_filter_map = {area_filter_key: admin_area} - return Station.objects.filter( - tally_id=tally_id, active=True, **area_filter_map - ) + return tally_stations_qs.filter(active=True, **area_filter_map) def get_result_forms_for_station_in_admin_area( - tally_id, admin_level, admin_area, station, station_obj + tally_id, admin_level, admin_area, station_id, station_number ): """ get distinct result forms for a given station @@ -83,14 +81,19 @@ def get_result_forms_for_station_in_admin_area( return ResultForm.objects.filter( tally__id=tally_id, **area_filter_map, - center__stations__id=station.get('id'), - station_number=station_obj.station_number, + center__stations__id=station_id, + station_number=station_number, ).values_list('form_state', flat=True).distinct() def get_station_votes_in_admin_area( - tally_id, admin_level, admin_area, station, station_obj - ): + tally_id, + admin_level, + admin_area, + station_id, + station_number, + station_races + ): """ Gets the station votes grouped by races for the given station. """ @@ -107,9 +110,9 @@ def get_station_votes_in_admin_area( return Result.objects.filter( result_form__tally__id=tally_id, **area_filter_map, - result_form__center__stations__id=station.get('id'), - result_form__station_number=station_obj.station_number, - result_form__ballot__electrol_race_id__in=station.get('races'), + result_form__center__stations__id=station_id, + result_form__station_number=station_number, + result_form__ballot__electrol_race_id__in=station_races, entry_version=EntryVersion.FINAL, active=True, ).annotate( @@ -163,8 +166,11 @@ def get_initial_queryset(self): # regions are not linked by their ids, so we use name area_code = area.get("area_name") - stations_in_area_qs = get_stations_in_admin_area( - tally_id, admin_level, area_code + stations_in_area_qs =\ + get_stations_in_admin_area( + tally_stations_qs, + admin_level, + area_code ) response['area_code_or_name'] = area.get("area_name") response['stations_expected'] = stations_in_area_qs.count() @@ -188,8 +194,8 @@ def get_initial_queryset(self): tally_id, admin_level, area_code, - station, - station_obj + station_obj.pk, + station_obj.station_number ) if form_states.count() == 1 and \ @@ -200,8 +206,13 @@ def get_initial_queryset(self): # Calculate voters voted in processed stations votes = get_station_votes_in_admin_area( - tally_id, admin_level, area_code, station, station_obj - ) + tally_id, + admin_level, + area_code, + station_obj.pk, + station_obj.station_number, + station.get('races'), + ) if votes.count() != 0: voters += votes[0].get('race_voters') From b859afcab7ab17b20709fcdeebe4d2503dd4858a Mon Sep 17 00:00:00 2001 From: JOHNMWASHUMA Date: Tue, 6 Aug 2024 20:48:57 +0300 Subject: [PATCH 6/6] Define pytest-cov package as a dev dependency --- .github/workflows/config.yml | 1 - requirements/dev.pip | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 23c40669f..a89924a40 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -57,7 +57,6 @@ jobs: . venv/bin/activate python -m pip install --upgrade pip pip install -r requirements/dev.pip - pip install pytest-cov ruff check . --exit-non-zero-on-fix pytest tally_ho --doctest-modules --junitxml=coverage.xml --cov=tally_ho --cov-report=xml --cov-report=html coverage xml diff --git a/requirements/dev.pip b/requirements/dev.pip index de3da1455..75d7372d3 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -13,3 +13,4 @@ locust ruff pre-commit beautifulsoup4 +pytest-cov