Over Limit Changes

commcare_connect/form_receiver/processor.py

@@ -235,15 +235,12 @@ def clean_form_submission(access: OpportunityAccess, user_visit: UserVisit, xfor
 def process_deliver_unit(user, xform: XForm, app: CommCareApp, opportunity: Opportunity, deliver_unit_block: dict):
     deliver_unit = get_or_create_deliver_unit(app, deliver_unit_block)
     access = OpportunityAccess.objects.get(opportunity=opportunity, user=user)
-
-    counts = (
-        UserVisit.objects.filter(opportunity_access=access, deliver_unit=deliver_unit)
-        .exclude(status__in=[VisitValidationStatus.over_limit, VisitValidationStatus.trial])
-        .aggregate(
-            daily=Count("pk", filter=Q(visit_date__date=xform.metadata.timeStart)),
-            total=Count("*"),
-            entity=Count("pk", filter=Q(entity_id=deliver_unit_block.get("entity_id"), deliver_unit=deliver_unit)),
-        )
+    counts = UserVisit.objects.filter(
+        opportunity_access=access, deliver_unit=deliver_unit, status=VisitValidationStatus.approved
+    ).aggregate(
+        daily=Count("pk", filter=Q(visit_date__date=xform.metadata.timeStart)),
+        total=Count("*"),
+        entity=Count("pk", filter=Q(entity_id=deliver_unit_block.get("entity_id"), deliver_unit=deliver_unit)),
     )
     payment_unit = deliver_unit.payment_unit
     claim = OpportunityClaim.objects.get(opportunity_access=access)

commcare_connect/form_receiver/tests/test_receiver_integration.py

@@ -1,4 +1,5 @@
 import datetime
+import random
 from copy import deepcopy
 from http import HTTPStatus
 from uuid import uuid4
@@ -179,6 +180,8 @@ def test_receiver_deliver_form_max_visits_reached(
     def submit_form_for_random_entity(form_json):
         duplicate_json = deepcopy(form_json)
         duplicate_json["form"]["deliver"]["entity_id"] = str(uuid4())
+        # generate random locations for form submissions
+        duplicate_json["metadata"]["location"] = " ".join([str(random.uniform(-90, 90)) for _ in range(4)])
         make_request(api_client, duplicate_json, mobile_user_with_connect_link)
 
     payment_units = opportunity.paymentunit_set.all()
@@ -193,7 +196,7 @@ def submit_form_for_random_entity(form_json):
     user_visits = UserVisit.objects.filter(user=mobile_user_with_connect_link)
     assert user_visits.count() == 5
     # First four are not over-limit
-    assert {u.status for u in user_visits[0:4]} == {VisitValidationStatus.pending, VisitValidationStatus.approved}
+    assert {u.status for u in user_visits[0:4]} == {VisitValidationStatus.approved}
     # Last one is over limit
     assert user_visits[4].status == VisitValidationStatus.over_limit
 

commcare_connect/opportunity/tests/test_visit_import.py

@@ -610,6 +610,55 @@ def test_review_completed_work_status(
     _validate_saved_fields(access)
 
 
+@pytest.mark.django_db
+def test_approved_visits_over_limit_count_daily(opportunity: Opportunity):
+    payment_unit = PaymentUnitFactory(opportunity=opportunity, max_daily=3, max_total=10)
+    deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app, payment_unit=payment_unit)
+    access_objects = OpportunityAccessFactory.create_batch(5, opportunity=opportunity)
+    dataset = Dataset(headers=["visit id", "status", "rejected reason"])
+    for access in access_objects:
+        visits = UserVisitFactory.create_batch(
+            5,
+            opportunity=opportunity,
+            status=VisitValidationStatus.pending.value,
+            user=access.user,
+            opportunity_access=access,
+            deliver_unit=deliver_unit,
+            visit_date=now(),
+        )
+        dataset.extend([[visit.xform_id, VisitValidationStatus.approved.value, ""] for visit in visits])
+    import_status = _bulk_update_visit_status(opportunity, dataset)
+
+    assert not import_status.missing_visits
+    assert import_status.over_limit_count
+
+    assert import_status.over_limit_count == 10
+
+
+@pytest.mark.django_db
+def test_approved_visits_over_limit_count_total(opportunity: Opportunity):
+    payment_unit = PaymentUnitFactory(opportunity=opportunity, max_daily=1, max_total=3)
+    deliver_unit = DeliverUnitFactory(app=opportunity.deliver_app, payment_unit=payment_unit)
+    access_objects = OpportunityAccessFactory.create_batch(5, opportunity=opportunity)
+    dataset = Dataset(headers=["visit id", "status", "rejected reason"])
+    for access in access_objects:
+        visits = UserVisitFactory.create_batch(
+            5,
+            opportunity=opportunity,
+            status=VisitValidationStatus.pending.value,
+            user=access.user,
+            opportunity_access=access,
+            deliver_unit=deliver_unit,
+        )
+        dataset.extend([[visit.xform_id, VisitValidationStatus.approved.value, ""] for visit in visits])
+    import_status = _bulk_update_visit_status(opportunity, dataset)
+
+    assert not import_status.missing_visits
+    assert import_status.over_limit_count
+
+    assert import_status.over_limit_count == 10
+
+
 def _validate_saved_fields(opportunity_access: OpportunityAccess):
     for completed_work in opportunity_access.completedwork_set.all():
         validate_saved_fields(completed_work)

commcare_connect/opportunity/views.py

@@ -437,6 +437,8 @@ def update_visit_status_import(request, org_slug=None, pk=None):
         message = f"Visit status updated successfully for {len(status)} visits."
         if status.missing_visits:
             message += status.get_missing_message()
+        if status.over_limit_count:
+            messages.warning(request, mark_safe(status.get_over_limit_count_message()))
         messages.success(request, mark_safe(message))
     if opportunity.managed:
         return redirect("opportunity:user_visit_review", org_slug, pk)

commcare_connect/opportunity/visit_import.py

@@ -2,13 +2,15 @@
 import json
 import textwrap
 import urllib
+from collections import defaultdict
 from dataclasses import astuple, dataclass
 from decimal import Decimal, InvalidOperation
 
 from django.conf import settings
 from django.core.cache import cache
 from django.core.files.uploadedfile import UploadedFile
 from django.db import transaction
+from django.db.models import Count
 from django.utils.timezone import now
 from tablib import Dataset
 
@@ -64,6 +66,7 @@ class InvalidValueError(RowDataError):
 class VisitImportStatus:
     seen_visits: set[str]
     missing_visits: set[str]
+    over_limit_count: int = 0
 
     def __len__(self):
         return len(self.seen_visits)
@@ -73,6 +76,9 @@ def get_missing_message(self):
         missing = textwrap.wrap(joined, width=115, break_long_words=False, break_on_hyphens=False)
         return f"<br>{len(self.missing_visits)} visits were not found:<br>{'<br>'.join(missing)}"
 
+    def get_over_limit_count_message(self):
+        return f"{self.over_limit_count} visits are marked as approved and are over the daily total visits allowed."
+
 
 @dataclass
 class PaymentImportStatus:
@@ -135,6 +141,21 @@ def _bulk_update_visit_status(opportunity: Opportunity, dataset: Dataset):
     missing_visits = set()
     seen_visits = set()
     user_ids = set()
+    deliver_units = opportunity.deliver_app.deliver_units.all().select_related("payment_unit")
+    deliver_unit_limits = {
+        d.id: (d.payment_unit.max_daily, d.payment_unit.max_total) for d in deliver_units if d.payment_unit is not None
+    }
+    counts = (
+        UserVisit.objects.filter(opportunity=opportunity, status=VisitValidationStatus.approved)
+        .values_list("user_id", "deliver_unit_id", "visit_date")
+        .annotate(count=Count("*"))
+    )
+    daily_approved_count = defaultdict(int)
+    total_approved_count = defaultdict(int)
+    for user_id, deliver_unit_id, visit_date, count in counts:
+        daily_approved_count[(user_id, deliver_unit_id, visit_date)] = count
+        total_approved_count[(user_id, deliver_unit_id)] += count
+    over_limit_count = 0
     with transaction.atomic():
         for visit_batch in batched(visit_ids, 100):
             to_update = []
@@ -146,9 +167,21 @@ def _bulk_update_visit_status(opportunity: Opportunity, dataset: Dataset):
                 changed = False
                 if visit.status != status:
                     visit.status = status
-                    if opportunity.managed and status == VisitValidationStatus.approved:
-                        visit.review_created_on = now()
                     changed = True
+                    if status == VisitValidationStatus.approved:
+                        d_counts = deliver_unit_limits.get(visit.deliver_unit_id)
+                        if d_counts is not None:
+                            d_max_daily, d_max_total = d_counts
+                            key = (visit.user_id, visit.deliver_unit_id)
+                            daily_approved_count[(*key, visit.visit_date)] += 1
+                            total_approved_count[key] += 1
+                            if (
+                                daily_approved_count[(*key, visit.visit_date)] > d_max_daily
+                                or total_approved_count[key] > d_max_total
+                            ):
+                                over_limit_count += 1
+                        if opportunity.managed:
+                            visit.review_created_on = now()
                 if status == VisitValidationStatus.rejected and reason and reason != visit.reason:
                     visit.reason = reason
                     changed = True
@@ -163,7 +196,7 @@ def _bulk_update_visit_status(opportunity: Opportunity, dataset: Dataset):
             )
             missing_visits |= set(visit_batch) - seen_visits
     update_payment_accrued(opportunity, users=user_ids)
-    return VisitImportStatus(seen_visits, missing_visits)
+    return VisitImportStatus(seen_visits, missing_visits, over_limit_count)
 
 
 def update_payment_accrued(opportunity: Opportunity, users):