CSV export optimisation

onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py

@@ -2,11 +2,12 @@
 """
 Test /orgs API endpoint implementation.
 """
+
 import json
 from builtins import str as text
 from unittest.mock import patch
 
-from django.contrib.auth.models import User, timezone, AnonymousUser
+from django.contrib.auth.models import AnonymousUser, User, timezone
 from django.core.cache import cache
 from django.test.utils import override_settings
 
@@ -15,8 +16,8 @@
 
 from onadata.apps.api.models.organization_profile import (
     OrganizationProfile,
-    get_organization_members_team,
     Team,
+    get_organization_members_team,
 )
 from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet
 from onadata.apps.api.tools import (
@@ -840,8 +841,13 @@ def test_add_members_to_owner_role(self):
         request = self.factory.get("/", **self.extra)
         response = view(request, user="denoinc")
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data["users"][1]["user"], "aboy")
-        self.assertEqual(response.data["users"][1]["role"], "owner")
+        aboy_data = {"user": "aboy", "role": "owner"}
+        self.assertTrue(
+            any(  # Order doesn't matter. aboy can be first or last
+                all(item.get(key) == value for key, value in aboy_data.items())
+                for item in response.data["users"]
+            )
+        )
 
         owner_team = get_or_create_organization_owners_team(self.organization)
 

onadata/apps/logger/models/instance.py

@@ -2,6 +2,9 @@
 """
 Instance model class
 """
+# pylint: disable=too-many-lines
+
+import importlib
 import math
 import sys
 from datetime import datetime
@@ -18,10 +21,10 @@
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext as _
-from multidb.pinning import use_master
 
 from celery import current_task
 from deprecated import deprecated
+from multidb.pinning import use_master
 from taggit.managers import TaggableManager
 
 from onadata.apps.logger.models.submission_review import SubmissionReview
@@ -35,11 +38,11 @@
 from onadata.celeryapp import app
 from onadata.libs.data.query import get_numeric_fields
 from onadata.libs.utils.cache_tools import (
-    PROJECT_DATE_MODIFIED_CACHE,
     DATAVIEW_COUNT,
     IS_ORG,
     PROJ_NUM_DATASET_CACHE,
     PROJ_SUB_DATE_CACHE,
+    PROJECT_DATE_MODIFIED_CACHE,
     XFORM_COUNT,
     XFORM_DATA_VERSIONS,
     XFORM_SUBMISSION_COUNT_FOR_DAY,
@@ -875,6 +878,14 @@ def permanently_delete_attachments(sender, instance=None, created=False, **kwarg
             )
 
 
+@use_master
+def register_export_repeats(sender, instance, created=False, **kwargs):
+    # Avoid cyclic dependency errors
+    logger_tasks = importlib.import_module("onadata.apps.logger.tasks")
+
+    logger_tasks.register_instance_export_repeats_async.delay(instance.pk)
+
+
 post_save.connect(
     post_save_submission, sender=Instance, dispatch_uid="post_save_submission"
 )
@@ -891,6 +902,12 @@ def permanently_delete_attachments(sender, instance=None, created=False, **kwarg
     dispatch_uid="permanently_delete_attachments",
 )
 
+post_save.connect(
+    register_export_repeats,
+    sender=Instance,
+    dispatch_uid="register_export_repeats",
+)
+
 
 class InstanceHistory(models.Model, InstanceBaseClass):
     """Stores deleted submission XML to maintain a history of edits."""

onadata/apps/logger/tasks.py

@@ -1,28 +1,27 @@
-# pylint: disable=import-error,ungrouped-imports
 """
 Asynchronous tasks for the logger app
 """
+
 import logging
 
-from django.core.cache import cache
 from django.contrib.auth import get_user_model
+from django.core.cache import cache
 from django.db import DatabaseError
+
 from multidb.pinning import use_master
 
-from onadata.apps.logger.models import Entity, EntityList, Project
+from onadata.apps.logger.models import Entity, EntityList, Instance, Project, XForm
 from onadata.celeryapp import app
-from onadata.libs.utils.cache_tools import (
-    PROJECT_DATE_MODIFIED_CACHE,
-    safe_delete,
-)
-from onadata.libs.utils.project_utils import set_project_perms_to_object
+from onadata.libs.utils.cache_tools import PROJECT_DATE_MODIFIED_CACHE, safe_delete
 from onadata.libs.utils.logger_tools import (
     commit_cached_elist_num_entities,
     dec_elist_num_entities,
     inc_elist_num_entities,
+    register_instance_export_repeats,
+    register_xform_export_repeats,
     soft_delete_entities_bulk,
 )
-
+from onadata.libs.utils.project_utils import set_project_perms_to_object
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
@@ -32,8 +31,7 @@
 def set_entity_list_perms_async(entity_list_id):
     """Set permissions for EntityList asynchronously
 
-    Args:
-        pk (int): Primary key for EntityList
+    :param entity_list_id: Primary key for EntityList
     """
     with use_master:
         try:
@@ -67,9 +65,8 @@ def apply_project_date_modified_async():
 def delete_entities_bulk_async(entity_pks: list[int], username: str | None = None):
     """Delete Entities asynchronously
 
-    Args:
-        entity_pks (list(int)): Primary keys of Entities to be deleted
-        username (str): Username of the user initiating the delete
+    :param entity_pks: Primary keys of Entities to be deleted
+    :param username: Username of the user initiating the delete operation
     """
     with use_master:
         entity_qs = Entity.objects.filter(pk__in=entity_pks, deleted_at__isnull=True)
@@ -104,8 +101,7 @@ def commit_cached_elist_num_entities_async():
 def inc_elist_num_entities_async(elist_pk: int):
     """Increment EntityList `num_entities` counter asynchronously
 
-    Args:
-        elist_pk (int): Primary key for EntityList
+    :param elist_pk: Primary key for EntityList
     """
     inc_elist_num_entities(elist_pk)
 
@@ -114,7 +110,38 @@ def inc_elist_num_entities_async(elist_pk: int):
 def dec_elist_num_entities_async(elist_pk: int) -> None:
     """Decrement EntityList `num_entities` counter asynchronously
 
-    Args:
-        elist_pk (int): Primary key for EntityList
+    :param elist_pk: Primary key for EntityList
     """
     dec_elist_num_entities(elist_pk)
+
+
+@app.task(retry_backoff=3, autoretry_for=(DatabaseError, ConnectionError))
+def register_instance_export_repeats_async(instance_pk: int) -> None:
+    """Register export repeats asynchronously
+
+    :param instance_pk: Primary key for Instance
+    """
+    try:
+        instance = Instance.objects.get(pk=instance_pk)
+
+    except Instance.DoesNotExist as exc:
+        logger.exception(exc)
+
+    else:
+        register_instance_export_repeats(instance)
+
+
+@app.task(retry_backoff=3, autoretry_for=(DatabaseError, ConnectionError))
+def register_xform_export_repeats_async(xform_id: int) -> None:
+    """Register export repeats for an XForm asynchronously
+
+    :param xform_id: Primary key for XForm
+    """
+    try:
+        xform = XForm.objects.get(pk=xform_id)
+
+    except XForm.DoesNotExist as exc:
+        logger.exception(exc)
+
+    else:
+        register_xform_export_repeats(xform)

onadata/apps/logger/tests/models/test_instance.py

@@ -1210,3 +1210,99 @@ def test_xml_entity_node_missing(self):
         Instance.objects.create(xml=xml, user=self.user, xform=xform)
 
         self.assertEqual(Entity.objects.count(), 0)
+
+    def test_register_repeats(self):
+        """Repeats are registered correctly"""
+        project = get_user_default_project(self.user)
+        md = """
+        | survey |
+        |        | type         | name            | label               |
+        |        | begin repeat | hospital_repeat |                     |
+        |        | text         | hospital        | Name of hospital    |
+        |        | begin repeat | child_repeat    |                     |
+        |        | text         | name            | Child's name        |
+        |        | decimal      | birthweight     | Child's birthweight |
+        |        | end_repeat   |                 |                     |
+        |        | end_repeat   |                 |                     |
+        | settings|             |                 |                     |
+        |         | form_title  | form_id         |                     |
+        |         | Births      | births          |                     |
+        """
+        self._publish_markdown(md, self.user, project)
+        xform = XForm.objects.all().order_by("-pk").first()
+        xml = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<data xmlns:jr="http://openrosa.org/javarosa" xmlns:orx='
+            '"http://openrosa.org/xforms" id="trees_update" version="2024050801">'
+            f"<formhub><uuid>{xform.uuid}</uuid></formhub>"
+            "<hospital_repeat>"
+            "<hospital>Aga Khan</hospital>"
+            "<child_repeat>"
+            "<name>Zakayo</name>"
+            "<birthweight>3.3</birthweight>"
+            "</child_repeat>"
+            "<child_repeat>"
+            "<name>Melania</name>"
+            "<birthweight>3.5</birthweight>"
+            "</child_repeat>"
+            "</hospital_repeat>"
+            "<hospital_repeat>"
+            "<hospital>Mama Lucy</hospital>"
+            "<child_repeat>"
+            "<name>Winnie</name>"
+            "<birthweight>3.1</birthweight>"
+            "</child_repeat>"
+            "</hospital_repeat>"
+            "<meta>"
+            "<instanceID>uuid:45d27780-48fd-4035-8655-9332649385bd</instanceID>"
+            "</meta>"
+            "</data>"
+        )
+        # Repeats are registered on creation
+        instance = Instance.objects.create(xml=xml, user=self.user, xform=xform)
+        metadata = MetaData.objects.get(data_type="export_repeat_register")
+        self.assertEqual(metadata.extra_data.get("hospital_repeat"), 2)
+        self.assertEqual(metadata.extra_data.get("child_repeat"), 2)
+        # Repeats are registered on update
+        xml = (
+            '<?xml version="1.0" encoding="UTF-8"?>'
+            '<data xmlns:jr="http://openrosa.org/javarosa" xmlns:orx='
+            '"http://openrosa.org/xforms" id="trees_update" version="2024050801">'
+            f"<formhub><uuid>{xform.uuid}</uuid></formhub>"
+            "<hospital_repeat>"
+            "<hospital>Aga Khan</hospital>"
+            "<child_repeat>"
+            "<name>Zakayo</name>"
+            "<birthweight>3.3</birthweight>"
+            "</child_repeat>"
+            "<child_repeat>"
+            "<name>Melania</name>"
+            "<birthweight>3.5</birthweight>"
+            "</child_repeat>"
+            "</hospital_repeat>"
+            "<hospital_repeat>"
+            "<hospital>Mama Lucy</hospital>"
+            "<child_repeat>"
+            "<name>Winnie</name>"
+            "<birthweight>3.1</birthweight>"
+            "</child_repeat>"
+            "</hospital_repeat>"
+            "<hospital_repeat>"
+            "<hospital>Mp Shah</hospital>"
+            "<child_repeat>"
+            "<name>Ada</name>"
+            "<birthweight>3.1</birthweight>"
+            "</child_repeat>"
+            "</hospital_repeat>"
+            "<meta>"
+            "<instanceID>uuid:51cb9e07-cfc7-413b-bc22-ee7adfa9dec4</instanceID>"
+            "<deprecatedID>uuid:45d27780-48fd-4035-8655-9332649385bd</deprecatedID>"
+            "</meta>"
+            "</data>"
+        )
+        instance.xml = xml
+        instance.uuid = "51cb9e07-cfc7-413b-bc22-ee7adfa9dec4"
+        instance.save()
+        metadata.refresh_from_db()
+        self.assertEqual(metadata.extra_data.get("hospital_repeat"), 3)
+        self.assertEqual(metadata.extra_data.get("child_repeat"), 2)