diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index 3ecedc4320..8f78f3e59a 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -2,31 +2,33 @@ """ Celery api.tasks module. """ + +import logging import os import sys -import logging from datetime import timedelta -from celery.result import AsyncResult from django.conf import settings -from django.core.files.uploadedfile import TemporaryUploadedFile -from django.core.files.storage import default_storage from django.contrib.auth import get_user_model +from django.core.files.storage import default_storage +from django.core.files.uploadedfile import TemporaryUploadedFile from django.db import DatabaseError from django.utils import timezone from django.utils.datastructures import MultiValueDict +from celery.result import AsyncResult + from onadata.apps.api import tools -from onadata.apps.logger.models import Instance, ProjectInvitation, XForm, Project +from onadata.apps.logger.models import Instance, Project, ProjectInvitation, XForm from onadata.celeryapp import app -from onadata.libs.utils.email import send_generic_email -from onadata.libs.utils.model_tools import queryset_iterator +from onadata.libs.models.share_project import ShareProject from onadata.libs.utils.cache_tools import ( - safe_delete, XFORM_REGENERATE_INSTANCE_JSON_TASK, + safe_delete, ) -from onadata.libs.models.share_project import ShareProject -from onadata.libs.utils.email import ProjectInvitationEmail +from onadata.libs.utils.email import ProjectInvitationEmail, send_generic_email +from onadata.libs.utils.logger_tools import delete_xform_submissions +from onadata.libs.utils.model_tools import queryset_iterator logger = logging.getLogger(__name__) @@ -200,3 +202,28 @@ def share_project_async(project_id, username, role, remove=False): else: share = ShareProject(project, username, role, remove) share.save() + + +@app.task(retry_backoff=3, autoretry_for=(DatabaseError, ConnectionError)) +def delete_xform_submissions_async( + xform_id: int, + deleted_by_id: int, + instance_ids: list[int] | None = None, + soft_delete: bool = True, +): + """Delete xform submissions asynchronously + + :param xform_id: XForm id + :param deleted_by_id: User id who deleted the instances + :param instance_ids: List of instance ids to delete, None to delete all + :param soft_delete: Soft delete instances if True, otherwise hard delete + """ + try: + xform = XForm.objects.get(pk=xform_id) + deleted_by = User.objects.get(pk=deleted_by_id) + + except (XForm.DoesNotExist, User.DoesNotExist) as err: + logger.exception(err) + + else: + delete_xform_submissions(xform, deleted_by, instance_ids, soft_delete) diff --git a/onadata/apps/api/tests/test_tasks.py b/onadata/apps/api/tests/test_tasks.py index d6c6f1c42f..8b7d15bc99 100644 --- a/onadata/apps/api/tests/test_tasks.py +++ b/onadata/apps/api/tests/test_tasks.py @@ -1,26 +1,26 @@ """Tests for module onadata.apps.api.tasks""" import sys - from unittest.mock import patch -from django.core.cache import cache from django.contrib.auth import get_user_model +from django.core.cache import cache from django.db import DatabaseError, OperationalError from onadata.apps.api.tasks import ( - send_project_invitation_email_async, + ShareProject, + delete_xform_submissions_async, regenerate_form_instance_json, + send_project_invitation_email_async, share_project_async, - ShareProject, ) -from onadata.apps.logger.models import ProjectInvitation, Instance +from onadata.apps.logger.models import Instance, ProjectInvitation from onadata.apps.main.tests.test_base import TestBase from onadata.libs.permissions import ManagerRole from onadata.libs.serializers.organization_serializer import OrganizationSerializer -from onadata.libs.utils.user_auth import get_user_default_project -from onadata.libs.utils.email import ProjectInvitationEmail from onadata.libs.utils.cache_tools import ORG_PROFILE_CACHE +from onadata.libs.utils.email import ProjectInvitationEmail +from onadata.libs.utils.user_auth import get_user_default_project User = get_user_model() @@ -185,3 +185,46 @@ def test_operation_error(self, mock_retry, mock_share): self.assertTrue(mock_retry.called) _, kwargs = mock_retry.call_args_list[0] self.assertTrue(isinstance(kwargs["exc"], OperationalError)) + + +@patch("onadata.apps.api.tasks.delete_xform_submissions") +class DeleteXFormSubmissionsAsyncTestCase(TestBase): + """Tests for delete_xform_submissions_async""" + + def setUp(self): + super().setUp() + + self._publish_transportation_form() + + def test_delete(self, mock_delete): + """Submissions are deleted""" + delete_xform_submissions_async.delay(self.xform.pk, self.user.pk, [1, 2], False) + mock_delete.assert_called_once_with(self.xform, self.user, [1, 2], False) + + @patch("onadata.apps.api.tasks.delete_xform_submissions_async.retry") + def test_database_error(self, mock_retry, mock_delete): + """We retry calls if DatabaseError is raised""" + mock_delete.side_effect = DatabaseError() + delete_xform_submissions_async.delay(self.xform.pk, self.user.pk) + self.assertTrue(mock_retry.called) + + @patch("onadata.apps.api.tasks.delete_xform_submissions_async.retry") + def test_connection_error(self, mock_retry, mock_delete): + """We retry calls if ConnectionError is raised""" + mock_delete.side_effect = ConnectionError() + delete_xform_submissions_async.delay(self.xform.pk, self.user.pk) + self.assertTrue(mock_retry.called) + + @patch("onadata.apps.api.tasks.logger.exception") + def test_xform_id_invalid(self, mock_logger, mock_delete): + """Invalid xform_id is handled""" + delete_xform_submissions_async.delay(sys.maxsize, self.user.pk) + self.assertFalse(mock_delete.called) + mock_logger.assert_called_once() + + @patch("onadata.apps.api.tasks.logger.exception") + def test_user_id_invalid(self, mock_logger, mock_delete): + """Invalid user_id is handled""" + delete_xform_submissions_async.delay(self.xform.pk, sys.maxsize) + self.assertFalse(mock_delete.called) + mock_logger.assert_called_once() diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index cadce3b8da..8889dc028c 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -2,6 +2,7 @@ """ Test /data API endpoint implementation. """ + from __future__ import unicode_literals import csv @@ -1671,8 +1672,7 @@ def test_data_w_attachment(self): self.assertIsInstance(response.data, dict) self.assertDictContainsSubset(data, response.data) - @patch("onadata.apps.api.viewsets.data_viewset.send_message") - def test_delete_submission(self, send_message_mock): + def test_delete_submission(self): self._make_submissions() formid = self.xform.pk dataid = self.xform.instances.all().order_by("id")[0].pk @@ -1691,15 +1691,6 @@ def test_delete_submission(self, send_message_mock): self.assertEqual(response.status_code, 204) first_xform_instance = self.xform.instances.filter(pk=dataid) self.assertEqual(first_xform_instance[0].deleted_by, request.user) - # message sent upon delete - self.assertTrue(send_message_mock.called) - send_message_mock.assert_called_with( - instance_id=dataid, - target_id=formid, - target_type=XFORM, - user=request.user, - message_verb=SUBMISSION_DELETED, - ) # second delete of same submission should return 404 request = self.factory.delete("/", **self.extra) @@ -1767,8 +1758,8 @@ def test_post_save_signal_on_submission_deletion(self, mock, send_message_mock): self.assertEqual(mock.call_count, 1) self.assertTrue(send_message_mock.called) - @patch("onadata.apps.api.viewsets.data_viewset.send_message") - def test_deletion_of_bulk_submissions(self, send_message_mock): + @patch("onadata.apps.api.viewsets.data_viewset.safe_cache_set") + def test_deletion_of_bulk_submissions(self, mock_cache_set): self._make_submissions() self.xform.refresh_from_db() formid = self.xform.pk @@ -1804,19 +1795,16 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): response.data.get("message"), "%d records were deleted" % len(records_to_be_deleted), ) - self.assertTrue(send_message_mock.called) - send_message_mock.assert_called_with( - instance_id=[str(i.pk) for i in records_to_be_deleted], - target_id=formid, - target_type=XFORM, - user=request.user, - message_verb=SUBMISSION_DELETED, - ) self.xform.refresh_from_db() current_count = self.xform.instances.filter(deleted_at=None).count() self.assertNotEqual(current_count, initial_count) self.assertEqual(current_count, 2) self.assertEqual(self.xform.num_of_submissions, 2) + mock_cache_set.assert_called_once_with( + f"xfm-submissions-deleting-{formid}", + [str(i.pk) for i in records_to_be_deleted], + 3600, + ) @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True) @patch("onadata.apps.api.viewsets.data_viewset.send_message") @@ -1879,8 +1867,7 @@ def test_submissions_permanent_deletion(self, send_message_mock): self.assertEqual(self.xform.num_of_submissions, 3) @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True) - @patch("onadata.apps.api.viewsets.data_viewset.send_message") - def test_permanent_deletions_bulk_submissions(self, send_message_mock): + def test_permanent_deletions_bulk_submissions(self): """ Test that permanent bulk submission deletions work """ @@ -1904,14 +1891,6 @@ def test_permanent_deletions_bulk_submissions(self, send_message_mock): response.data.get("message"), "%d records were deleted" % len(records_to_be_deleted), ) - self.assertTrue(send_message_mock.called) - send_message_mock.assert_called_with( - instance_id=[str(i.pk) for i in records_to_be_deleted], - target_id=formid, - target_type=XFORM, - user=request.user, - message_verb=SUBMISSION_DELETED, - ) self.xform.refresh_from_db() current_count = self.xform.num_of_submissions self.assertNotEqual(current_count, initial_count) @@ -2016,8 +1995,7 @@ def test_delete_submission_inactive_form(self, send_message_mock): self.assertEqual(response.status_code, 400) self.assertTrue(send_message_mock.called) - @patch("onadata.apps.api.viewsets.data_viewset.send_message") - def test_delete_submissions(self, send_message_mock): + def test_delete_submissions(self): xls_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../fixtures/tutorial/tutorial.xlsx", @@ -2057,14 +2035,6 @@ def test_delete_submissions(self, send_message_mock): response.data.get("message"), "%d records were deleted" % len(deleted_instances_subset), ) - self.assertTrue(send_message_mock.called) - send_message_mock.assert_called_with( - instance_id=[str(i.pk) for i in deleted_instances_subset], - target_id=formid, - target_type=XFORM, - user=request.user, - message_verb=SUBMISSION_DELETED, - ) # Test that num of submissions for the form is successfully updated self.xform.refresh_from_db() @@ -3825,6 +3795,81 @@ def test_merged_dataset_geojson(self): response.data, ) + def test_submissions_deletion_in_progress(self): + """Submissions whose deletion is in progress are excluded from list""" + self._make_submissions() + self.assertEqual(self.xform.instances.count(), 4) + view = DataViewSet.as_view({"get": "list"}) + formid = self.xform.pk + instances = self.xform.instances.all() + cache.set( + f"xfm-submissions-deleting-{self.xform.pk}", + [instances[0].pk, instances[1].pk], + ) + # No query + request = self.factory.get("/", **self.extra) + response = view(request, pk=formid) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) + # With query + data = {"query": '{"_submission_time":{"$gt":"2018-04-19"}}'} + request = self.factory.get("/", **self.extra, data=data) + response = view(request, pk=formid) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) + # With sort + data = {"sort": 1} + request = self.factory.get("/", **self.extra, data=data) + response = view(request, pk=formid) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) + # Cached submission ids saved as strings + cache.set( + f"xfm-submissions-deleting-{self.xform.pk}", + [str(instances[0].pk), str(instances[1].pk)], + ) + request = self.factory.get("/", **self.extra) + response = view(request, pk=formid) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data), 2) + + @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True) + @patch( + "onadata.apps.api.viewsets.data_viewset.delete_xform_submissions_async.delay" + ) + def test_deletion_of_bulk_submissions_async(self, mock_del_async): + """Deletion of bulk submissions is done asynchronously""" + self._make_submissions() + + view = DataViewSet.as_view({"delete": "destroy"}) + + records_to_be_deleted = self.xform.instances.all()[:2] + instance_ids = ",".join([str(i.pk) for i in records_to_be_deleted]) + data = {"instance_ids": instance_ids} + request = self.factory.delete("/", data=data, **self.extra) + response = view(request, pk=self.xform.pk) + + self.assertEqual(response.status_code, 200) + mock_del_async.assert_called_once_with( + self.xform.pk, + self.user.pk, + [str(records_to_be_deleted[0].pk), str(records_to_be_deleted[1].pk)], + True, + ) + # Permanent deletion + mock_del_async.reset_mock() # Reset mock + data = {"permanent_delete": True, "instance_ids": instance_ids} + request = self.factory.delete("/", data=data, **self.extra) + response = view(request, pk=self.xform.pk) + + self.assertEqual(response.status_code, 200) + mock_del_async.assert_called_once_with( + self.xform.pk, + self.user.pk, + [str(records_to_be_deleted[0].pk), str(records_to_be_deleted[1].pk)], + False, + ) + class TestOSM(TestAbstractViewSet): """ diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 534ff61cd0..de231880a8 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -2,6 +2,7 @@ """ The /data API endpoint. """ + import json import math import types @@ -24,12 +25,8 @@ from rest_framework.settings import api_settings from rest_framework.viewsets import ModelViewSet -from onadata.libs.serializers.geojson_serializer import GeoJsonSerializer -from onadata.libs.pagination import ( - CountOverridablePageNumberPagination, -) - from onadata.apps.api.permissions import ConnectViewsetPermissions, XFormPermissions +from onadata.apps.api.tasks import delete_xform_submissions_async from onadata.apps.api.tools import add_tags_to_instance, get_baseviewset_class from onadata.apps.logger.models import MergedXForm, OsmData from onadata.apps.logger.models.attachment import Attachment @@ -38,14 +35,15 @@ from onadata.apps.messaging.constants import SUBMISSION_DELETED, XFORM from onadata.apps.messaging.serializers import send_message from onadata.apps.viewer.models.parsed_instance import ( + ParsedInstance, + _get_sort_fields, + exclude_deleting_submissions_clause, get_etag_hash_from_query, get_sql_with_params, get_where_clause, + query_count, query_data, query_fields_data, - query_count, - _get_sort_fields, - ParsedInstance, ) from onadata.libs import filters from onadata.libs.data import parse_int, strtobool @@ -56,6 +54,7 @@ from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin +from onadata.libs.pagination import CountOverridablePageNumberPagination from onadata.libs.permissions import ( CAN_DELETE_SUBMISSION, filter_queryset_xform_meta_perms, @@ -70,7 +69,13 @@ JsonDataSerializer, OSMSerializer, ) +from onadata.libs.serializers.geojson_serializer import GeoJsonSerializer from onadata.libs.utils.api_export_tools import custom_response_handler +from onadata.libs.utils.cache_tools import ( + XFORM_SUBMISSIONS_DELETING, + XFORM_SUBMISSIONS_DELETING_TTL, + safe_cache_set, +) from onadata.libs.utils.common_tools import json_stream, str_to_bool from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url @@ -346,8 +351,6 @@ def enketo(self, request, *args, **kwargs): # pylint: disable=too-many-branches,too-many-locals def destroy(self, request, *args, **kwargs): """Deletes submissions data.""" - instance_ids = request.data.get("instance_ids") - delete_all_submissions = strtobool(request.data.get("delete_all", "False")) # get param to trigger permanent submission deletion permanent_delete = str_to_bool(request.data.get("permanent_delete")) enable_submission_permanent_delete = getattr( @@ -356,60 +359,45 @@ def destroy(self, request, *args, **kwargs): permanent_delete_disabled_msg = _( "Permanent submission deletion is not enabled for this server." ) + + if permanent_delete and not enable_submission_permanent_delete: + return Response( + {"error": permanent_delete_disabled_msg}, + status=status.HTTP_400_BAD_REQUEST, + ) + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() + instance_ids = request.data.get("instance_ids") + delete_all_submissions = strtobool(request.data.get("delete_all", "False")) if isinstance(self.object, XForm): if not instance_ids and not delete_all_submissions: raise ParseError(_("Data id(s) not provided.")) - initial_count = self.object.submission_count() - if delete_all_submissions: - # Update timestamp only for active records - queryset = self.object.instances.filter(deleted_at__isnull=True) - else: + + if not delete_all_submissions: instance_ids = [x for x in instance_ids.split(",") if x.isdigit()] + if not instance_ids: raise ParseError(_("Invalid data ids were provided.")) - queryset = self.object.instances.filter( - id__in=instance_ids, - xform=self.object, - # do not update this timestamp when the record have - # already been deleted. - deleted_at__isnull=True, - ) - - error_msg = None - for instance in queryset.iterator(): - if permanent_delete: - if enable_submission_permanent_delete: - instance.delete() - else: - error_msg = {"error": permanent_delete_disabled_msg} - break - else: - # enable soft deletion - delete_instance(instance, request.user) - - if error_msg: - # return error msg if permanent deletion not enabled - return Response(error_msg, status=status.HTTP_400_BAD_REQUEST) - - # updates the num_of_submissions for the form. - after_count = self.object.submission_count(force_update=True) - number_of_records_deleted = initial_count - after_count - - # update the date modified field of the project - self.object.project.date_modified = timezone.now() - self.object.project.save(update_fields=["date_modified"]) - - # send message - send_message( - instance_id=instance_ids, - target_id=self.object.id, - target_type=XFORM, - user=request.user, - message_verb=SUBMISSION_DELETED, + else: + instance_ids = None + + initial_num_of_submissions = self.object.num_of_submissions + delete_xform_submissions_async.delay( + self.object.id, + request.user.id, + instance_ids, + not permanent_delete, + ) + safe_cache_set( + f"{XFORM_SUBMISSIONS_DELETING}{self.object.id}", + instance_ids, + XFORM_SUBMISSIONS_DELETING_TTL, + ) + number_of_records_deleted = ( + len(instance_ids) if instance_ids else initial_num_of_submissions ) return Response( @@ -421,11 +409,7 @@ def destroy(self, request, *args, **kwargs): if request.user.has_perm(CAN_DELETE_SUBMISSION, self.object.xform): instance_id = self.object.pk if permanent_delete: - if enable_submission_permanent_delete: - self.object.delete() - else: - error_msg = {"error": permanent_delete_disabled_msg} - return Response(error_msg, status=status.HTTP_400_BAD_REQUEST) + self.object.delete() else: # enable soft deletion delete_instance(self.object, request.user) @@ -688,6 +672,16 @@ def set_object_list(self, query, fields, sort, start, limit, is_public_request): where, where_params = get_where_clause(query) + if not is_public_request: + # Exclude submissions whose deletion is in progress + exclude_del_sql, exclude_del_params = ( + exclude_deleting_submissions_clause(self.get_object().id) + ) + + if exclude_del_sql: + where.append(f" {exclude_del_sql}") + where_params.extend(exclude_del_params) + if where: # pylint: disable=attribute-defined-outside-init self.object_list = self.object_list.extra( diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index f50a218e77..988d7d1675 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -2,6 +2,7 @@ """ ParsedInstance model """ + import datetime from django.conf import settings @@ -21,6 +22,7 @@ json_order_by_params, sort_from_mongo_sort_str, ) +from onadata.libs.utils.cache_tools import XFORM_SUBMISSIONS_DELETING, safe_cache_get from onadata.libs.utils.common_tags import ( ATTACHMENTS, BAMBOO_DATASET_ID, @@ -179,6 +181,22 @@ def _get_sort_fields(sort): return list(_parse_sort_fields(sort)) +def exclude_deleting_submissions_clause(xform_id: int) -> tuple[str, list[int]]: + """Return SQL clause to exclude submissions whose deletion is in progress + + :param xform_id: XForm ID + :return: SQL and list of submission IDs under deletion + """ + instance_ids = safe_cache_get(f"{XFORM_SUBMISSIONS_DELETING}{xform_id}", []) + + if not instance_ids: + return ("", []) + + placeholders = ", ".join(["%s"] * len(instance_ids)) + return (f"id NOT IN ({placeholders})", instance_ids) + + +# pylint: disable=too-many-locals def build_sql_where(xform, query, start=None, end=None): """Build SQL WHERE clause""" known_integers = [ @@ -209,6 +227,13 @@ def build_sql_where(xform, query, start=None, end=None): sql_where += " AND date_created <= %s" where_params += [end.isoformat()] + exclude_sql, exclude_params = exclude_deleting_submissions_clause(xform.pk) + + if exclude_sql: + # Exclude submissions whose deletion is in progress + sql_where += f" AND {exclude_sql}" + where_params += exclude_params + xform_pks = [xform.pk] if xform.is_merged_dataset: diff --git a/onadata/libs/tests/utils/test_logger_tools.py b/onadata/libs/tests/utils/test_logger_tools.py index 825fa1de69..5a8c045a57 100644 --- a/onadata/libs/tests/utils/test_logger_tools.py +++ b/onadata/libs/tests/utils/test_logger_tools.py @@ -2,26 +2,28 @@ """ Test logger_tools utility functions. """ + import os import re from datetime import datetime, timedelta from io import BytesIO -from unittest.mock import patch, call +from unittest.mock import Mock, call, patch from django.conf import settings from django.core.cache import cache +from django.core.exceptions import PermissionDenied from django.core.files.uploadedfile import InMemoryUploadedFile from django.http.request import HttpRequest -from django.utils import timezone from django.test.utils import override_settings +from django.utils import timezone from defusedxml.ElementTree import ParseError from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models import ( - Instance, Entity, EntityList, + Instance, RegistrationForm, SurveyType, XForm, @@ -35,6 +37,7 @@ create_entity_from_instance, create_instance, dec_elist_num_entities, + delete_xform_submissions, generate_content_disposition_header, get_first_record, inc_elist_num_entities, @@ -1032,3 +1035,120 @@ def test_lock_already_acquired(self): self.assertIsNotNone(cache.get(self.ids_key)) self.assertIsNotNone(cache.get(self.counter_key)) self.assertIsNotNone(cache.get(self.created_at_key)) + + +class DeleteXFormSubmissionsTestCase(TestBase): + """Tests for method `delete_xform_submissions`""" + + def setUp(self): + super().setUp() + + self._publish_transportation_form() + self._make_submissions() + self.instances = self.xform.instances.all() + + def test_soft_delete_all(self): + """All submissions are soft deleted""" + delete_xform_submissions(self.xform, self.user) + + self.assertEqual(Instance.objects.filter(deleted_at__isnull=False).count(), 4) + self.xform.refresh_from_db() + self.assertEqual(self.xform.num_of_submissions, 0) + + @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True) + def test_hard_delete_all(self): + """All submissions are hard deleted""" + delete_xform_submissions(self.xform, self.user, soft_delete=False) + + self.assertEqual(Instance.objects.count(), 0) + self.xform.refresh_from_db() + self.assertEqual(self.xform.num_of_submissions, 0) + + def test_soft_delete_subset(self): + """Subset of submissions are soft deleted""" + delete_xform_submissions( + self.xform, self.user, instance_ids=[self.instances[0].pk] + ) + + self.assertEqual(Instance.objects.filter(deleted_at__isnull=False).count(), 1) + self.xform.refresh_from_db() + self.assertEqual(self.xform.num_of_submissions, 3) + + @override_settings(ENABLE_SUBMISSION_PERMANENT_DELETE=True) + def test_hard_delete_subset(self): + """Subset of submissions are hard deleted""" + delete_xform_submissions( + self.xform, + self.user, + instance_ids=[self.instances[0].pk], + soft_delete=False, + ) + + self.assertEqual(Instance.objects.count(), 3) + self.xform.refresh_from_db() + self.assertEqual(self.xform.num_of_submissions, 3) + + def test_sets_deleted_at(self): + """deleted_at is set to the current time""" + mocked_now = timezone.now() + + with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): + delete_xform_submissions(self.xform, self.user) + + self.assertTrue( + all(instance.deleted_at == mocked_now for instance in self.instances) + ) + + def test_sets_date_modified(self): + """date_modified is set to the current time""" + mocked_now = timezone.now() + + with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): + delete_xform_submissions(self.xform, self.user) + + self.assertTrue( + all(instance.date_modified == mocked_now for instance in self.instances) + ) + + def test_sets_deleted_by(self): + """Deleted_by is set to the user who initiated the deletion""" + delete_xform_submissions(self.xform, self.user) + + self.assertTrue( + all(instance.deleted_by == self.user for instance in self.instances) + ) + + def test_project_date_modified_updated(self): + """Project date_modified is updated to the current time""" + mocked_now = timezone.now() + + with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): + delete_xform_submissions(self.xform, self.user) + + self.project.refresh_from_db() + self.assertEqual(self.project.date_modified, mocked_now) + + @patch("onadata.libs.utils.logger_tools.send_message") + def test_action_recorded(self, mock_send_message): + """Action is recorded in the audit log""" + delete_xform_submissions(self.xform, self.user, [self.instances[0].pk]) + + mock_send_message.assert_called_once_with( + instance_id=[self.instances[0].pk], + target_id=self.xform.id, + target_type="xform", + user=self.user, + message_verb="submission_deleted", + ) + + def test_hard_delete_enabled(self): + """Hard delete should be enabled for hard delete to be successful""" + with self.assertRaises(PermissionDenied): + delete_xform_submissions(self.xform, self.user, soft_delete=False) + + def test_cache_deleted(self): + """Cache tracking submissions being deleted is cleared""" + cache.set(f"xfm-submissions-deleting-{self.xform.id}", [self.instances[0].pk]) + delete_xform_submissions(self.xform, self.user) + + self.assertIsNone(cache.get(f"xfm-submissions-deleting-{self.xform.id}")) diff --git a/onadata/libs/utils/cache_tools.py b/onadata/libs/utils/cache_tools.py index 6ed6a27201..ab980d37fc 100644 --- a/onadata/libs/utils/cache_tools.py +++ b/onadata/libs/utils/cache_tools.py @@ -66,6 +66,8 @@ XFORM_MANIFEST_CACHE = "xfm-manifest-" XFORM_LIST_CACHE = "xfm-list-" XFROM_LIST_CACHE_TTL = 10 * 60 # 10 minutes converted to seconds +XFORM_SUBMISSIONS_DELETING = "xfm-submissions-deleting-" +XFORM_SUBMISSIONS_DELETING_TTL = 60 * 60 # 1 hour converted to seconds # Cache timeouts used in XForm model XFORM_REGENERATE_INSTANCE_JSON_TASK_TTL = 24 * 60 * 60 # 24 hrs converted to seconds diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 0b22b451d7..0ee6f78c80 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -3,6 +3,7 @@ """ logger_tools - Logger app utility functions. """ + import json import logging import os @@ -13,12 +14,10 @@ from datetime import datetime, timedelta from hashlib import sha256 from http.client import BadStatusLine -from typing import NoReturn, Any +from typing import Any, NoReturn from wsgiref.util import FileWrapper from xml.dom import Node from xml.parsers.expat import ExpatError -import boto3 -from botocore.client import Config from django.conf import settings from django.contrib.auth import get_user_model @@ -30,12 +29,12 @@ ) from django.core.files.storage import get_storage_class from django.db import DataError, IntegrityError, transaction -from django.db.models import Q, F +from django.db.models import F, Q from django.db.models.query import QuerySet from django.http import ( HttpResponse, - HttpResponseRedirect, HttpResponseNotFound, + HttpResponseRedirect, StreamingHttpResponse, UnreadablePostError, ) @@ -44,7 +43,8 @@ from django.utils.encoding import DjangoUnicodeDecodeError from django.utils.translation import gettext as _ - +import boto3 +from botocore.client import Config from defusedxml.ElementTree import ParseError, fromstring from dict2xml import dict2xml from modilabs.utils.subprocess_timeout import ProcessTimedOut @@ -81,13 +81,14 @@ NonUniqueFormIdError, clean_and_parse_xml, get_deprecated_uuid_from_xml, - get_submission_date_from_xml, - get_uuid_from_xml, get_entity_uuid_from_xml, get_meta_from_xml, + get_submission_date_from_xml, + get_uuid_from_xml, ) from onadata.apps.messaging.constants import ( SUBMISSION_CREATED, + SUBMISSION_DELETED, SUBMISSION_EDITED, XFORM, ) @@ -97,20 +98,20 @@ from onadata.apps.viewer.signals import process_submission from onadata.libs.utils.analytics import TrackObjectEvent from onadata.libs.utils.cache_tools import ( + ELIST_FAILOVER_REPORT_SENT, ELIST_NUM_ENTITIES, + ELIST_NUM_ENTITIES_CREATED_AT, ELIST_NUM_ENTITIES_IDS, ELIST_NUM_ENTITIES_LOCK, - ELIST_NUM_ENTITIES_CREATED_AT, - ELIST_FAILOVER_REPORT_SENT, + XFORM_SUBMISSIONS_DELETING, safe_delete, set_cache_with_lock, ) from onadata.libs.utils.common_tags import METADATA_FIELDS from onadata.libs.utils.common_tools import get_uuid, report_exception -from onadata.libs.utils.model_tools import set_uuid, queryset_iterator +from onadata.libs.utils.model_tools import queryset_iterator, set_uuid from onadata.libs.utils.user_auth import get_user_default_project - OPEN_ROSA_VERSION_HEADER = "X-OpenRosa-Version" HTTP_OPEN_ROSA_VERSION_HEADER = "HTTP_X_OPENROSA_VERSION" OPEN_ROSA_VERSION = "1.0" @@ -1468,3 +1469,54 @@ def _exec_cached_elist_counter_commit_failover() -> None: ) report_exception(subject, msg) cache.set(ELIST_FAILOVER_REPORT_SENT, "sent", 86400) + + +def delete_xform_submissions( + xform: XForm, + deleted_by: User, + instance_ids: list[int] | None = None, + soft_delete: bool = True, +) -> None: + """ "Delete subset or all submissions of an XForm + + :param xform: XForm object + :param deleted_by: User initiating the delete + :param instance_ids: List of instance ids to delete, None to delete all + :param soft_delete: Flag to soft delete or hard delete + :return: None + """ + if not soft_delete and not getattr( + settings, "ENABLE_SUBMISSION_PERMANENT_DELETE", False + ): + raise PermissionDenied("Hard delete is not enabled") + + instance_qs = xform.instances.filter(deleted_at__isnull=True) + + if instance_ids: + instance_qs = instance_qs.filter(id__in=instance_ids) + + if soft_delete: + now = timezone.now() + instance_qs.update(deleted_at=now, date_modified=now, deleted_by=deleted_by) + else: + # Hard delete + instance_qs.delete() + + if instance_ids is None: + # Every submission has been deleted + xform.num_of_submissions = 0 + xform.save(update_fields=["num_of_submissions"]) + + else: + xform.submission_count(force_update=True) + + xform.project.date_modified = timezone.now() + xform.project.save(update_fields=["date_modified"]) + safe_delete(f"{XFORM_SUBMISSIONS_DELETING}{xform.pk}") + send_message( + instance_id=instance_ids, + target_id=xform.id, + target_type=XFORM, + user=deleted_by, + message_verb=SUBMISSION_DELETED, + )