From 06e9a47372fb7df1b7d97c747e061320d86dabc7 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Thu, 15 Jul 2021 10:22:29 -0700 Subject: [PATCH] Trigger update to last_activity_timestamp, related testing and formatting too --- morango/models/core.py | 21 ++- .../tests/integration/test_syncsession.py | 92 ++++++++-- tests/testapp/tests/models/test_core.py | 163 +++++++++++++++--- 3 files changed, 228 insertions(+), 48 deletions(-) diff --git a/morango/models/core.py b/morango/models/core.py index 2f28e995..ded47607 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -284,13 +284,22 @@ def update_state(self, stage=None, stage_status=None): :type stage_status: morango.constants.transfer_statuses.*|None """ if stage is not None: - if self.transfer_stage and transfer_stages.stage(self.transfer_stage) > transfer_stages.stage(stage): - raise ValueError("Update stage is behind current stage | current={}, new={}".format(self.transfer_stage, stage)) + if self.transfer_stage and transfer_stages.stage( + self.transfer_stage + ) > transfer_stages.stage(stage): + raise ValueError( + "Update stage is behind current stage | current={}, new={}".format( + self.transfer_stage, stage + ) + ) self.transfer_stage = stage if stage_status is not None: self.transfer_stage_status = stage_status if stage is not None or stage_status is not None: + self.last_activity_timestamp = timezone.now() self.save() + self.sync_session.last_activity_timestamp = timezone.now() + self.sync_session.save() def delete_buffers(self): """ @@ -311,7 +320,9 @@ def get_touched_record_ids_for_model(self, model): if isinstance(model, SyncableModel): model = model.morango_model_name assert isinstance(model, six.string_types) - return Store.objects.filter(model_name=model, last_transfer_session_id=self.id).values_list("id", flat=True) + return Store.objects.filter( + model_name=model, last_transfer_session_id=self.id + ).values_list("id", flat=True) class DeletedModels(models.Model): @@ -395,7 +406,9 @@ class Store(AbstractStore): dirty_bit = models.BooleanField(default=False) deserialization_error = models.TextField(blank=True) - last_transfer_session_id = UUIDField(blank=True, null=True, default=None, db_index=True) + last_transfer_session_id = UUIDField( + blank=True, null=True, default=None, db_index=True + ) objects = StoreManager() diff --git a/tests/testapp/tests/integration/test_syncsession.py b/tests/testapp/tests/integration/test_syncsession.py index a50109e3..1fe9f670 100644 --- a/tests/testapp/tests/integration/test_syncsession.py +++ b/tests/testapp/tests/integration/test_syncsession.py @@ -38,7 +38,9 @@ def second_environment(): assert instance1.id != instance2.id -@pytest.mark.skipif(getattr(settings, "MORANGO_TEST_POSTGRESQL", False), reason="Not supported") +@pytest.mark.skipif( + getattr(settings, "MORANGO_TEST_POSTGRESQL", False), reason="Not supported" +) class PushPullClientTestCase(LiveServerTestCase): multi_db = True profile = "facilitydata" @@ -46,15 +48,21 @@ class PushPullClientTestCase(LiveServerTestCase): def setUp(self): super(PushPullClientTestCase, self).setUp() self.profile_controller = MorangoProfileController(self.profile) - self.conn = self.profile_controller.create_network_connection(self.live_server_url) + self.conn = self.profile_controller.create_network_connection( + self.live_server_url + ) self.conn.chunk_size = 3 self.remote_user = self._setUpServer() self.filter = Filter("{}:user".format(self.remote_user.id)) self.client = self._setUpClient(self.remote_user.id) self.session = self.client.sync_session + self.last_session_activity = self.session.last_activity_timestamp + self.last_transfer_activity = None - self.local_user = MyUser.objects.create(id=self.remote_user.id, username="bob", is_superuser=True) + self.local_user = MyUser.objects.create( + id=self.remote_user.id, username="bob", is_superuser=True + ) def _setUpCertScopes(self): root_scope = ScopeDefinition.objects.create( @@ -85,7 +93,9 @@ def _setUpServer(self): root_scope, subset_scope = self._setUpCertScopes() root_cert = Certificate.generate_root_certificate(root_scope.id) - remote_user = MyUser.objects.create(id=root_cert.id, username="bob", is_superuser=True) + remote_user = MyUser.objects.create( + id=root_cert.id, username="bob", is_superuser=True + ) remote_user.set_password("password") remote_user.save() @@ -94,9 +104,7 @@ def _setUpServer(self): profile=self.profile, scope_definition=subset_scope, scope_version=subset_scope.version, - scope_params=json.dumps( - {"user": remote_user.id, "sub": "user"} - ), + scope_params=json.dumps({"user": remote_user.id, "sub": "user"}), private_key=Key(), ) root_cert.sign_certificate(subset_cert) @@ -106,11 +114,16 @@ def _setUpServer(self): def _setUpClient(self, primary_partition): root_scope, subset_scope = self._setUpCertScopes() - server_certs = self.conn.get_remote_certificates(primary_partition, root_scope.id) + server_certs = self.conn.get_remote_certificates( + primary_partition, root_scope.id + ) server_cert = server_certs[0] client_cert = self.conn.certificate_signing_request( - server_cert, subset_scope.id, {"user": primary_partition, "sub": "user"}, - userargs="bob", password="password" + server_cert, + subset_scope.id, + {"user": primary_partition, "sub": "user"}, + userargs="bob", + password="password", ) return self.conn.create_sync_session(client_cert, server_cert) @@ -118,7 +131,24 @@ def _setUpClient(self, primary_partition): def _create_server_thread(cls, connections_override): # override default to point to second environment database connections_override["default"] = connections["default2"] - return super(PushPullClientTestCase, cls)._create_server_thread(connections_override) + return super(PushPullClientTestCase, cls)._create_server_thread( + connections_override + ) + + def assertLastActivityUpdate(self, transfer_session=None): + """A signal callable that asserts `last_activity_timestamp`s are updated""" + if self.last_transfer_activity is not None: + self.assertLess( + self.last_transfer_activity, transfer_session.last_activity_timestamp + ) + self.assertLess( + self.last_session_activity, + transfer_session.sync_session.last_activity_timestamp, + ) + self.last_transfer_activity = transfer_session.last_activity_timestamp + self.last_session_activity = ( + transfer_session.sync_session.last_activity_timestamp + ) def test_push(self): for _ in range(5): @@ -127,26 +157,42 @@ def test_push(self): self.profile_controller.serialize_into_store(self.filter) with second_environment(): - self.assertEqual(0, SummaryLog.objects.filter(user=self.remote_user).count()) - self.assertEqual(0, InteractionLog.objects.filter(user=self.remote_user).count()) + self.assertEqual( + 0, SummaryLog.objects.filter(user=self.remote_user).count() + ) + self.assertEqual( + 0, InteractionLog.objects.filter(user=self.remote_user).count() + ) client = self.client.get_push_client() + client.signals.queuing.completed.connect(self.assertLastActivityUpdate) + client.signals.transferring.in_progress.connect(self.assertLastActivityUpdate) + client.signals.dequeuing.completed.connect(self.assertLastActivityUpdate) + self.assertEqual(0, TransferSession.objects.filter(active=True).count()) client.initialize(self.filter) self.assertEqual(1, TransferSession.objects.filter(active=True).count()) transfer_session = client.local_context.transfer_session self.assertNotEqual(0, transfer_session.records_total) self.assertEqual(0, transfer_session.records_transferred) - self.assertLessEqual(1, Buffer.objects.filter(transfer_session=transfer_session).count()) + self.assertLessEqual( + 1, Buffer.objects.filter(transfer_session=transfer_session).count() + ) client.run() self.assertNotEqual(0, transfer_session.records_transferred) client.finalize() - self.assertEqual(0, Buffer.objects.filter(transfer_session=transfer_session).count()) + self.assertEqual( + 0, Buffer.objects.filter(transfer_session=transfer_session).count() + ) self.assertEqual(0, TransferSession.objects.filter(active=True).count()) with second_environment(): - self.assertEqual(5, SummaryLog.objects.filter(user=self.remote_user).count()) - self.assertEqual(5, InteractionLog.objects.filter(user=self.remote_user).count()) + self.assertEqual( + 5, SummaryLog.objects.filter(user=self.remote_user).count() + ) + self.assertEqual( + 5, InteractionLog.objects.filter(user=self.remote_user).count() + ) def test_pull(self): with second_environment(): @@ -159,6 +205,10 @@ def test_pull(self): self.assertEqual(0, InteractionLog.objects.filter(user=self.local_user).count()) client = self.client.get_pull_client() + client.signals.queuing.completed.connect(self.assertLastActivityUpdate) + client.signals.transferring.in_progress.connect(self.assertLastActivityUpdate) + client.signals.dequeuing.completed.connect(self.assertLastActivityUpdate) + self.assertEqual(0, TransferSession.objects.filter(active=True).count()) client.initialize(self.filter) self.assertEqual(1, TransferSession.objects.filter(active=True).count()) @@ -167,9 +217,13 @@ def test_pull(self): self.assertEqual(0, transfer_session.records_transferred) client.run() self.assertNotEqual(0, transfer_session.records_transferred) - self.assertLessEqual(1, Buffer.objects.filter(transfer_session=transfer_session).count()) + self.assertLessEqual( + 1, Buffer.objects.filter(transfer_session=transfer_session).count() + ) client.finalize() - self.assertEqual(0, Buffer.objects.filter(transfer_session=transfer_session).count()) + self.assertEqual( + 0, Buffer.objects.filter(transfer_session=transfer_session).count() + ) self.assertEqual(0, TransferSession.objects.filter(active=True).count()) self.assertEqual(5, SummaryLog.objects.filter(user=self.local_user).count()) diff --git a/tests/testapp/tests/models/test_core.py b/tests/testapp/tests/models/test_core.py index 48d41a02..158597ff 100644 --- a/tests/testapp/tests/models/test_core.py +++ b/tests/testapp/tests/models/test_core.py @@ -1,19 +1,23 @@ import factory +import uuid from django.test import TestCase +from django.utils import timezone from django.utils.six import iteritems +from morango.constants import transfer_stages +from morango.constants import transfer_statuses from morango.models.certificates import Filter from morango.models.core import DatabaseMaxCounter +from morango.models.core import TransferSession +from morango.models.core import SyncSession class DatabaseMaxCounterFactory(factory.DjangoModelFactory): - class Meta: model = DatabaseMaxCounter class FilterMaxCounterTestCase(TestCase): - def setUp(self): self.instance_a = "a" * 32 self.prefix_a = "AAA" @@ -25,69 +29,178 @@ def setUp(self): self.user2_prefix_b = "BBB:user_id:emily" # instance A dmc - DatabaseMaxCounterFactory(instance_id=self.instance_a, partition=self.prefix_a, counter=15) - DatabaseMaxCounterFactory(instance_id=self.instance_a, partition=self.user_prefix_a, counter=20) - DatabaseMaxCounterFactory(instance_id=self.instance_a, partition=self.user2_prefix_b, counter=17) + DatabaseMaxCounterFactory( + instance_id=self.instance_a, partition=self.prefix_a, counter=15 + ) + DatabaseMaxCounterFactory( + instance_id=self.instance_a, partition=self.user_prefix_a, counter=20 + ) + DatabaseMaxCounterFactory( + instance_id=self.instance_a, partition=self.user2_prefix_b, counter=17 + ) # instance B dmc - DatabaseMaxCounterFactory(instance_id=self.instance_b, partition=self.user_prefix_a, counter=10) - DatabaseMaxCounterFactory(instance_id=self.instance_b, partition=self.prefix_b, counter=12) - DatabaseMaxCounterFactory(instance_id=self.instance_b, partition=self.user_prefix_b, counter=5) - DatabaseMaxCounterFactory(instance_id=self.instance_b, partition=self.user2_prefix_b, counter=2) + DatabaseMaxCounterFactory( + instance_id=self.instance_b, partition=self.user_prefix_a, counter=10 + ) + DatabaseMaxCounterFactory( + instance_id=self.instance_b, partition=self.prefix_b, counter=12 + ) + DatabaseMaxCounterFactory( + instance_id=self.instance_b, partition=self.user_prefix_b, counter=5 + ) + DatabaseMaxCounterFactory( + instance_id=self.instance_b, partition=self.user2_prefix_b, counter=2 + ) def test_filter_not_in_dmc(self): fmcs = DatabaseMaxCounter.calculate_filter_max_counters(Filter("ZZZ")) self.assertEqual(fmcs, {}) def test_instances_for_one_partition_but_not_other(self): - fmcs = DatabaseMaxCounter.calculate_filter_max_counters(Filter(self.user_prefix_a + "\n" + self.user_prefix_b)) + fmcs = DatabaseMaxCounter.calculate_filter_max_counters( + Filter(self.user_prefix_a + "\n" + self.user_prefix_b) + ) self.assertEqual(fmcs[self.instance_b], 10) def test_insufficient_instances_for_all_partitions(self): user_with_prefix = self.prefix_b + "user_id:richard" - fmcs = DatabaseMaxCounter.calculate_filter_max_counters(Filter(self.prefix_a + "\n" + user_with_prefix)) + fmcs = DatabaseMaxCounter.calculate_filter_max_counters( + Filter(self.prefix_a + "\n" + user_with_prefix) + ) self.assertFalse(fmcs) def test_single_partition_with_all_instances(self): - fmcs = DatabaseMaxCounter.calculate_filter_max_counters(Filter(self.user_prefix_a)) + fmcs = DatabaseMaxCounter.calculate_filter_max_counters( + Filter(self.user_prefix_a) + ) self.assertEqual(fmcs[self.instance_a], 20) self.assertEqual(fmcs[self.instance_b], 10) def test_all_partitions_have_all_instances(self): - fmcs = DatabaseMaxCounter.calculate_filter_max_counters(Filter(self.user_prefix_a + "\n" + self.user2_prefix_b)) + fmcs = DatabaseMaxCounter.calculate_filter_max_counters( + Filter(self.user_prefix_a + "\n" + self.user2_prefix_b) + ) self.assertEqual(fmcs[self.instance_a], 17) self.assertEqual(fmcs[self.instance_b], 10) class DatabaseMaxCounterUpdateCalculation(TestCase): - def setUp(self): self.filter = "filter" def test_update_all_fsics(self): - client_fsic = {'a' * 32: 2, 'b' * 32: 2, 'c' * 32: 2} - server_fsic = {'a' * 32: 1, 'b' * 32: 1, 'c' * 32: 1} + client_fsic = {"a" * 32: 2, "b" * 32: 2, "c" * 32: 2} + server_fsic = {"a" * 32: 1, "b" * 32: 1, "c" * 32: 1} self.assertFalse(DatabaseMaxCounter.objects.filter(counter=2).exists()) for instance_id, counter in iteritems(server_fsic): - DatabaseMaxCounter.objects.create(instance_id=instance_id, counter=counter, partition=self.filter) + DatabaseMaxCounter.objects.create( + instance_id=instance_id, counter=counter, partition=self.filter + ) DatabaseMaxCounter.update_fsics(client_fsic, Filter(self.filter)) self.assertTrue(DatabaseMaxCounter.objects.filter(counter=2).exists()) self.assertFalse(DatabaseMaxCounter.objects.filter(counter=1).exists()) def test_update_some_fsics(self): - client_fsic = {'a' * 32: 1, 'e' * 32: 2, 'c' * 32: 1} - server_fsic = {'a' * 32: 2, 'b' * 32: 1, 'c' * 32: 2} - self.assertFalse(DatabaseMaxCounter.objects.filter(instance_id='e' * 32).exists()) + client_fsic = {"a" * 32: 1, "e" * 32: 2, "c" * 32: 1} + server_fsic = {"a" * 32: 2, "b" * 32: 1, "c" * 32: 2} + self.assertFalse( + DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists() + ) for instance_id, counter in iteritems(server_fsic): - DatabaseMaxCounter.objects.create(instance_id=instance_id, counter=counter, partition=self.filter) + DatabaseMaxCounter.objects.create( + instance_id=instance_id, counter=counter, partition=self.filter + ) DatabaseMaxCounter.update_fsics(client_fsic, Filter(self.filter)) - self.assertTrue(DatabaseMaxCounter.objects.filter(instance_id='e' * 32).exists()) + self.assertTrue( + DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists() + ) def test_no_fsics_get_updated(self): - client_fsic = {'a' * 32: 1, 'b' * 32: 1, 'c' * 32: 1} - server_fsic = {'a' * 32: 2, 'b' * 32: 2, 'c' * 32: 2} + client_fsic = {"a" * 32: 1, "b" * 32: 1, "c" * 32: 1} + server_fsic = {"a" * 32: 2, "b" * 32: 2, "c" * 32: 2} self.assertFalse(DatabaseMaxCounter.objects.filter(counter=1).exists()) for instance_id, counter in iteritems(server_fsic): - DatabaseMaxCounter.objects.create(instance_id=instance_id, counter=counter, partition=self.filter) + DatabaseMaxCounter.objects.create( + instance_id=instance_id, counter=counter, partition=self.filter + ) DatabaseMaxCounter.update_fsics(client_fsic, Filter(self.filter)) self.assertFalse(DatabaseMaxCounter.objects.filter(counter=1).exists()) + + +class TransferSessionTestCase(TestCase): + def setUp(self): + super(TransferSessionTestCase, self).setUp() + self.sync_session = SyncSession.objects.create( + id=uuid.uuid4().hex, + profile="facilitydata", + last_activity_timestamp=timezone.now(), + ) + self.instance = TransferSession.objects.create( + id=uuid.uuid4().hex, + sync_session=self.sync_session, + push=True, + last_activity_timestamp=timezone.now(), + ) + + def test_update_state(self): + self.assertEqual("", self.instance.transfer_stage) + self.assertEqual("", self.instance.transfer_stage_status) + previous_activity = self.instance.last_activity_timestamp + previous_sync_activity = self.sync_session.last_activity_timestamp + + self.instance.update_state( + stage=transfer_stages.QUEUING, stage_status=transfer_statuses.PENDING + ) + + self.assertEqual(transfer_stages.QUEUING, self.instance.transfer_stage) + self.assertEqual(transfer_statuses.PENDING, self.instance.transfer_stage_status) + self.assertLess(previous_activity, self.instance.last_activity_timestamp) + self.assertLess( + previous_sync_activity, self.sync_session.last_activity_timestamp + ) + + def test_update_state__only_stage(self): + self.assertEqual("", self.instance.transfer_stage) + self.assertEqual("", self.instance.transfer_stage_status) + previous_activity = self.instance.last_activity_timestamp + previous_sync_activity = self.sync_session.last_activity_timestamp + + self.instance.update_state(stage=transfer_stages.QUEUING) + + self.assertEqual(transfer_stages.QUEUING, self.instance.transfer_stage) + self.assertEqual("", self.instance.transfer_stage_status) + self.assertLess(previous_activity, self.instance.last_activity_timestamp) + self.assertLess( + previous_sync_activity, self.sync_session.last_activity_timestamp + ) + + def test_update_state__only_status(self): + self.assertEqual("", self.instance.transfer_stage) + self.assertEqual("", self.instance.transfer_stage_status) + previous_activity = self.instance.last_activity_timestamp + previous_sync_activity = self.sync_session.last_activity_timestamp + + self.instance.update_state(stage_status=transfer_statuses.PENDING) + + self.assertEqual("", self.instance.transfer_stage) + self.assertEqual(transfer_statuses.PENDING, self.instance.transfer_stage_status) + self.assertLess(previous_activity, self.instance.last_activity_timestamp) + self.assertLess( + previous_sync_activity, self.sync_session.last_activity_timestamp + ) + + def test_update_state__none(self): + self.assertEqual("", self.instance.transfer_stage) + self.assertEqual("", self.instance.transfer_stage_status) + previous_activity = self.instance.last_activity_timestamp + previous_sync_activity = self.sync_session.last_activity_timestamp + + self.instance.update_state() + + self.assertEqual("", self.instance.transfer_stage) + self.assertEqual("", self.instance.transfer_stage_status) + self.assertEqual(previous_activity, self.instance.last_activity_timestamp) + self.assertEqual( + previous_sync_activity, self.sync_session.last_activity_timestamp + )