Merge remote-tracking branch 'origin/main' into reinout-project-tweaks

CHANGES.md

@@ -1,11 +1,17 @@
 # Changelog for BROStar API
 
-## 0.3 (unreleased)
+## 0.4 (unreleased)
 
 
 - Nothing changed yet.
 
 
+## 0.3 (2024-04-09)
+
+
+- Hotfix: savemethods of upload and import tasks were buggy.
+
+
 ## 0.2 (2024-04-09)
 
 - Made BRO portal (demo/production) configurable. The default is to use the demo portal, of course. Set `USE_BRO_PRODUCTION` environment variable to `true` for production.

api/bro_upload/delivery.py

@@ -69,7 +69,7 @@ def process(self) -> None:
         try:
             xml_file = self._generate_xml_file()
             self.upload_task_instance.progress = 25.00
-            self.upload_task_instance.progress.save()
+            self.upload_task_instance.save()
         except Exception as e:
             self.upload_task_instance.log = e
             self.upload_task_instance.status = "FAILED"
@@ -80,7 +80,7 @@ def process(self) -> None:
         try:
             self._validate_xml_file(xml_file)
             self.upload_task_instance.progress = 50.00
-            self.upload_task_instance.progress.save()
+            self.upload_task_instance.save()
         except Exception as e:
             self.upload_task_instance.log = e
             self.upload_task_instance.status = "FAILED"
@@ -92,7 +92,7 @@ def process(self) -> None:
             deliver_url = self._deliver_xml_file(xml_file)
             self.upload_task_instance.bro_delivery_url = deliver_url
             self.upload_task_instance.progress = 75.00
-            self.upload_task_instance.progress.save()
+            self.upload_task_instance.save()
         except Exception as e:
             self.upload_task_instance.log = e
             self.upload_task_instance.status = "FAILED"

api/models.py

@@ -46,27 +46,16 @@ class ImportTask(models.Model):
     )
     kvk_number = models.CharField(max_length=8, blank=True, null=True)
     status = models.CharField(
-        max_length=20, choices=choices.STATUS_CHOICES, default="PENDING", blank=True
+        max_length=20, choices=choices.STATUS_CHOICES, default="PENDING", blank=False
     )
     log = models.TextField(blank=True)
     progress = models.FloatField(blank=True, null=True)
 
-    def save(self, request=None, *args, **kwargs):
-        """
-        Initialize an upload task by posting the bro_domain, registration_type, request_type, and the sourcedocument_data
-        """
-        if not self.status:
-            super().save(*args, **kwargs)
-            # Update the status of the new task
-            self.status = "PENDING"
-            self.save()
-
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if self.status == "PENDING":
             # Start the celery task
             tasks.import_bro_data_task.delay(self.uuid)
-        else:
-            # This is an existing object being edited: no upload celery task required
-            super().save(*args, **kwargs)
-            pass
 
     def __str__(self):
         return f"{self.bro_domain} import - {self.data_owner}"
@@ -92,34 +81,23 @@ class UploadTask(models.Model):
     metadata = JSONField("Metadata", default=dict, blank=False)
     sourcedocument_data = JSONField("Sourcedocument data", default=dict, blank=False)
     status = models.CharField(
-        max_length=20, choices=choices.STATUS_CHOICES, default="PENDING", blank=True
+        max_length=20, choices=choices.STATUS_CHOICES, default="PENDING", blank=False
     )
     log = models.TextField(blank=True)
     bro_errors = models.TextField(blank=True)
     progress = models.FloatField(blank=True, null=True)
     bro_id = models.CharField(max_length=500, blank=True, null=True)
     bro_delivery_url = models.CharField(max_length=500, blank=True, null=True)
 
-    def save(self, request=None, *args, **kwargs):
-        """
-        Initialize an upload task by posting the bro_domain, registration_type, request_type, and the sourcedocument_data
-        """
-        if not self.status or self.status == "PENDING":
-            super().save(*args, **kwargs)
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if self.status == "PENDING":
             # Accessing the authenticated user's username and token
             username = self.data_owner.bro_user_token
             password = self.data_owner.bro_user_password
 
-            # Update the status of the new task
-            self.status = "PENDING"
-            self.save()
-
             # Start the celery task
             tasks.upload_bro_data_task.delay(self.uuid, username, password)
-        else:
-            # This is an existing object being edited: no upload celery task required
-            super().save(*args, **kwargs)
-            pass
 
     def __str__(self) -> str:
         return f"{self.data_owner}: {self.registration_type} ({self.request_type})"

api/tests/test_models.py

@@ -7,13 +7,38 @@
 
 @pytest.fixture
 def organisation():
-    return api_models.Organisation(name="Nieuwegein")
+    organisation = api_models.Organisation.objects.create(
+        name="Nieuwegein",
+        kvk_number="12345678",
+        bro_user_token="secret",
+        bro_user_password="secret",
+    )
+    organisation.save()
+    return organisation
+
+
+@pytest.fixture
+def gmn(organisation):
+    return gmn_models.GMN(
+        data_owner=organisation,
+        bro_id="GMN123456789",
+    )
 
 
+@pytest.fixture
+def gmw(organisation):
+    return gmw_models.GMW(
+        data_owner=organisation,
+        bro_id="GMW123456789",
+    )
+
+
+@pytest.mark.django_db
 def test_organisation_name(organisation):
     assert str(organisation) == "Nieuwegein"
 
 
+@pytest.mark.django_db
 def test_import_task_name(organisation):
     import_task = api_models.ImportTask(
         bro_domain="GMN",
@@ -24,6 +49,20 @@ def test_import_task_name(organisation):
     assert str(import_task) == "GMN import - Nieuwegein"
 
 
+@pytest.mark.django_db
+def test_import_task_save_method(organisation):
+    import_task = api_models.ImportTask.objects.create(
+        data_owner=organisation,
+        bro_domain="gmn",
+        kvk_number="12345678",
+        status="",
+    )
+    import_task.save()
+    import_task.refresh_from_db()
+    assert api_models.ImportTask.objects.count() == 1
+
+
+@pytest.mark.django_db
 def test_upload_task_name(organisation):
     upload_task = api_models.UploadTask(
         bro_domain="GMN",
@@ -36,18 +75,29 @@ def test_upload_task_name(organisation):
     assert str(upload_task) == "Nieuwegein: GMN_StartRegistration (registration)"
 
 
-@pytest.fixture
-def gmn(organisation):
-    return gmn_models.GMN(
+@pytest.mark.django_db
+def test_upload_task_save_method(organisation):
+    upload_task = api_models.UploadTask.objects.create(
         data_owner=organisation,
-        bro_id="GMN123456789",
+        bro_domain="gmn",
+        project_number="1",
+        status="PENDING",
+        registration_type="GMN_StartRegistration",
+        request_type="registration",
+        metadata={},
+        sourcedocument_data={},
     )
+    upload_task.save()
+    upload_task.refresh_from_db()
+    assert api_models.UploadTask.objects.count() == 1
 
 
+@pytest.mark.django_db
 def test_gmn_name(gmn):
     assert str(gmn) == "GMN123456789"
 
 
+@pytest.mark.django_db
 def test_measuringpoint_name(organisation, gmn):
     measuringpoint = gmn_models.Measuringpoint(
         data_owner=organisation, gmn=gmn, measuringpoint_code="MP1234"
@@ -56,18 +106,12 @@ def test_measuringpoint_name(organisation, gmn):
     assert str(measuringpoint) == "MP1234"
 
 
-@pytest.fixture
-def gmw(organisation):
-    return gmw_models.GMW(
-        data_owner=organisation,
-        bro_id="GMW123456789",
-    )
-
-
+@pytest.mark.django_db
 def test_gmw_name(gmw):
     assert str(gmw) == "GMW123456789"
 
 
+@pytest.mark.django_db
 def test_monitoringtube_name(organisation, gmw):
     monitoring_tube = gmw_models.MonitoringTube(
         data_owner=organisation, gmw=gmw, tube_number="1"

docker-compose.yml

@@ -8,6 +8,11 @@ services:
       POSTGRES_USER: 'brostar'
       POSTGRES_PASSWORD: 'brostar'
       POSTGRES_DB: 'brostar'
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
     volumes:
       - pgdata:/var/lib/postgresql/data
 
@@ -28,11 +33,11 @@ services:
     command: celery -A brostar_api worker --loglevel=INFO
     volumes:
       - .:/code
-    links:
-      - redis
     depends_on:
-      - db
-      - redis
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
 
   web:
     environment:
@@ -45,16 +50,17 @@ services:
     build: .
     # command: "bin/gunicorn -b 0.0.0.0:${PORT:-5000} --workers=3 --timeout 90 --preload --max-requests=10000 trs.wsgi"
     command: "python manage.py runserver 0.0.0.0:8000"
-    links:
-      - db
     ports:
       - "8000:8000"
     volumes:
       - .:/code
     depends_on:
-      - db
-      - redis
-      - celery
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+      celery:
+        condition: service_started
 
 
 volumes:

pyproject.toml

@@ -23,7 +23,7 @@ description = "Simplify data management for BRO"
 authors = []
 license = {file = "LICENSE.txt"}
 readme = "README.md"
-version = "0.3.dev0"
+version = "0.4.dev0"
 
 [project.optional-dependencies]
 test = ["pytest"] # pytest added by nens-meta