From 66ff06e342fc3b26fca5e47ed636738394cfc7df Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:50:44 +0100 Subject: [PATCH 01/23] removed old method to generate xml files. Added template method --- api/bro_upload/config.py | 25 --- api/bro_upload/delivery.py | 47 +++++- .../registration_GMN_StartRegistration.html | 31 ++++ .../xml_generators/base_generator.py | 82 ---------- .../xml_generators/frd_generators.py | 7 - .../xml_generators/gld_generators.py | 7 - .../xml_generators/gmn_generators.py | 148 ------------------ .../xml_generators/gmw_generators.py | 7 - api/tests/test_delivery.py | 77 +++++++++ bro_hub/settings.py | 2 +- 10 files changed, 151 insertions(+), 282 deletions(-) delete mode 100644 api/bro_upload/config.py create mode 100644 api/bro_upload/templates/registration_GMN_StartRegistration.html delete mode 100644 api/bro_upload/xml_generators/base_generator.py delete mode 100644 api/bro_upload/xml_generators/frd_generators.py delete mode 100644 api/bro_upload/xml_generators/gld_generators.py delete mode 100644 api/bro_upload/xml_generators/gmn_generators.py delete mode 100644 api/bro_upload/xml_generators/gmw_generators.py create mode 100644 api/tests/test_delivery.py diff --git a/api/bro_upload/config.py b/api/bro_upload/config.py deleted file mode 100644 index b1aeff2..0000000 --- a/api/bro_upload/config.py +++ /dev/null @@ -1,25 +0,0 @@ -from .xml_generators import gmn_generators - -# Mapping for the determination of which xml generator should be used for a given request_type -xml_generator_mapping = { - "GMN_StartRegistration": gmn_generators.GMNStartregistration, -} - - -# Declaration urls -brocom = "http://www.broservices.nl/xsd/brocommon/3.0" -gml = "http://www.opengis.net/gml/3.2" -xsi = "http://www.w3.org/2001/XMLSchema-instance" -gmn = "http://www.broservices.nl/xsd/isgmn/1.0" - -gmn_xsi_schemalocation = "http://www.broservices.nl/xsd/isgmn/1.0 https://schema.broservices.nl/xsd/isgmn/1.0/isgmn-messages.xsd" - -# Declaration configuration per request type -declaration_mapping = { - "GMN_StartRegistration": { - None: gmn, - "brocom": brocom, - "gml": gml, - "xsi": xsi, - }, -} diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 943b877..a94d2d0 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -1,8 +1,17 @@ import time import traceback +from typing import Any + +from django.template.loader import render_to_string from .. import models as api_models -from . import config, utils +from . import utils + + +class XMLGenerationError(Exception): + """Exception raised when XML generation fails.""" + + pass class XMLValidationError(Exception): @@ -41,9 +50,6 @@ def __init__( self.upload_task_instance = upload_task_instance self.bro_username = bro_username self.bro_password = bro_password - self.xml_generator_class = config.xml_generator_mapping.get( - self.upload_task_instance.registration_type - ) def process(self) -> None: # Generate the XML file. @@ -69,7 +75,7 @@ def process(self) -> None: def _generate_xml_file(self) -> str: try: - generator = self.xml_generator_class( + generator = XMLGenerator( self.upload_task_instance.registration_type, self.upload_task_instance.request_type, self.upload_task_instance.metadata, @@ -149,3 +155,34 @@ def _check_delivery(self, delivery_url: str) -> str | None: else: return False + + +class XMLGenerator: + """XML generator based on Django Templates.""" + + def __init__( + self, + registration_type: str, + request_type: str, + metadata: dict[str, Any], + sourcedocs_data: dict[str, Any], + ) -> None: + self.metadata = metadata + self.sourcedocs_data = sourcedocs_data + self.template_filepath = f"{request_type}_{registration_type}.html" + + def create_xml_file(self): + """Fills in the provided data into the templates""" + try: + rendered_xml = render_to_string( + self.template_filepath, + { + "metadata": self.metadata, + "sourcedocs_data": self.sourcedocs_data, + }, + ) + return rendered_xml + + except Exception as e: + traceback.print_exc() + raise XMLGenerationError(f"Error generating XML file: {e}") from e diff --git a/api/bro_upload/templates/registration_GMN_StartRegistration.html b/api/bro_upload/templates/registration_GMN_StartRegistration.html new file mode 100644 index 0000000..d8f9657 --- /dev/null +++ b/api/bro_upload/templates/registration_GMN_StartRegistration.html @@ -0,0 +1,31 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.qualityRegime }} + + + {{ sourcedocs_data.objectIdAccountableParty }} + {{ sourcedocs_data.name }} + {{ sourcedocs_data.deliveryContext }} + {{ sourcedocs_data.monitoringPurpose }} + {{ sourcedocs_data.groundwaterAspect }} + + {{ sourcedocs_data.startDateMonitoring }} + {% for measuringpoint in sourcedocs_data.measuringPoints %} + + + {{ measuringpoint.measuringPointCode }} + + + {{ measuringpoint.broId }} + {{ measuringpoint.tubeNumber }} + + + + {% endfor %} + + + \ No newline at end of file diff --git a/api/bro_upload/xml_generators/base_generator.py b/api/bro_upload/xml_generators/base_generator.py deleted file mode 100644 index 0bbb630..0000000 --- a/api/bro_upload/xml_generators/base_generator.py +++ /dev/null @@ -1,82 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - -from lxml import etree -from lxml.etree import tostring - -from .. import config - - -class XMLGenerator(ABC): - """Handles the creation of the XML files. - - This baseclass is inherited by the XML Generators per request type. - Only the _create_sourcedocument varies per request type. - - Returns: - XML in str format - """ - - def __init__( - self, - registration_type: str, - request_type: str, - metadata: dict[str, Any], - sourcedocs_data: dict[str, Any], - ) -> None: - self.registration_type = registration_type - self.request_type = request_type - self.metadata = metadata - self.sourcedocs_data = sourcedocs_data - self.namespace = None - self.xsi_schema_location = None - self.xml_tree = None - - def create_xml_file(self) -> str: - """Generates the XML file, based on the provided request_type, metadata and sourcedocs_data""" - self._setup_xml_tree() - self._add_metadata() - - self.source_document = etree.Element("sourceDocument") - self._create_sourcedocument() - self.xml_tree.append(self.source_document) - - xml_file = etree.ElementTree(self.xml_tree) - xml_string = tostring(xml_file, encoding="utf-8", method="xml") - - return xml_string - - def _setup_xml_tree(self) -> None: - """Sets up the basis of the XML File, consisting of the declaration urls.""" - self.xml_tree = etree.Element( - f"{self.request_type}Request", - nsmap=self.namespace, - ) - - if self.xsi_schema_location: - self.xml_tree.set( - "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation", - self.xsi_schema_location, - ) - - def _add_metadata(self): - """Fills in the metadata (all information between the declaration sourcedocs.""" - - metadata_options = [ - "requestReference", - "deliveryAccountableParty", - "broId", - "qualityRegime", - ] - for metadata_field in metadata_options: - if metadata_field in self.metadata.keys(): - metadata_element = etree.SubElement( - self.xml_tree, - f"{{{config.brocom}}}{metadata_field}", - ) - metadata_element.text = self.metadata[metadata_field] - - @abstractmethod - def _create_sourcedocument(self): - """Creates the sourcedocs XML structure.""" - pass diff --git a/api/bro_upload/xml_generators/frd_generators.py b/api/bro_upload/xml_generators/frd_generators.py deleted file mode 100644 index bbb0296..0000000 --- a/api/bro_upload/xml_generators/frd_generators.py +++ /dev/null @@ -1,7 +0,0 @@ -from .base_generator import XMLGenerator - - -class FRDStartregistration(XMLGenerator): - """Generates a XML file for the 10_FRD_StartRegistration.xml""" - - pass diff --git a/api/bro_upload/xml_generators/gld_generators.py b/api/bro_upload/xml_generators/gld_generators.py deleted file mode 100644 index cb57cf4..0000000 --- a/api/bro_upload/xml_generators/gld_generators.py +++ /dev/null @@ -1,7 +0,0 @@ -from .base_generator import XMLGenerator - - -class GLDStartregistration(XMLGenerator): - """Generates a XML file for the 10_GLD_StartRegistration.xml.""" - - pass diff --git a/api/bro_upload/xml_generators/gmn_generators.py b/api/bro_upload/xml_generators/gmn_generators.py deleted file mode 100644 index 100b87c..0000000 --- a/api/bro_upload/xml_generators/gmn_generators.py +++ /dev/null @@ -1,148 +0,0 @@ -from lxml import etree - -from .. import config -from .base_generator import XMLGenerator - - -class GMNStartregistration(XMLGenerator): - """Generates a XML file for the 01_GMN_StartRegistration.""" - - def __init__( - self, - registration_type: str, - request_type: str, - metadata: dict, - srcdocdata: dict, - ) -> None: - super().__init__(registration_type, request_type, metadata, srcdocdata) - self.namespace = config.declaration_mapping[self.registration_type] - self.xsi_schema_location = config.gmn_xsi_schemalocation - self.id_count = 1 - - def _create_sourcedocument(self): - """Example XML for register: - https://www.bro-productomgeving.nl/__attachments/159095997/01_GMN_StartRegistration.xml?inst-v=6362ee09-a77d-4052-8ab8-caf3a373ad16 - """ - # Create main sourcedocs element - main_element = etree.Element(f"{{{config.gmn}}}GMN_StartRegistration") - main_element.set(f"{{{config.gml}}}id", f"id_000{self.id_count}") - self.id_count += 1 - - # Create object id accountable party element - object_id_accountable_party_element = etree.SubElement( - main_element, - "objectIdAccountableParty", - ) - object_id_accountable_party_element.text = self.sourcedocs_data[ - "objectIdAccountableParty" - ] - - # Create name element - name_element = etree.SubElement( - main_element, - "name", - ) - name_element.text = self.sourcedocs_data["name"] - - # Create delivery context element - delivery_context_element = etree.SubElement( - main_element, - "deliveryContext", - attrib={"codeSpace": "urn:bro:gmn:DeliveryContext"}, - ) - delivery_context_element.text = self.sourcedocs_data["deliveryContext"] - - # Create monitoring purpose element - monitoring_purpose_element = etree.SubElement( - main_element, - "monitoringPurpose", - attrib={"codeSpace": "urn:bro:gmn:MonitoringPurpose"}, - ) - monitoring_purpose_element.text = self.sourcedocs_data["monitoringPurpose"] - - # Create groundwater aspect element - groundwater_aspect_element = etree.SubElement( - main_element, - "groundwaterAspect", - attrib={"codeSpace": "urn:bro:gmn:GroundwaterAspect"}, - ) - groundwater_aspect_element.text = self.sourcedocs_data["groundwaterAspect"] - - # Create startdate monitoring element - startdate_monitoring_element = etree.SubElement( - main_element, - "startDateMonitoring", - ) - - startdate_date_element = etree.SubElement( - startdate_monitoring_element, - f"{{{config.brocom}}}date", - ) - startdate_date_element.text = self.sourcedocs_data["startDateMonitoring"] - - # Add measuringpoints - for measuringpoint in self.sourcedocs_data["measuringPoints"]: - # Main element - measuringpoint_main_element = etree.SubElement( - main_element, - "measuringPoint", - ) - - # Sub element - measuringpoint_sub_element = etree.SubElement( - measuringpoint_main_element, - "MeasuringPoint", - ) - measuringpoint_sub_element.set( - f"{{{config.gml}}}id", f"id_000{self.id_count}" - ) - self.id_count += 1 - - measuringpoint_code_element = etree.SubElement( - measuringpoint_sub_element, - "measuringPointCode", - ) - measuringpoint_code_element.text = measuringpoint["measuringPointCode"] - - # monitoringTube element - monitoring_tube_element = etree.SubElement( - measuringpoint_sub_element, - "monitoringTube", - ) - - # GroundwaterMonitoringTube element - groundwater_monitoring_tube_element = etree.SubElement( - monitoring_tube_element, - "GroundwaterMonitoringTube", - ) - groundwater_monitoring_tube_element.set( - f"{{{config.gml}}}id", f"id_000{self.id_count}" - ) - self.id_count += 1 - - bro_id_element = etree.SubElement( - groundwater_monitoring_tube_element, - "broId", - ) - bro_id_element.text = measuringpoint["broId"] - - tube_number_element = etree.SubElement( - groundwater_monitoring_tube_element, - "tubeNumber", - ) - tube_number_element.text = measuringpoint["tubeNumber"] - - # Add startregistration to sourcedocs - self.source_document.append(main_element) - - -class GMNMeasuringPoint(XMLGenerator): - """Generates a XML file for the 02_GMN_MeasuringPoint.xml.""" - - pass - - -class GMNTubeReference(XMLGenerator): - """Generates a XML file for the 03_GMN_TubeReference.xml.""" - - pass diff --git a/api/bro_upload/xml_generators/gmw_generators.py b/api/bro_upload/xml_generators/gmw_generators.py deleted file mode 100644 index 75d36b2..0000000 --- a/api/bro_upload/xml_generators/gmw_generators.py +++ /dev/null @@ -1,7 +0,0 @@ -from .base_generator import XMLGenerator - - -class GMWStartregistration(XMLGenerator): - """Generates a XML file for the registrationRequestGMW.xml.""" - - pass diff --git a/api/tests/test_delivery.py b/api/tests/test_delivery.py new file mode 100644 index 0000000..40d2491 --- /dev/null +++ b/api/tests/test_delivery.py @@ -0,0 +1,77 @@ +from api.bro_upload import delivery + +expected_xml_str = """ + test + 27376655 + IMBRO/A + + + test + test + kaderrichtlijnWater + strategischBeheerKwaliteitRegionaal + kwantiteit + + 2024-01-01 + + + + GMW000000038946 + + + GMW000000038946 + 1 + + + + + + + GMW000000038946 + + + GMW000000038946 + 2 + + + + + + +""" + +def test_delivery1(): + generator = delivery.XMLGenerator( + registration_type = "GMN_StartRegistration", + request_type = "registration", + metadata= { + "requestReference":"test", + "deliveryAccountableParty":"27376655", + "qualityRegime":"IMBRO/A" + }, + sourcedocs_data= { + "objectIdAccountableParty":"test", + "name":"test", + "deliveryContext":"kaderrichtlijnWater", + "monitoringPurpose":"strategischBeheerKwaliteitRegionaal", + "groundwaterAspect":"kwantiteit", + "startDateMonitoring":"2024-01-01", + "measuringPoints":[ + { + "measuringPointCode":"GMW000000038946", + "broId":"GMW000000038946", + "tubeNumber":"1" + }, + { + "measuringPointCode":"GMW000000038946", + "broId":"GMW000000038946", + "tubeNumber":"2" + } + ] + } + ) + + assert generator.create_xml_file() == expected_xml_str \ No newline at end of file diff --git a/bro_hub/settings.py b/bro_hub/settings.py index e269e00..ea2a74d 100644 --- a/bro_hub/settings.py +++ b/bro_hub/settings.py @@ -60,7 +60,7 @@ TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [], + "DIRS": [os.path.join(BASE_DIR, "api", "bro_upload", "templates")], "APP_DIRS": True, "OPTIONS": { "context_processors": [ From a56fee5e6c97597528995a3c9bdb853988f837a9 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:58:57 +0100 Subject: [PATCH 02/23] updated delivery test: added empty line at end --- .../registration_GMN_StartRegistration.html | 2 +- api/tests/test_delivery.py | 52 ++++++++++--------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/api/bro_upload/templates/registration_GMN_StartRegistration.html b/api/bro_upload/templates/registration_GMN_StartRegistration.html index d8f9657..17dc409 100644 --- a/api/bro_upload/templates/registration_GMN_StartRegistration.html +++ b/api/bro_upload/templates/registration_GMN_StartRegistration.html @@ -28,4 +28,4 @@ {% endfor %} - \ No newline at end of file + diff --git a/api/tests/test_delivery.py b/api/tests/test_delivery.py index 40d2491..95cebce 100644 --- a/api/tests/test_delivery.py +++ b/api/tests/test_delivery.py @@ -41,37 +41,39 @@ -""" + +""" + def test_delivery1(): generator = delivery.XMLGenerator( - registration_type = "GMN_StartRegistration", - request_type = "registration", - metadata= { - "requestReference":"test", - "deliveryAccountableParty":"27376655", - "qualityRegime":"IMBRO/A" + registration_type="GMN_StartRegistration", + request_type="registration", + metadata={ + "requestReference": "test", + "deliveryAccountableParty": "27376655", + "qualityRegime": "IMBRO/A", }, - sourcedocs_data= { - "objectIdAccountableParty":"test", - "name":"test", - "deliveryContext":"kaderrichtlijnWater", - "monitoringPurpose":"strategischBeheerKwaliteitRegionaal", - "groundwaterAspect":"kwantiteit", - "startDateMonitoring":"2024-01-01", - "measuringPoints":[ + sourcedocs_data={ + "objectIdAccountableParty": "test", + "name": "test", + "deliveryContext": "kaderrichtlijnWater", + "monitoringPurpose": "strategischBeheerKwaliteitRegionaal", + "groundwaterAspect": "kwantiteit", + "startDateMonitoring": "2024-01-01", + "measuringPoints": [ { - "measuringPointCode":"GMW000000038946", - "broId":"GMW000000038946", - "tubeNumber":"1" + "measuringPointCode": "GMW000000038946", + "broId": "GMW000000038946", + "tubeNumber": "1", }, { - "measuringPointCode":"GMW000000038946", - "broId":"GMW000000038946", - "tubeNumber":"2" - } - ] - } + "measuringPointCode": "GMW000000038946", + "broId": "GMW000000038946", + "tubeNumber": "2", + }, + ], + }, ) - assert generator.create_xml_file() == expected_xml_str \ No newline at end of file + assert generator.create_xml_file() == expected_xml_str From b2b5cbbf1eeb7a292a5c07cb6ba71a9cf4d3eb16 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:01:52 +0100 Subject: [PATCH 03/23] removed lxml from reqs --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 06e9e39..d208dda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ markdown pytest pytest-django xmltodict -lxml python-dotenv django-encrypted-model-fields django-filter From 8eda0305b39e95a1254b334a3b9346253c40e876 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:17:28 +0100 Subject: [PATCH 04/23] removed default project number. Should be implemented different --- ...profile_default_project_number_and_more.py | 22 ++++++++++++++++ api/models.py | 4 +-- api/views.py | 25 ------------------- 3 files changed, 23 insertions(+), 28 deletions(-) create mode 100644 api/migrations/0021_remove_userprofile_default_project_number_and_more.py diff --git a/api/migrations/0021_remove_userprofile_default_project_number_and_more.py b/api/migrations/0021_remove_userprofile_default_project_number_and_more.py new file mode 100644 index 0000000..2130fe7 --- /dev/null +++ b/api/migrations/0021_remove_userprofile_default_project_number_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.1 on 2024-03-14 12:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0020_remove_userprofile_bro_user_password_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='userprofile', + name='default_project_number', + ), + migrations.AlterField( + model_name='uploadtask', + name='project_number', + field=models.CharField(max_length=20), + ), + ] diff --git a/api/models.py b/api/models.py index e1e2273..bff7b61 100644 --- a/api/models.py +++ b/api/models.py @@ -32,8 +32,6 @@ class UserProfile(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) - default_project_number = models.CharField(max_length=20, blank=True, null=True) - def __str__(self): return self.user.username @@ -68,7 +66,7 @@ class UploadTask(models.Model): bro_domain = models.CharField( max_length=3, choices=choices.BRO_DOMAIN_CHOICES, default=None ) - project_number = models.CharField(max_length=20, blank=True, null=True) + project_number = models.CharField(max_length=20, blank=False) registration_type = models.CharField( blank=False, max_length=235, choices=choices.REGISTRATION_TYPE_OPTIONS ) diff --git a/api/views.py b/api/views.py index b3826a4..3e913b7 100644 --- a/api/views.py +++ b/api/views.py @@ -70,27 +70,6 @@ class UserProfileDetailView(generics.RetrieveUpdateAPIView): lookup_field = "uuid" permission_classes = [permissions.IsAuthenticated] - # update makes sure only project number, token and password can be changed - def update(self, request, *args, **kwargs): - partial = kwargs.pop("partial", False) - instance = self.get_object() - - data = request.data - allowed_fields = { - "default_project_number", - } - for key in data.keys(): - if key not in allowed_fields: - return Response( - {"error": f"Cannot update field {key}"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - serializer = self.get_serializer(instance, data=request.data, partial=partial) - serializer.is_valid(raise_exception=True) - self.perform_update(serializer) - return Response(serializer.data) - class ImportTaskListView(mixins.UserOrganizationMixin, generics.ListAPIView): """ @@ -244,10 +223,6 @@ def post(self, request): # Update the instance of the new task upload_task_instance.status = "PENDING" upload_task_instance.data_owner = data_owner - upload_task_instance.project_number = ( - upload_task_instance.project_number - or user_profile.default_project_number - ) upload_task_instance.save() if not upload_task_instance.project_number: From 317ddea44cd1cfeec2de0904b6044c26039d8a59 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:18:46 +0100 Subject: [PATCH 05/23] removed default project number. Should be implemented different --- ...ove_userprofile_default_project_number_and_more.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/api/migrations/0021_remove_userprofile_default_project_number_and_more.py b/api/migrations/0021_remove_userprofile_default_project_number_and_more.py index 2130fe7..fd7de80 100644 --- a/api/migrations/0021_remove_userprofile_default_project_number_and_more.py +++ b/api/migrations/0021_remove_userprofile_default_project_number_and_more.py @@ -4,19 +4,18 @@ class Migration(migrations.Migration): - dependencies = [ - ('api', '0020_remove_userprofile_bro_user_password_and_more'), + ("api", "0020_remove_userprofile_bro_user_password_and_more"), ] operations = [ migrations.RemoveField( - model_name='userprofile', - name='default_project_number', + model_name="userprofile", + name="default_project_number", ), migrations.AlterField( - model_name='uploadtask', - name='project_number', + model_name="uploadtask", + name="project_number", field=models.CharField(max_length=20), ), ] From 88ef18ecdeae32a871284ff1f827b832c9cea5f2 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:48:48 +0100 Subject: [PATCH 06/23] added all model name tests --- api/models.py | 4 +- .../{test_delivery.py => test_bro_upload.py} | 0 api/tests/test_models.py | 74 ++++++++++++++++++- 3 files changed, 74 insertions(+), 4 deletions(-) rename api/tests/{test_delivery.py => test_bro_upload.py} (100%) diff --git a/api/models.py b/api/models.py index bff7b61..b24e55a 100644 --- a/api/models.py +++ b/api/models.py @@ -53,7 +53,7 @@ class ImportTask(models.Model): log = models.TextField(blank=True) def __str__(self): - return f"{self.bro_domain} import - {self.data_owner} ({self.created})" + return f"{self.bro_domain} import - {self.data_owner}" class UploadTask(models.Model): @@ -79,4 +79,4 @@ class UploadTask(models.Model): log = models.TextField(blank=True) def __str__(self) -> str: - return f"{self.request_type} {self.registration_type} {self.created}" + return f"{self.data_owner}: {self.registration_type} ({self.request_type})" diff --git a/api/tests/test_delivery.py b/api/tests/test_bro_upload.py similarity index 100% rename from api/tests/test_delivery.py rename to api/tests/test_bro_upload.py diff --git a/api/tests/test_models.py b/api/tests/test_models.py index f03c1a5..68fd07b 100644 --- a/api/tests/test_models.py +++ b/api/tests/test_models.py @@ -1,6 +1,76 @@ +import pytest + from api import models as api_models +from gmn import models as gmn_models +from gmw import models as gmw_models + + +@pytest.fixture +def organisation(): + return api_models.Organisation(name="Nieuwegein") -def test_organisation1(): - organisation = api_models.Organisation(name="Nieuwegein") +def test_organisation_name(organisation): assert str(organisation) == "Nieuwegein" + + +def test_import_task_name(organisation): + import_task = api_models.ImportTask( + bro_domain="GMN", + kvk_number="123456789", + data_owner=organisation, + ) + + assert str(import_task) == "GMN import - Nieuwegein" + + +def test_upload_task_name(organisation): + upload_task = api_models.UploadTask( + bro_domain="GMN", + data_owner=organisation, + project_number="1", + registration_type="GMN_StartRegistration", + request_type="registration", + ) + + assert str(upload_task) == "Nieuwegein: GMN_StartRegistration (registration)" + + +@pytest.fixture +def gmn(organisation): + return gmn_models.GMN( + data_owner=organisation, + bro_id="GMN123456789", + ) + + +def test_gmn_name(gmn): + assert str(gmn) == "GMN123456789" + + +def test_measuringpoint_name(organisation, gmn): + measuringpoint = gmn_models.Measuringpoint( + data_owner=organisation, gmn=gmn, measuringpoint_code="MP1234" + ) + + assert str(measuringpoint) == "MP1234" + + +@pytest.fixture +def gmw(organisation): + return gmw_models.GMW( + data_owner=organisation, + bro_id="GMW123456789", + ) + + +def test_gmw_name(gmw): + assert str(gmw) == "GMW123456789" + + +def test_monitoringtube_name(organisation, gmw): + monitoring_tube = gmw_models.MonitoringTube( + data_owner=organisation, gmw=gmw, tube_number="1" + ) + + assert str(monitoring_tube) == "GMW123456789-1" From 046f7746a1bad7eb590ebe2ae08951932b7d5914 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:57:13 +0100 Subject: [PATCH 07/23] changed name test --- api/tests/test_bro_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tests/test_bro_upload.py b/api/tests/test_bro_upload.py index 95cebce..f8c4c56 100644 --- a/api/tests/test_bro_upload.py +++ b/api/tests/test_bro_upload.py @@ -45,7 +45,7 @@ """ -def test_delivery1(): +def test_xml_generator(): generator = delivery.XMLGenerator( registration_type="GMN_StartRegistration", request_type="registration", From 224926b823db1c6b5cd9cf9accc6179a76e05219 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:35:28 +0100 Subject: [PATCH 08/23] fixed typehints --- api/bro_upload/delivery.py | 4 ++-- api/bro_upload/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index a94d2d0..ba42da2 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -129,7 +129,7 @@ def _deliver_xml_file(self, xml_file: str) -> str: return delivery_url - def _check_delivery(self, delivery_url: str) -> str | None: + def _check_delivery(self, delivery_url: str) -> bool: """Checks the delivery status.""" delivery_info = utils.check_delivery_status( @@ -171,7 +171,7 @@ def __init__( self.sourcedocs_data = sourcedocs_data self.template_filepath = f"{request_type}_{registration_type}.html" - def create_xml_file(self): + def create_xml_file(self) -> str: """Fills in the provided data into the templates""" try: rendered_xml = render_to_string( diff --git a/api/bro_upload/utils.py b/api/bro_upload/utils.py index 4685a91..5bd1c88 100644 --- a/api/bro_upload/utils.py +++ b/api/bro_upload/utils.py @@ -104,7 +104,7 @@ def create_delivery( def check_delivery_status( delivery_url: str, bro_username: str, bro_password: str -) -> str: +) -> requests.Response: """Checks the Delivery info. Step 4 of 4 in the upload process.""" try: r = requests.get( From c09a92121182fd2716b570c6eeb442f65de35a92 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:28:45 +0100 Subject: [PATCH 09/23] added celery and redis to docker-compose --- bro_hub/settings.py | 2 +- docker-compose.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/bro_hub/settings.py b/bro_hub/settings.py index ea2a74d..32da55f 100644 --- a/bro_hub/settings.py +++ b/bro_hub/settings.py @@ -146,7 +146,7 @@ # Automatically discover tasks in Django app CELERY_IMPORTS = ("api.tasks",) # TODO: fix celery env settings -CELERY_BROKER_URL = "redis://localhost:6379/0" +CELERY_BROKER_URL = "redis://redis:6379/0" # BRO SETTINGS BRO_UITGIFTE_SERVICE_URL = "https://publiek.broservices.nl" diff --git a/docker-compose.yml b/docker-compose.yml index e7a7ab9..644e8f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,33 @@ services: volumes: - pgdata:/var/lib/postgresql/data restart: unless-stopped + networks: + - backend + + redis: + image: redis:latest + expose: + - 6379 + ports: + - "6379:6379" + networks: + - backend + + celery: + environment: + - DJANGO_SETTINGS_MODULE=bro_hub.settings + - FIELD_ENCRYPTION_KEY="DUMMY-NEEDS-PROD-SETTING-Xgb1GczqZe909UMNc4= + build: . + command: celery -A bro_hub worker --loglevel=INFO + volumes: + - .:/code + links: + - redis + depends_on: + - db + - redis + networks: + - backend web: build: . @@ -32,6 +59,19 @@ services: # - NENS_AUTH_ISSUER # - NENS_AUTH_CLIENT_ID # - NENS_AUTH_CLIENT_SECRET + depends_on: + - db + - redis + - celery + networks: + - backend + + + volumes: pgdata: + +networks: + backend: + name: bro_hub \ No newline at end of file From 14f7d29e057985c2f707a1ccb775a7e893b13c36 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:30:09 +0100 Subject: [PATCH 10/23] forgot precommit again --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 644e8f0..24c27f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,12 +66,12 @@ services: networks: - backend - - + + volumes: pgdata: networks: backend: - name: bro_hub \ No newline at end of file + name: bro_hub From 7e34c677ee7d1b1b02669cdb5fef0b80dcfac793 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:45:26 +0100 Subject: [PATCH 11/23] added all relevant GMN models for now --- api/bro_upload/delivery.py | 4 ++- .../templates/registration_GMN_Closure.html | 16 +++++++++ .../registration_GMN_MeasuringPoint.html | 27 +++++++++++++++ ...egistration_GMN_MeasuringPointEndDate.html | 17 ++++++++++ .../registration_GMN_TubeReference.html | 27 +++++++++++++++ .../replace_GMN_StartRegistration.html | 33 +++++++++++++++++++ api/choices.py | 4 +++ 7 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 api/bro_upload/templates/registration_GMN_Closure.html create mode 100644 api/bro_upload/templates/registration_GMN_MeasuringPoint.html create mode 100644 api/bro_upload/templates/registration_GMN_MeasuringPointEndDate.html create mode 100644 api/bro_upload/templates/registration_GMN_TubeReference.html create mode 100644 api/bro_upload/templates/replace_GMN_StartRegistration.html diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index ba42da2..88ccdd7 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -185,4 +185,6 @@ def create_xml_file(self) -> str: except Exception as e: traceback.print_exc() - raise XMLGenerationError(f"Error generating XML file: {e}") from e + raise XMLGenerationError( + "De aangeleverde combinatie van request type en registratie type is niet mogelijk. Als de combinatie in de BRO wel mogelijk is, vraag dan deze combinatie aan bij Nelen & Schuurmans." + ) from e diff --git a/api/bro_upload/templates/registration_GMN_Closure.html b/api/bro_upload/templates/registration_GMN_Closure.html new file mode 100644 index 0000000..c9d038e --- /dev/null +++ b/api/bro_upload/templates/registration_GMN_Closure.html @@ -0,0 +1,16 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + + + + {{ measuringpoint.endDateMonitoring }} + + + + diff --git a/api/bro_upload/templates/registration_GMN_MeasuringPoint.html b/api/bro_upload/templates/registration_GMN_MeasuringPoint.html new file mode 100644 index 0000000..2138f6d --- /dev/null +++ b/api/bro_upload/templates/registration_GMN_MeasuringPoint.html @@ -0,0 +1,27 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + diff --git a/api/bro_upload/templates/registration_GMN_MeasuringPointEndDate.html b/api/bro_upload/templates/registration_GMN_MeasuringPointEndDate.html new file mode 100644 index 0000000..90dca5c --- /dev/null +++ b/api/bro_upload/templates/registration_GMN_MeasuringPointEndDate.html @@ -0,0 +1,17 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + + + + {{ measuringpoint.eventDate }} + + {{ measuringpoint.measuringPointCode }} + + + diff --git a/api/bro_upload/templates/registration_GMN_TubeReference.html b/api/bro_upload/templates/registration_GMN_TubeReference.html new file mode 100644 index 0000000..20de05d --- /dev/null +++ b/api/bro_upload/templates/registration_GMN_TubeReference.html @@ -0,0 +1,27 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + diff --git a/api/bro_upload/templates/replace_GMN_StartRegistration.html b/api/bro_upload/templates/replace_GMN_StartRegistration.html new file mode 100644 index 0000000..4d2b654 --- /dev/null +++ b/api/bro_upload/templates/replace_GMN_StartRegistration.html @@ -0,0 +1,33 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }}/A + {{ metadata.correctionReason }} + + + {{ sourcedocs_data.objectIdAccountableParty }} + {{ sourcedocs_data.name }} + {{ sourcedocs_data.deliveryContext }} + {{ sourcedocs_data.monitoringPurpose }} + {{ sourcedocs_data.groundwaterAspect }} + + {{ sourcedocs_data.startDateMonitoring }} + {% for measuringpoint in sourcedocs_data.measuringPoints %} + + + {{ measuringpoint.measuringPointCode }} + + + {{ measuringpoint.broId }} + {{ measuringpoint.tubeNumber }} + + + + {% endfor %} + + + diff --git a/api/choices.py b/api/choices.py index 7cff7e5..6036a33 100644 --- a/api/choices.py +++ b/api/choices.py @@ -14,6 +14,10 @@ REGISTRATION_TYPE_OPTIONS = [ ("GMN_StartRegistration", "GMN_StartRegistration"), + ("GMN_MeasuringPoint", "GMN_MeasuringPoint"), + ("GMN_MeasuringPointEndDate", "GMN_MeasuringPointEndDate"), + ("GMN_TubeReference", "GMN_TubeReference"), + ("GMN_Closure", "GMN_Closure"), ] REQUEST_TYPE_OPTIONS = [ From 7203902191711d2e0914ccdf3827648f19e92916 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:51:55 +0100 Subject: [PATCH 12/23] added test to check xml generator error raise --- api/tests/test_bro_upload.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/api/tests/test_bro_upload.py b/api/tests/test_bro_upload.py index f8c4c56..3ea5f31 100644 --- a/api/tests/test_bro_upload.py +++ b/api/tests/test_bro_upload.py @@ -1,3 +1,5 @@ +import pytest + from api.bro_upload import delivery expected_xml_str = """ Date: Fri, 15 Mar 2024 14:53:08 +0100 Subject: [PATCH 13/23] added test to check xml generator error raise --- api/tests/test_bro_upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/tests/test_bro_upload.py b/api/tests/test_bro_upload.py index 3ea5f31..f9f7d26 100644 --- a/api/tests/test_bro_upload.py +++ b/api/tests/test_bro_upload.py @@ -80,6 +80,7 @@ def test_xml_generator1(): assert generator.create_xml_file() == expected_xml_str + def test_xml_generator2(): """Tests a non existing combination of registration_type and request type.""" with pytest.raises(delivery.XMLGenerationError): From f18dd3f5f9d8951e0e374046bd6306c5f016fbc1 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:30:59 +0100 Subject: [PATCH 14/23] Null=trues back --- ...0022_alter_uploadtask_registration_type.py | 26 +++ ...gmn_delivery_accountable_party_and_more.py | 82 +++++++ gmn/models.py | 28 +-- ..._alter_monitoringtube_tube_top_diameter.py | 17 ++ .../0009_alter_gmw_well_stability.py | 17 ++ ...lter_gmw_construction_standard_and_more.py | 207 ++++++++++++++++++ gmw/models.py | 88 ++++---- 7 files changed, 404 insertions(+), 61 deletions(-) create mode 100644 api/migrations/0022_alter_uploadtask_registration_type.py create mode 100644 gmn/migrations/0015_alter_gmn_delivery_accountable_party_and_more.py create mode 100644 gmw/migrations/0008_alter_monitoringtube_tube_top_diameter.py create mode 100644 gmw/migrations/0009_alter_gmw_well_stability.py create mode 100644 gmw/migrations/0010_alter_gmw_construction_standard_and_more.py diff --git a/api/migrations/0022_alter_uploadtask_registration_type.py b/api/migrations/0022_alter_uploadtask_registration_type.py new file mode 100644 index 0000000..61078ef --- /dev/null +++ b/api/migrations/0022_alter_uploadtask_registration_type.py @@ -0,0 +1,26 @@ +# Generated by Django 5.0.1 on 2024-03-14 19:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("api", "0021_remove_userprofile_default_project_number_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="uploadtask", + name="registration_type", + field=models.CharField( + choices=[ + ("GMN_StartRegistration", "GMN_StartRegistration"), + ("GMN_MeasuringPoint", "GMN_MeasuringPoint"), + ("GMN_MeasuringPointEndDate", "GMN_MeasuringPointEndDate"), + ("GMN_TubeReference", "GMN_TubeReference"), + ("GMN_Closure", "GMN_Closure"), + ], + max_length=235, + ), + ), + ] diff --git a/gmn/migrations/0015_alter_gmn_delivery_accountable_party_and_more.py b/gmn/migrations/0015_alter_gmn_delivery_accountable_party_and_more.py new file mode 100644 index 0000000..9093762 --- /dev/null +++ b/gmn/migrations/0015_alter_gmn_delivery_accountable_party_and_more.py @@ -0,0 +1,82 @@ +# Generated by Django 5.0.1 on 2024-03-14 19:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gmn", "0014_alter_gmn_data_owner_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="gmn", + name="delivery_accountable_party", + field=models.CharField(max_length=8, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="delivery_context", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="groundwater_aspect", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="monitoring_purpose", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="name", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="object_registration_time", + field=models.DateTimeField(null=True), + ), + migrations.AlterField( + model_name="gmn", + name="quality_regime", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="registration_status", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="gmn", + name="start_date_monitoring", + field=models.DateField(null=True), + ), + migrations.AlterField( + model_name="measuringpoint", + name="gmw_bro_id", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="measuringpoint", + name="measuringpoint_code", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="measuringpoint", + name="measuringpoint_start_date", + field=models.DateField(null=True), + ), + migrations.AlterField( + model_name="measuringpoint", + name="tube_number", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="measuringpoint", + name="tube_start_date", + field=models.DateField(null=True), + ), + ] diff --git a/gmn/models.py b/gmn/models.py index 3f823ac..8a4f015 100644 --- a/gmn/models.py +++ b/gmn/models.py @@ -16,15 +16,15 @@ class GMN(models.Model): updated_at = models.DateTimeField(auto_now=True) data_owner = models.ForeignKey(Organisation, on_delete=models.CASCADE) bro_id = models.CharField(max_length=18) - delivery_accountable_party = models.CharField(max_length=8) - quality_regime = models.CharField(max_length=50) - name = models.CharField(max_length=100) - delivery_context = models.CharField(max_length=100) - monitoring_purpose = models.CharField(max_length=100) - groundwater_aspect = models.CharField(max_length=100) - start_date_monitoring = models.DateField() - object_registration_time = models.DateTimeField() - registration_status = models.CharField(max_length=50) + delivery_accountable_party = models.CharField(max_length=8, null=True) + quality_regime = models.CharField(max_length=50, null=True) + name = models.CharField(max_length=100, null=True) + delivery_context = models.CharField(max_length=100, null=True) + monitoring_purpose = models.CharField(max_length=100, null=True) + groundwater_aspect = models.CharField(max_length=100, null=True) + start_date_monitoring = models.DateField(null=True) + object_registration_time = models.DateTimeField(null=True) + registration_status = models.CharField(max_length=50, null=True) def __str__(self): return self.bro_id @@ -46,11 +46,11 @@ class Measuringpoint(models.Model): updated_at = models.DateTimeField(auto_now=True) data_owner = models.ForeignKey(Organisation, on_delete=models.CASCADE) gmn = models.ForeignKey(GMN, on_delete=models.CASCADE) - measuringpoint_code = models.CharField(max_length=50) - measuringpoint_start_date = models.DateField() - gmw_bro_id = models.CharField(max_length=50) - tube_number = models.CharField(max_length=50) - tube_start_date = models.DateField() + measuringpoint_code = models.CharField(max_length=50, null=True) + measuringpoint_start_date = models.DateField(null=True) + gmw_bro_id = models.CharField(max_length=50, null=True) + tube_number = models.CharField(max_length=50, null=True) + tube_start_date = models.DateField(null=True) def __str__(self): return self.measuringpoint_code diff --git a/gmw/migrations/0008_alter_monitoringtube_tube_top_diameter.py b/gmw/migrations/0008_alter_monitoringtube_tube_top_diameter.py new file mode 100644 index 0000000..c81c477 --- /dev/null +++ b/gmw/migrations/0008_alter_monitoringtube_tube_top_diameter.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.1 on 2024-03-14 19:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gmw", "0007_alter_gmw_construction_standard_alter_gmw_data_owner_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="monitoringtube", + name="tube_top_diameter", + field=models.CharField(max_length=100, null=True), + ), + ] diff --git a/gmw/migrations/0009_alter_gmw_well_stability.py b/gmw/migrations/0009_alter_gmw_well_stability.py new file mode 100644 index 0000000..b275f1f --- /dev/null +++ b/gmw/migrations/0009_alter_gmw_well_stability.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.1 on 2024-03-14 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gmw", "0008_alter_monitoringtube_tube_top_diameter"), + ] + + operations = [ + migrations.AlterField( + model_name="gmw", + name="well_stability", + field=models.CharField(max_length=100, null=True), + ), + ] diff --git a/gmw/migrations/0010_alter_gmw_construction_standard_and_more.py b/gmw/migrations/0010_alter_gmw_construction_standard_and_more.py new file mode 100644 index 0000000..4364ea6 --- /dev/null +++ b/gmw/migrations/0010_alter_gmw_construction_standard_and_more.py @@ -0,0 +1,207 @@ +# Generated by Django 5.0.1 on 2024-03-14 19:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("gmw", "0009_alter_gmw_well_stability"), + ] + + operations = [ + migrations.AlterField( + model_name="gmw", + name="construction_standard", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="delivered_location", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="delivery_accountable_party", + field=models.CharField(max_length=8, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="delivery_context", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="ground_level_position", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="ground_level_positioning_method", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="ground_level_stable", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="initial_function", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="local_vertical_reference_point", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="nitg_code", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="object_registration_time", + field=models.DateTimeField(null=True), + ), + migrations.AlterField( + model_name="gmw", + name="offset", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="owner", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="quality_regime", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="registration_status", + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="removed", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="standardized_location", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="vertical_datum", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="well_code", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="gmw", + name="well_head_protector", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="artesian_well_cap_present", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="glue", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="number_of_geo_ohm_cables", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="plain_tube_part_length", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="screen_bottom_position", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="screen_length", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="screen_top_position", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="sediment_sump_present", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="sock_material", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_in_use", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_material", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_number", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_packing_material", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_part_inserted", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_status", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_top_position", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_top_positioning_method", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="tube_type", + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name="monitoringtube", + name="variable_diameter", + field=models.CharField(max_length=100, null=True), + ), + ] diff --git a/gmw/models.py b/gmw/models.py index f13e6fc..2196c06 100644 --- a/gmw/models.py +++ b/gmw/models.py @@ -19,31 +19,27 @@ class GMW(models.Model): on_delete=models.CASCADE, ) bro_id = models.CharField(max_length=18) - delivery_accountable_party = models.CharField(max_length=8) - quality_regime = models.CharField(max_length=50) - delivery_context = models.CharField(max_length=100) - construction_standard = models.CharField(max_length=100) - initial_function = models.CharField(max_length=100) - removed = models.CharField(max_length=100) - ground_level_stable = models.CharField(max_length=100) - well_stability = models.CharField(max_length=100) - nitg_code = models.CharField(max_length=100) - well_code = models.CharField(max_length=100) - owner = models.CharField(max_length=100) - well_head_protector = models.CharField(max_length=100) - delivered_location = models.CharField(max_length=100) - local_vertical_reference_point = models.CharField( - max_length=100, - ) - offset = models.CharField(max_length=100) - vertical_datum = models.CharField(max_length=100) - ground_level_position = models.CharField(max_length=100) - ground_level_positioning_method = models.CharField( - max_length=100, - ) - standardized_location = models.CharField(max_length=100) - object_registration_time = models.DateTimeField() - registration_status = models.CharField(max_length=50) + delivery_accountable_party = models.CharField(max_length=8, null=True) + quality_regime = models.CharField(max_length=50, null=True) + delivery_context = models.CharField(max_length=100, null=True) + construction_standard = models.CharField(max_length=100, null=True) + initial_function = models.CharField(max_length=100, null=True) + removed = models.CharField(max_length=100, null=True) + ground_level_stable = models.CharField(max_length=100, null=True) + well_stability = models.CharField(max_length=100, null=True) + nitg_code = models.CharField(max_length=100, null=True) + well_code = models.CharField(max_length=100, null=True) + owner = models.CharField(max_length=100, null=True) + well_head_protector = models.CharField(max_length=100, null=True) + delivered_location = models.CharField(max_length=100, null=True) + local_vertical_reference_point = models.CharField(max_length=100, null=True) + offset = models.CharField(max_length=100, null=True) + vertical_datum = models.CharField(max_length=100, null=True) + ground_level_position = models.CharField(max_length=100, null=True) + ground_level_positioning_method = models.CharField(max_length=100, null=True) + standardized_location = models.CharField(max_length=100, null=True) + object_registration_time = models.DateTimeField(null=True) + registration_status = models.CharField(max_length=50, null=True) def __str__(self): return self.bro_id @@ -63,28 +59,26 @@ class MonitoringTube(models.Model): on_delete=models.CASCADE, ) gmw = models.ForeignKey(GMW, on_delete=models.CASCADE) - tube_number = models.CharField(max_length=100) - tube_type = models.CharField(max_length=100) - artesian_well_cap_present = models.CharField(max_length=100) - sediment_sump_present = models.CharField(max_length=100) - number_of_geo_ohm_cables = models.CharField(max_length=100) - tube_top_diameter = models.CharField(max_length=100) - variable_diameter = models.CharField(max_length=100) - tube_status = models.CharField(max_length=100) - tube_top_position = models.CharField(max_length=100) - tube_top_positioning_method = models.CharField( - max_length=100, - ) - tube_part_inserted = models.CharField(max_length=100) - tube_in_use = models.CharField(max_length=100) - tube_packing_material = models.CharField(max_length=100) - tube_material = models.CharField(max_length=100) - glue = models.CharField(max_length=100) - screen_length = models.CharField(max_length=100) - sock_material = models.CharField(max_length=100) - screen_top_position = models.CharField(max_length=100) - screen_bottom_position = models.CharField(max_length=100) - plain_tube_part_length = models.CharField(max_length=100) + tube_number = models.CharField(max_length=100, null=True) + tube_type = models.CharField(max_length=100, null=True) + artesian_well_cap_present = models.CharField(max_length=100, null=True) + sediment_sump_present = models.CharField(max_length=100, null=True) + number_of_geo_ohm_cables = models.CharField(max_length=100, null=True) + tube_top_diameter = models.CharField(max_length=100, null=True) + variable_diameter = models.CharField(max_length=100, null=True) + tube_status = models.CharField(max_length=100, null=True) + tube_top_position = models.CharField(max_length=100, null=True) + tube_top_positioning_method = models.CharField(max_length=100, null=True) + tube_part_inserted = models.CharField(max_length=100, null=True) + tube_in_use = models.CharField(max_length=100, null=True) + tube_packing_material = models.CharField(max_length=100, null=True) + tube_material = models.CharField(max_length=100, null=True) + glue = models.CharField(max_length=100, null=True) + screen_length = models.CharField(max_length=100, null=True) + sock_material = models.CharField(max_length=100, null=True) + screen_top_position = models.CharField(max_length=100, null=True) + screen_bottom_position = models.CharField(max_length=100, null=True) + plain_tube_part_length = models.CharField(max_length=100, null=True) def __str__(self): return f"{self.gmw}-{self.tube_number}" From 1f3cddb943e25f8edb3a1ef409ed940c1b8c4243 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:10:34 +0100 Subject: [PATCH 15/23] added all gmn templates --- .../templates/delete_GMN_Closure.html | 17 ++++++++ .../templates/delete_GMN_MeasuringPoint.html | 28 +++++++++++++ .../delete_GMN_MeasuringPointEndDate.html | 18 +++++++++ .../templates/delete_GMN_TubeReference.html | 28 +++++++++++++ .../templates/insert_GMN_MeasuringPoint.html | 28 +++++++++++++ .../templates/insert_GMN_TubeReference.html | 28 +++++++++++++ .../templates/move_GMN_Closure.html | 20 ++++++++++ .../templates/move_GMN_MeasuringPoint.html | 31 +++++++++++++++ .../move_GMN_MeasuringPointEndDate.html | 21 ++++++++++ .../templates/move_GMN_StartRegistration.html | 39 +++++++++++++++++++ .../templates/move_GMN_TubeReference.html | 31 +++++++++++++++ .../registration_GMN_StartRegistration.html | 4 +- .../templates/replace_GMN_MeasuringPoint.html | 28 +++++++++++++ .../replace_GMN_StartRegistration.html | 4 +- .../templates/replace_GMN_TubeReference.html | 28 +++++++++++++ api/tests/test_bro_upload.py | 10 +++-- 16 files changed, 355 insertions(+), 8 deletions(-) create mode 100644 api/bro_upload/templates/delete_GMN_Closure.html create mode 100644 api/bro_upload/templates/delete_GMN_MeasuringPoint.html create mode 100644 api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html create mode 100644 api/bro_upload/templates/delete_GMN_TubeReference.html create mode 100644 api/bro_upload/templates/insert_GMN_MeasuringPoint.html create mode 100644 api/bro_upload/templates/insert_GMN_TubeReference.html create mode 100644 api/bro_upload/templates/move_GMN_Closure.html create mode 100644 api/bro_upload/templates/move_GMN_MeasuringPoint.html create mode 100644 api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html create mode 100644 api/bro_upload/templates/move_GMN_StartRegistration.html create mode 100644 api/bro_upload/templates/move_GMN_TubeReference.html create mode 100644 api/bro_upload/templates/replace_GMN_MeasuringPoint.html create mode 100644 api/bro_upload/templates/replace_GMN_TubeReference.html diff --git a/api/bro_upload/templates/delete_GMN_Closure.html b/api/bro_upload/templates/delete_GMN_Closure.html new file mode 100644 index 0000000..5c05bb6 --- /dev/null +++ b/api/bro_upload/templates/delete_GMN_Closure.html @@ -0,0 +1,17 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + \ No newline at end of file diff --git a/api/bro_upload/templates/delete_GMN_MeasuringPoint.html b/api/bro_upload/templates/delete_GMN_MeasuringPoint.html new file mode 100644 index 0000000..23c99f3 --- /dev/null +++ b/api/bro_upload/templates/delete_GMN_MeasuringPoint.html @@ -0,0 +1,28 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + \ No newline at end of file diff --git a/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html b/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html new file mode 100644 index 0000000..7b8ab13 --- /dev/null +++ b/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html @@ -0,0 +1,18 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + {{ sourcedocs_data.measuringPointCode }} + + + \ No newline at end of file diff --git a/api/bro_upload/templates/delete_GMN_TubeReference.html b/api/bro_upload/templates/delete_GMN_TubeReference.html new file mode 100644 index 0000000..cc9fb11 --- /dev/null +++ b/api/bro_upload/templates/delete_GMN_TubeReference.html @@ -0,0 +1,28 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.evemeasuringPointCodentDate }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + \ No newline at end of file diff --git a/api/bro_upload/templates/insert_GMN_MeasuringPoint.html b/api/bro_upload/templates/insert_GMN_MeasuringPoint.html new file mode 100644 index 0000000..79df4a4 --- /dev/null +++ b/api/bro_upload/templates/insert_GMN_MeasuringPoint.html @@ -0,0 +1,28 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.evenmeasuringPointCodetDate }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + \ No newline at end of file diff --git a/api/bro_upload/templates/insert_GMN_TubeReference.html b/api/bro_upload/templates/insert_GMN_TubeReference.html new file mode 100644 index 0000000..f889155 --- /dev/null +++ b/api/bro_upload/templates/insert_GMN_TubeReference.html @@ -0,0 +1,28 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + \ No newline at end of file diff --git a/api/bro_upload/templates/move_GMN_Closure.html b/api/bro_upload/templates/move_GMN_Closure.html new file mode 100644 index 0000000..5323552 --- /dev/null +++ b/api/bro_upload/templates/move_GMN_Closure.html @@ -0,0 +1,20 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + + {{ sourcedocs_data.dateToBeCorrected }} + + \ No newline at end of file diff --git a/api/bro_upload/templates/move_GMN_MeasuringPoint.html b/api/bro_upload/templates/move_GMN_MeasuringPoint.html new file mode 100644 index 0000000..45074ae --- /dev/null +++ b/api/bro_upload/templates/move_GMN_MeasuringPoint.html @@ -0,0 +1,31 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + + {{ sourcedocs_data.dateToBeCorrected }} + + \ No newline at end of file diff --git a/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html b/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html new file mode 100644 index 0000000..2a08fc5 --- /dev/null +++ b/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html @@ -0,0 +1,21 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + {{ sourcedocs_data.measuringPointCode }} + + + + {{ sourcedocs_data.dateToBeCorrected }} + + \ No newline at end of file diff --git a/api/bro_upload/templates/move_GMN_StartRegistration.html b/api/bro_upload/templates/move_GMN_StartRegistration.html new file mode 100644 index 0000000..1cc9ac4 --- /dev/null +++ b/api/bro_upload/templates/move_GMN_StartRegistration.html @@ -0,0 +1,39 @@ +This XML file does not appear to have any style information associated with it. The document tree is shown below. + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + {{ sourcedocs_data.objectIdAccountableParty }} + {{ sourcedocs_data.name }} + {{ sourcedocs_data.deliveryContext }} + {{ sourcedocs_data.monitoringPurpose }} + {{ sourcedocs_data.groundwaterAspect }} + + {{ sourcedocs_data.startDateMonitoring }} + + {% for measuringpoint in sourcedocs_data.measuringPoints %} + + + {{ measuringpoint.measuringPointCode }} + + + {{ measuringpoint.broId }} + {{ measuringpoint.tubeNumber }} + + + + + {% endfor %} + + + + {{ sourcedocs_data.dateToBeCorrected }} + + \ No newline at end of file diff --git a/api/bro_upload/templates/move_GMN_TubeReference.html b/api/bro_upload/templates/move_GMN_TubeReference.html new file mode 100644 index 0000000..97cb164 --- /dev/null +++ b/api/bro_upload/templates/move_GMN_TubeReference.html @@ -0,0 +1,31 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + + {{ sourcedocs_data.dateToBeCorrected }} + + \ No newline at end of file diff --git a/api/bro_upload/templates/registration_GMN_StartRegistration.html b/api/bro_upload/templates/registration_GMN_StartRegistration.html index 17dc409..a83f679 100644 --- a/api/bro_upload/templates/registration_GMN_StartRegistration.html +++ b/api/bro_upload/templates/registration_GMN_StartRegistration.html @@ -16,10 +16,10 @@ {{ sourcedocs_data.startDateMonitoring }} {% for measuringpoint in sourcedocs_data.measuringPoints %} - + {{ measuringpoint.measuringPointCode }} - + {{ measuringpoint.broId }} {{ measuringpoint.tubeNumber }} diff --git a/api/bro_upload/templates/replace_GMN_MeasuringPoint.html b/api/bro_upload/templates/replace_GMN_MeasuringPoint.html new file mode 100644 index 0000000..9092887 --- /dev/null +++ b/api/bro_upload/templates/replace_GMN_MeasuringPoint.html @@ -0,0 +1,28 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + \ No newline at end of file diff --git a/api/bro_upload/templates/replace_GMN_StartRegistration.html b/api/bro_upload/templates/replace_GMN_StartRegistration.html index 4d2b654..d2c1320 100644 --- a/api/bro_upload/templates/replace_GMN_StartRegistration.html +++ b/api/bro_upload/templates/replace_GMN_StartRegistration.html @@ -18,10 +18,10 @@ {{ sourcedocs_data.startDateMonitoring }} {% for measuringpoint in sourcedocs_data.measuringPoints %} - + {{ measuringpoint.measuringPointCode }} - + {{ measuringpoint.broId }} {{ measuringpoint.tubeNumber }} diff --git a/api/bro_upload/templates/replace_GMN_TubeReference.html b/api/bro_upload/templates/replace_GMN_TubeReference.html new file mode 100644 index 0000000..2082737 --- /dev/null +++ b/api/bro_upload/templates/replace_GMN_TubeReference.html @@ -0,0 +1,28 @@ + + {{ metadata.requestReference }} + {{ metadata.deliveryAccountableParty }} + {{ metadata.broId }} + {{ metadata.qualityRegime }} + {{ metadata.correctionReason }} + + + + {{ sourcedocs_data.eventDate }} + + + + {{ sourcedocs_data.measuringPointCode }} + + + {{ sourcedocs_data.broId }} + {{ sourcedocs_data.tubeNumber }} + + + + + + + \ No newline at end of file diff --git a/api/tests/test_bro_upload.py b/api/tests/test_bro_upload.py index f9f7d26..177bab7 100644 --- a/api/tests/test_bro_upload.py +++ b/api/tests/test_bro_upload.py @@ -20,10 +20,10 @@ 2024-01-01 - + GMW000000038946 - + GMW000000038946 1 @@ -31,10 +31,10 @@ - + GMW000000038946 - + GMW000000038946 2 @@ -78,6 +78,8 @@ def test_xml_generator1(): }, ) + print(generator.create_xml_file()) + assert generator.create_xml_file() == expected_xml_str From 5115135965b172f5ed9ea3c7bc9d785e8ac4f3f2 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:12:16 +0100 Subject: [PATCH 16/23] traceback -> logger --- api/bro_upload/delivery.py | 6 +++--- api/bro_upload/utils.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 88ccdd7..4ad054d 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -1,5 +1,5 @@ import time -import traceback +import logger from typing import Any from django.template.loader import render_to_string @@ -84,7 +84,7 @@ def _generate_xml_file(self) -> str: return generator.create_xml_file() except Exception as e: - traceback.print_exc() + logger.exception() raise RuntimeError(f"Error generating XML file: {e}") from e def _validate_xml_file(self, xml_file: str) -> None: @@ -184,7 +184,7 @@ def create_xml_file(self) -> str: return rendered_xml except Exception as e: - traceback.print_exc() + logger.exception() raise XMLGenerationError( "De aangeleverde combinatie van request type en registratie type is niet mogelijk. Als de combinatie in de BRO wel mogelijk is, vraag dan deze combinatie aan bij Nelen & Schuurmans." ) from e diff --git a/api/bro_upload/utils.py b/api/bro_upload/utils.py index 5bd1c88..25d42fc 100644 --- a/api/bro_upload/utils.py +++ b/api/bro_upload/utils.py @@ -1,5 +1,5 @@ import json -import traceback +import logger from typing import Any import requests @@ -24,7 +24,7 @@ def validate_xml_file( return r.json() except requests.RequestException as e: - traceback.print_exc() + logger.exception() raise RuntimeError(f"Validate xml error: {e}") @@ -44,7 +44,7 @@ def create_upload_url(bro_username: str, bro_password: str, project_number: str) return upload_url except requests.RequestException as e: - traceback.print_exc() + logger.exception() raise RuntimeError(f"Create upload url error: {e}") @@ -70,7 +70,7 @@ def add_xml_to_upload( return r.headers["Location"] except requests.RequestException as e: - traceback.print_exc() + logger.exception() raise RuntimeError(f"Add XML to upload error: {e}") @@ -98,7 +98,7 @@ def create_delivery( return r.headers["Location"] except requests.RequestException as e: - traceback.print_exc() + logger.exception() raise RuntimeError(f"Deliver uploaded XML error: {e}") @@ -115,5 +115,5 @@ def check_delivery_status( return r except requests.RequestException as e: - traceback.print_exc() + logger.exception() raise RuntimeError(f"Delivery info check error: {e}") From 8ac6baeb946e48c7f97c59743977fe9f9b82be3a Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:18:00 +0100 Subject: [PATCH 17/23] traceback -> logger --- api/bro_import/bulk_import.py | 5 +++-- api/bro_upload/delivery.py | 3 ++- api/bro_upload/utils.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/bro_import/bulk_import.py b/api/bro_import/bulk_import.py index 0378734..4fb2474 100644 --- a/api/bro_import/bulk_import.py +++ b/api/bro_import/bulk_import.py @@ -1,4 +1,4 @@ -import traceback +import logging import requests from django.conf import settings @@ -6,6 +6,7 @@ from api import models from api.bro_import import config +logger = logging.getLogger(__name__) class FetchBROIDsError(Exception): """Custom exception for errors during BRO IDs fetching.""" @@ -43,7 +44,7 @@ def run(self) -> None: ) data_importer.run() except requests.RequestException as e: - traceback.print_exc() + logger.exception() raise DataImportError(f"Error fetching BRO IDs from {url}: {e}") from e def _create_bro_ids_import_url(self) -> str: diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 4ad054d..56b7b9f 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -1,5 +1,5 @@ import time -import logger +import logging from typing import Any from django.template.loader import render_to_string @@ -7,6 +7,7 @@ from .. import models as api_models from . import utils +logger = logging.getLogger(__name__) class XMLGenerationError(Exception): """Exception raised when XML generation fails.""" diff --git a/api/bro_upload/utils.py b/api/bro_upload/utils.py index 25d42fc..97a4a31 100644 --- a/api/bro_upload/utils.py +++ b/api/bro_upload/utils.py @@ -1,10 +1,11 @@ import json -import logger +import logging from typing import Any import requests from django.conf import settings +logger = logging.getLogger(__name__) def validate_xml_file( xml_file: bytes, bro_username: str, bro_password: str, project_number: str From d05eb6e2abc698175dab96400cafbf1e7be9f381 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:26:31 +0100 Subject: [PATCH 18/23] specified xml generation error --- api/bro_import/bulk_import.py | 2 +- api/bro_upload/delivery.py | 11 ++++++++--- api/bro_upload/utils.py | 10 +++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/api/bro_import/bulk_import.py b/api/bro_import/bulk_import.py index 4fb2474..82a11e5 100644 --- a/api/bro_import/bulk_import.py +++ b/api/bro_import/bulk_import.py @@ -44,7 +44,7 @@ def run(self) -> None: ) data_importer.run() except requests.RequestException as e: - logger.exception() + logger.exception(e) raise DataImportError(f"Error fetching BRO IDs from {url}: {e}") from e def _create_bro_ids_import_url(self) -> str: diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 56b7b9f..76ec5d8 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -3,6 +3,7 @@ from typing import Any from django.template.loader import render_to_string +from django.template.exceptions import TemplateDoesNotExist from .. import models as api_models from . import utils @@ -85,7 +86,7 @@ def _generate_xml_file(self) -> str: return generator.create_xml_file() except Exception as e: - logger.exception() + logger.exception(e) raise RuntimeError(f"Error generating XML file: {e}") from e def _validate_xml_file(self, xml_file: str) -> None: @@ -184,8 +185,12 @@ def create_xml_file(self) -> str: ) return rendered_xml - except Exception as e: - logger.exception() + except TemplateDoesNotExist as e: + logger.exception(e) raise XMLGenerationError( "De aangeleverde combinatie van request type en registratie type is niet mogelijk. Als de combinatie in de BRO wel mogelijk is, vraag dan deze combinatie aan bij Nelen & Schuurmans." ) from e + + except Exception as e: + logger.exception(e) + raise XMLGenerationError(e) from e diff --git a/api/bro_upload/utils.py b/api/bro_upload/utils.py index 97a4a31..54bcd6a 100644 --- a/api/bro_upload/utils.py +++ b/api/bro_upload/utils.py @@ -25,7 +25,7 @@ def validate_xml_file( return r.json() except requests.RequestException as e: - logger.exception() + logger.exception(e) raise RuntimeError(f"Validate xml error: {e}") @@ -45,7 +45,7 @@ def create_upload_url(bro_username: str, bro_password: str, project_number: str) return upload_url except requests.RequestException as e: - logger.exception() + logger.exception(e) raise RuntimeError(f"Create upload url error: {e}") @@ -71,7 +71,7 @@ def add_xml_to_upload( return r.headers["Location"] except requests.RequestException as e: - logger.exception() + logger.exception(e) raise RuntimeError(f"Add XML to upload error: {e}") @@ -99,7 +99,7 @@ def create_delivery( return r.headers["Location"] except requests.RequestException as e: - logger.exception() + logger.exception(e) raise RuntimeError(f"Deliver uploaded XML error: {e}") @@ -116,5 +116,5 @@ def check_delivery_status( return r except requests.RequestException as e: - logger.exception() + logger.exception(e) raise RuntimeError(f"Delivery info check error: {e}") From 0f15346599a65a14b0bcd02c1acc9b128118241f Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:29:13 +0100 Subject: [PATCH 19/23] removed redundant docker-compose elements --- docker-compose.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 24c27f4..0a14ee7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,17 +11,11 @@ services: volumes: - pgdata:/var/lib/postgresql/data restart: unless-stopped - networks: - - backend redis: image: redis:latest expose: - 6379 - ports: - - "6379:6379" - networks: - - backend celery: environment: @@ -36,8 +30,6 @@ services: depends_on: - db - redis - networks: - - backend web: build: . @@ -63,15 +55,8 @@ services: - db - redis - celery - networks: - - backend - volumes: pgdata: - -networks: - backend: - name: bro_hub From c2379742d1b074b889fb3dedd4d34afe6e75319a Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:29:52 +0100 Subject: [PATCH 20/23] precommit --- api/bro_import/bulk_import.py | 1 + api/bro_upload/delivery.py | 7 ++++--- api/bro_upload/templates/delete_GMN_Closure.html | 2 +- api/bro_upload/templates/delete_GMN_MeasuringPoint.html | 2 +- .../templates/delete_GMN_MeasuringPointEndDate.html | 2 +- api/bro_upload/templates/delete_GMN_TubeReference.html | 2 +- api/bro_upload/templates/insert_GMN_MeasuringPoint.html | 2 +- api/bro_upload/templates/insert_GMN_TubeReference.html | 2 +- api/bro_upload/templates/move_GMN_Closure.html | 2 +- api/bro_upload/templates/move_GMN_MeasuringPoint.html | 2 +- .../templates/move_GMN_MeasuringPointEndDate.html | 2 +- api/bro_upload/templates/move_GMN_StartRegistration.html | 2 +- api/bro_upload/templates/move_GMN_TubeReference.html | 2 +- api/bro_upload/templates/replace_GMN_MeasuringPoint.html | 2 +- api/bro_upload/templates/replace_GMN_TubeReference.html | 2 +- api/bro_upload/utils.py | 1 + 16 files changed, 19 insertions(+), 16 deletions(-) diff --git a/api/bro_import/bulk_import.py b/api/bro_import/bulk_import.py index 82a11e5..14a22a4 100644 --- a/api/bro_import/bulk_import.py +++ b/api/bro_import/bulk_import.py @@ -8,6 +8,7 @@ logger = logging.getLogger(__name__) + class FetchBROIDsError(Exception): """Custom exception for errors during BRO IDs fetching.""" diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 76ec5d8..a4dbe9f 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -1,15 +1,16 @@ -import time import logging +import time from typing import Any -from django.template.loader import render_to_string from django.template.exceptions import TemplateDoesNotExist +from django.template.loader import render_to_string from .. import models as api_models from . import utils logger = logging.getLogger(__name__) + class XMLGenerationError(Exception): """Exception raised when XML generation fails.""" @@ -190,7 +191,7 @@ def create_xml_file(self) -> str: raise XMLGenerationError( "De aangeleverde combinatie van request type en registratie type is niet mogelijk. Als de combinatie in de BRO wel mogelijk is, vraag dan deze combinatie aan bij Nelen & Schuurmans." ) from e - + except Exception as e: logger.exception(e) raise XMLGenerationError(e) from e diff --git a/api/bro_upload/templates/delete_GMN_Closure.html b/api/bro_upload/templates/delete_GMN_Closure.html index 5c05bb6..b876e57 100644 --- a/api/bro_upload/templates/delete_GMN_Closure.html +++ b/api/bro_upload/templates/delete_GMN_Closure.html @@ -14,4 +14,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/templates/delete_GMN_MeasuringPoint.html b/api/bro_upload/templates/delete_GMN_MeasuringPoint.html index 23c99f3..c902cc9 100644 --- a/api/bro_upload/templates/delete_GMN_MeasuringPoint.html +++ b/api/bro_upload/templates/delete_GMN_MeasuringPoint.html @@ -25,4 +25,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html b/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html index 7b8ab13..d6e4228 100644 --- a/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html +++ b/api/bro_upload/templates/delete_GMN_MeasuringPointEndDate.html @@ -15,4 +15,4 @@ {{ sourcedocs_data.measuringPointCode }} - \ No newline at end of file + diff --git a/api/bro_upload/templates/delete_GMN_TubeReference.html b/api/bro_upload/templates/delete_GMN_TubeReference.html index cc9fb11..ef2d6f7 100644 --- a/api/bro_upload/templates/delete_GMN_TubeReference.html +++ b/api/bro_upload/templates/delete_GMN_TubeReference.html @@ -25,4 +25,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/templates/insert_GMN_MeasuringPoint.html b/api/bro_upload/templates/insert_GMN_MeasuringPoint.html index 79df4a4..fa1b858 100644 --- a/api/bro_upload/templates/insert_GMN_MeasuringPoint.html +++ b/api/bro_upload/templates/insert_GMN_MeasuringPoint.html @@ -25,4 +25,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/templates/insert_GMN_TubeReference.html b/api/bro_upload/templates/insert_GMN_TubeReference.html index f889155..a00c7ff 100644 --- a/api/bro_upload/templates/insert_GMN_TubeReference.html +++ b/api/bro_upload/templates/insert_GMN_TubeReference.html @@ -25,4 +25,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/templates/move_GMN_Closure.html b/api/bro_upload/templates/move_GMN_Closure.html index 5323552..99df738 100644 --- a/api/bro_upload/templates/move_GMN_Closure.html +++ b/api/bro_upload/templates/move_GMN_Closure.html @@ -17,4 +17,4 @@ {{ sourcedocs_data.dateToBeCorrected }} - \ No newline at end of file + diff --git a/api/bro_upload/templates/move_GMN_MeasuringPoint.html b/api/bro_upload/templates/move_GMN_MeasuringPoint.html index 45074ae..18ec00d 100644 --- a/api/bro_upload/templates/move_GMN_MeasuringPoint.html +++ b/api/bro_upload/templates/move_GMN_MeasuringPoint.html @@ -28,4 +28,4 @@ {{ sourcedocs_data.dateToBeCorrected }} - \ No newline at end of file + diff --git a/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html b/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html index 2a08fc5..cf9e13c 100644 --- a/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html +++ b/api/bro_upload/templates/move_GMN_MeasuringPointEndDate.html @@ -18,4 +18,4 @@ {{ sourcedocs_data.dateToBeCorrected }} - \ No newline at end of file + diff --git a/api/bro_upload/templates/move_GMN_StartRegistration.html b/api/bro_upload/templates/move_GMN_StartRegistration.html index 1cc9ac4..1fe6d47 100644 --- a/api/bro_upload/templates/move_GMN_StartRegistration.html +++ b/api/bro_upload/templates/move_GMN_StartRegistration.html @@ -36,4 +36,4 @@ {{ sourcedocs_data.dateToBeCorrected }} - \ No newline at end of file + diff --git a/api/bro_upload/templates/move_GMN_TubeReference.html b/api/bro_upload/templates/move_GMN_TubeReference.html index 97cb164..1d8abef 100644 --- a/api/bro_upload/templates/move_GMN_TubeReference.html +++ b/api/bro_upload/templates/move_GMN_TubeReference.html @@ -28,4 +28,4 @@ {{ sourcedocs_data.dateToBeCorrected }} - \ No newline at end of file + diff --git a/api/bro_upload/templates/replace_GMN_MeasuringPoint.html b/api/bro_upload/templates/replace_GMN_MeasuringPoint.html index 9092887..21bbb3a 100644 --- a/api/bro_upload/templates/replace_GMN_MeasuringPoint.html +++ b/api/bro_upload/templates/replace_GMN_MeasuringPoint.html @@ -25,4 +25,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/templates/replace_GMN_TubeReference.html b/api/bro_upload/templates/replace_GMN_TubeReference.html index 2082737..366f9b2 100644 --- a/api/bro_upload/templates/replace_GMN_TubeReference.html +++ b/api/bro_upload/templates/replace_GMN_TubeReference.html @@ -25,4 +25,4 @@ - \ No newline at end of file + diff --git a/api/bro_upload/utils.py b/api/bro_upload/utils.py index 54bcd6a..a51108c 100644 --- a/api/bro_upload/utils.py +++ b/api/bro_upload/utils.py @@ -7,6 +7,7 @@ logger = logging.getLogger(__name__) + def validate_xml_file( xml_file: bytes, bro_username: str, bro_password: str, project_number: str ) -> dict[str, Any]: From c920ce89d1ca79963ed6584db42f4a6b8b2cb71b Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:02:54 +0100 Subject: [PATCH 21/23] template for loop more clearly --- api/bro_upload/templates/move_GMN_StartRegistration.html | 4 ++-- .../templates/registration_GMN_StartRegistration.html | 6 ++++-- api/bro_upload/templates/replace_GMN_StartRegistration.html | 6 ++++-- api/tests/test_bro_upload.py | 3 +++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/api/bro_upload/templates/move_GMN_StartRegistration.html b/api/bro_upload/templates/move_GMN_StartRegistration.html index 1fe6d47..2f00c05 100644 --- a/api/bro_upload/templates/move_GMN_StartRegistration.html +++ b/api/bro_upload/templates/move_GMN_StartRegistration.html @@ -18,7 +18,7 @@ {{ sourcedocs_data.startDateMonitoring }} - {% for measuringpoint in sourcedocs_data.measuringPoints %} +{% for measuringpoint in sourcedocs_data.measuringPoints %} {{ measuringpoint.measuringPointCode }} @@ -30,7 +30,7 @@ - {% endfor %} +{% endfor %} diff --git a/api/bro_upload/templates/registration_GMN_StartRegistration.html b/api/bro_upload/templates/registration_GMN_StartRegistration.html index a83f679..a0cdd5c 100644 --- a/api/bro_upload/templates/registration_GMN_StartRegistration.html +++ b/api/bro_upload/templates/registration_GMN_StartRegistration.html @@ -14,7 +14,8 @@ {{ sourcedocs_data.groundwaterAspect }} {{ sourcedocs_data.startDateMonitoring }} - {% for measuringpoint in sourcedocs_data.measuringPoints %} + +{% for measuringpoint in sourcedocs_data.measuringPoints %} {{ measuringpoint.measuringPointCode }} @@ -25,7 +26,8 @@ - {% endfor %} + +{% endfor %} diff --git a/api/bro_upload/templates/replace_GMN_StartRegistration.html b/api/bro_upload/templates/replace_GMN_StartRegistration.html index d2c1320..68c50be 100644 --- a/api/bro_upload/templates/replace_GMN_StartRegistration.html +++ b/api/bro_upload/templates/replace_GMN_StartRegistration.html @@ -16,7 +16,8 @@ {{ sourcedocs_data.groundwaterAspect }} {{ sourcedocs_data.startDateMonitoring }} - {% for measuringpoint in sourcedocs_data.measuringPoints %} + +{% for measuringpoint in sourcedocs_data.measuringPoints %} {{ measuringpoint.measuringPointCode }} @@ -27,7 +28,8 @@ - {% endfor %} + +{% endfor %} diff --git a/api/tests/test_bro_upload.py b/api/tests/test_bro_upload.py index 177bab7..80f9675 100644 --- a/api/tests/test_bro_upload.py +++ b/api/tests/test_bro_upload.py @@ -19,6 +19,7 @@ 2024-01-01 + GMW000000038946 @@ -30,6 +31,7 @@ + GMW000000038946 @@ -41,6 +43,7 @@ + From ea6ac2f7360fdaa7e729dd867db7569e2bc4ec52 Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:14:30 +0100 Subject: [PATCH 22/23] returned json iso response --- api/bro_upload/delivery.py | 6 +++--- api/bro_upload/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index a4dbe9f..142282c 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -139,14 +139,14 @@ def _check_delivery(self, delivery_url: str) -> bool: delivery_url, self.bro_username, self.bro_password ) - errors = delivery_info.json()["brondocuments"][0]["errors"] + errors = delivery_info["brondocuments"][0]["errors"] if errors: raise DeliveryError(f"Errors found after delivering the XML file: {errors}") else: - delivery_status = delivery_info.json()["status"] - delivery_brondocument_status = delivery_info.json()["brondocuments"][0][ + delivery_status = delivery_info["status"] + delivery_brondocument_status = delivery_info["brondocuments"][0][ "status" ] diff --git a/api/bro_upload/utils.py b/api/bro_upload/utils.py index a51108c..f077a35 100644 --- a/api/bro_upload/utils.py +++ b/api/bro_upload/utils.py @@ -106,7 +106,7 @@ def create_delivery( def check_delivery_status( delivery_url: str, bro_username: str, bro_password: str -) -> requests.Response: +) -> dict[str, Any]: """Checks the Delivery info. Step 4 of 4 in the upload process.""" try: r = requests.get( @@ -114,7 +114,7 @@ def check_delivery_status( auth=(bro_username, bro_password), ) - return r + return r.json() except requests.RequestException as e: logger.exception(e) From e9573f1530bc4620d0c1a035c807e840863537ce Mon Sep 17 00:00:00 2001 From: Florian Knappers <73856313+JJFlorian@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:14:47 +0100 Subject: [PATCH 23/23] format --- api/bro_upload/delivery.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/bro_upload/delivery.py b/api/bro_upload/delivery.py index 142282c..488b153 100644 --- a/api/bro_upload/delivery.py +++ b/api/bro_upload/delivery.py @@ -146,9 +146,7 @@ def _check_delivery(self, delivery_url: str) -> bool: else: delivery_status = delivery_info["status"] - delivery_brondocument_status = delivery_info["brondocuments"][0][ - "status" - ] + delivery_brondocument_status = delivery_info["brondocuments"][0]["status"] if ( delivery_status == "DOORGELEVERD"