Fix issue api data (#313)

* fix exception when None is returned * fix issue s3 upload config
kartoza · Dec 15, 2024 · c0c9f49 · c0c9f49
1 parent 3d1323f
commit c0c9f49
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 7 deletions.
diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py
@@ -53,7 +53,7 @@
 AWS_PRODUCTS_TRANSFER_CONFIG = TransferConfig(
     multipart_chunksize=300 * MB,
     use_threads=True,
-    max_concurrency=4
+    max_concurrency=2
 )
 MINIO_AWS_ACCESS_KEY_ID = os.environ.get("MINIO_AWS_ACCESS_KEY_ID")
 MINIO_AWS_SECRET_ACCESS_KEY = os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY")
@@ -84,7 +84,6 @@
             "secret_key": MINIO_AWS_SECRET_ACCESS_KEY,
             "bucket_name": os.environ.get("MINIO_GAP_AWS_BUCKET_NAME"),
             "file_overwrite": False,
-            "max_memory_size": 500 * MB,  # 500MB
             "transfer_config": AWS_PRODUCTS_TRANSFER_CONFIG,
             "endpoint_url": MINIO_AWS_ENDPOINT_URL
         },

diff --git a/django_project/gap/models/preferences.py b/django_project/gap/models/preferences.py
@@ -10,6 +10,7 @@
 
 from django.contrib.gis.db import models
 from django.contrib.gis.geos import Polygon
+from boto3.s3.transfer import TransferConfig
 
 from core.models.singleton import SingletonModel
 from gap.utils.dms import dms_string_to_point
@@ -184,3 +185,17 @@ def east_africa_timezone() -> tzinfo:
             crop_plan_config_default()['tz']
         )
         return datetime.strptime(timezone, "%z").tzinfo
+
+    @staticmethod
+    def user_file_s3_transfer_config() -> TransferConfig:
+        """Get S3 transfer config for GAP Products."""
+        conf = Preferences.load().user_file_uploader_config
+        return TransferConfig(
+            multipart_chunksize=(
+                conf.get('default_block_size', 500) * 1024 * 1024
+            ),
+            use_threads=True,
+            max_concurrency=(
+                conf.get('max_concurrency', 2)
+            )
+        )
diff --git a/django_project/gap/providers/airborne_observation.py b/django_project/gap/providers/airborne_observation.py
@@ -89,7 +89,7 @@ def get_measurements(self, start_date: datetime, end_date: datetime):
             nearest_histories is None or
             self._get_count(nearest_histories) == 0
         ):
-            return None
+            return Measurement.objects.none()
 
         return Measurement.objects.annotate(
             geom=F('station_history__geometry'),

diff --git a/django_project/gap/providers/observation.py b/django_project/gap/providers/observation.py
@@ -22,7 +22,8 @@
     Dataset,
     DatasetAttribute,
     Station,
-    Measurement
+    Measurement,
+    Preferences
 )
 from gap.utils.reader import (
     LocationInputType,
@@ -243,6 +244,10 @@ def to_csv(self, suffix='.csv', separator=','):
                 if write_headers:
                     write_headers = False
 
+            # upload to s3
+            s3_storage.transfer_config = (
+                Preferences.user_file_s3_transfer_config()
+            )
             s3_storage.save(output, tmp_file)
 
         return output
@@ -312,6 +317,10 @@ def to_netcdf(self):
             )
             execute_dask_compute(x)
 
+            # upload to s3
+            s3_storage.transfer_config = (
+                Preferences.user_file_s3_transfer_config()
+            )
             s3_storage.save(output_url, tmp_file)
 
         return output_url
@@ -485,7 +494,7 @@ def get_measurements(self, start_date: datetime, end_date: datetime):
             self.nearest_stations is None or
             self._get_count(self.nearest_stations) == 0
         ):
-            return
+            return Measurement.objects.none()
 
         return Measurement.objects.annotate(
             geom=F('station__geometry'),

diff --git a/django_project/gap/tests/providers/test_observation.py b/django_project/gap/tests/providers/test_observation.py
@@ -30,7 +30,7 @@
 )
 
 
-class TestObsrvationReader(TestCase):
+class TestObservationReader(TestCase):
     """Unit test for ObservationDatasetReader class."""
 
     def setUp(self):
@@ -58,6 +58,16 @@ def setUp(self):
             self.start_date, self.end_date
         )
 
+    def test_find_nearest_station_empty(self):
+        """Test find nearest station empty."""
+        new_provider = ProviderFactory(name='test_provider')
+        self.station.provider = new_provider
+        self.station.save()
+
+        self.reader.read_historical_data(self.start_date, self.end_date)
+        data_value = self.reader.get_data_values()
+        self.assertTrue(data_value.is_empty())
+
     def test_find_nearest_station_by_point(self):
         """Test find nearest station from single point."""
         MeasurementFactory.create(

diff --git a/django_project/gap/utils/reader.py b/django_project/gap/utils/reader.py
@@ -31,7 +31,8 @@
     Dataset,
     DatasetAttribute,
     DatasetTimeStep,
-    DatasetObservationType
+    DatasetObservationType,
+    Preferences
 )
 from gap.utils.dask import execute_dask_compute, get_num_of_threads
 
@@ -495,6 +496,10 @@ def to_netcdf(self):
             )
             execute_dask_compute(x, is_api=True)
 
+            # upload to s3
+            s3_storage.transfer_config = (
+                Preferences.user_file_s3_transfer_config()
+            )
             s3_storage.save(output_url, tmp_file)
 
         return output_url
@@ -684,6 +689,9 @@ def to_csv(self, suffix='.csv', separator=','):
                             write_headers = False
 
                 # save to s3
+                s3_storage.transfer_config = (
+                    Preferences.user_file_s3_transfer_config()
+                )
                 s3_storage.save(output, tmp_file)
 
         return output