diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index 67f703e1..345d212f 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -53,7 +53,7 @@ AWS_PRODUCTS_TRANSFER_CONFIG = TransferConfig( multipart_chunksize=300 * MB, use_threads=True, - max_concurrency=4 + max_concurrency=2 ) MINIO_AWS_ACCESS_KEY_ID = os.environ.get("MINIO_AWS_ACCESS_KEY_ID") MINIO_AWS_SECRET_ACCESS_KEY = os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY") @@ -84,7 +84,6 @@ "secret_key": MINIO_AWS_SECRET_ACCESS_KEY, "bucket_name": os.environ.get("MINIO_GAP_AWS_BUCKET_NAME"), "file_overwrite": False, - "max_memory_size": 500 * MB, # 500MB "transfer_config": AWS_PRODUCTS_TRANSFER_CONFIG, "endpoint_url": MINIO_AWS_ENDPOINT_URL }, diff --git a/django_project/gap/models/preferences.py b/django_project/gap/models/preferences.py index f23e4ad3..ebfc7616 100644 --- a/django_project/gap/models/preferences.py +++ b/django_project/gap/models/preferences.py @@ -10,6 +10,7 @@ from django.contrib.gis.db import models from django.contrib.gis.geos import Polygon +from boto3.s3.transfer import TransferConfig from core.models.singleton import SingletonModel from gap.utils.dms import dms_string_to_point @@ -184,3 +185,17 @@ def east_africa_timezone() -> tzinfo: crop_plan_config_default()['tz'] ) return datetime.strptime(timezone, "%z").tzinfo + + @staticmethod + def user_file_s3_transfer_config() -> TransferConfig: + """Get S3 transfer config for GAP Products.""" + conf = Preferences.load().user_file_uploader_config + return TransferConfig( + multipart_chunksize=( + conf.get('default_block_size', 500) * 1024 * 1024 + ), + use_threads=True, + max_concurrency=( + conf.get('max_concurrency', 2) + ) + ) diff --git a/django_project/gap/providers/airborne_observation.py b/django_project/gap/providers/airborne_observation.py index 85139c57..9c5305f1 100644 --- a/django_project/gap/providers/airborne_observation.py +++ b/django_project/gap/providers/airborne_observation.py @@ -89,7 +89,7 @@ def get_measurements(self, start_date: datetime, end_date: datetime): nearest_histories is None or self._get_count(nearest_histories) == 0 ): - return None + return Measurement.objects.none() return Measurement.objects.annotate( geom=F('station_history__geometry'), diff --git a/django_project/gap/providers/observation.py b/django_project/gap/providers/observation.py index 794f0bb9..5ee45062 100644 --- a/django_project/gap/providers/observation.py +++ b/django_project/gap/providers/observation.py @@ -22,7 +22,8 @@ Dataset, DatasetAttribute, Station, - Measurement + Measurement, + Preferences ) from gap.utils.reader import ( LocationInputType, @@ -243,6 +244,10 @@ def to_csv(self, suffix='.csv', separator=','): if write_headers: write_headers = False + # upload to s3 + s3_storage.transfer_config = ( + Preferences.user_file_s3_transfer_config() + ) s3_storage.save(output, tmp_file) return output @@ -312,6 +317,10 @@ def to_netcdf(self): ) execute_dask_compute(x) + # upload to s3 + s3_storage.transfer_config = ( + Preferences.user_file_s3_transfer_config() + ) s3_storage.save(output_url, tmp_file) return output_url @@ -485,7 +494,7 @@ def get_measurements(self, start_date: datetime, end_date: datetime): self.nearest_stations is None or self._get_count(self.nearest_stations) == 0 ): - return + return Measurement.objects.none() return Measurement.objects.annotate( geom=F('station__geometry'), diff --git a/django_project/gap/tests/providers/test_observation.py b/django_project/gap/tests/providers/test_observation.py index a875b316..7482fd22 100644 --- a/django_project/gap/tests/providers/test_observation.py +++ b/django_project/gap/tests/providers/test_observation.py @@ -30,7 +30,7 @@ ) -class TestObsrvationReader(TestCase): +class TestObservationReader(TestCase): """Unit test for ObservationDatasetReader class.""" def setUp(self): @@ -58,6 +58,16 @@ def setUp(self): self.start_date, self.end_date ) + def test_find_nearest_station_empty(self): + """Test find nearest station empty.""" + new_provider = ProviderFactory(name='test_provider') + self.station.provider = new_provider + self.station.save() + + self.reader.read_historical_data(self.start_date, self.end_date) + data_value = self.reader.get_data_values() + self.assertTrue(data_value.is_empty()) + def test_find_nearest_station_by_point(self): """Test find nearest station from single point.""" MeasurementFactory.create( diff --git a/django_project/gap/utils/reader.py b/django_project/gap/utils/reader.py index 27cc0638..7c386f5f 100644 --- a/django_project/gap/utils/reader.py +++ b/django_project/gap/utils/reader.py @@ -31,7 +31,8 @@ Dataset, DatasetAttribute, DatasetTimeStep, - DatasetObservationType + DatasetObservationType, + Preferences ) from gap.utils.dask import execute_dask_compute, get_num_of_threads @@ -495,6 +496,10 @@ def to_netcdf(self): ) execute_dask_compute(x, is_api=True) + # upload to s3 + s3_storage.transfer_config = ( + Preferences.user_file_s3_transfer_config() + ) s3_storage.save(output_url, tmp_file) return output_url @@ -684,6 +689,9 @@ def to_csv(self, suffix='.csv', separator=','): write_headers = False # save to s3 + s3_storage.transfer_config = ( + Preferences.user_file_s3_transfer_config() + ) s3_storage.save(output, tmp_file) return output