Fix observation api response time (#261)

* add index to measurement table

* fix query measurements to_csv

* refactor to_netcdf_stream

* fix to json for observation data

* update uwsgi harakiri value

* fix tests

* remove debug

* update version

deployment/docker/uwsgi.conf

@@ -17,5 +17,5 @@ env = DJANGO_SETTINGS_MODULE=core.settings.prod
 #uid = 1000
 #gid = 1000
 memory-report = true
-harakiri = 120
+harakiri = 600
 

django_project/_version.txt

@@ -1 +1 @@
-0.0.10
+0.0.11

django_project/gap/admin/preferences.py

@@ -60,8 +60,9 @@ class PreferencesAdmin(admin.ModelAdmin):
             }
         ),
         (
-            'Logging', {
+            'API Config', {
                 'fields': (
+                    'dask_threads_num_api',
                     'api_log_batch_size',
                 )
             }

django_project/gap/migrations/0038_measurement_gap_measure_dataset_c4f74a_idx_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.7 on 2024-11-14 19:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0037_alter_collectorsession_ingestor_type_and_more'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='measurement',
+            index=models.Index(fields=['dataset_attribute', 'date_time'], name='gap_measure_dataset_c4f74a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='measurement',
+            index=models.Index(fields=['station_history', 'dataset_attribute', 'date_time'], name='gap_measure_station_694206_idx'),
+        ),
+    ]

django_project/gap/migrations/0039_preferences_dask_threads_num_api.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-11-15 06:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0038_measurement_gap_measure_dataset_c4f74a_idx_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='preferences',
+            name='dask_threads_num_api',
+            field=models.IntegerField(default=2, help_text='Number of threads for dask parallel computation in API, higher number will use more memory.'),
+        ),
+    ]

django_project/gap/models/measurement.py

@@ -91,3 +91,9 @@ def __str__(self):
 
     class Meta:  # noqa
         unique_together = ('station', 'dataset_attribute', 'date_time')
+        indexes = [
+            models.Index(fields=['dataset_attribute', 'date_time']),
+            models.Index(
+                fields=['station_history', 'dataset_attribute', 'date_time']
+            ),
+        ]

django_project/gap/models/preferences.py

@@ -109,6 +109,14 @@ class Preferences(SingletonModel):
         )
     )
 
+    dask_threads_num_api = models.IntegerField(
+        default=2,
+        help_text=(
+            'Number of threads for dask parallel computation in API, '
+            'higher number will use more memory.'
+        )
+    )
+
     # ingestor config
     ingestor_config = models.JSONField(
         default=dict,

django_project/gap/providers/airborne_observation.py

@@ -8,6 +8,7 @@
 
 from datetime import datetime
 
+from django.db.models import F, QuerySet
 from django.contrib.gis.db.models.functions import Distance
 from django.contrib.gis.geos import Polygon, Point
 
@@ -79,15 +80,23 @@ def _find_nearest_station_by_points(self):
     def get_measurements(self, start_date: datetime, end_date: datetime):
         """Return measurements."""
         nearest_histories = self.get_nearest_stations()
-        if nearest_histories is None or len(nearest_histories) == 0:
+        if isinstance(nearest_histories, QuerySet):
+            nearest_histories = nearest_histories.filter(
+                date_time__gte=start_date,
+                date_time__lte=end_date
+            )
+        if (
+            nearest_histories is None or
+            self._get_count(nearest_histories) == 0
+        ):
             return None
 
-        return Measurement.objects.select_related(
-            'dataset_attribute', 'dataset_attribute__attribute',
-            'station', 'station_history'
+        return Measurement.objects.annotate(
+            geom=F('station_history__geometry'),
+            alt=F('station_history__altitude')
         ).filter(
             date_time__gte=start_date,
             date_time__lte=end_date,
             dataset_attribute__in=self.attributes,
             station_history__in=nearest_histories
-        ).order_by('date_time', 'station', 'dataset_attribute')
+        ).order_by('date_time')