Fix API issues (#269)

* fix issue non dictionary in request_params

* change terminology of bbox

* add zarr cache size checker

* increase read and send timeouts of uwsgi and nginx

deployment/docker/uwsgi.conf

@@ -17,5 +17,11 @@ env = DJANGO_SETTINGS_MODULE=core.settings.prod
 #uid = 1000
 #gid = 1000
 memory-report = true
-harakiri = 600
+harakiri = 620
 
+# Restart workers after this condition
+max-requests = 1000
+reload-on-rss = 2048
+
+# increase timeout
+socket-timeout = 600

deployment/nginx/sites-enabled/default.conf

@@ -70,5 +70,10 @@ server {
         uwsgi_param  REMOTE_PORT        $remote_port;
         uwsgi_param  SERVER_PORT        $server_port;
         uwsgi_param  SERVER_NAME        $server_name;
+
+        # Increase timeout values
+        uwsgi_read_timeout 610s;
+        uwsgi_send_timeout 610s;
+        client_body_timeout 610s;
     }
 }

django_project/core/tests/test_file.py

@@ -0,0 +1,65 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Unit test for file utils.
+"""
+import os
+import tempfile
+
+from django.test import TestCase
+
+from core.utils.file import get_directory_size, format_size
+
+
+class TestFileUtilities(TestCase):
+    """Test File utilities."""
+
+    def setUp(self):
+        """Set test class."""
+        # Create a temporary directory
+        self.test_dir = tempfile.TemporaryDirectory()
+        self.test_dir_path = self.test_dir.name
+
+        # Create files and subdirectories for testing
+        self.file1 = os.path.join(self.test_dir_path, "file1.txt")
+        self.file2 = os.path.join(self.test_dir_path, "file2.txt")
+        self.sub_dir = os.path.join(self.test_dir_path, "subdir")
+        os.mkdir(self.sub_dir)
+
+        self.sub_file = os.path.join(self.sub_dir, "subfile.txt")
+
+        # Write data to files
+        with open(self.file1, "wb") as f:
+            f.write(b"a" * 1024)  # 1 KB
+        with open(self.file2, "wb") as f:
+            f.write(b"b" * 2048)  # 2 KB
+        with open(self.sub_file, "wb") as f:
+            f.write(b"c" * 4096)  # 4 KB
+
+    def tearDown(self):
+        """Clean up temporary directory."""
+        self.test_dir.cleanup()
+
+    def test_get_directory_size(self):
+        """Test get directory size."""
+        # Expected total size: 1 KB + 2 KB + 4 KB = 7 KB
+        expected_size = 1024 + 2048 + 4096
+        calculated_size = get_directory_size(self.test_dir_path)
+        self.assertEqual(calculated_size, expected_size)
+
+    def test_format_size(self):
+        """Test format_size function."""
+        test_cases = [
+            (512, "512.00 B"),
+            (1024, "1.00 KB"),
+            (1536, "1.50 KB"),
+            (1048576, "1.00 MB"),
+            (1073741824, "1.00 GB"),
+            (1099511627776, "1.00 TB"),
+            (1125899906842624, "1.00 PB"),
+        ]
+
+        for size, expected in test_cases:
+            with self.subTest(size=size):
+                self.assertEqual(format_size(size), expected)

django_project/core/utils/file.py

@@ -0,0 +1,40 @@
+# coding=utf-8
+"""
+Tomorrow Now GAP.
+
+.. note:: Utilities for file.
+"""
+import os
+
+
+def get_directory_size(path):
+    """Get size of directory.
+
+    :param path: directory path
+    :type path: string
+    :return: size in bytes
+    :rtype: int
+    """
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(path):
+        for file in filenames:
+            file_path = os.path.join(dirpath, file)
+            # Add file size, skipping broken symbolic links
+            if os.path.exists(file_path):
+                total_size += os.path.getsize(file_path)
+    return total_size
+
+
+def format_size(size_in_bytes):
+    """Format size to human readable.
+
+    :param size_in_bytes: size
+    :type size_in_bytes: int
+    :return: human readable size
+    :rtype: str
+    """
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if size_in_bytes < 1024:
+            return f"{size_in_bytes:.2f} {unit}"
+        size_in_bytes /= 1024
+    return f"{size_in_bytes:.2f} PB"

django_project/gap/admin/main.py

@@ -11,6 +11,7 @@
 from django_admin_inline_paginator.admin import TabularInlinePaginated
 
 from core.admin import AbstractDefinitionAdmin
+from core.utils.file import get_directory_size, format_size
 from gap.models import (
     Attribute, Country, Provider, Measurement, IngestorSession,
     IngestorSessionProgress, Dataset, DatasetAttribute, DataSourceFile,
@@ -175,30 +176,39 @@ def load_source_zarr_cache(modeladmin, request, queryset):
         )
 
 
-@admin.action(description='Clear zarr cache')
-def clear_source_zarr_cache(modeladmin, request, queryset):
-    """Clear DataSourceFile zarr cache."""
-    name = None
-    for query in queryset:
-        if query.format != DatasetStore.ZARR:
-            continue
-        name = query.name
-        zarr_path = BaseZarrReader.get_zarr_cache_dir(name)
-        if os.path.exists(zarr_path):
-            shutil.rmtree(zarr_path)
-        break
-    if name is not None:
-        modeladmin.message_user(
-            request,
-            f'{zarr_path} has been cleared!',
-            messages.SUCCESS
-        )
-    else:
+def _clear_zarr_cache(source: DataSourceFile, modeladmin, request):
+    """Clear zarr cache directory."""
+    if source.format != DatasetStore.ZARR:
         modeladmin.message_user(
             request,
             'Please select zarr data source!',
             messages.WARNING
         )
+        return
+
+    name = source.name
+    zarr_path = BaseZarrReader.get_zarr_cache_dir(name)
+    if os.path.exists(zarr_path):
+        shutil.rmtree(zarr_path)
+
+    modeladmin.message_user(
+        request,
+        f'{zarr_path} has been cleared!',
+        messages.SUCCESS
+    )
+
+    # update cache size to 0
+    DataSourceFileCache.objects.filter(
+        source_file=source
+    ).update(size=0)
+
+
+@admin.action(description='Clear zarr cache')
+def clear_source_zarr_cache(modeladmin, request, queryset):
+    """Clear single DataSourceFile zarr cache."""
+    for query in queryset:
+        _clear_zarr_cache(query, modeladmin, request)
+        break
 
 
 @admin.register(DataSourceFile)
@@ -213,15 +223,43 @@ class DataSourceFileAdmin(admin.ModelAdmin):
     actions = (load_source_zarr_cache, clear_source_zarr_cache,)
 
 
+@admin.action(description='Calculate zarr cache size')
+def calculate_zarr_cache_size(modeladmin, request, queryset):
+    """Calculate the size of zarr cache."""
+    for query in queryset:
+        name = query.source_file.name
+        zarr_path = BaseZarrReader.get_zarr_cache_dir(name)
+        if not os.path.exists(zarr_path):
+            continue
+
+        query.size = get_directory_size(zarr_path)
+        query.save()
+
+    modeladmin.message_user(
+        request,
+        'Calculate zarr cache size successful!',
+        messages.SUCCESS
+    )
+
+
+@admin.action(description='Clear zarr cache')
+def clear_zarr_dir_cache(modeladmin, request, queryset):
+    """Clear DataSourceFile zarr cache."""
+    for query in queryset:
+        _clear_zarr_cache(query.source_file, modeladmin, request)
+        break
+
+
 @admin.register(DataSourceFileCache)
 class DataSourceFileCacheAdmin(admin.ModelAdmin):
     """DataSourceFileCache admin."""
 
     list_display = (
         'get_name', 'get_dataset', 'hostname',
-        'created_on', 'expired_on'
+        'get_cache_size', 'created_on', 'expired_on'
     )
     list_filter = ('hostname',)
+    actions = (calculate_zarr_cache_size, clear_zarr_dir_cache,)
 
     def get_name(self, obj: DataSourceFileCache):
         """Get name of data source.
@@ -243,10 +281,16 @@ def get_dataset(self, obj: DataSourceFileCache):
         """
         return obj.source_file.dataset.name
 
+    def get_cache_size(self, obj: DataSourceFileCache):
+        """Get the cache size."""
+        return format_size(obj.size)
+
     get_name.short_description = 'Name'
     get_name.admin_order_field = 'source_file__name'
     get_dataset.short_description = 'Dataset'
     get_dataset.admin_order_field = 'source_file__dataset__name'
+    get_cache_size.short_description = 'Cache Size'
+    get_cache_size.admin_order_field = 'size'
 
 
 @admin.register(Village)

django_project/gap/factories/main.py

@@ -141,6 +141,7 @@ class Meta:  # noqa
     source_file = factory.SubFactory(DataSourceFileFactory)
     hostname = factory.Faker('text')
     created_on = factory.Faker('date_time')
+    size = 0
 
 
 class VillageFactory(DjangoModelFactory):

django_project/gap/migrations/0040_datasourcefilecache_size.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-11-19 20:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0039_preferences_dask_threads_num_api'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datasourcefilecache',
+            name='size',
+            field=models.PositiveIntegerField(default=0),
+        ),
+    ]

django_project/gap/models/dataset.py

@@ -174,6 +174,7 @@ class DataSourceFileCache(models.Model):
         null=True,
         blank=True
     )
+    size = models.PositiveIntegerField(default=0)
 
     class Meta:
         """Meta class for DataSourceFileCache."""

django_project/gap/tests/utils/test_zarr.py

@@ -27,7 +27,9 @@
 )
 from gap.admin.main import (
     load_source_zarr_cache,
-    clear_source_zarr_cache
+    clear_source_zarr_cache,
+    clear_zarr_dir_cache,
+    calculate_zarr_cache_size
 )
 from gap.factories import (
     DataSourceFileFactory,
@@ -327,3 +329,76 @@ def test_clear_source_zarr_cache(self, mock_os_path_exists, mock_rmtree):
             '/tmp/test_zarr has been cleared!',
             messages.SUCCESS
         )
+
+    @patch('gap.admin.main.get_directory_size')
+    @patch('os.path.exists')
+    def test_calculate_zarr_cache_size(
+        self, mock_os_path_exists, mock_calculate
+    ):
+        """Test calculate zarr cache size."""
+        # Mock the queryset with a Zarr file
+        data_source = DataSourceFileFactory.create(
+            format=DatasetStore.ZARR,
+            name='test.zarr'
+        )
+        cache_file = DataSourceFileCacheFactory.create(
+            source_file=data_source
+        )
+
+        mock_queryset = [cache_file]
+
+        # Mock os.path.exists to return True
+        mock_os_path_exists.return_value = True
+        # Mock get_directory_size to return 10000
+        mock_calculate.return_value = 10000
+
+        # Mock the modeladmin and request objects
+        mock_modeladmin = MagicMock()
+        mock_request = MagicMock()
+
+        calculate_zarr_cache_size(mock_modeladmin, mock_request, mock_queryset)
+
+        # Assertions
+        mock_os_path_exists.assert_called_once_with('/tmp/test_zarr')
+        mock_calculate.assert_called_once_with('/tmp/test_zarr')
+        mock_modeladmin.message_user.assert_called_once_with(
+            mock_request,
+            'Calculate zarr cache size successful!',
+            messages.SUCCESS
+        )
+        cache_file.refresh_from_db()
+        self.assertEqual(cache_file.size, 10000)
+
+    @patch('shutil.rmtree')
+    @patch('os.path.exists')
+    def test_clear_zarr_dir_cache(self, mock_os_path_exists, mock_rmtree):
+        """Test clear_zarr_dir_cache."""
+        # Mock the queryset with a Zarr file
+        data_source = DataSourceFileFactory.create(
+            format=DatasetStore.ZARR,
+            name='test.zarr'
+        )
+        cache_file = DataSourceFileCacheFactory.create(
+            source_file=data_source
+        )
+
+        mock_queryset = [cache_file]
+
+        # Mock os.path.exists to return True
+        mock_os_path_exists.return_value = True
+
+        # Mock the modeladmin and request objects
+        mock_modeladmin = MagicMock()
+        mock_request = MagicMock()
+
+        # Call the clear_zarr_dir_cache function
+        clear_zarr_dir_cache(mock_modeladmin, mock_request, mock_queryset)
+
+        # Assertions
+        mock_os_path_exists.assert_called_once_with('/tmp/test_zarr')
+        mock_rmtree.assert_called_once_with('/tmp/test_zarr')
+        mock_modeladmin.message_user.assert_called_once_with(
+            mock_request,
+            '/tmp/test_zarr has been cleared!',
+            messages.SUCCESS
+        )

django_project/gap_api/admin.py

@@ -79,6 +79,11 @@ def product_type(self, obj: APIRequestLog):
         :return: product in json query_params
         :rtype: str
         """
+        if obj.query_params is None:
+            return '-'
+        if not isinstance(obj.query_params, dict):
+            return '-'
+
         return obj.query_params.get('product', '-')
 
     product_type.short_description = 'Product Type'