Feat validate upload

deployment/docker/requirements.txt

@@ -58,3 +58,6 @@ pyjwt
 cryptography
 
 Pillow
+
+# fiona
+fiona==1.10.1

django_project/frontend/api_views/layers.py

@@ -10,12 +10,21 @@
 from django.http import FileResponse
 from django.conf import settings
 from django.urls import reverse
+from django.core.files.uploadedfile import TemporaryUploadedFile
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.views import APIView
+from rest_framework.exceptions import ValidationError
 from cloud_native_gis.models import Layer, LayerUpload
 from cloud_native_gis.utils.main import id_generator
 from django.shortcuts import get_object_or_404
+from cloud_native_gis.utils.fiona import (
+    FileType,
+    validate_shapefile_zip,
+    validate_collection_crs,
+    delete_tmp_shapefile,
+    open_fiona_collection
+)
 
 from layers.models import InputLayer, DataProvider, LayerGroupType
 from frontend.serializers.layers import LayerSerializer
@@ -58,8 +67,99 @@ class UploadLayerAPI(APIView):
 
     permission_classes = [IsAuthenticated]
 
+    def _check_file_type(self, filename: str) -> str:
+        """Check file type from upload filename.
+
+        :param filename: filename of uploaded file
+        :type filename: str
+        :return: file type
+        :rtype: str
+        """
+        if filename.lower().endswith('.zip'):
+            return FileType.SHAPEFILE
+        return ''
+
+    def _check_shapefile_zip(self, file_obj: any) -> str:
+        """Validate if zip shapefile has complete files.
+
+        :param file_obj: file object
+        :type file_obj: file
+        :return: list of error
+        :rtype: str
+        """
+        _, error = validate_shapefile_zip(file_obj)
+        if error:
+            return (
+                'Missing required file(s) inside zip file: \n- ' +
+                '\n- '.join(error)
+            )
+        return ''
+
+    def _remove_temp_files(self, file_obj_list: list) -> None:
+        """Remove temporary files.
+
+        :param file_obj: list temporary files
+        :type file_obj: list
+        """
+        for file_obj in file_obj_list:
+            if isinstance(file_obj, TemporaryUploadedFile):
+                if os.path.exists(file_obj.temporary_file_path()):
+                    os.remove(file_obj.temporary_file_path())
+            elif isinstance(file_obj, str):
+                delete_tmp_shapefile(file_obj)
+
+    def _on_validation_error(self, error: str, file_obj_list: list):
+        """Handle when there is error on validation."""
+        self._remove_temp_files(file_obj_list)
+        raise ValidationError({
+            'Invalid uploaded file': error
+        })
+
     def post(self, request):
         """Post file."""
+        file = None
+        file_url = request.data.get('file_url', None)
+        if request.FILES:
+            file = request.FILES['file']
+        elif file_url is None:
+            raise ValidationError({
+                'Invalid uploaded file': 'Missing required file!'
+            })
+
+        tmp_file_obj_list = [file]
+
+        # validate uploaded file
+        file_type = self._check_file_type(file.name)
+        if file_type == '':
+            self._on_validation_error(
+                'Unrecognized file type! Please upload the zip of shapefile!',
+                tmp_file_obj_list
+            )
+
+        if file_type == FileType.SHAPEFILE:
+            validate_shp_file = self._check_shapefile_zip(file)
+            if validate_shp_file != '':
+                self._on_validation_error(
+                    validate_shp_file, tmp_file_obj_list)
+
+        # open fiona collection
+        collection = open_fiona_collection(file, file_type)
+        tmp_file_obj_list.append(collection.path)
+
+        is_valid_crs, crs = validate_collection_crs(collection)
+        if not is_valid_crs:
+            collection.close()
+            self._on_validation_error(
+                f'Incorrect CRS type: {crs}! Please use epsg:4326 (WGS84)!',
+                tmp_file_obj_list
+            )
+
+        # close collection
+        collection.close()
+
+        # remove temporary uploaded file if any
+        self._remove_temp_files(tmp_file_obj_list)
+
         # create layer
         layer = Layer.objects.create(
             created_by=request.user
@@ -74,16 +174,13 @@ def post(self, request):
             updated_by=request.user
         )
 
-        file_url = request.data.get('file_url', None)
-
         instance = LayerUpload(
             created_by=request.user, layer=layer
         )
         instance.emptying_folder()
 
         # Save files
-        if request.FILES:
-            file = request.FILES['file']
+        if file:
             FileSystemStorage(
                 location=instance.folder
             ).save(file.name, file)

django_project/frontend/src/store/uploadSlice.ts

@@ -48,7 +48,11 @@ export const uploadFile = createAsyncThunk(
       });
       return response.data;
     } catch (error: any) {
-      return rejectWithValue(error.response?.data || 'Upload failed');
+      let error_msg = error.response?.data || 'Upload failed';
+      if (typeof error_msg === 'object') {
+        error_msg = Object.values(error_msg)[0];
+      }
+      return rejectWithValue(error_msg);
     }
   }
 );

django_project/frontend/tests/api_views/test_layers.py

@@ -13,7 +13,6 @@
 from cloud_native_gis.models.layer_upload import LayerUpload
 
 from core.settings.utils import absolute_path
-from core.factories import UserF
 from core.tests.common import BaseAPIViewTest
 from layers.models import (
     InputLayer, InputLayerType,
@@ -86,17 +85,12 @@ def test_upload_layer_no_auth(self):
         response = view(request)
         self.assertEqual(response.status_code, 401)
 
-    @mock.patch('layers.tasks.import_layer.import_layer.delay')
-    def test_upload_layer(self, mock_import_layer):
-        """Test upload layer."""
-        view = UploadLayerAPI.as_view()
-        file_path = absolute_path(
-            'frontend', 'tests', 'data', 'polygons.zip'
-        )
+    def _get_request(self, file_path, file_name=None):
+        """Get request for test upload."""
         with open(file_path, 'rb') as data:
             file = SimpleUploadedFile(
                 content=data.read(),
-                name=data.name,
+                name=file_name if file_name else data.name,
                 content_type='multipart/form-data'
             )
         request = self.factory.post(
@@ -106,6 +100,25 @@ def test_upload_layer(self, mock_import_layer):
             }
         )
         request.user = self.superuser
+        return request
+
+    def _check_error(
+            self, response, error_detail, status_code = 400,
+            error_key='Invalid uploaded file'
+        ):
+        """Check for error in the response."""
+        self.assertEqual(response.status_code, status_code)
+        self.assertIn(error_key, response.data)
+        self.assertEqual(str(response.data[error_key]), error_detail)
+
+    @mock.patch('layers.tasks.import_layer.import_layer.delay')
+    def test_upload_layer(self, mock_import_layer):
+        """Test upload layer."""
+        view = UploadLayerAPI.as_view()
+        file_path = absolute_path(
+            'frontend', 'tests', 'data', 'polygons.zip'
+        )
+        request = self._get_request(file_path)
         response = view(request)
         self.assertEqual(response.status_code, 200)
         self.assertIn('id', response.data)
@@ -173,3 +186,43 @@ def test_upload_pmtile(self):
         self.assertIsNotNone(self.input_layer.url)
         self.layer.refresh_from_db()
         self.assertIsNotNone(self.layer.pmtile)
+
+    def test_upload_invalid_type(self):
+        """Test upload with invalid file type."""
+        view = UploadLayerAPI.as_view()
+        file_path = absolute_path(
+            'frontend', 'tests', 'data', 'polygons.zip'
+        )
+        request = self._get_request(file_path, 'test.txt')
+        response = view(request)
+        self._check_error(
+            response,
+            'Unrecognized file type! Please upload the zip of shapefile!',
+            400
+        )
+
+    def test_upload_invalid_shapefile(self):
+        """Test upload with invalid shapefile."""
+        view = UploadLayerAPI.as_view()
+        file_path = absolute_path(
+            'frontend', 'tests', 'data', 'shp_no_shp.zip')
+        request = self._get_request(file_path)
+        response = view(request)
+        self._check_error(
+            response,
+            'Missing required file(s) inside zip file: \n- shp_1_1.shp',
+            400
+        )
+
+    def test_upload_invalid_crs(self):
+        """Test upload with invalid crs."""
+        view = UploadLayerAPI.as_view()
+        file_path = absolute_path(
+            'frontend', 'tests', 'data', 'shp_3857.zip')
+        request = self._get_request(file_path)
+        response = view(request)
+        self._check_error(
+            response,
+            'Incorrect CRS type: epsg:3857! Please use epsg:4326 (WGS84)!',
+            400
+        )