Merge pull request #980 from mapswipe/feature/street

feat: add project for street level imagery

.github/workflows/actions.yml

@@ -80,6 +80,7 @@ jobs:
         POSTGRES_DB: postgres
         OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }}
         DJANGO_SECRET_KEY: test-django-secret-key
+        MAPILLARY_API_KEY: ${{ secrets.MAPILLARY_API_KEY }}
         COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml
       run: |
         docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/

django/apps/existing_database/models.py

@@ -66,6 +66,9 @@ class Type(models.IntegerChoices):
         FOOTPRINT = 2, "Validate"
         CHANGE_DETECTION = 3, "Compare"
         COMPLETENESS = 4, "Completeness"
+        MEDIA = 5, "Media"
+        DIGITIZATION = 6, "Digitization"
+        STREET = 7, "Street"
 
     project_id = models.CharField(primary_key=True, max_length=999)
     created = models.DateTimeField(blank=True, null=True)
@@ -127,7 +130,7 @@ class Task(Model):
     project = models.ForeignKey(Project, models.DO_NOTHING, related_name="+")
     group_id = models.CharField(max_length=999)
     task_id = models.CharField(max_length=999)
-    geom = gis_models.MultiPolygonField(blank=True, null=True)
+    geom = gis_models.GeometryField(blank=True, null=True)
     # Database uses JSON instead of JSONB (not supported by django)
     project_type_specifics = models.TextField(blank=True, null=True)
 

django/schema.graphql

@@ -97,6 +97,9 @@ enum ProjectTypeEnum {
   FOOTPRINT
   CHANGE_DETECTION
   COMPLETENESS
+  MEDIA
+  DIGITIZATION
+  STREET
 }
 
 type ProjectTypeSwipeStatsType {

docker-compose.yaml

@@ -76,6 +76,7 @@ x-mapswipe-workers: &base_mapswipe_workers
         SLACK_CHANNEL: '${SLACK_CHANNEL}'
         SENTRY_DSN: '${SENTRY_DSN}'
         OSMCHA_API_KEY: '${OSMCHA_API_KEY}'
+        MAPILLARY_API_KEY: '${MAPILLARY_API_KEY}'
     depends_on:
         - postgres
     volumes:

example.env

@@ -75,3 +75,6 @@ COMMUNITY_DASHBOARD_GRAPHQL_ENDPOINT=https://api.example.com/graphql/
 COMMUNITY_DASHBOARD_SENTRY_DSN=
 COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE=
 COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org
+
+# Mapillary
+MAPILLARY_API_KEY=

mapswipe_workers/mapswipe_workers/definitions.py

@@ -16,6 +16,8 @@
 OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/"
 OSMCHA_API_LINK = "https://osmcha.org/api/v1/"
 OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"]
+MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/"
+MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"]
 
 # number of geometries for project geometries
 MAX_INPUT_GEOMETRIES = 10
@@ -134,6 +136,7 @@ class ProjectType(Enum):
     COMPLETENESS = 4
     MEDIA_CLASSIFICATION = 5
     DIGITIZATION = 6
+    STREET = 7
 
     @property
     def constructor(self):
@@ -145,6 +148,7 @@ def constructor(self):
             DigitizationProject,
             FootprintProject,
             MediaClassificationProject,
+            StreetProject,
         )
 
         project_type_classes = {
@@ -154,6 +158,7 @@ def constructor(self):
             4: CompletenessProject,
             5: MediaClassificationProject,
             6: DigitizationProject,
+            7: StreetProject,
         }
         return project_type_classes[self.value]
 

mapswipe_workers/mapswipe_workers/project_types/__init__.py

@@ -2,6 +2,7 @@
 from .arbitrary_geometry.footprint.project import FootprintProject
 from .arbitrary_geometry.footprint.tutorial import FootprintTutorial
 from .media_classification.project import MediaClassificationProject
+from .street.project import StreetProject
 from .tile_map_service.change_detection.project import ChangeDetectionProject
 from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial
 from .tile_map_service.classification.project import ClassificationProject
@@ -20,4 +21,5 @@
     "FootprintProject",
     "FootprintTutorial",
     "DigitizationProject",
+    "StreetProject",
 ]

mapswipe_workers/mapswipe_workers/project_types/street/project.py

@@ -0,0 +1,134 @@
+import math
+from dataclasses import dataclass
+from typing import Dict, List
+
+from mapswipe_workers.definitions import logger
+from mapswipe_workers.firebase.firebase import Firebase
+from mapswipe_workers.firebase_to_postgres.transfer_results import (
+    results_to_file,
+    save_results_to_postgres,
+    truncate_temp_results,
+)
+from mapswipe_workers.generate_stats.project_stats import (
+    get_statistics_for_integer_result_project,
+)
+from mapswipe_workers.project_types.project import BaseGroup, BaseProject, BaseTask
+from mapswipe_workers.utils.process_mapillary import get_image_metadata
+from mapswipe_workers.utils.validate_input import (
+    build_multipolygon_from_layer_geometries,
+    check_if_layer_has_too_many_geometries,
+    check_if_layer_is_empty,
+    load_geojson_to_ogr,
+    multipolygon_to_wkt,
+    save_geojson_to_file,
+)
+
+
+@dataclass
+class StreetGroup(BaseGroup):
+    # todo: does client use this, or only for the implementation of project creation?
+    pass
+
+
+@dataclass
+class StreetTask(BaseTask):
+    geometry: str
+
+
+class StreetProject(BaseProject):
+    def __init__(self, project_draft):
+        super().__init__(project_draft)
+        self.groups: Dict[str, StreetGroup] = {}
+        self.tasks: Dict[str, List[StreetTask]] = {}
+
+        self.geometry = project_draft["geometry"]
+
+        # TODO: validate inputs
+        ImageMetadata = get_image_metadata(
+            self.geometry,
+            is_pano=project_draft.get("isPano", None),
+            start_time=project_draft.get("startTimestamp", None),
+            end_time=project_draft.get("endTimestamp", None),
+            organization_id=project_draft.get("organizationId", None),
+            sampling_threshold=project_draft.get("samplingThreshold", None),
+        )
+
+        self.imageIds = ImageMetadata["ids"]
+        self.imageGeometries = ImageMetadata["geometries"]
+
+    def save_tasks_to_firebase(self, projectId: str, tasks: dict):
+        firebase = Firebase()
+        firebase.save_tasks_to_firebase(projectId, tasks, useCompression=False)
+
+    @staticmethod
+    def results_to_postgres(results: dict, project_id: str, filter_mode: bool):
+        """How to move the result data from firebase to postgres."""
+        results_file, user_group_results_file = results_to_file(results, project_id)
+        truncate_temp_results()
+        save_results_to_postgres(results_file, project_id, filter_mode)
+        return user_group_results_file
+
+    @staticmethod
+    def get_per_project_statistics(project_id, project_info):
+        """How to aggregate the project results."""
+        return get_statistics_for_integer_result_project(
+            project_id, project_info, generate_hot_tm_geometries=False
+        )
+
+    def validate_geometries(self):
+        self.inputGeometriesFileName = save_geojson_to_file(
+            self.projectId, self.geometry
+        )
+        layer, datasource = load_geojson_to_ogr(
+            self.projectId, self.inputGeometriesFileName
+        )
+
+        # check if inputs fit constraints
+        check_if_layer_is_empty(self.projectId, layer)
+
+        multi_polygon, project_area = build_multipolygon_from_layer_geometries(
+            self.projectId, layer
+        )
+
+        check_if_layer_has_too_many_geometries(self.projectId, multi_polygon)
+
+        del datasource
+        del layer
+
+        logger.info(
+            f"{self.projectId}" f" - validate geometry - " f"input geometry is correct."
+        )
+        wkt_geometry = multipolygon_to_wkt(multi_polygon)
+        return wkt_geometry
+
+    def create_groups(self):
+        self.numberOfGroups = math.ceil(len(self.imageIds) / self.groupSize)
+        for group_id in range(self.numberOfGroups):
+            self.groups[f"g{group_id}"] = StreetGroup(
+                projectId=self.projectId,
+                groupId=f"g{group_id}",
+                progress=0,
+                finishedCount=0,
+                requiredCount=0,
+                numberOfTasks=self.groupSize,
+            )
+
+    def create_tasks(self):
+        if len(self.groups) == 0:
+            raise ValueError("Groups needs to be created before tasks can be created.")
+        for group_id, group in self.groups.items():
+            self.tasks[group_id] = []
+            for i in range(self.groupSize):
+                task = StreetTask(
+                    projectId=self.projectId,
+                    groupId=group_id,
+                    geometry=self.imageGeometries.pop(),
+                    taskId=self.imageIds.pop(),
+                )
+                self.tasks[group_id].append(task)
+
+                # list now empty? if usual group size is not reached
+                # the actual number of tasks for the group is updated
+                if not self.imageIds:
+                    group.numberOfTasks = i + 1
+                    break

mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py

@@ -0,0 +1,14 @@
+from mapswipe_workers.project_types.tutorial import BaseTutorial
+
+
+class StreetTutorial(BaseTutorial):
+    """The subclass for an TMS Grid based Tutorial."""
+
+    def save_tutorial(self):
+        raise NotImplementedError("Currently Street has no Tutorial")
+
+    def create_tutorial_groups(self):
+        raise NotImplementedError("Currently Street has no Tutorial")
+
+    def create_tutorial_tasks(self):
+        raise NotImplementedError("Currently Street has no Tutorial")