refactor: Multiple custom visualizations for custom/HDX exports

hotosm · Jan 29, 2025 · 22c20f1 · 22c20f1
1 parent 1eeb47c
commit 22c20f1
Show file tree

Hide file tree

Showing 10 changed files with 100 additions and 89 deletions.
diff --git a/API/api_worker.py b/API/api_worker.py
@@ -54,7 +54,7 @@
 # celery.conf.result_serializer = "json"
 # celery.conf.accept_content = ["application/json", "application/x-python-serialize"]
 celery.conf.task_track_started = True
-celery.conf.update(result_extended=True)
+celery.conf.update(result_extended=True, worker_pool_restarts=True)
 # celery.conf.task_reject_on_worker_lost = True
 # celery.conf.task_acks_late = True
 

diff --git a/requirements.txt b/requirements.txt
@@ -27,7 +27,7 @@ humanize==4.9.0
 python-slugify==8.0.1
 geomet==1.1.0
 PyYAML==6.0.1
-geojson-stats==0.2.5
+geojson-stats==0.2.6
 transliterate==1.10.2
 
 ## documentation

diff --git a/src/app.py b/src/app.py
@@ -1299,6 +1299,7 @@ def __init__(self, params, uid=None):
         self.default_export_base_name = (
             self.iso3.upper() if self.iso3 else self.params.dataset.dataset_prefix
         )
+
         self.default_export_path = os.path.join(
             export_path,
             self.uuid,
@@ -1307,6 +1308,7 @@ def __init__(self, params, uid=None):
         )
         if os.path.exists(self.default_export_path):
             shutil.rmtree(self.default_export_path, ignore_errors=True)
+
         os.makedirs(self.default_export_path)
 
         if USE_DUCK_DB_FOR_CUSTOM_EXPORTS is True:
@@ -1941,8 +1943,13 @@ def add_resource(self, resource_meta):
 
             # Add customviz if available
             if resource_meta.get("stats_html"):
+                dataset_customviz = self.dataset.get("customviz")
+                if not dataset_customviz:
+                    dataset_customviz = [{"name": resource_meta["name"], "url": resource_meta["stats_html"]}]
+                else:
+                    dataset_customviz.append({"name": resource_meta["name"], "url": resource_meta["stats_html"]})
                 self.dataset.update(
-                    {"customviz": [{"url": resource_meta["stats_html"]}]}
+                    {"customviz": dataset_customviz}
                 )
 
     def upload_dataset(self, dump_config_to_s3=False):

diff --git a/src/post_processing/geojson_stats.py b/src/post_processing/geojson_stats.py
@@ -1,61 +1,23 @@
 from geojson_stats.stats import Stats
 from geojson_stats.html import Html
 
-CONFIG_AREA = ["building"]
-CONFIG_LENGTH = ["highway", "waterway"]
-
-
 class GeoJSONStats(Stats):
-    """Used for collecting stats while processing GeoJSON files line by line"""
+    """Used for collecting stats while processing GeoJSON files"""
 
     def __init__(self, filters, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.config.clean = True
         self.config.properties_prop = "properties.tags"
 
-        if filters and filters.tags:
-            for tag in CONFIG_AREA:
-                if self.check_filter(filters.tags, tag):
-                    self.config.keys.append(tag)
-                    self.config.value_keys.append(tag)
-                    self.config.area = True
-
-            for tag in CONFIG_LENGTH:
-                if self.check_filter(filters.tags, tag):
-                    self.config.keys.append(tag)
-                    self.config.value_keys.append(tag)
-                    self.config.length = True
-
-    def check_filter(self, tags, tag):
-        """
-        Check if a tag is present in tag filters
-        """
-
-        if tags.all_geometry:
-            if tags.all_geometry.join_or and tag in tags.all_geometry.join_or:
-                return True
-            if tags.all_geometry.join_and and tag in tags.all_geometry.join_and:
-                return True
-        if tags.polygon:
-            if tags.polygon.join_or and tag in tags.polygon.join_or:
-                return True
-            if tags.polygon.join_and and tag in tags.polygon.join_and:
-                return True
-        if tags.line:
-            if tags.line.join_or and tag in tags.line.join_or:
-                return True
-            if tags.line.join_and and tag in tags.line.join_and:
-                return True
-
     def raw_data_line_stats(self, json_object: dict):
         """
         Process a GeoJSON line (for getting stats) and return that line
         """
         self.get_object_stats(json_object)
 
-    def html(self, tpl):
+    def html(self, tpl, tpl_params):
         """
         Returns stats Html object, generated from stats data using a template
         """
-        return Html(tpl, self)
+        return Html(tpl, self, tpl_params)
diff --git a/src/post_processing/processor.py b/src/post_processing/processor.py
@@ -4,9 +4,26 @@
 import os
 import pathlib
 
+CATEGORIES_CONFIG = {
+    "roads": {
+        "tag": "highway", "length": True, "area": False
+    },
+    "buildings": {
+        "tag": "building", "length": False, "area": True
+    },
+    "waterways": {
+        "tag": "waterway", "length": True, "area": False
+    },
+    "railways": {
+        "tag": "railway", "length": True, "area": False
+    },
+    "default": {
+        "tag": None, "length": False, "area": False
+    },
+}
 
 class PostProcessor:
-    """Used for posst-process data while processing GeoJSON files line by line"""
+    """Used for post-process GeoJSON files"""
 
     options = {}
     filters = {}
@@ -26,6 +43,10 @@ def post_process_line(self, line: str):
             fn(line_object)
 
         return json.dumps(line_object)
+
+    def get_categories_config(self, category_name):
+        config = CATEGORIES_CONFIG.get(category_name)
+        return config if config else CATEGORIES_CONFIG["default"]
 
     def custom(
         self, category_name, export_format_path, export_filename, file_export_path
@@ -35,19 +56,10 @@ def custom(
         """
         self.geoJSONStats.config.properties_prop = "properties"
 
-        category_tag = ""
-        if category_name == "roads":
-            category_tag = "highway"
-            self.geoJSONStats.config.length = True
-        elif category_name == "buildings":
-            category_tag = "building"
-            self.geoJSONStats.config.area = True
-        elif category_name == "waterways":
-            category_tag = "waterway"
-            self.geoJSONStats.config.length = True
-        elif category_name == "railways":
-            category_tag = "railway"
-            self.geoJSONStats.config.length = True
+        category_config = self.get_categories_config(category_name)
+        category_tag = category_config["tag"]
+        self.geoJSONStats.config.length = category_config["length"]
+        self.geoJSONStats.config.area = category_config["area"]
 
         if self.options["include_stats"]:
             if category_tag:
@@ -102,7 +114,7 @@ def custom(
                     project_root,
                     "{tpl}_tpl.html".format(tpl=tpl),
                 )
-                geojson_stats_html = self.geoJSONStats.html(tpl_path).build()
+                geojson_stats_html = self.geoJSONStats.html(tpl_path, {"title": f"{export_filename}.geojson"}).build()
                 upload_html_path = os.path.join(file_export_path, "stats-summary.html")
                 with open(upload_html_path, "w") as f:
                     f.write(geojson_stats_html)

diff --git a/src/post_processing/stats_building_tpl.html b/src/post_processing/stats_building_tpl.html
@@ -3,31 +3,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
-
         :root,
         :host,
         .hot-theme-light {
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
+            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-gray-200: #C4C3C5;
+            --hot-color-red-700: #C53639;
             --hot-color-gray-50: #F3F3F3;
+            --hot-color-gray-200: #C4C3C5;
+            --hot-color-gray-100: #E1E0E1;
+            --hot-color-gray-400: #9A969B;
+            --hot-color-gray-950: #2C3038;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
-            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -42,6 +42,11 @@
             vertical-align: baseline;
             background: transparent;
         }
+        h2 {
+            margin-left: var(--hot-spacing-medium);
+            color: var(--hot-color-gray-400);
+            font-weight: var(--hot-font-weight-normal);
+        }
         .container {
             display: flex;
             width: 100%;
@@ -96,6 +101,7 @@
   </head>
   <body>
     <div id="root">
+        <h2>{title}</h2>
         <div class="container">
             <div class="box featured">
                 <h3>${key_building_area}</h3>

diff --git a/src/post_processing/stats_highway_tpl.html b/src/post_processing/stats_highway_tpl.html
@@ -2,31 +2,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
-
         :root,
         :host,
         .hot-theme-light {
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
+            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-gray-200: #C4C3C5;
+            --hot-color-red-700: #C53639;
             --hot-color-gray-50: #F3F3F3;
+            --hot-color-gray-200: #C4C3C5;
+            --hot-color-gray-100: #E1E0E1;
+            --hot-color-gray-400: #9A969B;
+            --hot-color-gray-950: #2C3038;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
-            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -41,6 +41,11 @@
             vertical-align: baseline;
             background: transparent;
         }
+        h2 {
+            margin-left: var(--hot-spacing-medium);
+            color: var(--hot-color-gray-400);
+            font-weight: var(--hot-font-weight-normal);
+        }
         .container {
             display: flex;
             width: 100%;
@@ -95,6 +100,7 @@
   </head>
   <body>
     <div id="root">
+        <h2>{title}</h2>
         <div class="container">
             <div class="box featured">
                 <h3>${key_highway_length}</h3>

diff --git a/src/post_processing/stats_railway_tpl.html b/src/post_processing/stats_railway_tpl.html
@@ -3,31 +3,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
-
         :root,
         :host,
         .hot-theme-light {
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
+            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-gray-200: #C4C3C5;
+            --hot-color-red-700: #C53639;
             --hot-color-gray-50: #F3F3F3;
+            --hot-color-gray-200: #C4C3C5;
+            --hot-color-gray-100: #E1E0E1;
+            --hot-color-gray-400: #9A969B;
+            --hot-color-gray-950: #2C3038;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
-            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -42,6 +42,11 @@
             vertical-align: baseline;
             background: transparent;
         }
+        h2 {
+            margin-left: var(--hot-spacing-medium);
+            color: var(--hot-color-gray-400);
+            font-weight: var(--hot-font-weight-normal);
+        }
         .container {
             display: flex;
             width: 100%;
@@ -96,6 +101,7 @@
   </head>
   <body>
     <div id="root">
+        <h2>{title}</h2>
         <div class="container">
             <div class="box featured">
                 <h3>${key_railway_length}</h3>