Merge pull request #291 from hotosm/refactor/multipleCustomviz

Refactor for custom visualizations for HDX custom exports
hotosm · Jan 30, 2025 · 144b898 · 144b898
2 parents 1eeb47c + 1a38f9a
commit 144b898
Show file tree

Hide file tree

Showing 11 changed files with 115 additions and 96 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,10 @@
+## 1.5.12 (2025-01-29)
+
+### Refactor
+
+- Remove unused stats collection, fix stats templates
+- Multiple custom visualizations for custom/HDX exports
+
 ## 1.5.11 (2025-01-28)
 
 ### Fix

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "raw-data-api"
-version = "1.5.11"
+version = "1.5.12"
 description = "Set of high-performant APIs for transforming and exporting OpenStreetMap (OSM) data in different GIS file formats."
 readme = "README.md"
 authors = [

diff --git a/requirements.txt b/requirements.txt
@@ -27,7 +27,7 @@ humanize==4.9.0
 python-slugify==8.0.1
 geomet==1.1.0
 PyYAML==6.0.1
-geojson-stats==0.2.5
+geojson-stats==0.2.6
 transliterate==1.10.2
 
 ## documentation

diff --git a/src/app.py b/src/app.py
@@ -1299,6 +1299,7 @@ def __init__(self, params, uid=None):
         self.default_export_base_name = (
             self.iso3.upper() if self.iso3 else self.params.dataset.dataset_prefix
         )
+
         self.default_export_path = os.path.join(
             export_path,
             self.uuid,
@@ -1307,6 +1308,7 @@ def __init__(self, params, uid=None):
         )
         if os.path.exists(self.default_export_path):
             shutil.rmtree(self.default_export_path, ignore_errors=True)
+
         os.makedirs(self.default_export_path)
 
         if USE_DUCK_DB_FOR_CUSTOM_EXPORTS is True:
@@ -1941,9 +1943,22 @@ def add_resource(self, resource_meta):
 
             # Add customviz if available
             if resource_meta.get("stats_html"):
-                self.dataset.update(
-                    {"customviz": [{"url": resource_meta["stats_html"]}]}
-                )
+                dataset_customviz = self.dataset.get("customviz")
+                if not dataset_customviz:
+                    dataset_customviz = [
+                        {
+                            "name": resource_meta["name"],
+                            "url": resource_meta["stats_html"],
+                        }
+                    ]
+                else:
+                    dataset_customviz.append(
+                        {
+                            "name": resource_meta["name"],
+                            "url": resource_meta["stats_html"],
+                        }
+                    )
+                self.dataset.update({"customviz": dataset_customviz})
 
     def upload_dataset(self, dump_config_to_s3=False):
         """

diff --git a/src/post_processing/geojson_stats.py b/src/post_processing/geojson_stats.py
@@ -1,61 +1,24 @@
 from geojson_stats.stats import Stats
 from geojson_stats.html import Html
 
-CONFIG_AREA = ["building"]
-CONFIG_LENGTH = ["highway", "waterway"]
-
 
 class GeoJSONStats(Stats):
-    """Used for collecting stats while processing GeoJSON files line by line"""
+    """Used for collecting stats while processing GeoJSON files"""
 
     def __init__(self, filters, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         self.config.clean = True
         self.config.properties_prop = "properties.tags"
 
-        if filters and filters.tags:
-            for tag in CONFIG_AREA:
-                if self.check_filter(filters.tags, tag):
-                    self.config.keys.append(tag)
-                    self.config.value_keys.append(tag)
-                    self.config.area = True
-
-            for tag in CONFIG_LENGTH:
-                if self.check_filter(filters.tags, tag):
-                    self.config.keys.append(tag)
-                    self.config.value_keys.append(tag)
-                    self.config.length = True
-
-    def check_filter(self, tags, tag):
-        """
-        Check if a tag is present in tag filters
-        """
-
-        if tags.all_geometry:
-            if tags.all_geometry.join_or and tag in tags.all_geometry.join_or:
-                return True
-            if tags.all_geometry.join_and and tag in tags.all_geometry.join_and:
-                return True
-        if tags.polygon:
-            if tags.polygon.join_or and tag in tags.polygon.join_or:
-                return True
-            if tags.polygon.join_and and tag in tags.polygon.join_and:
-                return True
-        if tags.line:
-            if tags.line.join_or and tag in tags.line.join_or:
-                return True
-            if tags.line.join_and and tag in tags.line.join_and:
-                return True
-
     def raw_data_line_stats(self, json_object: dict):
         """
         Process a GeoJSON line (for getting stats) and return that line
         """
         self.get_object_stats(json_object)
 
-    def html(self, tpl):
+    def html(self, tpl, tpl_params):
         """
         Returns stats Html object, generated from stats data using a template
         """
-        return Html(tpl, self)
+        return Html(tpl, self, tpl_params)
diff --git a/src/post_processing/processor.py b/src/post_processing/processor.py
@@ -4,9 +4,17 @@
 import os
 import pathlib
 
+CATEGORIES_CONFIG = {
+    "roads": {"tag": "highway", "length": True, "area": False},
+    "buildings": {"tag": "building", "length": False, "area": True},
+    "waterways": {"tag": "waterway", "length": True, "area": False},
+    "railways": {"tag": "railway", "length": True, "area": False},
+    "default": {"tag": None, "length": False, "area": False},
+}
+
 
 class PostProcessor:
-    """Used for posst-process data while processing GeoJSON files line by line"""
+    """Used for post-process GeoJSON files"""
 
     options = {}
     filters = {}
@@ -27,6 +35,13 @@ def post_process_line(self, line: str):
 
         return json.dumps(line_object)
 
+    def get_categories_config(self, category_name):
+        """
+        Get configuration for categories
+        """
+        config = CATEGORIES_CONFIG.get(category_name)
+        return config if config else CATEGORIES_CONFIG["default"]
+
     def custom(
         self, category_name, export_format_path, export_filename, file_export_path
     ):
@@ -35,25 +50,12 @@ def custom(
         """
         self.geoJSONStats.config.properties_prop = "properties"
 
-        category_tag = ""
-        if category_name == "roads":
-            category_tag = "highway"
-            self.geoJSONStats.config.length = True
-        elif category_name == "buildings":
-            category_tag = "building"
-            self.geoJSONStats.config.area = True
-        elif category_name == "waterways":
-            category_tag = "waterway"
-            self.geoJSONStats.config.length = True
-        elif category_name == "railways":
-            category_tag = "railway"
-            self.geoJSONStats.config.length = True
+        category_config = self.get_categories_config(category_name)
+        category_tag = category_config["tag"]
+        self.geoJSONStats.config.length = category_config["length"]
+        self.geoJSONStats.config.area = category_config["area"]
 
         if self.options["include_stats"]:
-            if category_tag:
-                self.geoJSONStats.config.keys.append(category_tag)
-                self.geoJSONStats.config.value_keys.append(category_tag)
-
             path_input = os.path.join(export_format_path, f"{export_filename}.geojson")
             path_output = os.path.join(
                 export_format_path, f"{export_filename}-post.geojson"
@@ -102,7 +104,9 @@ def custom(
                     project_root,
                     "{tpl}_tpl.html".format(tpl=tpl),
                 )
-                geojson_stats_html = self.geoJSONStats.html(tpl_path).build()
+                geojson_stats_html = self.geoJSONStats.html(
+                    tpl_path, {"title": f"{export_filename}.geojson"}
+                ).build()
                 upload_html_path = os.path.join(file_export_path, "stats-summary.html")
                 with open(upload_html_path, "w") as f:
                     f.write(geojson_stats_html)

diff --git a/src/post_processing/stats_building_tpl.html b/src/post_processing/stats_building_tpl.html
@@ -3,31 +3,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
-
         :root,
         :host,
         .hot-theme-light {
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
+            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-gray-200: #C4C3C5;
+            --hot-color-red-700: #C53639;
             --hot-color-gray-50: #F3F3F3;
+            --hot-color-gray-200: #C4C3C5;
+            --hot-color-gray-100: #E1E0E1;
+            --hot-color-gray-400: #9A969B;
+            --hot-color-gray-950: #2C3038;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
-            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -42,6 +42,11 @@
             vertical-align: baseline;
             background: transparent;
         }
+        h2 {
+            margin-left: var(--hot-spacing-medium);
+            color: var(--hot-color-gray-400);
+            font-weight: var(--hot-font-weight-normal);
+        }
         .container {
             display: flex;
             width: 100%;
@@ -96,9 +101,10 @@
   </head>
   <body>
     <div id="root">
+        <h2>${title}</h2>
         <div class="container">
             <div class="box featured">
-                <h3>${key_building_area}</h3>
+                <h3>${area}</h3>
                 <h4>Km2 of buildings</h4>
             </div>
             <div class="box">

diff --git a/src/post_processing/stats_highway_tpl.html b/src/post_processing/stats_highway_tpl.html
@@ -2,31 +2,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
-
         :root,
         :host,
         .hot-theme-light {
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
+            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-gray-200: #C4C3C5;
+            --hot-color-red-700: #C53639;
             --hot-color-gray-50: #F3F3F3;
+            --hot-color-gray-200: #C4C3C5;
+            --hot-color-gray-100: #E1E0E1;
+            --hot-color-gray-400: #9A969B;
+            --hot-color-gray-950: #2C3038;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
-            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -41,6 +41,11 @@
             vertical-align: baseline;
             background: transparent;
         }
+        h2 {
+            margin-left: var(--hot-spacing-medium);
+            color: var(--hot-color-gray-400);
+            font-weight: var(--hot-font-weight-normal);
+        }
         .container {
             display: flex;
             width: 100%;
@@ -95,9 +100,10 @@
   </head>
   <body>
     <div id="root">
+        <h2>${title}</h2>
         <div class="container">
             <div class="box featured">
-                <h3>${key_highway_length}</h3>
+                <h3>${length}</h3>
                 <h4>Km of roads</h4>
             </div>
             <div class="box">

diff --git a/src/post_processing/stats_railway_tpl.html b/src/post_processing/stats_railway_tpl.html
@@ -3,31 +3,31 @@
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/hot.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Archivo:ital,wght@0,100..900;1,100..900&display=swap" rel="stylesheet">
     <title>HOT Export Stats</title>
     <style type="text/css">
-
         :root,
         :host,
         .hot-theme-light {
-            --hot-color-red-700: #C53639;
-            --hot-color-gray-950: #2C3038;
             --hot-font-sans: Archivo, -apple-system, Roboto, Helvetica, Arial, sans-serif,
   'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
             --hot-font-size-2x-large: 2.25rem; /* 36px */
             --hot-font-size-medium: 1rem; /* 16px */
             --hot-font-size-small: 0.875rem; /* 14px */
+            --hot-font-weight-normal: 400;
             --hot-border-radius-medium: 0.25rem; /* 4px */
             --hot-color-neutral-0: #fff;
-            --hot-color-gray-200: #C4C3C5;
+            --hot-color-red-700: #C53639;
             --hot-color-gray-50: #F3F3F3;
+            --hot-color-gray-200: #C4C3C5;
+            --hot-color-gray-100: #E1E0E1;
+            --hot-color-gray-400: #9A969B;
+            --hot-color-gray-950: #2C3038;
             --hot-spacing-medium: 1rem;
             --hot-spacing-x-small: 0.5rem; /* 8px */
-            --hot-color-gray-100: #E1E0E1;
         }
         body {
             font-family: var(--hot-font-sans);
@@ -42,6 +42,11 @@
             vertical-align: baseline;
             background: transparent;
         }
+        h2 {
+            margin-left: var(--hot-spacing-medium);
+            color: var(--hot-color-gray-400);
+            font-weight: var(--hot-font-weight-normal);
+        }
         .container {
             display: flex;
             width: 100%;
@@ -96,6 +101,7 @@
   </head>
   <body>
     <div id="root">
+        <h2>${title}</h2>
         <div class="container">
             <div class="box featured">
                 <h3>${key_railway_length}</h3>