feat: Add export to Mapnik

tagmaps/__main__.py

@@ -70,7 +70,8 @@ def main():
         topic_modeling=cfg.topic_modeling,
         local_saturation_check=cfg.local_saturation_check,
         max_items=cfg.max_items,
-        logging_level=cfg.logging_level)
+        logging_level=cfg.logging_level,
+        mapnik_export=cfg.mapnik_export,)
 
     if cfg.load_from_intermediate or input_data.is_intermediate():
         # load data from intermediate (already filtered) results

tagmaps/classes/compile_output.py

@@ -28,7 +28,7 @@ class Compile():
     def write_shapes(cls,
                      bounds: AnalysisBounds,
                      shapes_and_meta_list,
-                     output_folder: Path):
+                     output_folder: Path, mapnik_export):
         """Main wrapper for writing
         all results to output
 
@@ -49,11 +49,11 @@ def write_shapes(cls,
         __, epsg_code = tagmaps.classes.utils.Utils.get_best_utmzone(
             bound_points_shapely)
         cls._compile_merge_shapes(
-            shapes_and_meta_list, epsg_code, output_folder)
+            shapes_and_meta_list, epsg_code, output_folder, mapnik_export)
 
     @classmethod
     def _compile_merge_shapes(cls, shapes_and_meta_list,
-                              epsg_code, output_folder):
+                              epsg_code, output_folder, mapnik_export):
         all_itemized_shapes = list()
         all_non_itemized_shapes = list()
         contains_emoji_output = False
@@ -79,7 +79,7 @@ def _compile_merge_shapes(cls, shapes_and_meta_list,
         if all_itemized_shapes:
             cls._write_all(
                 all_itemized_shapes, True,
-                contains_emoji_output, epsg_code, output_folder)
+                contains_emoji_output, epsg_code, output_folder, mapnik_export)
         if all_non_itemized_shapes:
             cls._write_all(
                 all_non_itemized_shapes, False,
@@ -145,7 +145,8 @@ def _getcompile_nonitemized_shapes(
 
     @classmethod
     def _write_all(cls, shapes, itemized,
-                   contains_emoji_output, epsg_code, output_folder):
+                   contains_emoji_output, epsg_code, output_folder,
+                   mapnik_export: bool = None):
         schema = cls._get_shape_schema(itemized)
         # update for emoji only run
         if itemized:
@@ -164,23 +165,26 @@ def _write_all(cls, shapes, itemized,
                 encoding='UTF-8', driver='ESRI Shapefile',
                 schema=schema, crs=from_epsg(epsg_code)) as shapefile:
             cls._attach_emojitable_handler(
-                shapefile,
-                shapes,
-                contains_emoji_output,
-                itemized, output_folder)
+                shapefile, shapes, contains_emoji_output,
+                itemized, output_folder, mapnik_export)
 
     @classmethod
-    def _attach_emojitable_handler(cls, shapefile,
-                                   shapes,
-                                   contains_emoji_output,
-                                   itemized, output_folder):
+    def _attach_emojitable_handler(
+            cls, shapefile, shapes, contains_emoji_output,
+            itemized, output_folder, mapnik_export):
         """If Emoji Output present, open csv for writing
         Note: refactor as optional property!
         """
         if contains_emoji_output:
-            # If newline is '', no translation takes place on write
+            if mapnik_export:
+                cls._write_all_shapes_emoji(
+                    shapefile, shapes, itemized)
+                return
+            # ArcGIS Bug: Emoji must be written to separate CSV file
+            # and joined back to the shapefile inside ArcGIS;
+            # Note: If newline is '', no translation takes place on write
             # that means: \n (LF) is written, not CRLF
-            with open(output_folder / f'emojiTable.csv',
+            with open(output_folder / 'emojiTable.csv',
                       "w", newline='', encoding='utf-8') as emoji_table:
                 emoji_table.write("FID,Emoji\n")
                 if itemized:
@@ -215,6 +219,20 @@ def _write_all_shapes(cls, shapefile, shapes,
                 cls._write_nonitemized_shape(
                     shapefile, shape)
 
+    @classmethod
+    def _write_all_shapes_emoji(
+            cls, shapefile, shapes,
+            itemized: bool):
+        for shape in shapes:
+            if itemized:
+                # check if colum 10 is set to 1
+                # == emoji record
+                cls._write_itemized_shape(
+                    shapefile, shape, False)
+            else:
+                cls._write_nonitemized_shape(
+                    shapefile, shape)
+
     @staticmethod
     def _write_nonitemized_shape(shapefile, shape):
         shapefile.write({
@@ -269,7 +287,7 @@ def _getcompile_itemized_shapes(
 
     @staticmethod
     def _get_himp(idx, shapes):
-        """check if current cluster is most
+        """Check if current cluster is most
         used for item
         Compares item str to previous item,
         if different, then himp=1
@@ -283,7 +301,7 @@ def _get_himp(idx, shapes):
             # if item is different to
             # previous item and
             # user count is not 1
-            # TODO: check - prviously and not shapes[idx][2]
+            # TODO: check - previously and not shapes[idx][2]
             h_imp = 1
         else:
             h_imp = 0

tagmaps/config/config.py

@@ -59,6 +59,7 @@ def __init__(self):
         self.output_folder = None
         self.config_folder = None
         self.cluster_cut_distance = None
+        self.mapnik_export = False
 
         # additional auto settings
         self.sort_out_always_set = set()
@@ -307,6 +308,13 @@ def parse_args(self):
                             help="A (relative) path to load from "
                             "intermediate (cleaned) data",
                             type=Path)
+        parser.add_argument("--mapnik_export",
+            help="If enabled, emoji and tags will be exported together, in a single "
+                "shapefile, to be used in the Mapnik renderer. Unlike the default "
+                "output, for ESRI ArcGIS/ArcPro, where "
+                "emoji need to be written to a separate file, to be joined later - "
+                "due to a bug in the ESRI software that continues to exists. ",
+            action="store_true")
 
         args = parser.parse_args()
         if args.verbose:
@@ -388,6 +396,8 @@ def parse_args(self):
                 self.resource_path / args.load_intermediate)
         if args.cluster_cut_distance:
             self.cluster_cut_distance = args.cluster_cut_distance
+        if args.mapnik_export:
+            self.mapnik_export = True
 
     def load_filterlists(self):
         """Load filterlists for filtering terms (in-string and full match)

tagmaps/tagmaps_.py

@@ -91,6 +91,13 @@ class TagMaps():
         Provide a cluster cut distance (in meters) where the clustering
         will be stopped. This will override the auto detection of cluster
         distance.
+
+    mapnik_export : bool (default=None)
+        If enabled, emoji and tags will be exported together, in a single
+        shapefile, to be used in the Mapnik renderer - unlike the default
+        output, for ESRI ArcGIS/ArcPro, where
+        emoji need to be written to a separate file, to be joined later -
+        due to a bug in the ESRI software that continues to exists.
     """
 
     class TMDec():
@@ -156,7 +163,7 @@ def __init__(
             limit_bottom_user_count: int = 5, topic_modeling: bool = False,
             local_saturation_check: bool = False, max_items: int = None,
             logging_level=None, topic_cluster: bool = None,
-            cluster_cut_distance: float = None):
+            cluster_cut_distance: float = None, mapnik_export: bool = None):
         """Init settings for Tag Maps Clustering"""
         if output_folder is None:
             output_folder = Path.cwd() / "02_Output"
@@ -166,6 +173,7 @@ def __init__(
         self.remove_long_tail = remove_long_tail
         self.limit_bottom_user_count = limit_bottom_user_count
         self.topic_modeling = topic_modeling
+        self.mapnik_export = mapnik_export
         if max_items is None:
             max_items = 1000
         if topic_cluster is None:
@@ -416,7 +424,8 @@ def _write_shapes(self, itemized=True):
         Compile.write_shapes(
             bounds=self.lbsn_data.bounds,
             shapes_and_meta_list=shapelist,
-            output_folder=self.output_folder
+            output_folder=self.output_folder,
+            mapnik_export=self.mapnik_export,
         )
 
     @TMDec.prepare_clustering_check