Add plugin for detecting values in Burmese Zawgyi encoding

Background: [Zawgyi](https://en.wikipedia.org/wiki/Zawgyi_font) is an
obsolete font encoding that is incompatible with proper Unicode.
Structurally, Zawgyi strings look like Unicode (they can be passed
around and stored as UTF-8, UTF-16, etc.), but when rendered,
the result get displayed as garbled characters unless the user happens
to install a non-standard font. With a non-standard font, the system
can render Zawgyi, but not properly encoded Unicode strings. Also,
because the Zawgyi encoding abuses codepoints intended for Myanmar’s
minority languages, installing a Zawgyi font breaks the display of
those minority languages.  The situation is a bit like with ISO 8859
in the 1980s, but worse because Zawgyi text and fonts pretend to be
Unicode. Because of the structural similarity to Unicode, detecting
Zawgyi is non-trivial and can only be done probabilistically.  As of
early 2025, Zawgyi is on the decline on the general Internet.
However, OpenStreetMap still contains thousands of objects with tag
values that are encoded in Burmese Zawgyi instead of proper Unicode.

This Osmose plugin makes use of Google’s open-source Zawgyi detector,
which uses Markov chains to estimate the likelihood of a Burmese
string being Zawgyi-encoded versus proper Unicode. The Osmose plugin
suggests fixes by converting mis-encoded strings from Zawgyi to
Unicode using the Unicode ICU library, which comes with a built-in
converter for this purpose.

Fixes osm-fr#2442.

docker/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get update && \
         git \
         libarchive-dev \
         libboost-python-dev \
+        libicu-dev \
         libosmpbf-dev \
         libprotobuf-dev \
         locales \
@@ -23,6 +24,7 @@ RUN apt-get update && \
         postgresql-client \
         protobuf-compiler \
         python3-dev \
+        python3-icu \
         python3-pip \
         python3-setuptools \
         python3-wheel \

mypy.ini

@@ -10,12 +10,16 @@ ignore_errors = True
 # External
 [mypy-pandas.*]
 ignore_missing_imports = True
+[mypy-icu.*]
+ignore_missing_imports = True
 [mypy-ipyleaflet.*]
 ignore_missing_imports = True
 [mypy-ipywidgets.*]
 ignore_missing_imports = True
 [mypy-antlr4.*]
 ignore_missing_imports = True
+[mypy-myanmartools.*]
+ignore_missing_imports = True
 [mypy-Pyro.*]
 ignore_missing_imports = True
 [mypy-shapely.*]

plugins/TagFix_ZawgyiBurmese.py

@@ -0,0 +1,102 @@
+#-*- coding: utf-8 -*-
+
+###########################################################################
+##                                                                       ##
+## Copyrights Sascha Brawer 2025                                         ##
+##                                                                       ##
+## This program is free software: you can redistribute it and/or modify  ##
+## it under the terms of the GNU General Public License as published by  ##
+## the Free Software Foundation, either version 3 of the License, or     ##
+## (at your option) any later version.                                   ##
+##                                                                       ##
+## This program is distributed in the hope that it will be useful,       ##
+## but WITHOUT ANY WARRANTY; without even the implied warranty of        ##
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         ##
+## GNU General Public License for more details.                          ##
+##                                                                       ##
+## You should have received a copy of the GNU General Public License     ##
+## along with this program.  If not, see <http://www.gnu.org/licenses/>. ##
+##                                                                       ##
+###########################################################################
+
+from modules.OsmoseTranslation import T_
+from plugins.Plugin import Plugin
+
+import myanmartools
+import icu
+
+
+# https://en.wikipedia.org/wiki/Zawgyi_font
+
+class TagFix_ZawgyiBurmese(Plugin):
+
+    only_for = ['MM']
+
+    def init(self, logger):
+        Plugin.init(self, logger)
+        self.errors[50706] = self.def_class(
+            item = 5070,
+            level = 2,
+            tags = ['value', 'fix:chair'],
+            title = T_('Value contains Zawgyi-encoded Burmese characters'),
+            detail = T_(
+'''Tag values in OpenStreetMap should be stored in Unicode. However, this
+value contains Burmese characters in the obsolete “Zawgyi” font encoding.
+As long as this value is stored in a non-standard way, modern devices cannot
+display it correctly. Please change the text to be encoded in Unicode.'''),
+        )
+        self.detector = myanmartools.ZawgyiDetector()
+        self.converter = icu.Transliterator.createInstance('Zawgyi-my')
+
+    def node(self, data, tags):
+        errs = []
+        for key, value in tags.items():
+            if not any(0x1000 <= ord(c) <= 0x109F for c in value):
+                continue
+            score = self.detector.get_zawgyi_probability(value)
+            if score < 0.8:
+                continue
+            fixed_value = self.converter.transliterate(value)
+            if value == fixed_value:
+                continue
+            errs.append({'class': 50706, 'subclass': 0, 'fix': {key: fixed_value}})
+        return errs
+
+    def way(self, data, tags, nodes):
+        return self.node(data, tags)
+
+    def relation(self, data, tags, members):
+        return self.node(data, tags)
+
+
+###########################################################################
+from plugins.Plugin import TestPluginCommon
+
+
+class Test(TestPluginCommon):
+    def test(self):
+        a = TagFix_ZawgyiBurmese(None)
+        a.init(None)
+        for name in [
+                '',
+                'foo',
+                'ဘားအံ',
+                'ကျိုက်မရော အဝေးပြေးလမ်း',
+        ]:
+            assert not a.node(None, {'name': name}), name
+            assert not a.way(None, {'name': name}, nodes=None), name
+            assert not a.relation(None, {'name': name}, members=None), name
+
+        for zawgyi, uni in [('မ္း', 'မ်း'), ('က္ေ', 'က်ေ')]:
+            self.check_err(
+                a.node(None, {'addr:street': zawgyi}),
+                {'class': 50706, 'subclass': 0, 'fix': {'addr:street': uni}},
+            )
+            self.check_err(
+                a.way(None, {'addr:city': zawgyi}, nodes=None),
+                {'class': 50706, 'subclass': 0, 'fix': {'addr:street': uni}},
+            )
+            self.check_err(
+                a.relation(None, {'fixme': zawgyi}, members=None),
+                {'class': 50706, 'subclass': 0, 'fix': {'addr:street': uni}},
+            )

requirements.txt

@@ -17,6 +17,8 @@ tiletanic
 sentry-sdk
 wikitextparser
 pycountry
+myanmartools
+PyICU
 
 # Tests
 pytest == 7.4.4 # In v8 it skips the plugins folder, see our issue #2266 and https://github.com/pytest-dev/pytest/issues/12605