Merge pull request #470 from creativecommons/add-translation

App: improve translation tooling

docs/translation.md

@@ -60,21 +60,29 @@ Documentation:
 1. Add language to appropriate resource in Transifex
 2. Ensure language is present in Django
    - If not, update `cc_legal_tools/settings/base.py`
-3. Add objects for new language translation using the `add_objects` management
+3. Add objects for new language translation using the `add_translation`
+   management
    command.
    - Examples:
         ```shell
-        docker compose exec app ./manage.py add_objects -v2 --licenses -l tlh
+        docker compose exec app ./manage.py add_translation -v2 --licenses -l tlh
         ```
         ```shell
-        docker compose exec app ./manage.py add_objects -v2 --zero -l tlh
+        docker compose exec app ./manage.py add_translation -v2 --zero -l tlh
         ```
 4. Synchronize repository Gettext files with Transifex
 5. Compile `.mo` machine object Gettext files:
     ```shell
     docker compose exec app ./manage.py compilemessages
     ```
 
+Documentation:
+- [Quick start guide — polib documentation][polibdocs]
+- Also see How the tool translation is implemented documentation, above
+
+[polibdocs]: https://polib.readthedocs.io/en/latest/quickstart.html
+
+
 ## Synchronize repository Gettext files with Transifex
 
 - **TODO** document processes of synchronizing the repository Gettext files

legal_tools/management/commands/add_objects.py → i18n/management/commands/add_translation.py

@@ -1,12 +1,19 @@
 # Standard library
+import datetime
 import logging
+import os.path
 from argparse import ArgumentParser
 
 # Third-party
+import polib
 from django.conf import settings
 from django.core.management import BaseCommand, CommandError
 
 # First-party/Local
+from i18n.utils import (
+    map_django_to_transifex_language_code,
+    save_pofile_as_pofile_and_mofile,
+)
 from legal_tools.models import LegalCode, Tool
 
 LOG = logging.getLogger(__name__)
@@ -16,6 +23,7 @@
     2: logging.INFO,
     3: logging.DEBUG,
 }
+NOW = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S+0000")
 
 
 class Command(BaseCommand):
@@ -54,6 +62,49 @@ def add_arguments(self, parser: ArgumentParser):
             help="dry run: do not make any changes",
         )
 
+    def write_po_files(
+        self,
+        legal_code,
+        language_code,
+    ):
+        po_filename = legal_code.translation_filename()
+        pofile = polib.POFile()
+        transifex_language = map_django_to_transifex_language_code(
+            language_code
+        )
+
+        # Use the English message text as the message key
+        en_pofile_path = legal_code.get_english_pofile_path()
+        en_pofile_obj = polib.pofile(en_pofile_path)
+        for entry in en_pofile_obj:
+            pofile.append(polib.POEntry(msgid=entry.msgid, msgstr=""))
+
+        # noqa: E501
+        # https://www.gnu.org/software/gettext/manual/html_node/Header-Entry.html
+        pofile.metadata = {
+            "Content-Transfer-Encoding": "8bit",
+            "Content-Type": "text/plain; charset=UTF-8",
+            "Language": transifex_language,
+            "Language-Django": language_code,
+            "Language-Transifex": transifex_language,
+            "Language-Team": "https://www.transifex.com/creativecommons/CC/",
+            "MIME-Version": "1.0",
+            "PO-Revision-Date": NOW,
+            "Percent-Translated": pofile.percent_translated(),
+            "Project-Id-Version": legal_code.tool.resource_slug,
+        }
+
+        directory = os.path.dirname(po_filename)
+        if not os.path.isdir(directory):
+            os.makedirs(directory)
+        # Save mofile ourself. We could call 'compilemessages' but
+        # it wants to compile everything, which is both overkill
+        # and can fail if the venv or project source is not
+        # writable. We know this dir is writable, so just save this
+        # pofile and mofile ourselves.
+        LOG.info(f"Writing {po_filename.replace('.po', '')}.(mo|po)")
+        save_pofile_as_pofile_and_mofile(pofile, po_filename)
+
     def add_legal_code(self, options, category, version, unit=None):
         tool_parameters = {"category": category, "version": version}
         if unit is not None:
@@ -67,10 +118,19 @@ def add_legal_code(self, options, category, version, unit=None):
             }
             if LegalCode.objects.filter(**legal_code_parameters).exists():
                 LOG.warn(f"LegalCode object already exists: {title}")
+                legal_code = LegalCode.objects.get(**legal_code_parameters)
             else:
                 LOG.info(f"Creating LeglCode object: {title}")
                 if not options["dryrun"]:
-                    _ = LegalCode.objects.create(**legal_code_parameters)
+                    legal_code = LegalCode.objects.create(
+                        **legal_code_parameters
+                    )
+            if not options["dryrun"]:
+                po_filename = legal_code.translation_filename()
+                if os.path.isfile(po_filename):
+                    LOG.debug(f"File already exists: {po_filename}")
+                else:
+                    self.write_po_files(legal_code, options["language"])
 
     def handle(self, **options):
         LOG.setLevel(LOG_LEVELS[int(options["verbosity"])])

i18n/tests/test_transifex.py

@@ -2236,42 +2236,19 @@ def test_upload_translation_to_transifex_resource_present(self):
         api.Resource.get.assert_not_called()
         api.ResourceTranslationsAsyncUpload.upload.assert_not_called()
 
-    def test_upload_translation_to_transifex_resource_dryrun(self):
+    def test_upload_translation_to_transifex_resource_present_forced(self):
         api = self.helper.api
-        self.helper.dryrun = True
         resource_slug = "x_slug_x"
         language_code = "x_lang_code_x"
         transifex_code = "x_trans_code_x"
         pofile_path = "x_path_x"
         pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
-        push_overwrite = False
-        self.helper._resource_stats = {resource_slug: None}
-        self.helper._translation_stats = {resource_slug: {}}
-
-        self.helper.upload_translation_to_transifex_resource(
-            resource_slug,
-            language_code,
-            transifex_code,
-            pofile_path,
-            pofile_obj,
-            push_overwrite,
-        )
-
-        api.Language.get.assert_called_once()
-        api.Resource.get.assert_called_once()
-        api.ResourceTranslationsAsyncUpload.upload.assert_not_called()
-
-    def test_upload_translation_to_transifex_resource_miss_with_changes(self):
-        api = self.helper.api
-        resource_slug = "x_slug_x"
-        language_code = "x_lang_code_x"
-        transifex_code = "x_trans_code_x"
-        pofile_path = "x_path_x"
-        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
-        push_overwrite = False
+        push_overwrite = True
         pofile_content = get_pofile_content(pofile_obj)
         self.helper._resource_stats = {resource_slug: {}}
-        self.helper._translation_stats = {resource_slug: {}}
+        self.helper._translation_stats = {
+            resource_slug: {transifex_code: {"translated_strings": 99}}
+        }
         language = mock.Mock(
             id=f"l:{transifex_code}",
         )
@@ -2306,17 +2283,42 @@ def test_upload_translation_to_transifex_resource_miss_with_changes(self):
         )
         self.helper.clear_transifex_stats.assert_called_once()
 
-    def test_upload_translation_to_transifex_resource_push(self):
+    def test_upload_translation_to_transifex_resource_dryrun(self):
         api = self.helper.api
+        self.helper.dryrun = True
         resource_slug = "x_slug_x"
         language_code = "x_lang_code_x"
         transifex_code = "x_trans_code_x"
         pofile_path = "x_path_x"
         pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
-        push_overwrite = True
+        push_overwrite = False
+        self.helper._resource_stats = {resource_slug: None}
+        self.helper._translation_stats = {resource_slug: {}}
+
+        self.helper.upload_translation_to_transifex_resource(
+            resource_slug,
+            language_code,
+            transifex_code,
+            pofile_path,
+            pofile_obj,
+            push_overwrite,
+        )
+
+        api.Language.get.assert_called_once()
+        api.Resource.get.assert_called_once()
+        api.ResourceTranslationsAsyncUpload.upload.assert_not_called()
+
+    def test_upload_translation_to_transifex_resource_miss_with_changes(self):
+        api = self.helper.api
+        resource_slug = "x_slug_x"
+        language_code = "x_lang_code_x"
+        transifex_code = "x_trans_code_x"
+        pofile_path = "x_path_x"
+        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
+        push_overwrite = False
         pofile_content = get_pofile_content(pofile_obj)
-        self.helper._resource_stats = {}
-        self.helper._translation_stats = {}
+        self.helper._resource_stats = {resource_slug: {}}
+        self.helper._translation_stats = {resource_slug: {}}
         language = mock.Mock(
             id=f"l:{transifex_code}",
         )
@@ -2399,6 +2401,37 @@ def test_upload_translation_to_transifex_resource_no_changes(self):
         self.assertIn("Translation upload failed", log_context.output[2])
         self.helper.clear_transifex_stats.assert_not_called()
 
+    def test_upload_translation_to_transifex_resource_local_empty(self):
+        api = self.helper.api
+        resource_slug = "x_slug_x"
+        language_code = "x_lang_code_x"
+        transifex_code = "x_trans_code_x"
+        pofile_path = "x_path_x"
+        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
+        for entry in pofile_obj:
+            entry.msgstr = ""
+        push_overwrite = False
+        self.helper._resource_stats = {resource_slug: None}
+        self.helper._translation_stats = {resource_slug: {}}
+        self.helper.clear_transifex_stats = mock.Mock()
+
+        with self.assertLogs(self.helper.log, level="DEBUG") as log_context:
+            self.helper.upload_translation_to_transifex_resource(
+                resource_slug,
+                language_code,
+                transifex_code,
+                pofile_path,
+                pofile_obj,
+                push_overwrite,
+            )
+
+        api.Language.get.assert_not_called()
+        api.Resource.get.assert_not_called()
+        api.ResourceTranslationsAsyncUpload.upload.assert_not_called()
+        self.assertTrue(log_context.output[0].startswith("DEBUG:"))
+        self.assertIn("Skipping upload of 0% complete", log_context.output[0])
+        self.helper.clear_transifex_stats.assert_not_called()
+
     # Test: normalize_pofile_language ########################################
 
     def test_noramalize_pofile_language_correct(self):
@@ -2672,6 +2705,94 @@ def test_normalize_pofile_last_translator_incorrect(self):
         mock_pofile_save.assert_called()
         self.assertNotIn("Last-Translator", new_pofile_obj.metadata)
 
+    # Test: normalize_pofile_percent_translated ##############################
+
+    def test_normalize_pofile_percent_translated_resource_language(self):
+        transifex_code = settings.LANGUAGE_CODE
+        resource_slug = "x_slug_x"
+        resource_name = "x_name_x"
+        pofile_path = "x_path_x"
+        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
+        pofile_obj.metadata["Percent-Translated"] = (
+            pofile_obj.percent_translated()
+        )
+
+        with mock.patch.object(polib.POFile, "save") as mock_pofile_save:
+            self.helper.normalize_pofile_percent_translated(
+                transifex_code,
+                resource_slug,
+                resource_name,
+                pofile_path,
+                pofile_obj,
+            )
+
+        mock_pofile_save.assert_not_called()
+
+    def test_normalize_pofile_percent_translated_correct(self):
+        transifex_code = "x_trans_code_x"
+        resource_slug = "x_slug_x"
+        resource_name = "x_name_x"
+        pofile_path = "x_path_x"
+        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
+        pofile_obj.metadata["Percent-Translated"] = (
+            pofile_obj.percent_translated()
+        )
+
+        with mock.patch.object(polib.POFile, "save") as mock_pofile_save:
+            self.helper.normalize_pofile_percent_translated(
+                transifex_code,
+                resource_slug,
+                resource_name,
+                pofile_path,
+                pofile_obj,
+            )
+
+        mock_pofile_save.assert_not_called()
+
+    def test_normalize_pofile_percent_translated_dryrun(self):
+        self.helper.dryrun = True
+        transifex_code = "x_trans_code_x"
+        resource_slug = "x_slug_x"
+        resource_name = "x_name_x"
+        pofile_path = "x_path_x"
+        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
+        pofile_obj.metadata["Percent-Translated"] = 37
+
+        with mock.patch.object(polib.POFile, "save") as mock_pofile_save:
+            self.helper.normalize_pofile_percent_translated(
+                transifex_code,
+                resource_slug,
+                resource_name,
+                pofile_path,
+                pofile_obj,
+            )
+
+        mock_pofile_save.assert_not_called()
+
+    def test_normalize_pofile_percent_translated_incorrect(self):
+        transifex_code = "x_trans_code_x"
+        resource_slug = "x_slug_x"
+        resource_name = "x_name_x"
+        pofile_path = "x_path_x"
+        pofile_obj = polib.pofile(pofile=POFILE_CONTENT)
+        pofile_obj.metadata["Percent-Translated"] = 37
+
+        with mock.patch.object(polib.POFile, "save") as mock_pofile_save:
+            new_pofile_obj = self.helper.normalize_pofile_percent_translated(
+                transifex_code,
+                resource_slug,
+                resource_name,
+                pofile_path,
+                pofile_obj,
+            )
+
+        mock_pofile_save.assert_called()
+        self.assertIn("Percent-Translated", new_pofile_obj.metadata)
+        self.assertEqual(
+            new_pofile_obj.percent_translated(),
+            new_pofile_obj.metadata["Percent-Translated"],
+        )
+
     # Test: normalize_pofile_project_id ######################################
 
     def test_normalize_pofile_project_id_correct(self):