diff --git a/docs/translation.md b/docs/translation.md index 85706a8a..3b8d934a 100644 --- a/docs/translation.md +++ b/docs/translation.md @@ -60,14 +60,15 @@ Documentation: 1. Add language to appropriate resource in Transifex 2. Ensure language is present in Django - If not, update `cc_legal_tools/settings/base.py` -3. Add objects for new language translation using the `add_objects` management +3. Add objects for new language translation using the `add_translation` + management command. - Examples: ```shell - docker compose exec app ./manage.py add_objects -v2 --licenses -l tlh + docker compose exec app ./manage.py add_translation -v2 --licenses -l tlh ``` ```shell - docker compose exec app ./manage.py add_objects -v2 --zero -l tlh + docker compose exec app ./manage.py add_translation -v2 --zero -l tlh ``` 4. Synchronize repository Gettext files with Transifex 5. Compile `.mo` machine object Gettext files: @@ -75,6 +76,13 @@ Documentation: docker compose exec app ./manage.py compilemessages ``` +Documentation: +- [Quick start guide — polib documentation][polibdocs] +- Also see How the tool translation is implemented documentation, above + +[polibdocs]: https://polib.readthedocs.io/en/latest/quickstart.html + + ## Synchronize repository Gettext files with Transifex - **TODO** document processes of synchronizing the repository Gettext files diff --git a/legal_tools/management/commands/add_objects.py b/i18n/management/commands/add_translation.py similarity index 51% rename from legal_tools/management/commands/add_objects.py rename to i18n/management/commands/add_translation.py index 4ab6d9e5..5320edcf 100644 --- a/legal_tools/management/commands/add_objects.py +++ b/i18n/management/commands/add_translation.py @@ -1,12 +1,19 @@ # Standard library +import datetime import logging +import os.path from argparse import ArgumentParser # Third-party +import polib from django.conf import settings from django.core.management import BaseCommand, CommandError # First-party/Local +from i18n.utils import ( + map_django_to_transifex_language_code, + save_pofile_as_pofile_and_mofile, +) from legal_tools.models import LegalCode, Tool LOG = logging.getLogger(__name__) @@ -16,6 +23,7 @@ 2: logging.INFO, 3: logging.DEBUG, } +NOW = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S+0000") class Command(BaseCommand): @@ -54,6 +62,49 @@ def add_arguments(self, parser: ArgumentParser): help="dry run: do not make any changes", ) + def write_po_files( + self, + legal_code, + language_code, + ): + po_filename = legal_code.translation_filename() + pofile = polib.POFile() + transifex_language = map_django_to_transifex_language_code( + language_code + ) + + # Use the English message text as the message key + en_pofile_path = legal_code.get_english_pofile_path() + en_pofile_obj = polib.pofile(en_pofile_path) + for entry in en_pofile_obj: + pofile.append(polib.POEntry(msgid=entry.msgid, msgstr="")) + + # noqa: E501 + # https://www.gnu.org/software/gettext/manual/html_node/Header-Entry.html + pofile.metadata = { + "Content-Transfer-Encoding": "8bit", + "Content-Type": "text/plain; charset=UTF-8", + "Language": transifex_language, + "Language-Django": language_code, + "Language-Transifex": transifex_language, + "Language-Team": "https://www.transifex.com/creativecommons/CC/", + "MIME-Version": "1.0", + "PO-Revision-Date": NOW, + "Percent-Translated": pofile.percent_translated(), + "Project-Id-Version": legal_code.tool.resource_slug, + } + + directory = os.path.dirname(po_filename) + if not os.path.isdir(directory): + os.makedirs(directory) + # Save mofile ourself. We could call 'compilemessages' but + # it wants to compile everything, which is both overkill + # and can fail if the venv or project source is not + # writable. We know this dir is writable, so just save this + # pofile and mofile ourselves. + LOG.info(f"Writing {po_filename.replace('.po', '')}.(mo|po)") + save_pofile_as_pofile_and_mofile(pofile, po_filename) + def add_legal_code(self, options, category, version, unit=None): tool_parameters = {"category": category, "version": version} if unit is not None: @@ -67,10 +118,19 @@ def add_legal_code(self, options, category, version, unit=None): } if LegalCode.objects.filter(**legal_code_parameters).exists(): LOG.warn(f"LegalCode object already exists: {title}") + legal_code = LegalCode.objects.get(**legal_code_parameters) else: LOG.info(f"Creating LeglCode object: {title}") if not options["dryrun"]: - _ = LegalCode.objects.create(**legal_code_parameters) + legal_code = LegalCode.objects.create( + **legal_code_parameters + ) + if not options["dryrun"]: + po_filename = legal_code.translation_filename() + if os.path.isfile(po_filename): + LOG.debug(f"File already exists: {po_filename}") + else: + self.write_po_files(legal_code, options["language"]) def handle(self, **options): LOG.setLevel(LOG_LEVELS[int(options["verbosity"])]) diff --git a/i18n/tests/test_transifex.py b/i18n/tests/test_transifex.py index 01458d45..75ea654c 100644 --- a/i18n/tests/test_transifex.py +++ b/i18n/tests/test_transifex.py @@ -2236,42 +2236,19 @@ def test_upload_translation_to_transifex_resource_present(self): api.Resource.get.assert_not_called() api.ResourceTranslationsAsyncUpload.upload.assert_not_called() - def test_upload_translation_to_transifex_resource_dryrun(self): + def test_upload_translation_to_transifex_resource_present_forced(self): api = self.helper.api - self.helper.dryrun = True resource_slug = "x_slug_x" language_code = "x_lang_code_x" transifex_code = "x_trans_code_x" pofile_path = "x_path_x" pofile_obj = polib.pofile(pofile=POFILE_CONTENT) - push_overwrite = False - self.helper._resource_stats = {resource_slug: None} - self.helper._translation_stats = {resource_slug: {}} - - self.helper.upload_translation_to_transifex_resource( - resource_slug, - language_code, - transifex_code, - pofile_path, - pofile_obj, - push_overwrite, - ) - - api.Language.get.assert_called_once() - api.Resource.get.assert_called_once() - api.ResourceTranslationsAsyncUpload.upload.assert_not_called() - - def test_upload_translation_to_transifex_resource_miss_with_changes(self): - api = self.helper.api - resource_slug = "x_slug_x" - language_code = "x_lang_code_x" - transifex_code = "x_trans_code_x" - pofile_path = "x_path_x" - pofile_obj = polib.pofile(pofile=POFILE_CONTENT) - push_overwrite = False + push_overwrite = True pofile_content = get_pofile_content(pofile_obj) self.helper._resource_stats = {resource_slug: {}} - self.helper._translation_stats = {resource_slug: {}} + self.helper._translation_stats = { + resource_slug: {transifex_code: {"translated_strings": 99}} + } language = mock.Mock( id=f"l:{transifex_code}", ) @@ -2306,17 +2283,42 @@ def test_upload_translation_to_transifex_resource_miss_with_changes(self): ) self.helper.clear_transifex_stats.assert_called_once() - def test_upload_translation_to_transifex_resource_push(self): + def test_upload_translation_to_transifex_resource_dryrun(self): api = self.helper.api + self.helper.dryrun = True resource_slug = "x_slug_x" language_code = "x_lang_code_x" transifex_code = "x_trans_code_x" pofile_path = "x_path_x" pofile_obj = polib.pofile(pofile=POFILE_CONTENT) - push_overwrite = True + push_overwrite = False + self.helper._resource_stats = {resource_slug: None} + self.helper._translation_stats = {resource_slug: {}} + + self.helper.upload_translation_to_transifex_resource( + resource_slug, + language_code, + transifex_code, + pofile_path, + pofile_obj, + push_overwrite, + ) + + api.Language.get.assert_called_once() + api.Resource.get.assert_called_once() + api.ResourceTranslationsAsyncUpload.upload.assert_not_called() + + def test_upload_translation_to_transifex_resource_miss_with_changes(self): + api = self.helper.api + resource_slug = "x_slug_x" + language_code = "x_lang_code_x" + transifex_code = "x_trans_code_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + push_overwrite = False pofile_content = get_pofile_content(pofile_obj) - self.helper._resource_stats = {} - self.helper._translation_stats = {} + self.helper._resource_stats = {resource_slug: {}} + self.helper._translation_stats = {resource_slug: {}} language = mock.Mock( id=f"l:{transifex_code}", ) @@ -2399,6 +2401,37 @@ def test_upload_translation_to_transifex_resource_no_changes(self): self.assertIn("Translation upload failed", log_context.output[2]) self.helper.clear_transifex_stats.assert_not_called() + def test_upload_translation_to_transifex_resource_local_empty(self): + api = self.helper.api + resource_slug = "x_slug_x" + language_code = "x_lang_code_x" + transifex_code = "x_trans_code_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + for entry in pofile_obj: + entry.msgstr = "" + push_overwrite = False + self.helper._resource_stats = {resource_slug: None} + self.helper._translation_stats = {resource_slug: {}} + self.helper.clear_transifex_stats = mock.Mock() + + with self.assertLogs(self.helper.log, level="DEBUG") as log_context: + self.helper.upload_translation_to_transifex_resource( + resource_slug, + language_code, + transifex_code, + pofile_path, + pofile_obj, + push_overwrite, + ) + + api.Language.get.assert_not_called() + api.Resource.get.assert_not_called() + api.ResourceTranslationsAsyncUpload.upload.assert_not_called() + self.assertTrue(log_context.output[0].startswith("DEBUG:")) + self.assertIn("Skipping upload of 0% complete", log_context.output[0]) + self.helper.clear_transifex_stats.assert_not_called() + # Test: normalize_pofile_language ######################################## def test_noramalize_pofile_language_correct(self): @@ -2672,6 +2705,94 @@ def test_normalize_pofile_last_translator_incorrect(self): mock_pofile_save.assert_called() self.assertNotIn("Last-Translator", new_pofile_obj.metadata) + # Test: normalize_pofile_percent_translated ############################## + + def test_normalize_pofile_percent_translated_resource_language(self): + transifex_code = settings.LANGUAGE_CODE + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = ( + pofile_obj.percent_translated() + ) + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_not_called() + + def test_normalize_pofile_percent_translated_correct(self): + transifex_code = "x_trans_code_x" + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = ( + pofile_obj.percent_translated() + ) + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_not_called() + + def test_normalize_pofile_percent_translated_dryrun(self): + self.helper.dryrun = True + transifex_code = "x_trans_code_x" + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = 37 + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_not_called() + + def test_normalize_pofile_percent_translated_incorrect(self): + transifex_code = "x_trans_code_x" + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = 37 + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + new_pofile_obj = self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_called() + self.assertIn("Percent-Translated", new_pofile_obj.metadata) + self.assertEqual( + new_pofile_obj.percent_translated(), + new_pofile_obj.metadata["Percent-Translated"], + ) + # Test: normalize_pofile_project_id ###################################### def test_normalize_pofile_project_id_correct(self): diff --git a/i18n/transifex.py b/i18n/transifex.py index ff39e977..59ac1df6 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -317,23 +317,38 @@ def upload_translation_to_transifex_resource( """ project_api = self.resource_to_api[resource_slug] + # Always perform following tests (regardless of push_overwrite) + # + # Raise error if attempting to push resource + if language_code == settings.LANGUAGE_CODE: + raise ValueError( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): This function," + " upload_translation_to_transifex_resource(), is for" + " translations, not sources." + ) + # Raise error if related resource is missing from Transifex + elif resource_slug not in self.resource_stats.keys(): + raise ValueError( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Transifex does not yet contain" + " resource. The upload_resource_to_transifex() function" + " must be called before this one " + " [upload_translation_to_transifex_resource()]." + ) + # Skip push if there is nothing to push (local translation empty) + elif pofile_obj.percent_translated() == 0: + self.log.debug( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Skipping upload of 0% complete" + f" translation: {pofile_path}" + ) + return + + # Only perform the follwoing tests if push_oversite is False if not push_overwrite: - if language_code == settings.LANGUAGE_CODE: - raise ValueError( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): This function," - " upload_translation_to_transifex_resource(), is for" - " translations, not sources." - ) - elif resource_slug not in self.resource_stats.keys(): - raise ValueError( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Transifex does not yet contain" - " resource. The upload_resource_to_transifex() function" - " must be called before this one " - " [upload_translation_to_transifex_resource()]." - ) - elif ( + # Skip push if Transifex translation isn't empty + if ( resource_slug in self.translation_stats and transifex_code in self.translation_stats[resource_slug] and self.translation_stats[resource_slug][transifex_code].get( @@ -343,8 +358,8 @@ def upload_translation_to_transifex_resource( ): self.log.debug( f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Transifex already contains" - " translation." + f" ({transifex_code}): Skipping upload of translation" + " already present on Transifex." ) return @@ -602,6 +617,34 @@ def normalize_pofile_last_translator( pofile_obj.save(pofile_path) return pofile_obj + def normalize_pofile_percent_translated( + self, + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ): + if transifex_code == settings.LANGUAGE_CODE: + return pofile_obj + + key = "Percent-Translated" + percent_translated = pofile_obj.percent_translated() + + if int(pofile_obj.metadata.get(key, 0)) == percent_translated: + return pofile_obj + + self.log.info( + f"{self.nop}{resource_name} ({resource_slug}) {transifex_code}:" + f" Correcting PO file '{key}':" + f"\n{pofile_path}: New value: '{percent_translated}'" + ) + if self.dryrun: + return pofile_obj + pofile_obj.metadata[key] = percent_translated + pofile_obj.save(pofile_path) + return pofile_obj + def normalize_pofile_project_id( self, transifex_code, @@ -656,6 +699,13 @@ def normalize_pofile_metadata( pofile_path, pofile_obj, ) + pofile_obj = self.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) pofile_obj = self.normalize_pofile_project_id( transifex_code, resource_slug, @@ -750,7 +800,7 @@ def normalize_pofile_dates( ) # Process revision date - if pofile_revision is None: + if pofile_revision is None and transifex_revision is not None: # Normalize Local PO File revision date if its empty or invalid pofile_obj = self.update_pofile_revision_datetime( resource_slug, @@ -1158,6 +1208,13 @@ def save_transifex_to_pofile( pofile=transifex_pofile_content.decode(), encoding="utf-8" ) + # Ensure correct metadata values for items unsupported by Transifex + transifex_obj.metadata["Language-Django"] = language_code + transifex_obj.metadata["Language-Transifex"] = transifex_code + transifex_obj.metadata["Percent-Translated"] = ( + transifex_obj.percent_translated() + ) + # Overrite local PO File self.log.info( f"{self.nop}{resource_slug} {language_code} ({transifex_code}):" @@ -1478,6 +1535,15 @@ def normalize_translations( ) transifex_translated = t_stats["translated_strings"] + # Normalize percent translated + pofile_obj = self.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + # Normalize Creation and Revision dates in local PO File pofile_obj = self.normalize_pofile_dates( resource_slug,