From ff68e9e02e197f36a897afea90f30d61a1320491 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Thu, 11 Jul 2024 08:07:58 -0700 Subject: [PATCH 01/15] rename management command to add_translation --- docs/translation.md | 7 ++++--- .../management/commands/add_translation.py | 0 2 files changed, 4 insertions(+), 3 deletions(-) rename legal_tools/management/commands/add_objects.py => i18n/management/commands/add_translation.py (100%) diff --git a/docs/translation.md b/docs/translation.md index 85706a8a..477bb139 100644 --- a/docs/translation.md +++ b/docs/translation.md @@ -60,14 +60,15 @@ Documentation: 1. Add language to appropriate resource in Transifex 2. Ensure language is present in Django - If not, update `cc_legal_tools/settings/base.py` -3. Add objects for new language translation using the `add_objects` management +3. Add objects for new language translation using the `add_translation` + management command. - Examples: ```shell - docker compose exec app ./manage.py add_objects -v2 --licenses -l tlh + docker compose exec app ./manage.py add_translation -v2 --licenses -l tlh ``` ```shell - docker compose exec app ./manage.py add_objects -v2 --zero -l tlh + docker compose exec app ./manage.py add_translation -v2 --zero -l tlh ``` 4. Synchronize repository Gettext files with Transifex 5. Compile `.mo` machine object Gettext files: diff --git a/legal_tools/management/commands/add_objects.py b/i18n/management/commands/add_translation.py similarity index 100% rename from legal_tools/management/commands/add_objects.py rename to i18n/management/commands/add_translation.py From 6ae28bef94c4df14e7072c0788c4d68bd70521f5 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Thu, 11 Jul 2024 13:43:58 -0700 Subject: [PATCH 02/15] add .po portable object Gettext file and .mo machine object Gettext file generation --- i18n/management/commands/add_translation.py | 56 ++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/i18n/management/commands/add_translation.py b/i18n/management/commands/add_translation.py index 4ab6d9e5..bf9672dd 100644 --- a/i18n/management/commands/add_translation.py +++ b/i18n/management/commands/add_translation.py @@ -1,12 +1,19 @@ # Standard library +import datetime import logging +import os.path from argparse import ArgumentParser # Third-party +import polib from django.conf import settings from django.core.management import BaseCommand, CommandError # First-party/Local +from i18n.utils import ( + map_django_to_transifex_language_code, + save_pofile_as_pofile_and_mofile, +) from legal_tools.models import LegalCode, Tool LOG = logging.getLogger(__name__) @@ -16,6 +23,7 @@ 2: logging.INFO, 3: logging.DEBUG, } +NOW = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S+0000") class Command(BaseCommand): @@ -54,6 +62,49 @@ def add_arguments(self, parser: ArgumentParser): help="dry run: do not make any changes", ) + def write_po_files( + self, + legal_code, + language_code, + ): + po_filename = legal_code.translation_filename() + pofile = polib.POFile() + transifex_language = map_django_to_transifex_language_code( + language_code + ) + + # Use the English message text as the message key + en_pofile_path = legal_code.get_english_pofile_path() + en_pofile_obj = polib.pofile(en_pofile_path) + for entry in en_pofile_obj: + pofile.append(polib.POEntry(msgid=entry.msgid, msgstr="")) + + # noqa: E501 + # https://www.gnu.org/software/gettext/manual/html_node/Header-Entry.html + pofile.metadata = { + "Content-Transfer-Encoding": "8bit", + "Content-Type": "text/plain; charset=utf-8", + "Language": transifex_language, + "Language-Django": language_code, + "Language-Transifex": transifex_language, + "Language-Team": "https://www.transifex.com/creativecommons/CC/", + "MIME-Version": "1.0", + "PO-Revision-Date": NOW, + "Percent-Translated": pofile.percent_translated(), + "Project-Id-Version": legal_code.tool.resource_slug, + } + + directory = os.path.dirname(po_filename) + if not os.path.isdir(directory): + os.makedirs(directory) + # Save mofile ourself. We could call 'compilemessages' but + # it wants to compile everything, which is both overkill + # and can fail if the venv or project source is not + # writable. We know this dir is writable, so just save this + # pofile and mofile ourselves. + LOG.info(f"Writing {po_filename.replace('.po', '')}.(mo|po)") + save_pofile_as_pofile_and_mofile(pofile, po_filename) + def add_legal_code(self, options, category, version, unit=None): tool_parameters = {"category": category, "version": version} if unit is not None: @@ -70,7 +121,10 @@ def add_legal_code(self, options, category, version, unit=None): else: LOG.info(f"Creating LeglCode object: {title}") if not options["dryrun"]: - _ = LegalCode.objects.create(**legal_code_parameters) + legal_code = LegalCode.objects.create( + **legal_code_parameters + ) + self.write_po_files(legal_code, options["language"]) def handle(self, **options): LOG.setLevel(LOG_LEVELS[int(options["verbosity"])]) From b10e1fff34172342a2ee0ca39ae89b9872ce94c2 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Thu, 11 Jul 2024 14:15:26 -0700 Subject: [PATCH 03/15] add polib documentation --- docs/translation.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/translation.md b/docs/translation.md index 477bb139..3b8d934a 100644 --- a/docs/translation.md +++ b/docs/translation.md @@ -76,6 +76,13 @@ Documentation: docker compose exec app ./manage.py compilemessages ``` +Documentation: +- [Quick start guide — polib documentation][polibdocs] +- Also see How the tool translation is implemented documentation, above + +[polibdocs]: https://polib.readthedocs.io/en/latest/quickstart.html + + ## Synchronize repository Gettext files with Transifex - **TODO** document processes of synchronizing the repository Gettext files From 7f2eb03037d8d22d0264191975d3b56dfe79f8e5 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Thu, 11 Jul 2024 14:15:45 -0700 Subject: [PATCH 04/15] skip upload of 0% complete translation --- i18n/transifex.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/i18n/transifex.py b/i18n/transifex.py index ff39e977..03e7fc2c 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -343,8 +343,15 @@ def upload_translation_to_transifex_resource( ): self.log.debug( f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Transifex already contains" - " translation." + f" ({transifex_code}): Skipping upload of translation" + " already present on Transifex." + ) + return + elif pofile_obj.percent_translated() == 0: + self.log.debug( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Skipping upload of 0% complete" + f" translation: {pofile_path}" ) return From c194560cc65b05794d5c6348d982fbb57f963cfb Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Thu, 11 Jul 2024 14:56:34 -0700 Subject: [PATCH 05/15] improve upload_translation_to_transifex_resource logic and related tests. add check for empty translations --- i18n/tests/test_transifex.py | 70 +++++++++++++------------------- i18n/transifex.py | 77 +++++++++++++++++++----------------- 2 files changed, 68 insertions(+), 79 deletions(-) diff --git a/i18n/tests/test_transifex.py b/i18n/tests/test_transifex.py index 01458d45..755ce4e2 100644 --- a/i18n/tests/test_transifex.py +++ b/i18n/tests/test_transifex.py @@ -2306,17 +2306,17 @@ def test_upload_translation_to_transifex_resource_miss_with_changes(self): ) self.helper.clear_transifex_stats.assert_called_once() - def test_upload_translation_to_transifex_resource_push(self): + def test_upload_translation_to_transifex_resource_no_changes(self): api = self.helper.api resource_slug = "x_slug_x" language_code = "x_lang_code_x" transifex_code = "x_trans_code_x" pofile_path = "x_path_x" pofile_obj = polib.pofile(pofile=POFILE_CONTENT) - push_overwrite = True + push_overwrite = False pofile_content = get_pofile_content(pofile_obj) - self.helper._resource_stats = {} - self.helper._translation_stats = {} + self.helper._resource_stats = {resource_slug: {}} + self.helper._translation_stats = {resource_slug: {}} language = mock.Mock( id=f"l:{transifex_code}", ) @@ -2327,19 +2327,20 @@ def test_upload_translation_to_transifex_resource_push(self): ) self.helper.api.Resource.get = mock.Mock(return_value=resource) api.ResourceTranslationsAsyncUpload.upload.return_value = { - "translations_created": 1, - "translations_updated": 1, + "translations_created": 0, + "translations_updated": 0, } self.helper.clear_transifex_stats = mock.Mock() - self.helper.upload_translation_to_transifex_resource( - resource_slug, - language_code, - transifex_code, - pofile_path, - pofile_obj, - push_overwrite, - ) + with self.assertLogs(self.helper.log) as log_context: + self.helper.upload_translation_to_transifex_resource( + resource_slug, + language_code, + transifex_code, + pofile_path, + pofile_obj, + push_overwrite, + ) api.Language.get.assert_called_once() api.Resource.get.assert_called_once() @@ -2349,35 +2350,25 @@ def test_upload_translation_to_transifex_resource_push(self): content=pofile_content, language=language.id, ) - self.helper.clear_transifex_stats.assert_called_once() + self.assertTrue(log_context.output[2].startswith("CRITICAL:")) + self.assertIn("Translation upload failed", log_context.output[2]) + self.helper.clear_transifex_stats.assert_not_called() - def test_upload_translation_to_transifex_resource_no_changes(self): + def test_upload_translation_to_transifex_resource_local_empty(self): api = self.helper.api resource_slug = "x_slug_x" language_code = "x_lang_code_x" transifex_code = "x_trans_code_x" pofile_path = "x_path_x" pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + for entry in pofile_obj: + entry.msgstr = "" push_overwrite = False - pofile_content = get_pofile_content(pofile_obj) - self.helper._resource_stats = {resource_slug: {}} + self.helper._resource_stats = {resource_slug: None} self.helper._translation_stats = {resource_slug: {}} - language = mock.Mock( - id=f"l:{transifex_code}", - ) - self.helper.api.Language.get = mock.Mock(return_value=language) - resource = mock.Mock( - id=f"o:{TEST_ORG_SLUG}:p:{TEST_PROJ_SLUG}:r:{resource_slug}", - attributes={"i18n_type": "PO"}, - ) - self.helper.api.Resource.get = mock.Mock(return_value=resource) - api.ResourceTranslationsAsyncUpload.upload.return_value = { - "translations_created": 0, - "translations_updated": 0, - } self.helper.clear_transifex_stats = mock.Mock() - with self.assertLogs(self.helper.log) as log_context: + with self.assertLogs(self.helper.log, level="DEBUG") as log_context: self.helper.upload_translation_to_transifex_resource( resource_slug, language_code, @@ -2387,16 +2378,11 @@ def test_upload_translation_to_transifex_resource_no_changes(self): push_overwrite, ) - api.Language.get.assert_called_once() - api.Resource.get.assert_called_once() - api.ResourceTranslationsAsyncUpload.upload.assert_called_once() - api.ResourceTranslationsAsyncUpload.upload.assert_called_with( - resource=resource, - content=pofile_content, - language=language.id, - ) - self.assertTrue(log_context.output[2].startswith("CRITICAL:")) - self.assertIn("Translation upload failed", log_context.output[2]) + api.Language.get.assert_not_called() + api.Resource.get.assert_not_called() + api.ResourceTranslationsAsyncUpload.upload.assert_not_called() + self.assertTrue(log_context.output[0].startswith("DEBUG:")) + self.assertIn("Skipping upload of 0% complete", log_context.output[0]) self.helper.clear_transifex_stats.assert_not_called() # Test: normalize_pofile_language ######################################## diff --git a/i18n/transifex.py b/i18n/transifex.py index 03e7fc2c..e22262e5 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -317,43 +317,46 @@ def upload_translation_to_transifex_resource( """ project_api = self.resource_to_api[resource_slug] - if not push_overwrite: - if language_code == settings.LANGUAGE_CODE: - raise ValueError( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): This function," - " upload_translation_to_transifex_resource(), is for" - " translations, not sources." - ) - elif resource_slug not in self.resource_stats.keys(): - raise ValueError( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Transifex does not yet contain" - " resource. The upload_resource_to_transifex() function" - " must be called before this one " - " [upload_translation_to_transifex_resource()]." - ) - elif ( - resource_slug in self.translation_stats - and transifex_code in self.translation_stats[resource_slug] - and self.translation_stats[resource_slug][transifex_code].get( - "translated_strings", 0 - ) - > 0 - ): - self.log.debug( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Skipping upload of translation" - " already present on Transifex." - ) - return - elif pofile_obj.percent_translated() == 0: - self.log.debug( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Skipping upload of 0% complete" - f" translation: {pofile_path}" - ) - return + # Always perform following tests (regardless of push_overwrite) + if language_code == settings.LANGUAGE_CODE: + raise ValueError( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): This function," + " upload_translation_to_transifex_resource(), is for" + " translations, not sources." + ) + elif resource_slug not in self.resource_stats.keys(): + raise ValueError( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Transifex does not yet contain" + " resource. The upload_resource_to_transifex() function" + " must be called before this one " + " [upload_translation_to_transifex_resource()]." + ) + elif pofile_obj.percent_translated() == 0: + self.log.debug( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Skipping upload of 0% complete" + f" translation: {pofile_path}" + ) + return + + # Only perform tests if push_oversite is False + if ( + not push_overwrite + and resource_slug in self.translation_stats + and transifex_code in self.translation_stats[resource_slug] + and self.translation_stats[resource_slug][transifex_code].get( + "translated_strings", 0 + ) + > 0 + ): + self.log.debug( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Skipping upload of translation" + " already present on Transifex." + ) + return pofile_content = get_pofile_content(pofile_obj) language = self.api.Language.get(code=transifex_code) From 2c8917efbf2e37139973f1267d2b6287eb8e5a60 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Sat, 13 Jul 2024 00:31:34 -0700 Subject: [PATCH 06/15] add update_is_replaced_by and update_source --- i18n/management/commands/add_translation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/i18n/management/commands/add_translation.py b/i18n/management/commands/add_translation.py index bf9672dd..033239e5 100644 --- a/i18n/management/commands/add_translation.py +++ b/i18n/management/commands/add_translation.py @@ -7,7 +7,7 @@ # Third-party import polib from django.conf import settings -from django.core.management import BaseCommand, CommandError +from django.core.management import BaseCommand, CommandError, call_command # First-party/Local from i18n.utils import ( @@ -134,3 +134,5 @@ def handle(self, **options): self.add_legal_code(options, "licenses", "4.0") elif options["domains"] == "zero": self.add_legal_code(options, "publicdomain", "1.0", "zero") + call_command("update_is_replaced_by", verbosity=options["verbosity"]) + call_command("update_source", verbosity=options["verbosity"]) From 6980014cbded5cfe5bc0b9bb34c9cd93996e17a1 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 08:14:00 -0700 Subject: [PATCH 07/15] no need to invoke tools that act on Tool objects as this script only impacts LegalCode objects --- i18n/management/commands/add_translation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/i18n/management/commands/add_translation.py b/i18n/management/commands/add_translation.py index 033239e5..bf9672dd 100644 --- a/i18n/management/commands/add_translation.py +++ b/i18n/management/commands/add_translation.py @@ -7,7 +7,7 @@ # Third-party import polib from django.conf import settings -from django.core.management import BaseCommand, CommandError, call_command +from django.core.management import BaseCommand, CommandError # First-party/Local from i18n.utils import ( @@ -134,5 +134,3 @@ def handle(self, **options): self.add_legal_code(options, "licenses", "4.0") elif options["domains"] == "zero": self.add_legal_code(options, "publicdomain", "1.0", "zero") - call_command("update_is_replaced_by", verbosity=options["verbosity"]) - call_command("update_source", verbosity=options["verbosity"]) From f70c61b0e0ee02d0b1eb72134124366f5744fefc Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 08:36:25 -0700 Subject: [PATCH 08/15] improve logic and documentation (comments) --- i18n/transifex.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/i18n/transifex.py b/i18n/transifex.py index e22262e5..4bc07306 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -318,6 +318,8 @@ def upload_translation_to_transifex_resource( project_api = self.resource_to_api[resource_slug] # Always perform following tests (regardless of push_overwrite) + # + # Raise error if attempting to push resource if language_code == settings.LANGUAGE_CODE: raise ValueError( f"{self.nop}{resource_slug} {language_code}" @@ -325,6 +327,7 @@ def upload_translation_to_transifex_resource( " upload_translation_to_transifex_resource(), is for" " translations, not sources." ) + # Raise error if related resource is missing from Transifex elif resource_slug not in self.resource_stats.keys(): raise ValueError( f"{self.nop}{resource_slug} {language_code}" @@ -333,6 +336,7 @@ def upload_translation_to_transifex_resource( " must be called before this one " " [upload_translation_to_transifex_resource()]." ) + # Skip push if there is nothing to push (local translation empty) elif pofile_obj.percent_translated() == 0: self.log.debug( f"{self.nop}{resource_slug} {language_code}" @@ -341,22 +345,23 @@ def upload_translation_to_transifex_resource( ) return - # Only perform tests if push_oversite is False - if ( - not push_overwrite - and resource_slug in self.translation_stats - and transifex_code in self.translation_stats[resource_slug] - and self.translation_stats[resource_slug][transifex_code].get( - "translated_strings", 0 - ) - > 0 - ): - self.log.debug( - f"{self.nop}{resource_slug} {language_code}" - f" ({transifex_code}): Skipping upload of translation" - " already present on Transifex." - ) - return + # Only perform the follwoing tests if push_oversite is False + if not push_overwrite: + # Skip push if Transifex translation isn't empty + if ( + resource_slug in self.translation_stats + and transifex_code in self.translation_stats[resource_slug] + and self.translation_stats[resource_slug][transifex_code].get( + "translated_strings", 0 + ) + > 0 + ): + self.log.debug( + f"{self.nop}{resource_slug} {language_code}" + f" ({transifex_code}): Skipping upload of translation" + " already present on Transifex." + ) + return pofile_content = get_pofile_content(pofile_obj) language = self.api.Language.get(code=transifex_code) From d0a23ea9e32c4a20c37bfd10e90d5e0896a76203 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 08:55:05 -0700 Subject: [PATCH 09/15] improve po files support --- i18n/management/commands/add_translation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/i18n/management/commands/add_translation.py b/i18n/management/commands/add_translation.py index bf9672dd..5320edcf 100644 --- a/i18n/management/commands/add_translation.py +++ b/i18n/management/commands/add_translation.py @@ -83,7 +83,7 @@ def write_po_files( # https://www.gnu.org/software/gettext/manual/html_node/Header-Entry.html pofile.metadata = { "Content-Transfer-Encoding": "8bit", - "Content-Type": "text/plain; charset=utf-8", + "Content-Type": "text/plain; charset=UTF-8", "Language": transifex_language, "Language-Django": language_code, "Language-Transifex": transifex_language, @@ -118,12 +118,18 @@ def add_legal_code(self, options, category, version, unit=None): } if LegalCode.objects.filter(**legal_code_parameters).exists(): LOG.warn(f"LegalCode object already exists: {title}") + legal_code = LegalCode.objects.get(**legal_code_parameters) else: LOG.info(f"Creating LeglCode object: {title}") if not options["dryrun"]: legal_code = LegalCode.objects.create( **legal_code_parameters ) + if not options["dryrun"]: + po_filename = legal_code.translation_filename() + if os.path.isfile(po_filename): + LOG.debug(f"File already exists: {po_filename}") + else: self.write_po_files(legal_code, options["language"]) def handle(self, **options): From 59e284c8cecc97b14919f4d493c942fb32e2340c Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 09:24:05 -0700 Subject: [PATCH 10/15] add to pull: Ensure correct metadata values for items unsupported by Transifex --- i18n/transifex.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/i18n/transifex.py b/i18n/transifex.py index 4bc07306..5901fbea 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -1173,6 +1173,13 @@ def save_transifex_to_pofile( pofile=transifex_pofile_content.decode(), encoding="utf-8" ) + # Ensure correct metadata values for items unsupported by Transifex + transifex_obj.metadata["Language-Django"] = language_code + transifex_obj.metadata["Language-Transifex"] = transifex_code + transifex_obj.metadata["Percent-Translated"] = ( + transifex_obj.percent_translated() + ) + # Overrite local PO File self.log.info( f"{self.nop}{resource_slug} {language_code} ({transifex_code}):" From f7d7dd7df8cfb7132c82a16fc801e182f2ef2eba Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 09:51:22 -0700 Subject: [PATCH 11/15] add normalize_pofile_percent_translated and improve date logic (so that None value is not always "replaced" by None value) --- i18n/transifex.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/i18n/transifex.py b/i18n/transifex.py index 5901fbea..be5cb78c 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -617,6 +617,34 @@ def normalize_pofile_last_translator( pofile_obj.save(pofile_path) return pofile_obj + def normalize_pofile_percent_translated( + self, + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ): + if transifex_code == settings.LANGUAGE_CODE: + return pofile_obj + + key = "Percent-Translated" + percent_translated = pofile_obj.percent_translated() + + if int(pofile_obj.metadata.get(key, None)) == percent_translated: + return pofile_obj + + self.log.info( + f"{self.nop}{resource_name} ({resource_slug}) {transifex_code}:" + f" Correcting PO file '{key}':" + f"\n{pofile_path}: New value: '{percent_translated}'" + ) + if self.dryrun: + return pofile_obj + pofile_obj.metadata[key] = percent_translated + pofile_obj.save(pofile_path) + return pofile_obj + def normalize_pofile_project_id( self, transifex_code, @@ -671,6 +699,13 @@ def normalize_pofile_metadata( pofile_path, pofile_obj, ) + pofile_obj = self.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) pofile_obj = self.normalize_pofile_project_id( transifex_code, resource_slug, @@ -765,7 +800,7 @@ def normalize_pofile_dates( ) # Process revision date - if pofile_revision is None: + if pofile_revision is None and transifex_revision is not None: # Normalize Local PO File revision date if its empty or invalid pofile_obj = self.update_pofile_revision_datetime( resource_slug, From 5a6435269a7acc4249723803680f0058d0456d0d Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 09:52:41 -0700 Subject: [PATCH 12/15] fix type --- i18n/transifex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/i18n/transifex.py b/i18n/transifex.py index be5cb78c..eb6fafb7 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -631,7 +631,7 @@ def normalize_pofile_percent_translated( key = "Percent-Translated" percent_translated = pofile_obj.percent_translated() - if int(pofile_obj.metadata.get(key, None)) == percent_translated: + if int(pofile_obj.metadata.get(key, 0)) == percent_translated: return pofile_obj self.log.info( From dfeeef4076c271c6a7c3264dd34f9b19a5511637 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 10:03:53 -0700 Subject: [PATCH 13/15] add test_upload_translation_to_transifex_resource_present_forced --- i18n/tests/test_transifex.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/i18n/tests/test_transifex.py b/i18n/tests/test_transifex.py index 755ce4e2..7c384a57 100644 --- a/i18n/tests/test_transifex.py +++ b/i18n/tests/test_transifex.py @@ -2236,6 +2236,53 @@ def test_upload_translation_to_transifex_resource_present(self): api.Resource.get.assert_not_called() api.ResourceTranslationsAsyncUpload.upload.assert_not_called() + def test_upload_translation_to_transifex_resource_present_forced(self): + api = self.helper.api + resource_slug = "x_slug_x" + language_code = "x_lang_code_x" + transifex_code = "x_trans_code_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + push_overwrite = True + pofile_content = get_pofile_content(pofile_obj) + self.helper._resource_stats = {resource_slug: {}} + self.helper._translation_stats = { + resource_slug: {transifex_code: {"translated_strings": 99}} + } + language = mock.Mock( + id=f"l:{transifex_code}", + ) + self.helper.api.Language.get = mock.Mock(return_value=language) + resource = mock.Mock( + id=f"o:{TEST_ORG_SLUG}:p:{TEST_PROJ_SLUG}:r:{resource_slug}", + attributes={"i18n_type": "PO"}, + ) + self.helper.api.Resource.get = mock.Mock(return_value=resource) + api.ResourceTranslationsAsyncUpload.upload.return_value = { + "translations_created": 1, + "translations_updated": 1, + } + self.helper.clear_transifex_stats = mock.Mock() + + self.helper.upload_translation_to_transifex_resource( + resource_slug, + language_code, + transifex_code, + pofile_path, + pofile_obj, + push_overwrite, + ) + + api.Language.get.assert_called_once() + api.Resource.get.assert_called_once() + api.ResourceTranslationsAsyncUpload.upload.assert_called_once() + api.ResourceTranslationsAsyncUpload.upload.assert_called_with( + resource=resource, + content=pofile_content, + language=language.id, + ) + self.helper.clear_transifex_stats.assert_called_once() + def test_upload_translation_to_transifex_resource_dryrun(self): api = self.helper.api self.helper.dryrun = True From 9bf782f43b2b11513e2896ba3ab68609816d6b97 Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 10:18:32 -0700 Subject: [PATCH 14/15] add tests for normalize_pofile_percent_translated --- i18n/tests/test_transifex.py | 88 ++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/i18n/tests/test_transifex.py b/i18n/tests/test_transifex.py index 7c384a57..75ea654c 100644 --- a/i18n/tests/test_transifex.py +++ b/i18n/tests/test_transifex.py @@ -2705,6 +2705,94 @@ def test_normalize_pofile_last_translator_incorrect(self): mock_pofile_save.assert_called() self.assertNotIn("Last-Translator", new_pofile_obj.metadata) + # Test: normalize_pofile_percent_translated ############################## + + def test_normalize_pofile_percent_translated_resource_language(self): + transifex_code = settings.LANGUAGE_CODE + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = ( + pofile_obj.percent_translated() + ) + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_not_called() + + def test_normalize_pofile_percent_translated_correct(self): + transifex_code = "x_trans_code_x" + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = ( + pofile_obj.percent_translated() + ) + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_not_called() + + def test_normalize_pofile_percent_translated_dryrun(self): + self.helper.dryrun = True + transifex_code = "x_trans_code_x" + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = 37 + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_not_called() + + def test_normalize_pofile_percent_translated_incorrect(self): + transifex_code = "x_trans_code_x" + resource_slug = "x_slug_x" + resource_name = "x_name_x" + pofile_path = "x_path_x" + pofile_obj = polib.pofile(pofile=POFILE_CONTENT) + pofile_obj.metadata["Percent-Translated"] = 37 + + with mock.patch.object(polib.POFile, "save") as mock_pofile_save: + new_pofile_obj = self.helper.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + + mock_pofile_save.assert_called() + self.assertIn("Percent-Translated", new_pofile_obj.metadata) + self.assertEqual( + new_pofile_obj.percent_translated(), + new_pofile_obj.metadata["Percent-Translated"], + ) + # Test: normalize_pofile_project_id ###################################### def test_normalize_pofile_project_id_correct(self): From 6213d5200fc24831a0208a85d6508349766e232b Mon Sep 17 00:00:00 2001 From: Timid Robot Zehta Date: Mon, 15 Jul 2024 10:30:27 -0700 Subject: [PATCH 15/15] Normalize percent translated after update --- i18n/transifex.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/i18n/transifex.py b/i18n/transifex.py index eb6fafb7..59ac1df6 100644 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -1535,6 +1535,15 @@ def normalize_translations( ) transifex_translated = t_stats["translated_strings"] + # Normalize percent translated + pofile_obj = self.normalize_pofile_percent_translated( + transifex_code, + resource_slug, + resource_name, + pofile_path, + pofile_obj, + ) + # Normalize Creation and Revision dates in local PO File pofile_obj = self.normalize_pofile_dates( resource_slug,