-
Notifications
You must be signed in to change notification settings - Fork 91
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add LibreTranslate machine translator, documentation and its test cases.
- Loading branch information
Showing
4 changed files
with
241 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| import json | ||
|
|
||
| import requests | ||
|
|
||
| from wagtail_localize.machine_translators.base import BaseMachineTranslator | ||
| from wagtail_localize.strings import StringValue | ||
|
|
||
|
|
||
| class LibreTranslator(BaseMachineTranslator): | ||
| """ | ||
| A machine translator that uses the LibreTranslate API. | ||
| API Documentation: | ||
| https://libretranslate.com/docs/ | ||
| """ | ||
|
|
||
| display_name = "LibreTranslate" | ||
|
|
||
| def get_api_endpoint(self): | ||
| return self.options["LIBRETRANSLATE_URL"] | ||
|
|
||
| def language_code(self, code): | ||
| return code.split("-")[0] | ||
|
|
||
| def translate(self, source_locale, target_locale, strings): | ||
| translations = [item.data for item in list(strings)] | ||
| response = requests.post( | ||
| self.get_api_endpoint() + "/translate", | ||
| data=json.dumps( | ||
| { | ||
| "q": translations, | ||
| "source": self.language_code(source_locale.language_code), | ||
| "target": self.language_code(target_locale.language_code), | ||
| "api_key": self.options["API_KEY"], | ||
| } | ||
| ), | ||
| headers={"Content-Type": "application/json"}, | ||
| timeout=10, | ||
| ) | ||
| response.raise_for_status() | ||
|
|
||
| return { | ||
| string: StringValue(translation) | ||
| for string, translation in zip(strings, response.json()["translatedText"]) | ||
| } | ||
|
|
||
| def can_translate(self, source_locale, target_locale): | ||
| return self.language_code(source_locale.language_code) != self.language_code( | ||
| target_locale.language_code | ||
| ) |
168 changes: 168 additions & 0 deletions
168
wagtail_localize/machine_translators/tests/test_libretranslate_translator.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| import json | ||
|
|
||
| from unittest import mock | ||
|
|
||
| from django.test import TestCase, override_settings | ||
| from wagtail.models import Locale | ||
|
|
||
| from wagtail_localize.machine_translators import get_machine_translator | ||
| from wagtail_localize.machine_translators.libretranslate import LibreTranslator | ||
| from wagtail_localize.strings import StringValue | ||
|
|
||
|
|
||
| LIBRETRANSLATE_SETTINGS_ENDPOINT = { | ||
| "CLASS": "wagtail_localize.machine_translators.libretranslate.LibreTranslator", | ||
| "OPTIONS": { | ||
| "LIBRETRANSLATE_URL": "https://libretranslate.org", | ||
| "API_KEY": "test-api-key", | ||
| }, | ||
| } | ||
|
|
||
|
|
||
| class TestLibreTranslator(TestCase): | ||
| @override_settings( | ||
| WAGTAILLOCALIZE_MACHINE_TRANSLATOR=LIBRETRANSLATE_SETTINGS_ENDPOINT | ||
| ) | ||
| def setUp(self): | ||
| self.english_locale = Locale.objects.get() | ||
| self.french_locale = Locale.objects.create(language_code="fr-fr") | ||
| self.translator = get_machine_translator() | ||
|
|
||
| def test_api_endpoint(self): | ||
| self.assertIsInstance(self.translator, LibreTranslator) | ||
| api_endpoint = self.translator.get_api_endpoint() | ||
| self.assertEqual(api_endpoint, "https://libretranslate.org") | ||
|
|
||
| def test_language_code(self): | ||
| self.assertEqual( | ||
| self.translator.language_code(self.english_locale.language_code), "en" | ||
| ) | ||
| self.assertEqual( | ||
| self.translator.language_code(self.french_locale.language_code), "fr" | ||
| ) | ||
| self.assertEqual(self.translator.language_code("foo-bar-baz"), "foo") | ||
|
|
||
| @mock.patch("wagtail_localize.machine_translators.libretranslate.requests.post") | ||
| def test_translate_text(self, mock_post): | ||
| # Mock the response of requests.post | ||
| mock_response = mock.Mock() | ||
| mock_response.json.return_value = { | ||
| "translatedText": [ | ||
| "Bonjour le monde!", | ||
| "Ceci est une phrase. Ceci est une autre phrase.", | ||
| ] | ||
| } | ||
| mock_response.raise_for_status = mock.Mock() | ||
| mock_post.return_value = mock_response | ||
|
|
||
| input_strings = [ | ||
| StringValue("Hello world!"), | ||
| StringValue("This is a sentence. This is another sentence."), | ||
| ] | ||
|
|
||
| translations = self.translator.translate( | ||
| self.english_locale, self.french_locale, input_strings | ||
| ) | ||
|
|
||
| expected_translations = { | ||
| StringValue("Hello world!"): StringValue("Bonjour le monde!"), | ||
| StringValue("This is a sentence. This is another sentence."): StringValue( | ||
| "Ceci est une phrase. Ceci est une autre phrase." | ||
| ), | ||
| } | ||
|
|
||
| # Assertions to check if the translation is as expected | ||
| self.assertEqual(translations, expected_translations) | ||
|
|
||
| # Assert that requests.post was called with the correct arguments | ||
| mock_post.assert_called_once_with( | ||
| LIBRETRANSLATE_SETTINGS_ENDPOINT["OPTIONS"]["LIBRETRANSLATE_URL"] | ||
| + "/translate", | ||
| data=json.dumps( | ||
| { | ||
| "q": [ | ||
| "Hello world!", | ||
| "This is a sentence. This is another sentence.", | ||
| ], | ||
| "source": "en", | ||
| "target": "fr", | ||
| "api_key": LIBRETRANSLATE_SETTINGS_ENDPOINT["OPTIONS"]["API_KEY"], | ||
| } | ||
| ), | ||
| headers={"Content-Type": "application/json"}, | ||
| timeout=10, | ||
| ) | ||
|
|
||
| @mock.patch("wagtail_localize.machine_translators.libretranslate.requests.post") | ||
| def test_translate_html(self, mock_post): | ||
| # Mock the response of requests.post | ||
| mock_response = mock.Mock() | ||
| mock_response.json.return_value = { | ||
| "translatedText": ["""<a id="a1">Bonjour !</a>. <b>C'est un test</b>."""] | ||
| } | ||
| mock_response.raise_for_status = mock.Mock() | ||
| mock_post.return_value = mock_response | ||
|
|
||
| input_string, attrs = StringValue.from_source_html( | ||
| '<a href="https://en.wikipedia.org/wiki/World">Hello !</a>. <b>This is a test</b>.' | ||
| ) | ||
|
|
||
| translations = self.translator.translate( | ||
| self.english_locale, self.french_locale, [input_string] | ||
| ) | ||
|
|
||
| expected_translation = { | ||
| input_string: StringValue( | ||
| """<a id="a1">Bonjour !</a>. <b>C'est un test</b>.""" | ||
| ) | ||
| } | ||
|
|
||
| # Assertions to check if the translation is as expected | ||
| self.assertEqual(translations, expected_translation) | ||
|
|
||
| # Additional assertion to check the rendered HTML | ||
| translated_string = translations[input_string] | ||
| rendered_html = translated_string.render_html(attrs) | ||
| expected_rendered_html = '<a href="https://en.wikipedia.org/wiki/World">Bonjour !</a>. <b>C\'est un test</b>.' | ||
|
|
||
| self.assertEqual(rendered_html, expected_rendered_html) | ||
|
|
||
| # Assert that requests.post was called with the correct arguments | ||
| mock_post.assert_called_once_with( | ||
| LIBRETRANSLATE_SETTINGS_ENDPOINT["OPTIONS"]["LIBRETRANSLATE_URL"] | ||
| + "/translate", | ||
| data=json.dumps( | ||
| { | ||
| "q": [ | ||
| '<a id="a1">Hello !</a>. <b>This is a test</b>.' | ||
| ], # Use the string from StringValue | ||
| "source": "en", | ||
| "target": "fr", | ||
| "api_key": LIBRETRANSLATE_SETTINGS_ENDPOINT["OPTIONS"]["API_KEY"], | ||
| } | ||
| ), | ||
| headers={"Content-Type": "application/json"}, | ||
| timeout=10, | ||
| ) | ||
|
|
||
| def test_can_translate(self): | ||
| self.assertIsInstance(self.translator, LibreTranslator) | ||
|
|
||
| french_locale = Locale.objects.create(language_code="fr") | ||
|
|
||
| self.assertTrue( | ||
| self.translator.can_translate(self.english_locale, self.french_locale) | ||
| ) | ||
| self.assertTrue( | ||
| self.translator.can_translate(self.english_locale, french_locale) | ||
| ) | ||
|
|
||
| # Can't translate the same language | ||
| self.assertFalse( | ||
| self.translator.can_translate(self.english_locale, self.english_locale) | ||
| ) | ||
|
|
||
| # Can't translate two variants of the same language | ||
| self.assertFalse( | ||
| self.translator.can_translate(self.french_locale, french_locale) | ||
| ) |