VPN-5175 - Get languages, currencies and server name translations fro…

…m the l10n repository strings (#9448)

.github/workflows/linters.yaml

@@ -95,23 +95,3 @@ jobs:
         run: |
           conda info
           REPOSITORY_ROOT=$(pwd)/.. ./gradlew ktlint
-
-  check-language-json:
-    name: Check if languages.json is valid
-    runs-on: ubuntu-latest
-    steps:
-      - name: Clone repository
-        uses: actions/checkout@v3
-        with:
-          submodules: "true"
-
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
-          cache: "npm"
-      - run: npm install
-
-      - name: Run languagelocalizer
-        run: |
-          cd tools/languagelocalizer
-          node languageLocalizer.js --check

scripts/cmake/generate_translations_target.cmake

@@ -41,6 +41,7 @@ function(generate_translations_target TARGET_NAME ASSETS_DIRECTORY TRANSLATIONS_
     target_include_directories(${TARGET_NAME} PUBLIC ${GENERATED_DIR})
 
     target_sources(${TARGET_NAME} PRIVATE
+        ${GENERATED_DIR}/i18nlanguagenames.h
         ${GENERATED_DIR}/i18nstrings_p.cpp
         ${GENERATED_DIR}/i18nstrings.h
         ${GENERATED_DIR}/translations.qrc
@@ -56,15 +57,26 @@ function(generate_translations_target TARGET_NAME ASSETS_DIRECTORY TRANSLATIONS_
         OUTPUT ${GENERATED_DIR}/i18nstrings_p.cpp ${GENERATED_DIR}/i18nstrings.h
         DEPENDS
             ${ASSETS_DIRECTORY}/strings.yaml
+            ${ASSETS_DIRECTORY}/extras/extras.xliff
             ${MVPN_SCRIPT_DIR}/utils/generate_strings.py
         COMMAND ${PYTHON_EXECUTABLE} ${MVPN_SCRIPT_DIR}/utils/generate_strings.py
             -o ${GENERATED_DIR}
             ${ASSETS_DIRECTORY}/strings.yaml
+            ${ASSETS_DIRECTORY}/extras/extras.xliff
+    )
+
+    add_custom_command(
+        OUTPUT ${GENERATED_DIR}/i18nlanguagenames.h
+        DEPENDS
+            ${ASSETS_DIRECTORY}/extras/extras.xliff
+            ${MVPN_SCRIPT_DIR}/utils/generate_language_names_map.py
+        COMMAND ${PYTHON_EXECUTABLE} ${MVPN_SCRIPT_DIR}/utils/generate_language_names_map.py
+            ${TRANSLATIONS_DIRECTORY}
+            ${GENERATED_DIR}/i18nlanguagenames.h
     )
 
     ## Build the list of supported locales and add rules to build them.
     file(GLOB I18N_LOCALES LIST_DIRECTORIES true RELATIVE ${TRANSLATIONS_DIRECTORY} ${TRANSLATIONS_DIRECTORY}/*)
-    message(${I18N_LOCALES})
     list(FILTER I18N_LOCALES EXCLUDE REGEX "^\\..+")
     foreach(LOCALE ${I18N_LOCALES})
         if(NOT EXISTS ${TRANSLATIONS_DIRECTORY}/${LOCALE}/mozillavpn.xliff)
@@ -81,11 +93,13 @@ function(generate_translations_target TARGET_NAME ASSETS_DIRECTORY TRANSLATIONS_
         add_custom_command(
             OUTPUT ${GENERATED_DIR}/mozillavpn_${LOCALE}.ts
             MAIN_DEPENDENCY ${TRANSLATIONS_DIRECTORY}/${LOCALE}/mozillavpn.xliff
-            DEPENDS ${GENERATED_DIR}/i18nstrings_p.cpp
+            DEPENDS
+                ${GENERATED_DIR}/i18nstrings_p.cpp
+                ${TRANSLATIONS_DIRECTORY}/${LOCALE}/extras.xliff
             COMMAND ${QT_LUPDATE_EXECUTABLE} -target-language ${LOCALE} ${GENERATED_DIR}/i18nstrings_p.cpp -ts ${GENERATED_DIR}/mozillavpn_${LOCALE}.ts
             COMMAND ${QT_LCONVERT_EXECUTABLE} -verbose -o ${GENERATED_DIR}/mozillavpn_${LOCALE}.ts
                             -if ts -i ${GENERATED_DIR}/mozillavpn_${LOCALE}.ts ${INCLUDE_UNTRANSLATED}
-                            -if xlf -i ${TRANSLATIONS_DIRECTORY}/${LOCALE}/mozillavpn.xliff
+                            -if xlf -i ${TRANSLATIONS_DIRECTORY}/${LOCALE}/mozillavpn.xliff -i ${TRANSLATIONS_DIRECTORY}/${LOCALE}/extras.xliff
         )
 
         add_custom_command(

scripts/utils/generate_language_names_map.py

@@ -0,0 +1,58 @@
+#! /usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import argparse
+import xml.etree.ElementTree as etree
+
+def extract_strings_with_id(directory):
+    language_strings = {}
+
+    for root, dirs, files in os.walk(directory):
+        if "extras.xliff" in files:
+            extras_xliff_path = os.path.join(root, "extras.xliff")
+            lang_code = os.path.basename(root)
+            if lang_code not in language_strings:
+                language_strings[lang_code] = {}
+
+            tree = etree.parse(extras_xliff_path)
+            root = tree.getroot()
+
+            for trans_unit in root.findall('.//{urn:oasis:names:tc:xliff:document:1.2}trans-unit'):
+                unit_id = trans_unit.get('id')
+                if unit_id.startswith('languages.'):
+                    if lang_code == 'en':
+                        value = trans_unit.find('.//{urn:oasis:names:tc:xliff:document:1.2}source')
+                    else:
+                        value = trans_unit.find('.//{urn:oasis:names:tc:xliff:document:1.2}target')
+
+                    if value is not None and value.text is not None:
+                        source_text = value.text.strip()
+                        language_strings[lang_code][unit_id.replace('languages.', '')] = source_text
+
+    return language_strings
+
+def generate_cpp_header(language_strings, output_file):
+    with open(output_file, 'w', encoding="utf-8") as f:
+        f.write('#ifndef LANGUAGE_STRINGS_H\n')
+        f.write('#define LANGUAGE_STRINGS_H\n\n')
+        f.write('#include <QMap>\n#include <QString>\n\n')
+        f.write('namespace LanguageStrings {\n')
+        f.write('const QMap<QString, QString> NATIVE_LANGUAGE_NAMES = {\n')
+        for lang_code, strings in language_strings.items():
+            f.write('    {"' + lang_code + '", "' + strings.get(lang_code, strings.get("en", "")) + '"},\n')
+        f.write('};\n}\n\n#endif\n')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Extract language name strings from "extras.xliff" files.')
+    parser.add_argument('directory', type=str, help='Path to the i18n directory')
+    parser.add_argument('output_file', type=str, help='Output file name')
+    args = parser.parse_args()
+
+    directory_path = args.directory
+    output_file = args.output_file
+
+    language_strings = extract_strings_with_id(directory_path)
+    generate_cpp_header(language_strings, output_file)

scripts/utils/generate_strings.py

@@ -6,7 +6,8 @@
 import os
 import yaml
 import argparse
-
+import xml.etree.ElementTree as etree
+import re
 
 def stop(string_id):
     exit(
@@ -36,7 +37,31 @@ def construct_mapping(self, node, deep=False):
         return super().construct_mapping(node, deep)
 
 
-def parseTranslationStrings(yamlfile):
+def parseXLIFFTranslationStrings(xliff_file):
+    if not os.path.isfile(xliff_file):
+        exit(f"Unable to find {xliff_file}")
+
+    strings = {}
+
+    tree = etree.parse(xliff_file)
+    root = tree.getroot()
+
+    for node in root.findall('.//{urn:oasis:names:tc:xliff:document:1.2}trans-unit'):
+        # Remove any unexpected characters e.g. São Paulo -> SoPaulo
+        id = re.sub(r'[^a-zA-Z._]', '', node.get('id'))
+        cpp_id = pascalize(id.replace('.', '_'))
+        value = node.findall('.//{urn:oasis:names:tc:xliff:document:1.2}source')[0].text
+
+        strings[cpp_id] = {
+            "string_id": id,
+            "value": [value],
+            "comments": [],
+        }
+
+    return strings
+
+
+def parseYAMLTranslationStrings(yamlfile):
     if not os.path.isfile(yamlfile):
         exit(f"Unable to find {yamlfile}")
 
@@ -147,6 +172,10 @@ class I18nStrings final : public QQmlPropertyMap {
             """    __Last,
   };
 
+  static String getString(const QString& s) {
+    return s_stringIdMap.value(s, I18nStrings::Empty);
+  }
+
   static I18nStrings* instance();
   static void initialize();
 
@@ -157,10 +186,20 @@ class I18nStrings final : public QQmlPropertyMap {
 
   const char* id(I18nStrings::String) const;
 
-  QString t(String) const;
+  QString t(I18nStrings::String) const;
 
  private:
   static const char* const _ids[];
+
+  static inline const QHash<QString, I18nStrings::String> s_stringIdMap = {
+"""
+    )
+
+        for i, key in enumerate(strings):
+            output.write(f"    {{\"{key}\", I18nStrings::{key}}}, \n")
+
+        output.write("""
+  };
 };
 
 #endif  // I18NSTRINGS_H
@@ -218,15 +257,15 @@ def serialize(string):
 if __name__ == "__main__":
     # Parse arguments to locate the input and output files.
     parser = argparse.ArgumentParser(
-        description="Generate internationalization strings database from a YAML source"
+        description="Generate internationalization strings database from a YAML and/or XLIFF sources"
     )
     parser.add_argument(
-        "source",
+        "sources",
         metavar="SOURCE",
         type=str,
         action="store",
         nargs='+',
-        help="YAML strings file to process",
+        help="Comma separated list of YAML and/or XLIFF sources to parse",
     )
     parser.add_argument(
         "-o",
@@ -238,18 +277,25 @@ def serialize(string):
     )
     args = parser.parse_args()
 
-    if not args.source:
+    if not args.sources:
         exit("No source argument.")
 
     # If no output directory was provided, use the current directory.
     if args.output is None:
         args.output = os.getcwd()
 
-    # Parse the inputs for their sweet juicy strings.
+    # Parse the inputs
     strings = {}
-    for source in args.source:
-        substrings = parseTranslationStrings(source)
-        strings.update(substrings)
+    for source in args.sources:
+        _, ext = os.path.splitext(source)
+        if ext == '.yaml':
+            substrings = parseYAMLTranslationStrings(source)
+            strings.update(substrings)
+        elif ext == '.xliff':
+            substrings = parseXLIFFTranslationStrings(source)
+            strings.update(substrings)
+        else:
+            raise f'Unknown file format provided: {source}'
 
     # Render the strings into generated content.
     generateStrings(strings, args.output)

src/cmake/shared-sources.cmake

@@ -124,8 +124,6 @@ target_sources(shared-sources INTERFACE
     ${CMAKE_SOURCE_DIR}/src/ipaddress.h
     ${CMAKE_SOURCE_DIR}/src/itempicker.cpp
     ${CMAKE_SOURCE_DIR}/src/itempicker.h
-    ${CMAKE_SOURCE_DIR}/src/languagei18n.cpp
-    ${CMAKE_SOURCE_DIR}/src/languagei18n.h
     ${CMAKE_SOURCE_DIR}/src/leakdetector.cpp
     ${CMAKE_SOURCE_DIR}/src/leakdetector.h
     ${CMAKE_SOURCE_DIR}/src/localizer.cpp

src/cmake/sources.cmake

@@ -151,8 +151,6 @@ target_sources(mozillavpn-sources INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR}/purchasewebhandler.h
     ${CMAKE_CURRENT_SOURCE_DIR}/releasemonitor.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/releasemonitor.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/serveri18n.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/serveri18n.h
     ${CMAKE_CURRENT_SOURCE_DIR}/serverlatency.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/serverlatency.h
     ${CMAKE_CURRENT_SOURCE_DIR}/settingswatcher.cpp
@@ -230,7 +228,7 @@ endif()
 
 # Creates Target (mozillavpn-sources_clang_tidy_report)
 mz_add_clang_tidy(mozillavpn-sources)
-# we need to make sure those are up to date before we build. 
+# we need to make sure those are up to date before we build.
 # Those targets generate code we #include, therefore
 
 mz_optional_dependency(mozillavpn-sources_clang_tidy_report qtglean)

src/controller.cpp

@@ -28,7 +28,6 @@
 #include "rfc/rfc1918.h"
 #include "rfc/rfc4193.h"
 #include "rfc/rfc4291.h"
-#include "serveri18n.h"
 #include "serverlatency.h"
 #include "settingsholder.h"
 #include "tasks/controlleraction/taskcontrolleraction.h"