diff --git a/.gitignore b/.gitignore index 31494f2f..28734dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ build/ # Unit test / coverage reports .coverage* +htmlcov # Django development */settings/local.py diff --git a/cc_legal_tools/settings/base.py b/cc_legal_tools/settings/base.py index 012a43da..4a78ce54 100644 --- a/cc_legal_tools/settings/base.py +++ b/cc_legal_tools/settings/base.py @@ -116,13 +116,13 @@ }, ] -# template_fragments -# For our use case, caching doesn't appear to offer any benefits and only adds -# complexity. This was determined by testing both publishing speed and page -# speed. +# The caching API is used, but there are no caching MIDDLEWARE, above, as we +# are not using site caching (which adds overhead without benefit for our uses) CACHES = { "default": { - "BACKEND": "django.core.cache.backends.dummy.DummyCache", + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": 600, + "OPTIONS": {"CULL_FREQUENCY": 0, "MAX_ENTRIES": 3000}, }, } diff --git a/cc_legal_tools/settings/dev.py b/cc_legal_tools/settings/dev.py index be538573..4bb152fe 100644 --- a/cc_legal_tools/settings/dev.py +++ b/cc_legal_tools/settings/dev.py @@ -54,9 +54,10 @@ # can set DEBUG_TOOLBAR_CONFIG['IS_RUNNING_TESTS'] = False to bypass # this check. DEBUG = True + DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda request: DEBUG} INSTALLED_APPS += [ # noqa: F405 "debug_toolbar", ] - MIDDLEWARE += ( # noqa: F405 + MIDDLEWARE += [ # noqa: F405 "debug_toolbar.middleware.DebugToolbarMiddleware", - ) + ] diff --git a/cc_legal_tools/urls.py b/cc_legal_tools/urls.py index 37d47e47..3e739010 100644 --- a/cc_legal_tools/urls.py +++ b/cc_legal_tools/urls.py @@ -48,5 +48,5 @@ def custom_page_not_found(request): import debug_toolbar urlpatterns += [ - re_path(r"^__debug__/", include(debug_toolbar.urls)), + path("__debug__/", include(debug_toolbar.urls)), ] diff --git a/dev/coverage.sh b/dev/coverage.sh index 5b4c7d7e..f75991f3 100755 --- a/dev/coverage.sh +++ b/dev/coverage.sh @@ -2,38 +2,67 @@ # # Run coverage tests and report # -set -o errtrace -set -o nounset - # This script passes all arguments to Coverage Tests. For example, it can be # called from the cc-legal-tools-app directory like so: # # ./dev/coverage.sh --failfast -# Change directory to cc-legal-tools-app (grandparent directory of this script) -cd ${0%/*}/../ +set -o errexit +set -o errtrace +set -o nounset + +# shellcheck disable=SC2154 +trap '_es=${?}; + printf "${0}: line ${LINENO}: \"${BASH_COMMAND}\""; + printf " exited with a status of ${_es}\n"; + exit ${_es}' ERR + +DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)" +# https://en.wikipedia.org/wiki/ANSI_escape_code +E0="$(printf "\e[0m")" # reset +E30="$(printf "\e[30m")" # black foreground +E31="$(printf "\e[31m")" # red foreground +E107="$(printf "\e[107m")" # bright white background -if ! docker compose exec app true 2>/dev/null; then - echo 'The app container/services is not avaialable.' 1>&2 - echo 'First run `docker compose up`.' 1>&2 +#### FUNCTIONS ################################################################ + +error_exit() { + # Echo error message and exit with error + echo -e "${E31}ERROR:${E0} ${*}" 1>&2 exit 1 -fi +} + +print_header() { + # Print 80 character wide black on white heading with time + printf "${E30}${E107}# %-69s$(date '+%T') ${E0}\n" "${@}" +} + +#### MAIN ##################################################################### + +cd "${DIR_REPO}" -printf "\e[1m\e[7m %-80s\e[0m\n" 'Coverage Erase' -docker compose exec app coverage erase -echo 'done.' +docker compose exec app true 2>/dev/null \ + || error_exit \ + 'The Docker app container/service is not avaialable. See README.md' + +print_header 'Coverage erase' +docker compose exec app coverage erase --debug=dataio echo -printf "\e[1m\e[7m %-80s\e[0m\n" 'Coverage Tests' -docker compose exec app coverage run \ +print_header 'Coverage tests' +docker compose exec app coverage run --debug=pytest \ manage.py test --noinput --parallel 4 ${@:-} \ || exit echo -printf "\e[1m\e[7m %-80s\e[0m\n" 'Coverage Combine' +print_header 'Coverage combine' docker compose exec app coverage combine echo -printf "\e[1m\e[7m %-80s\e[0m\n" 'Coverage Report' +print_header 'Coverage html' +docker compose exec app coverage html +echo + +print_header 'Coverage report' docker compose exec app coverage report echo diff --git a/dev/stats.sh b/dev/stats.sh index 543ed680..39fc90df 100755 --- a/dev/stats.sh +++ b/dev/stats.sh @@ -53,9 +53,9 @@ error_exit() { } -header() { +print_header() { # Print 80 character wide black on white heading with time - printf "${E30}${E107}# %-70s$(date '+%T') ${E0}\n" "${@}" + printf "${E30}${E107}# %-69s$(date '+%T') ${E0}\n" "${@}" } @@ -78,7 +78,7 @@ print_var() { published_documents() { local _count _subtotal _ver - header 'Published' + print_header 'Published' print_var DIR_PUB_LICENSES print_var DIR_PUB_PUBLICDOMAIN echo @@ -185,7 +185,7 @@ published_documents() { source_code() { - header 'Source code' + print_header 'Source code' print_var DIR_REPO cd "${DIR_REPO}" scc \ @@ -197,10 +197,10 @@ source_code() { todo() { - header 'Deeds & UX translation' + print_header 'Deeds & UX translation' echo "${E33}TODO${E0}" echo - header 'Legal Code translation' + print_header 'Legal Code translation' echo "${E33}TODO${E0}" echo } diff --git a/i18n/__init__.py b/i18n/__init__.py index 8f3de749..2a051866 100644 --- a/i18n/__init__.py +++ b/i18n/__init__.py @@ -191,6 +191,8 @@ "za": gettext_lazy("South Africa"), } UNIT_NAMES = { + # 4.0 licenses use "NoDerivatives" instead of "NoDerivs". When appropriate, + # the following values are updated by legal_tools.views.get_tool_title() "by": gettext_lazy("Attribution"), "by-nc": gettext_lazy("Attribution-NonCommercial"), "by-nc-nd": gettext_lazy("Attribution-NonCommercial-NoDerivs"), diff --git a/legal_tools/tests/test_views.py b/legal_tools/tests/test_views.py index cc3d4c74..50c1ab69 100644 --- a/legal_tools/tests/test_views.py +++ b/legal_tools/tests/test_views.py @@ -4,12 +4,14 @@ # Third-party from django.conf import settings +from django.core.cache import cache from django.test import TestCase, override_settings from django.urls import reverse from django.utils import timezone from django.utils.translation.trans_real import DjangoTranslation # First-party/Local +from i18n.utils import get_default_language_for_jurisdiction_deed from legal_tools.models import UNITS_LICENSES, LegalCode, Tool, build_path from legal_tools.rdf_utils import ( convert_https_to_http, @@ -25,6 +27,7 @@ branch_status_helper, get_category_and_category_title, get_deed_rel_path, + get_legal_code_replaced_rel_path, normalize_path_and_lang, render_redirect, ) @@ -389,6 +392,104 @@ def setUp(self): super().setUp() +class ViewHelperFunctionsTest(ToolsTestsMixin, TestCase): + def test_get_category_and_category_title_category_tool(self): + category, category_title = get_category_and_category_title( + category=None, + tool=None, + ) + self.assertEqual(category, "licenses") + self.assertEqual(category_title, "Licenses") + + tool = Tool.objects.get(unit="by", version="4.0") + category, category_title = get_category_and_category_title( + category=None, + tool=tool, + ) + self.assertEqual(category, "licenses") + self.assertEqual(category_title, "Licenses") + + def test_get_category_and_category_title_category_publicdomain(self): + category, category_title = get_category_and_category_title( + category="publicdomain", + tool=None, + ) + self.assertEqual(category, "publicdomain") + self.assertEqual(category_title, "Public Domain") + + @override_settings(LANGUAGES_MOSTLY_TRANSLATED=["x1", "x2"]) + def test_get_deed_rel_path_mostly_translated_language_code(self): + expected_deed_rel_path = "deed.x1" + deed_rel_path = get_deed_rel_path( + deed_url="/deed.x1", + path_start="/", + language_code="x1", + language_default="x2", + ) + self.assertEqual(expected_deed_rel_path, deed_rel_path) + + @override_settings(LANGUAGES_MOSTLY_TRANSLATED=["x1", "x2"]) + def test_get_deed_rel_path_less_translated_language_code(self): + expected_deed_rel_path = "deed.x2" + deed_rel_path = get_deed_rel_path( + deed_url="/deed.x3", + path_start="/", + language_code="x3", + language_default="x2", + ) + self.assertEqual(expected_deed_rel_path, deed_rel_path) + + @override_settings( + LANGUAGE_CODE="x1", + LANGUAGES_MOSTLY_TRANSLATED=[], + ) + def test_get_deed_rel_path_less_translated_language_default(self): + expected_deed_rel_path = "deed.x1" + deed_rel_path = get_deed_rel_path( + deed_url="/deed.x3", + path_start="/", + language_code="x3", + language_default="x2", + ) + self.assertEqual(expected_deed_rel_path, deed_rel_path) + + def test_get_legal_code_replaced_rel_path_cache_miss(self): + tool = Tool.objects.get( + unit="by", + version="3.0", + jurisdiction_code="", + ) + path_start = "/licenses/by/3.0" + language_code = "en" + language_default = get_default_language_for_jurisdiction_deed(None) + + cache.clear() + self.assertFalse(cache.has_key("by-4.0--en-replaced_deed_str")) + self.assertFalse(cache.has_key("by-4.0--en-replaced_legal_code_title")) + _, _, _, _ = get_legal_code_replaced_rel_path( + tool.is_replaced_by, path_start, language_code, language_default + ) + self.assertEqual( + cache.get("by-4.0--en-replaced_deed_title"), + "Deed - Attribution 4.0 International", + ) + self.assertEqual( + cache.get("by-4.0--en-replaced_legal_code_title"), + "Legal Code - Attribution 4.0 International", + ) + + def test_normalize_path_and_lang(self): + request_path = "/licenses/by/3.0/de/legalcode" + jurisdiction = "de" + norm_request_path, norm_language_code = normalize_path_and_lang( + request_path, + jurisdiction, + language_code=None, + ) + self.assertEqual(norm_request_path, f"{request_path}.de") + self.assertEqual(norm_language_code, "de") + + class ViewDevHomeTest(ToolsTestsMixin, TestCase): def test_view_dev_index_view(self): url = reverse("dev_index") @@ -786,81 +887,6 @@ class ViewLegalCodeTest(TestCase): # self.assertContains(rsp, 'lang="de"') # self.assertEqual(lc, context["legal_code"]) - def test_get_category_and_category_title_category_tool(self): - category, category_title = get_category_and_category_title( - category=None, - tool=None, - ) - self.assertEqual(category, "licenses") - self.assertEqual(category_title, "Licenses") - - tool = ToolFactory( - category="licenses", - base_url="https://creativecommons.org/licenses/by/4.0/", - version="4.0", - ) - category, category_title = get_category_and_category_title( - category=None, - tool=tool, - ) - self.assertEqual(category, "licenses") - self.assertEqual(category_title, "Licenses") - - def test_get_category_and_category_title_category_publicdomain(self): - category, category_title = get_category_and_category_title( - category="publicdomain", - tool=None, - ) - self.assertEqual(category, "publicdomain") - self.assertEqual(category_title, "Public Domain") - - def test_normalize_path_and_lang(self): - request_path = "/licenses/by/3.0/de/legalcode" - jurisdiction = "de" - norm_request_path, norm_language_code = normalize_path_and_lang( - request_path, - jurisdiction, - language_code=None, - ) - self.assertEqual(norm_request_path, f"{request_path}.de") - self.assertEqual(norm_language_code, "de") - - @override_settings(LANGUAGES_MOSTLY_TRANSLATED=["x1", "x2"]) - def test_get_deed_rel_path_mostly_translated_language_code(self): - expected_deed_rel_path = "deed.x1" - deed_rel_path = get_deed_rel_path( - deed_url="/deed.x1", - path_start="/", - language_code="x1", - language_default="x2", - ) - self.assertEqual(expected_deed_rel_path, deed_rel_path) - - @override_settings(LANGUAGES_MOSTLY_TRANSLATED=["x1", "x2"]) - def test_get_deed_rel_path_less_translated_language_code(self): - expected_deed_rel_path = "deed.x2" - deed_rel_path = get_deed_rel_path( - deed_url="/deed.x3", - path_start="/", - language_code="x3", - language_default="x2", - ) - self.assertEqual(expected_deed_rel_path, deed_rel_path) - - @override_settings( - LANGUAGE_CODE="x1", - LANGUAGES_MOSTLY_TRANSLATED=[], - ) - def test_get_deed_rel_path_less_translated_language_default(self): - expected_deed_rel_path = "deed.x1" - deed_rel_path = get_deed_rel_path( - deed_url="/deed.x3", - path_start="/", - language_code="x3", - language_default="x2", - ) - self.assertEqual(expected_deed_rel_path, deed_rel_path) - def test_view_legal_code_identifying_jurisdiction_default_language(self): language_code = "de" lc = LegalCodeFactory( diff --git a/legal_tools/views.py b/legal_tools/views.py index 20a7abb2..2ce57bc5 100644 --- a/legal_tools/views.py +++ b/legal_tools/views.py @@ -10,6 +10,7 @@ from bs4.dammit import EntitySubstitution from bs4.formatter import HTMLFormatter from django.conf import settings +from django.core.cache import cache from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, render from django.template.loader import render_to_string @@ -22,6 +23,7 @@ get_default_language_for_jurisdiction_deed, get_default_language_for_jurisdiction_naive, get_jurisdiction_name, + get_translation_object, load_deeds_ux_translations, map_django_to_transifex_language_code, ) @@ -83,12 +85,73 @@ def get_category_and_category_title(category=None, tool=None): return category, category_title -def get_tool_title(tool): - tool_name = UNIT_NAMES.get(tool.unit, "UNIMPLEMENTED") - jurisdiction_name = get_jurisdiction_name( - tool.category, tool.unit, tool.version, tool.jurisdiction_code - ) - tool_title = f"{tool_name} {tool.version} {jurisdiction_name}" +def get_tool_title_en(unit, version, category, jurisdiction): + prefix = f"{unit}-{version}-{jurisdiction}-en-" + tool_title_en = cache.get(f"{prefix}title", "") + if tool_title_en: + return tool_title_en + + # Retrieve title parts untranslated (English) + with translation.override(None): + tool_name = str(UNIT_NAMES.get(unit, "UNIMPLEMENTED")) + jurisdiction_name = str( + get_jurisdiction_name(category, unit, version, jurisdiction) + ) + # Licenses before 4.0 use "NoDerivs" instead of "NoDerivatives" + if version not in ("1.0", "2.0", "2.1", "2.5", "3.0"): + tool_name = tool_name.replace("NoDerivs", "NoDerivatives") + tool_title_en = f"{tool_name} {version} {jurisdiction_name}".strip() + + cache.add(f"{prefix}title", tool_title_en) + return tool_title_en + + +def get_tool_title(unit, version, category, jurisdiction, language_code): + prefix = f"{unit}-{version}-{jurisdiction}-{language_code}-" + tool_title = cache.get(f"{prefix}title", "") + if tool_title: + return tool_title + + # English is easy given it is the default + tool_title_en = get_tool_title_en(unit, version, category, jurisdiction) + if language_code == "en": + tool_title = tool_title_en + cache.add(f"{prefix}title", tool_title) + return tool_title + + # Translate title using legal code translation domain for legal code that + # is in Transifex (ex. CC0, Licenses 4.0) + if ( + category == "licenses" + and version not in ("1.0", "2.0", "2.1", "2.5", "3.0") + ) or unit == "zero": + slug = f"{unit}_{version}".replace(".", "") + language_default = get_default_language_for_jurisdiction_naive( + jurisdiction + ) + current_translation = get_translation_object( + slug, language_code, language_default + ) + tool_title_lc = "" + with active_translation(current_translation): + tool_title_lc = translation.gettext(tool_title_en) + # Only use legal code translation domain version if translation + # was successful (does not match English). There are deed translations + # in languages for which we do not yet have legal code translations. + if tool_title_lc != tool_title_en: + tool_title = tool_title_lc + cache.add(f"{prefix}title", tool_title) + return tool_title + + # Translate title using Deeds & UX translation domain + with translation.override(language_code): + tool_name = UNIT_NAMES.get(unit, "UNIMPLEMENTED") + jurisdiction_name = get_jurisdiction_name( + category, unit, version, jurisdiction + ) + tool_title = f"{tool_name} {version} {jurisdiction_name}" + + cache.add(f"{prefix}title", tool_title) return tool_title @@ -200,34 +263,52 @@ def get_legal_code_replaced_rel_path( try: # Same language legal_code = LegalCode.objects.valid().get( - tool=tool, - language_code=language_code, + tool=tool, language_code=language_code ) except LegalCode.DoesNotExist: try: # Jurisdiction default language legal_code = LegalCode.objects.valid().get( - tool=tool, - language_code=language_default, + tool=tool, language_code=language_default ) except LegalCode.DoesNotExist: # Global default language legal_code = LegalCode.objects.valid().get( - tool=tool, - language_code=settings.LANGUAGE_CODE, + tool=tool, language_code=settings.LANGUAGE_CODE ) - identifier = legal_code.tool.identifier() - lazy_deed = translation.gettext_lazy("Deed") - title = get_tool_title(legal_code.tool) - replaced_deed_title = f"{identifier} {lazy_deed} | {title}" + title = get_tool_title( + tool.unit, + tool.version, + tool.category, + tool.jurisdiction_code, + legal_code.language_code, + ) + prefix = ( + f"{tool.unit}-{tool.version}-" + f"{tool.jurisdiction_code}-{legal_code.language_code}-" + ) + replaced_deed_title = cache.get(f"{prefix}replaced_deed_title", "") + if not replaced_deed_title: + with translation.override(legal_code.language_code): + deed_str = translation.gettext("Deed") + replaced_deed_title = f"{deed_str} - {title}" + cache.add(f"{prefix}replaced_deed_title", replaced_deed_title) replaced_deed_path = get_deed_rel_path( legal_code.deed_url, path_start, language_code, language_default, ) - lazy_legal_code = translation.gettext_lazy("Legal Code") - replaced_legal_code_title = f"{identifier} {lazy_legal_code} | {title}" + replaced_legal_code_title = cache.get( + f"{prefix}replaced_legal_code_title", "" + ) + if not replaced_legal_code_title: + with translation.override(legal_code.language_code): + legal_code_str = translation.gettext("Legal Code") + replaced_legal_code_title = f"{legal_code_str} - {title}" + cache.add( + f"{prefix}replaced_legal_code_title", replaced_legal_code_title + ) replaced_legal_code_path = os.path.relpath( legal_code.legal_code_url, path_start ) @@ -369,7 +450,9 @@ def view_list(request, category, language_code=None): ) if language_code not in settings.LANGUAGES_MOSTLY_TRANSLATED: raise Http404(f"invalid language: {language_code}") + translation.activate(language_code) + list_licenses, list_publicdomain = get_list_paths(language_code, None) # Get the list of units and languages that occur among the tools # to let the template iterate over them as it likes. @@ -489,22 +572,17 @@ def view_deed( return view_page_not_found( request, Http404(f"invalid language: {language_code}") ) - translation.activate(language_code) path_start = os.path.dirname(request.path) language_default = get_default_language_for_jurisdiction_deed(jurisdiction) - list_licenses, list_publicdomain = get_list_paths( - language_code, language_default - ) - try: tool = Tool.objects.get( unit=unit, version=version, jurisdiction_code=jurisdiction ) except Tool.DoesNotExist as e: + translation.activate(language_code) return view_page_not_found(request, e) - tool_title = get_tool_title(tool) try: # Try to load legal code with specified language @@ -522,14 +600,25 @@ def view_deed( settings.LANGUAGE_CODE ) + tool_title = get_tool_title( + unit, version, category, jurisdiction, language_code + ) + legal_code_rel_path = os.path.relpath( legal_code.legal_code_url, path_start ) + translation.activate(language_code) + + list_licenses, list_publicdomain = get_list_paths( + language_code, language_default + ) + category, category_title = get_category_and_category_title( category, tool, ) + languages_and_links = get_languages_and_links_for_deeds_ux( request_path=request.path, selected_language_code=language_code, @@ -636,7 +725,19 @@ def view_legal_code( else: translation.activate(settings.LANGUAGE_CODE) tool = legal_code.tool - tool_title = get_tool_title(tool) + + # get_tool_title manipulates the translation domain and, therefore, MUST + # be called before we Activate Legal Code translation + tool_title = get_tool_title( + unit, version, category, jurisdiction, language_code + ) + # get_legal_code_replaced_rel_path calls get_tool_title, see note above + _, _, replaced_title, replaced_path = get_legal_code_replaced_rel_path( + tool.is_replaced_by, + path_start, + language_code, + language_default, + ) # Activate Legal Code translation current_translation = legal_code.get_translation_object() @@ -659,13 +760,6 @@ def view_legal_code( language_default, ) - _, _, replaced_title, replaced_path = get_legal_code_replaced_rel_path( - tool.is_replaced_by, - path_start, - language_code, - language_default, - ) - if tool.identifier() in PLAIN_TEXT_TOOL_IDENTIFIERS: plain_text_url = "legalcode.txt" diff --git a/manage.py b/manage.py index b8e6650b..8f10f98c 100755 --- a/manage.py +++ b/manage.py @@ -1,25 +1,15 @@ #!/usr/bin/env python # Standard library import logging -import os import sys +# Third-party +from django.core.management import execute_from_command_line + LOG = logging.getLogger("management.commands") def main(): - if "DATABASE_URL" in os.environ: - os.environ.setdefault( - "DJANGO_SETTINGS_MODULE", "cc_legal_tools.settings.deploy" - ) - else: - os.environ.setdefault( - "DJANGO_SETTINGS_MODULE", "cc_legal_tools.settings.local" - ) - - # Third-party - from django.core.management import execute_from_command_line - execute_from_command_line(sys.argv) diff --git a/templates/includes/legalcode_contextual_menu.html b/templates/includes/legalcode_contextual_menu.html index 12b7c583..1921b397 100644 --- a/templates/includes/legalcode_contextual_menu.html +++ b/templates/includes/legalcode_contextual_menu.html @@ -3,7 +3,7 @@