From cb32fe1121340cc64aca887d726c0e6890c0615f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Tue, 23 Apr 2024 10:00:32 +0200 Subject: [PATCH] Document custom Jinja template filters (#2769) Includes a small refactoring of man page building to overcome rst2man handling of `:ref:` directives. --- .gitignore | 1 + docs/Makefile | 11 +- docs/code/index.rst | 1 + docs/scripts/generate-template-filters.py | 28 +++ docs/templates/plugins.rst.j2 | 6 +- docs/templates/story.rst.j2 | 10 +- docs/templates/template-filters.rst.j2 | 34 +++ tmt/export/templates/default-story.rst.j2 | 10 +- tmt/utils.py | 275 +++++++++++++++++++++- 9 files changed, 349 insertions(+), 27 deletions(-) create mode 100755 docs/scripts/generate-template-filters.py create mode 100644 docs/templates/template-filters.rst.j2 diff --git a/.gitignore b/.gitignore index 0a45047285..bece243ca8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /tmp/ docs/code/autodocs/*.rst +docs/code/template-filters.rst docs/plugins/discover.rst docs/plugins/execute.rst docs/plugins/finish.rst diff --git a/docs/Makefile b/docs/Makefile index 3519f30081..90b57f548f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,10 +16,10 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help generate-plugins plugins/*.rst generate-stories generate-autodocs clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext +.PHONY: help generate-plugins plugins/*.rst generate-stories generate-template-filters generate-autodocs clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext clean: - rm -rf $(BUILDDIR) stories spec code/autodocs/*.rst + rm -rf $(BUILDDIR) stories spec code/autodocs/*.rst code/template-filters.rst ## ## Building documentation @@ -55,7 +55,7 @@ TEMPLATESDIR = templates PLUGINS_TEMPLATE := $(TEMPLATESDIR)/plugins.rst.j2 -generate: spec stories generate-lint-checks generate-plugins generate-stories generate-autodocs ## Refresh all generated documentation sources +generate: spec stories generate-lint-checks generate-template-filters generate-plugins generate-stories generate-autodocs ## Refresh all generated documentation sources spec: mkdir -p spec @@ -66,6 +66,9 @@ stories: spec/lint.rst: $(SCRIPTSDIR)/generate-lint-checks.py $(TEMPLATESDIR)/lint-checks.rst.j2 $(TMTDIR)/base.py $(SCRIPTSDIR)/generate-lint-checks.py $(TEMPLATESDIR)/lint-checks.rst.j2 $@ +code/template-filters.rst: $(SCRIPTSDIR)/generate-template-filters.py $(TEMPLATESDIR)/template-filters.rst.j2 $(TMTDIR)/utils.py + $(SCRIPTSDIR)/generate-template-filters.py $(TEMPLATESDIR)/template-filters.rst.j2 $@ + plugins/discover.rst: $(SCRIPTSDIR)/generate-plugins.py $(PLUGINS_TEMPLATE) $(TMTDIR)/steps/discover/*.py $(SCRIPTSDIR)/generate-plugins.py discover $(PLUGINS_TEMPLATE) $@ @@ -89,6 +92,8 @@ plugins/test-checks.rst: $(SCRIPTSDIR)/generate-plugins.py $(PLUGINS_TEMPLATE) $ generate-lint-checks: spec spec/lint.rst ## Generate documentation sources for lint checks +generate-template-filters: code/template-filters.rst ## Generate documentation sources for Jinja2 template filters + generate-stories: stories $(TEMPLATESDIR)/story.rst.j2 ## Generate documentation sources for stories $(SCRIPTSDIR)/generate-stories.py $(TEMPLATESDIR)/story.rst.j2 diff --git a/docs/code/index.rst b/docs/code/index.rst index 54f042103e..54fb4a8307 100644 --- a/docs/code/index.rst +++ b/docs/code/index.rst @@ -14,6 +14,7 @@ the documentation generated from sources linked below. :maxdepth: 2 Class Overview + Template Filters Plugin Introduction tmt diff --git a/docs/scripts/generate-template-filters.py b/docs/scripts/generate-template-filters.py new file mode 100755 index 0000000000..ebaef04591 --- /dev/null +++ b/docs/scripts/generate-template-filters.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import sys +import textwrap + +from tmt.utils import TEMPLATE_FILTERS, Path, render_template_file + +HELP = textwrap.dedent(""" +Usage: generate-template-filters.py + +Generate docs for all known Jinja2 template filters. +""").strip() + + +def main() -> None: + if len(sys.argv) != 3: + print(HELP) + + sys.exit(1) + + template_filepath = Path(sys.argv[1]) + output_filepath = Path(sys.argv[2]) + + output_filepath.write_text(render_template_file(template_filepath, TEMPLATES=TEMPLATE_FILTERS)) + + +if __name__ == '__main__': + main() diff --git a/docs/templates/plugins.rst.j2 b/docs/templates/plugins.rst.j2 index 3bd51c8d86..ad2173597c 100644 --- a/docs/templates/plugins.rst.j2 +++ b/docs/templates/plugins.rst.j2 @@ -7,7 +7,7 @@ {% for PLUGIN_ID, PLUGIN, PLUGIN_DATA_CLASS in PLUGINS() %} -.. _plugins/{{ STEP }}/{{ PLUGIN_ID | strip }}: +.. _plugins/{{ STEP }}/{{ PLUGIN_ID | trim }}: {{ PLUGIN_ID }} {{ '-' * (PLUGIN_ID | length)}} @@ -25,7 +25,7 @@ formatting is often suboptional. {% if PLUGIN.__doc__ %} -{{ PLUGIN.__doc__ | dedent | strip }} +{{ PLUGIN.__doc__ | dedent | trim }} {% endif %} Configuration @@ -42,7 +42,7 @@ Configuration {{ option }}: {% endif %} {% if metadata.help %} -{{ metadata.help | strip | indent(4, first=true) }} +{{ metadata.help | trim | indent(4, first=true) }} {% endif %} {% if metadata.has_default %} {% set actual_default = metadata.materialized_default %} diff --git a/docs/templates/story.rst.j2 b/docs/templates/story.rst.j2 index 2301f32ae6..fd2ea8393e 100644 --- a/docs/templates/story.rst.j2 +++ b/docs/templates/story.rst.j2 @@ -121,7 +121,7 @@ {% endmacro %} {% if INCLUDE_TITLE %} -{% set depth = STORY.name | findall('/') | length - 1 %} +{% set depth = STORY.name | regex_findall('/') | length - 1 %} {% set title_underline = '=~^:-><'[depth] %} {% if STORY.title and STORY.title != STORY.node.parent.get('title') %} {% set title = STORY.title %} @@ -129,19 +129,19 @@ {% set title = STORY.name | regex_replace('.*/', '') %} {% endif %} -.. _{{ STORY.name | strip }}: +.. _{{ STORY.name | trim }}: -{{ title | strip }} +{{ title | trim }} {{ title_underline * title | length }} {% endif %} {# Summary, story and description #} {% if STORY.summary and STORY.summary != STORY.node.parent.get('summary') %} -{{ STORY.summary | strip }} +{{ STORY.summary | trim }} {% endif %} {% if STORY.story != STORY.node.parent.get('story') %} -*{{ STORY.story | strip }}* +*{{ STORY.story | trim }}* {% endif %} {# Insert note about unimplemented feature (leaf nodes only) #} diff --git a/docs/templates/template-filters.rst.j2 b/docs/templates/template-filters.rst.j2 new file mode 100644 index 0000000000..ee0e2756bb --- /dev/null +++ b/docs/templates/template-filters.rst.j2 @@ -0,0 +1,34 @@ +{# + This template is rendered by conf.py & Sphinx when building documentation. + + The reST comment below should land in the rendered file, to discourage people + from editing it instead of this template. +#} + +.. + Please, do not edit this file, is is rendered from template-filters.rst.j2, + and all your changes would be overwritten. + +Template filters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When it comes to creating text documents, like +:ref:`custom_templates` or exporting tests, plans +and stories into ReST or Markdown, tmt relies on the well-known +`Jinja`__ package. All Jinja templating features are supported, and, on +top of `filters`__ shipped with Jinja, tmt adds several custom ones. + +__ https://palletsprojects.com/p/jinja/ +__ https://jinja.palletsprojects.com/en/3.1.x/templates/#filters + +{% for filter_name in TEMPLATES.keys() | sort %} + {% set filter_callable = TEMPLATES[filter_name] %} + +{{ filter_name }} +{{ '-' * (filter_name | length)}} + +{% if filter_callable.__doc__ %} +{{ filter_callable.__doc__ | dedent | trim }} +{% endif %} + +{% endfor %} diff --git a/tmt/export/templates/default-story.rst.j2 b/tmt/export/templates/default-story.rst.j2 index c80f2a0b0c..32ec8283d6 100644 --- a/tmt/export/templates/default-story.rst.j2 +++ b/tmt/export/templates/default-story.rst.j2 @@ -14,7 +14,7 @@ {% endmacro %} {% if INCLUDE_TITLE %} -{% set depth = STORY.name | findall('/') | length - 1 %} +{% set depth = STORY.name | regex_findall('/') | length - 1 %} {% set title_underline = '=~^:-><'[depth] %} {% if STORY.title and STORY.title != STORY.node.parent.get('title') %} {% set title = STORY.title %} @@ -22,19 +22,19 @@ {% set title = STORY.name | regex_replace('.*/', '') %} {% endif %} -.. _{{ STORY.name | strip }}: +.. _{{ STORY.name | trim }}: -{{ title | strip }} +{{ title | trim }} {{ title_underline * title | length }} {% endif %} {# Summary, story and description #} {% if STORY.summary and STORY.summary != STORY.node.parent.get('summary') %} -{{ STORY.summary | strip }} +{{ STORY.summary | trim }} {% endif %} {% if STORY.story != STORY.node.parent.get('story') %} -*{{ STORY.story | strip }}* +*{{ STORY.story | trim }}* {% endif %} {# Insert note about unimplemented feature (leaf nodes only) #} diff --git a/tmt/utils.py b/tmt/utils.py index 8e37c6a7fc..9bd94edc0d 100644 --- a/tmt/utils.py +++ b/tmt/utils.py @@ -27,7 +27,7 @@ from collections import Counter, OrderedDict from collections.abc import Iterable, Iterator, Sequence from contextlib import suppress -from re import Pattern +from re import Match, Pattern from threading import Thread from types import ModuleType from typing import ( @@ -6553,6 +6553,268 @@ def field( ) +# +# Jinja2 template rendering +# +def _template_filter_basename(pathlike: str) -> str: + """ + Return the last component of the given path. + + .. code-block:: jinja + + # "/etc/fstab" -> "fstab" + {{ "/etc/fstab" | basename }} + + # "/var/log/" -> "log" + {{ "/var/log/" | basename }} + """ + + return Path(pathlike).name + + +def _template_filter_match(s: str, pattern: str) -> Optional[Match[str]]: + """ + Return `re.Match`__ if the string matches a given pattern. + + Pattern is tested in "match" mode, i.e. it must match from the + beginning of the string. See :ref:`regular-expressions` description + for more details. + + __ https://docs.python.org/3.9/library/re.html#match-objects + + .. code-block:: jinja + + # 'foo/bar' -> 'foo/bar' + {{ 'foo/bar' | match('foo/.*').group() }} + + # 'foo/bar' -> '' + {{ 'foo/bar' | regex_match('foo/(.+?)/(.*)') }} + + # 'foo/bar/baz' -> 'bar' + {{ 'foo/bar' | regex_match('foo/(.+?)/.*').group(1) }} + """ + + return re.match(pattern, s) + + +def _template_filter_search(s: str, pattern: str) -> Optional[Match[str]]: + """ + Return `re.Match`__ if the string matches a given pattern. + + Pattern is tested in "search" mode, i.e. it can match anywhere + in the string. See :ref:`regular-expressions` description for more + details. + + __ https://docs.python.org/3.9/library/re.html#match-objects + + .. code-block:: jinja + + # ' foo/bar' -> 'foo/bar' + {{ ' foo/bar' | match('foo/.*').group() }} + + # ' foo/bar' -> '' + {{ ' foo/bar' | regex_match('foo/(.+?)/(.*)') }} + + # ' foo/bar/baz' -> 'bar' + {{ ' foo/bar' | regex_match('foo/(.+?)/.*').group(1) }} + """ + + return re.search(pattern, s) + + +def _template_filter_regex_findall(s: str, pattern: str) -> list[str]: + """ + Return a list of all non-overlapping matches in the string. + + If one or more capturing groups are present in the pattern, return + a list of groups; this will be a list of tuples if the pattern + has more than one group. + + Empty matches are included in the result. + + .. code-block:: jinja + + # '/var/log/mail.log' => ['/', '/', '/'] + {{ '/var/log/mail.log' | regex_findall('/') }} + """ + + return re.findall(pattern, s) + + +def _template_filter_regex_match(s: str, pattern: str) -> str: + """ + Return string matching a given pattern. + + Pattern is tested in "match" mode, i.e. it must match from the + beginning of the string. See :ref:`regular-expressions` description + for more details. + + If the string matches and pattern contains capture groups, the + first group is returned. If the string matches, but patterns + contains no capture group, the whole match is returned. + Otherwise, an empty string is returned. + + .. code-block:: jinja + + # 'foo/bar' -> 'foo/bar' + {{ 'foo/bar' | regex_match('foo/.*') }} + + # 'foo/bar' -> '' + {{ 'foo/bar' | regex_match('foo/(.+?)/(.*)') }} + + # 'foo/bar/baz' -> 'bar' + {{ 'foo/bar/baz' | regex_match('foo/(.+?)/.*') }} + """ + + match = re.match(pattern, s) + + if match is None: + return '' + + if not match.groups(): + return match.group() + + return match.group(1) + + +def _template_filter_regex_search(s: str, pattern: str) -> str: + """ + Return string matching a given pattern. + + Pattern is tested in "search" mode, i.e. it can match anywhere + in the string. See :ref:`regular-expressions` description for more + details. + + If the string matches and pattern contains capture groups, the + first group is returned. If the string matches, but patterns + contains no capture group, the whole match is returned. + Otherwise, an empty string is returned. + + .. code-block:: jinja + + # ' foo/bar' -> 'foo/bar' + {{ ' foo/bar' | regex_search('foo/.*') }} + + # ' foo/bar' -> '' + {{ ' foo/bar' | regex_search('foo/(.+?)/(.*)') }} + + # ' foo/bar/baz' -> 'bar' + {{ ' foo/bar/baz' | regex_search('foo/(.+?)/.*') }} + """ + + match = re.search(pattern, s) + + if match is None: + return '' + + if not match.groups(): + return match.group() + + return match.group(1) + + +def _template_filter_regex_replace(s: str, pattern: str, repl: str) -> str: + """ + Replace a substring defined by a regular expression with another string. + + Return the string obtained by replacing the leftmost + non-overlapping occurrences of pattern in string by the replacement. + If the pattern isn't found, string is returned unchanged. + + Backreferences in the replacement string are replaced with the + substring matched by a group in the pattern. + + .. code-block:: jinja + + # 'foo/bar' -> 'foo/baz' + {{ 'foo/bar' | regex_replace('(.+)/bar', '\1/baz') }} + + # 'foo/bar' -> 'foo/bar' + {{ 'foo/bar' | regex_replace('(.+)/baz', '\1/') }} + """ + + return re.sub(pattern, repl, s) + + +def _template_filter_dedent(s: str) -> str: + """ + Remove any common leading whitespace from every line in the string. + + .. code-block:: jinja + + # "foo bar" -> "foo bar" + {{ "foo bar" | dedent }} + + # ''' + # foo + # bar + # baz + # ''' + # -> + # ''' + # foo + # bar + # baz + # ''' + {{ "\\n foo\\n bar\\n baz\\n" | dedent }} + """ + + return textwrap.dedent(s) + + +def _template_filter_listed( + items: list[Any], + singular: Optional[str] = None, + plural: Optional[str] = None, + max: Optional[int] = None, + quote: str = "", + join: str = "and") -> str: + """ + Return a nice, human readable description of an iterable. + + .. code-block:: jinja + + # [0] -> "0" + {{ [0] | listed }} + + # [0, 1] -> "0 and 1" + {{ [0, 1] | listed }} + + # [0, 1, 2] -> "0, 1, or 2" + {{ [0, 1, 2] | listed(join='or') }} + + # [0, 1, 2] -> '"0", "1" and "2"' + {{ [0, 1, 2] | listed(quote='"') }} + + # [0, 1, 2, 3] -> "0, 1, 2 and 1 more" + {{ [0, 1, 2, 3] | listed(max=3) }} + + # [0, 1, 2, 3, 4] -> "0, 1, 2 and 2 more numbers" + {{ [0, 1, 2, 3, 4, 5] | listed('number', max=3) }} + + # [0, 1, 2, 3, 4, 5] -> "6 categories" + {{ [0, 1, 2, 3, 4, 5] | listed('category') }} + + # [0, 1, 2, 3, 4, 5, 6] -> "7 leaves" + {{ [0, 1, 2, 3, 4, 5, 6] | listed("leaf", "leaves") }} + """ + + # cast: for some reason, mypy sees `listed` as `Any` + return cast(str, fmf.utils.listed( + items, + singular=singular, + plural=plural, + max=max, + quote=quote, + join=join)) + + +TEMPLATE_FILTERS: dict[str, Callable[..., Any]] = { + _name.replace('_template_filter_', ''): _obj + for _name, _obj in locals().items() if callable(_obj) and _name.startswith('_template_filter_') + } + + def default_template_environment() -> jinja2.Environment: """ Create a Jinja2 environment with default settings. @@ -6572,16 +6834,7 @@ def regex_search( return match.groups() - environment.filters['basename'] = lambda x: Path(x).name - environment.filters['dedent'] = textwrap.dedent - environment.filters['findall'] = lambda s, pattern: re.findall(pattern, s) - environment.filters['listed'] = fmf.utils.listed - environment.filters['strip'] = lambda x: x.strip() - environment.filters['search'] = lambda string, pattern: re.search(pattern, string) - environment.filters['match'] = lambda string, pattern: re.search(pattern, string) - environment.filters['regex_search'] = regex_search - environment.filters['regex_replace'] = lambda string, pattern, repl: re.sub( - pattern, repl, string) + environment.filters.update(TEMPLATE_FILTERS) environment.trim_blocks = True environment.lstrip_blocks = True