From aaa391c465f0e7466ea510465e1359a80fe5babd Mon Sep 17 00:00:00 2001 From: michalpokusa <72110769+michalpokusa@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:22:52 +0000 Subject: [PATCH 01/17] Replaced FileNotFoundError with OSError --- adafruit_templateengine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adafruit_templateengine.py b/adafruit_templateengine.py index b5c6394..eaab02a 100644 --- a/adafruit_templateengine.py +++ b/adafruit_templateengine.py @@ -192,7 +192,7 @@ def _resolve_includes(template: str): # TODO: Restrict include to specific directory if not _exists_and_is_file(template_path): - raise FileNotFoundError(f"Include template not found: {template_path}") + raise OSError(f"Include template not found: {template_path}") # Replace the include with the template content with open(template_path, "rt", encoding="utf-8") as template_file: From 9bb2220ca5ae1a14ff0f6219e8e1208a047cc8d6 Mon Sep 17 00:00:00 2001 From: michalpokusa <72110769+michalpokusa@users.noreply.github.com> Date: Thu, 26 Oct 2023 10:27:24 +0000 Subject: [PATCH 02/17] Replaced .title() with a workaround --- examples/templateengine_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/templateengine_exec.py b/examples/templateengine_exec.py index 1169482..1b71483 100644 --- a/examples/templateengine_exec.py +++ b/examples/templateengine_exec.py @@ -16,7 +16,7 @@ {% exec name = "jake" %} We defined a name: {{ name }}
- {% exec name = name.title() %} + {% exec name = (name[0].upper() + name[1:]) if name else "" %} First letter was capitalized: {{ name }}
{% exec name = list(name) %} From 331bbbfb7ab5d4d60849a592006a58cacde1cb98 Mon Sep 17 00:00:00 2001 From: michalpokusa <72110769+michalpokusa@users.noreply.github.com> Date: Thu, 26 Oct 2023 10:41:43 +0000 Subject: [PATCH 03/17] Fix for incorrect [\s\S] matching by re module --- adafruit_templateengine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/adafruit_templateengine.py b/adafruit_templateengine.py index eaab02a..02161c7 100644 --- a/adafruit_templateengine.py +++ b/adafruit_templateengine.py @@ -156,8 +156,11 @@ def safe_markdown(value: Any) -> str: _PRECOMPILED_BLOCK_PATTERN = re.compile(r"{% block \w+? %}") _PRECOMPILED_INCLUDE_PATTERN = re.compile(r"{% include '.+?' %}|{% include \".+?\" %}") _PRECOMPILED_HASH_COMMENT_PATTERN = re.compile(r"{# .+? #}") +# Workaround for bug in re module https://github.com/adafruit/circuitpython/issues/8525 _PRECOMPILED_BLOCK_COMMENT_PATTERN = re.compile( - r"{% comment ('.*?' |\".*?\" )?%}[\s\S]*?{% endcomment %}" + # TODO: Use r"{% comment ('.*?' |\".*?\" )?%}[\s\S]*?{% endcomment %}" without flags when fixed + r"{% comment ('.*?' |\".*?\" )?%}.*?{% endcomment %}", + 16, # re.DOTALL flag ) _PRECOMPILED_TOKEN_PATTERN = re.compile(r"{{ .+? }}|{% .+? %}") From f5651788d3385c517c4dad9fea0c247f3d011a99 Mon Sep 17 00:00:00 2001 From: michalpokusa <72110769+michalpokusa@users.noreply.github.com> Date: Thu, 26 Oct 2023 10:46:41 +0000 Subject: [PATCH 04/17] Redone fix for incorrect detecting unicode characters by re module --- adafruit_templateengine.py | 67 ++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/adafruit_templateengine.py b/adafruit_templateengine.py index 02161c7..e87441d 100644 --- a/adafruit_templateengine.py +++ b/adafruit_templateengine.py @@ -165,32 +165,34 @@ def safe_markdown(value: Any) -> str: _PRECOMPILED_TOKEN_PATTERN = re.compile(r"{{ .+? }}|{% .+? %}") -def _find_next_extends(template: str): +def _find_next_extends(template: bytes) -> "re.Match[bytes]": return _PRECOMPILED_EXTENDS_PATTERN.search(template) -def _find_next_block(template: str): +def _find_next_block(template: bytes) -> "re.Match[bytes]": return _PRECOMPILED_BLOCK_PATTERN.search(template) -def _find_next_include(template: str): +def _find_next_include(template: bytes) -> "re.Match[bytes]": return _PRECOMPILED_INCLUDE_PATTERN.search(template) -def _find_named_endblock(template: str, name: str): - return re.search(r"{% endblock " + name + r" %}", template) +def _find_named_endblock(template: bytes, name: bytes) -> "re.Match[bytes]": + return re.search( + r"{% endblock ".encode("utf-8") + name + r" %}".encode("utf-8"), template + ) -def _exists_and_is_file(path: str): +def _exists_and_is_file(path: str) -> bool: try: return (os.stat(path)[0] & 0b_11110000_00000000) == 0b_10000000_00000000 except OSError: return False -def _resolve_includes(template: str): +def _resolve_includes(template: bytes) -> bytes: while (include_match := _find_next_include(template)) is not None: - template_path = include_match.group(0)[12:-4] + template_path = include_match.group(0)[12:-4].decode("utf-8") # TODO: Restrict include to specific directory @@ -201,19 +203,19 @@ def _resolve_includes(template: str): with open(template_path, "rt", encoding="utf-8") as template_file: template = ( template[: include_match.start()] - + template_file.read() + + template_file.read().encode("utf-8") + template[include_match.end() :] ) return template -def _check_for_unsupported_nested_blocks(template: str): +def _check_for_unsupported_nested_blocks(template: bytes): if _find_next_block(template) is not None: raise ValueError("Nested blocks are not supported") -def _resolve_includes_blocks_and_extends(template: str): - block_replacements: "dict[str, str]" = {} +def _resolve_includes_blocks_and_extends(template: bytes) -> bytes: + block_replacements: "dict[bytes, bytes]" = {} # Processing nested child templates while (extends_match := _find_next_extends(template)) is not None: @@ -223,7 +225,7 @@ def _resolve_includes_blocks_and_extends(template: str): with open( extended_template_name, "rt", encoding="utf-8" ) as extended_template_file: - extended_template = extended_template_file.read() + extended_template = extended_template_file.read().encode("utf-8") # Removed the extend tag template = template[extends_match.end() :] @@ -240,18 +242,13 @@ def _resolve_includes_blocks_and_extends(template: str): if endblock_match is None: raise ValueError(r"Missing {% endblock %} for block: " + block_name) - # Workaround for bug in re module https://github.com/adafruit/circuitpython/issues/6860 - block_content = template.encode("utf-8")[ - block_match.end() : endblock_match.start() - ].decode("utf-8") - # TODO: Uncomment when bug is fixed - # block_content = template[block_match.end() : endblock_match.start()] + block_content = template[block_match.end() : endblock_match.start()] _check_for_unsupported_nested_blocks(block_content) if block_name in block_replacements: block_replacements[block_name] = block_replacements[block_name].replace( - r"{{ block.super }}", block_content + r"{{ block.super }}".encode("utf-8"), block_content ) else: block_replacements.setdefault(block_name, block_content) @@ -268,14 +265,16 @@ def _resolve_includes_blocks_and_extends(template: str): return _replace_blocks_with_replacements(template, block_replacements) -def _replace_blocks_with_replacements(template: str, replacements: "dict[str, str]"): +def _replace_blocks_with_replacements( + template: bytes, replacements: "dict[bytes, bytes]" +) -> bytes: # Replace blocks in top-level template while (block_match := _find_next_block(template)) is not None: block_name = block_match.group(0)[9:-3] # Self-closing block tag without default content if (endblock_match := _find_named_endblock(template, block_name)) is None: - replacement = replacements.get(block_name, "") + replacement = replacements.get(block_name, "".encode("utf-8")) template = ( template[: block_match.start()] @@ -300,7 +299,7 @@ def _replace_blocks_with_replacements(template: str, replacements: "dict[str, st # Replace default content with replacement else: replacement = replacements[block_name].replace( - r"{{ block.super }}", block_content + r"{{ block.super }}".encode("utf-8"), block_content ) template = ( @@ -312,15 +311,15 @@ def _replace_blocks_with_replacements(template: str, replacements: "dict[str, st return template -def _find_next_hash_comment(template: str): +def _find_next_hash_comment(template: bytes) -> "re.Match[bytes]": return _PRECOMPILED_HASH_COMMENT_PATTERN.search(template) -def _find_next_block_comment(template: str): +def _find_next_block_comment(template: bytes) -> "re.Match[bytes]": return _PRECOMPILED_BLOCK_COMMENT_PATTERN.search(template) -def _remove_comments(template: str): +def _remove_comments(template: bytes) -> bytes: # Remove hash comments: {# ... #} while (comment_match := _find_next_hash_comment(template)) is not None: template = template[: comment_match.start()] + template[comment_match.end() :] @@ -332,7 +331,7 @@ def _remove_comments(template: str): return template -def _find_next_token(template: str): +def _find_next_token(template: bytes) -> "re.Match[bytes]": return _PRECOMPILED_TOKEN_PATTERN.search(template) @@ -344,6 +343,10 @@ def _create_template_function( # pylint: disable=,too-many-locals,too-many-bran context_name: str = "context", dry_run: bool = False, ) -> "Generator[str] | str": + # Workaround for bug in re module https://github.com/adafruit/circuitpython/issues/6860 + # TODO: Remove .encode() and .decode() when bug is fixed + template: bytes = template.encode("utf-8") + # Resolve includes, blocks and extends template = _resolve_includes_blocks_and_extends(template) @@ -360,10 +363,10 @@ def _create_template_function( # pylint: disable=,too-many-locals,too-many-bran # Resolve tokens while (token_match := _find_next_token(template)) is not None: - token = token_match.group(0) + token: str = token_match.group(0).decode("utf-8") # Add the text before the token - if text_before_token := template[: token_match.start()]: + if text_before_token := template[: token_match.start()].decode("utf-8"): function_string += ( indent * indentation_level + f"yield {repr(text_before_token)}\n" ) @@ -452,9 +455,11 @@ def _create_template_function( # pylint: disable=,too-many-locals,too-many-bran # Continue with the rest of the template template = template[token_match.end() :] - # Add the text after the last token (if any) and return + # Add the text after the last token (if any) if template: - function_string += indent * indentation_level + f"yield {repr(template)}\n" + function_string += ( + indent * indentation_level + f"yield {repr(template.decode('utf-8'))}\n" # + ) # If dry run, return the template function string if dry_run: From 74e4842acb6d78c79464552c85e206eb6f22cc30 Mon Sep 17 00:00:00 2001 From: michalpokusa <72110769+michalpokusa@users.noreply.github.com> Date: Tue, 7 Nov 2023 20:37:47 +0000 Subject: [PATCH 05/17] Replaced hash coments with HTML comments in template files --- examples/autoescape.html | 8 +++++--- examples/base_without_footer.html | 8 +++++--- examples/child.html | 8 +++++--- examples/comments.html | 8 +++++--- examples/footer.html | 8 +++++--- examples/parent_layout.html | 8 +++++--- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/examples/autoescape.html b/examples/autoescape.html index 4abf2b2..78791c0 100644 --- a/examples/autoescape.html +++ b/examples/autoescape.html @@ -1,6 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 Michał Pokusa -# -# SPDX-License-Identifier: Unlicense + diff --git a/examples/base_without_footer.html b/examples/base_without_footer.html index cc7b2c4..59dce0b 100644 --- a/examples/base_without_footer.html +++ b/examples/base_without_footer.html @@ -1,6 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 Michał Pokusa -# -# SPDX-License-Identifier: Unlicense + diff --git a/examples/child.html b/examples/child.html index ed9cf67..5abc6e9 100644 --- a/examples/child.html +++ b/examples/child.html @@ -1,6 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 Michał Pokusa -# -# SPDX-License-Identifier: Unlicense + {% extends "./examples/parent_layout.html" %} diff --git a/examples/comments.html b/examples/comments.html index 9173aad..e376157 100644 --- a/examples/comments.html +++ b/examples/comments.html @@ -1,6 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 Michał Pokusa -# -# SPDX-License-Identifier: Unlicense + diff --git a/examples/footer.html b/examples/footer.html index c32d42b..e6563d0 100644 --- a/examples/footer.html +++ b/examples/footer.html @@ -1,6 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 Michał Pokusa -# -# SPDX-License-Identifier: Unlicense +