From b5c42093d5be496e279a849eb4a723664d6750b5 Mon Sep 17 00:00:00 2001 From: Scott Date: Sat, 21 Dec 2019 22:08:21 -0800 Subject: [PATCH] Issue #91 incremental progress 1. refactor to support embedding support code in nunavut distribution. 2. refactor to support optional inclusion of serialization support and routines. Retaining ability to generate POD types. 3. Simplification of some CMake stuff to deduplicate what TOX does for us. 4. Unification of test fixtures between Sybil and our regular unit tests. --- .buildkite/verify_cpp.sh | 2 +- .../cpp/.clang-format => .clang-format | 0 .vscode/launch.json | 14 +- .vscode/settings.json | 7 +- CONTRIBUTING.rst | 8 +- test/conftest.py => conftest.py | 129 +++++++++++++++- docs/templates.rst | 13 +- setup.py | 2 +- src/conftest.py | 129 ---------------- src/nunavut/__init__.py | 81 +++++++++- src/nunavut/cli.py | 33 ++++- src/nunavut/generators.py | 66 +++++++-- src/nunavut/jinja/__init__.py | 71 ++++++--- src/nunavut/lang/__init__.py | 139 +++++++++++++++++- src/nunavut/lang/cpp/__init__.py | 100 +++++++++++-- .../lang/cpp/{_support.py => _utilities.py} | 13 +- src/nunavut/lang/cpp/support/__init__.py | 107 ++++++++++++++ .../lang/cpp/support/serialization.hpp | 29 ++-- src/nunavut/lang/cpp/templates/Header.j2 | 3 +- .../lang/cpp/templates/_composite_type.j2 | 15 +- src/nunavut/lang/properties.ini | 7 + src/nunavut/lang/py.py | 2 +- src/nunavut/postprocessors.py | 1 - src/nunavut/version.py | 2 +- test/gentest_any/test_any.py | 4 +- test/gentest_dsdl/test_dsdl.py | 18 +-- test/gentest_filters/test_filters.py | 14 +- test/gentest_json/test_json.py | 4 +- test/gentest_lang/test_lang.py | 19 +-- test/gentest_lookup/test_lookup.py | 7 +- test/gentest_multiple/test_multiple.py | 2 +- test/gentest_namespaces/test_namespaces.py | 11 +- .../test_postprocessors.py | 81 ++++++---- .../test_serialization.py | 19 +++ test/gentest_tests/templates/Any.j2 | 3 +- test/gentest_tests/test_tests.py | 4 +- tox.ini | 2 +- verification/cpp/CMakeLists.txt | 23 +-- verification/cpp/cmake/modules/Findnnvg.cmake | 6 +- .../{Findpython3.cmake => Findtox.cmake} | 50 +++---- .../cpp/cmake/modules/Findvirtualenv.cmake | 41 ------ verification/cpp/suite/test_support.cpp | 13 +- 42 files changed, 881 insertions(+), 413 deletions(-) rename verification/cpp/.clang-format => .clang-format (100%) rename test/conftest.py => conftest.py (53%) delete mode 100644 src/conftest.py rename src/nunavut/lang/cpp/{_support.py => _utilities.py} (84%) create mode 100644 src/nunavut/lang/cpp/support/__init__.py rename verification/cpp/include/nunavut/support.hpp => src/nunavut/lang/cpp/support/serialization.hpp (56%) create mode 100644 test/gentest_serialization/test_serialization.py rename verification/cpp/cmake/modules/{Findpython3.cmake => Findtox.cmake} (51%) delete mode 100644 verification/cpp/cmake/modules/Findvirtualenv.cmake diff --git a/.buildkite/verify_cpp.sh b/.buildkite/verify_cpp.sh index 33d9a413..bcfdc0b8 100755 --- a/.buildkite/verify_cpp.sh +++ b/.buildkite/verify_cpp.sh @@ -29,6 +29,6 @@ if [ ! -d build ]; then mkdir build fi cd build -cmake -DNUNAVUT_CPP_FLAG_SET=linux -DVIRTUALENV_OUTPUT="$PWD/.pyenv" .. +cmake -DNUNAVUT_CPP_FLAG_SET=linux .. cmake --build . --target all -- -j4 cmake --build . --target cov_all diff --git a/verification/cpp/.clang-format b/.clang-format similarity index 100% rename from verification/cpp/.clang-format rename to .clang-format diff --git a/.vscode/launch.json b/.vscode/launch.json index d2e750bd..e385282b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,7 +24,7 @@ "module": "pytest", "args": ["${file}"], "console": "internalConsole", - "cwd": "${workspaceFolder}/test" + "cwd": "${workspaceFolder}" }, { "name": "Pytest: all tests", @@ -32,15 +32,7 @@ "request": "launch", "module": "pytest", "console": "internalConsole", - "cwd": "${workspaceFolder}/test" - }, - { - "name": "Pytest: all doctests", - "type": "python", - "request": "launch", - "module": "pytest", - "console": "internalConsole", - "cwd": "${workspaceFolder}/src" - }, + "cwd": "${workspaceFolder}" + } ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index e8160cf4..240727a6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -42,21 +42,26 @@ "apidoc", "autofunction", "bitor", + "buncho", "chmod", "compl", "conftest", "datetime", "deque", + "deserialize", "docstring", "docstrings", "doctests", "dsdl", "endblock", + "esmeinc", "fnmatch", + "fspath", "functools", "functors", "gentest", "getenv", + "huckco", "ifdef", "ifndef", "inout", @@ -88,11 +93,11 @@ "rtype", "scotec", "serializable", + "serializables", "sonarcloud", "struct", "submodule", "uint", - "virtualenv", "wchar", "xargs", "yamlfy" diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 71551679..8a5a7b37 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -24,7 +24,7 @@ your dev environment setup. Tools ************************************************ -tox -e local (virtualenv) +tox -e local ================================================ I highly recommend using the local tox environment when doing python development. It'll save you hours @@ -68,6 +68,12 @@ and running do:: docker run --rm -it -v $PWD:/repo uavcan/toxic:py35-py38-sq tox +To run the c++ verification build you'll need to use a different docker container: + + docker pull uavcan/libuavcan:1.0 + docker run --rm -it -v $PWD:/repo uavcan/libuavcan:1.0 + ./.buildkite/verify_cpp.sh + Files Generated by the Tests ================================================ diff --git a/test/conftest.py b/conftest.py similarity index 53% rename from test/conftest.py rename to conftest.py index ee0b59d5..21f46ece 100644 --- a/test/conftest.py +++ b/conftest.py @@ -6,6 +6,7 @@ Fixtures for our tests. """ +import functools import os import pathlib import re @@ -13,10 +14,22 @@ import tempfile import textwrap import typing +from doctest import ELLIPSIS +from fnmatch import fnmatch +from unittest.mock import MagicMock import pytest +from sybil import Sybil +from sybil.integration.pytest import SybilFile +from sybil.parsers.codeblock import CodeBlockParser +from sybil.parsers.doctest import DocTestParser from nunavut import Namespace +from nunavut.jinja.jinja2 import DictLoader, Environment +from nunavut.lang import LanguageContext +from nunavut.templates import (CONTEXT_FILTER_ATTRIBUTE_NAME, + ENVIRONMENT_FILTER_ATTRIBUTE_NAME, + LANGUAGE_FILTER_ATTRIBUTE_NAME) @pytest.fixture @@ -42,11 +55,14 @@ def _run_nnvg(gen_paths: typing.Any, class GenTestPaths: """Helper to generate common paths used in our unit tests.""" - def __init__(self, test_file: str, keep_temporaries: bool, param_index: int): + def __init__(self, test_file: str, keep_temporaries: bool, node_name: str): test_file_path = pathlib.Path(test_file) - self.test_name = '{}_{}'.format(test_file_path.parent.stem, param_index) + self.test_name = '{}_{}'.format(test_file_path.parent.stem, node_name) self.test_dir = test_file_path.parent - self.root_dir = self.test_dir.resolve().parent.parent + search_dir = self.test_dir.resolve() + while search_dir.is_dir() and not (search_dir / pathlib.Path('src')).is_dir(): + search_dir = search_dir.parent + self.root_dir = search_dir self.templates_dir = self.test_dir / pathlib.Path('templates') self.dsdl_dir = self.test_dir / pathlib.Path('dsdl') @@ -97,7 +113,7 @@ def _ensure_dir(path_dir: pathlib.Path) -> pathlib.Path: @pytest.fixture(scope='function') def gen_paths(request): # type: ignore - return GenTestPaths(request.module.__file__, request.config.option.keep_generated, request.param_index) + return GenTestPaths(str(request.fspath), request.config.option.keep_generated, request.node.name) def pytest_addoption(parser): # type: ignore @@ -145,3 +161,108 @@ def test_is_unique(unique_name_evaluator) -> None: """ return _UniqueNameEvaluator() + + +@pytest.fixture +def jinja_filter_tester(request): # type: ignore + """ + Use to create fluent but testable documentation for Jinja filters. + + Example:: + + .. invisible-code-block: python + + from nunavut.templates import template_environment_filter + + @template_environment_filter + def filter_dummy(env, input): + return input + + .. code-block:: python + + # Given + I = 'foo' + + # and + template = '{{ I | dummy }}' + + # then + rendered = I + + + .. invisible-code-block: python + + jinja_filter_tester(filter_dummy, template, rendered, 'c', I=I) + + """ + def _make_filter_test_template(filter: typing.Callable, + body: str, + expected: str, + target_language: typing.Union[typing.Optional[str], LanguageContext], + **globals: typing.Optional[typing.Dict[str, typing.Any]]) -> str: + e = Environment(loader=DictLoader({'test': body})) + filter_name = filter.__name__[7:] + if hasattr(filter, ENVIRONMENT_FILTER_ATTRIBUTE_NAME) and getattr(filter, ENVIRONMENT_FILTER_ATTRIBUTE_NAME): + e.filters[filter_name] = functools.partial(filter, e) + else: + e.filters[filter_name] = filter + + if hasattr(filter, CONTEXT_FILTER_ATTRIBUTE_NAME) and getattr(filter, CONTEXT_FILTER_ATTRIBUTE_NAME): + context = MagicMock() + e.filters[filter_name] = functools.partial(filter, context) + else: + e.filters[filter_name] = filter + + if globals is not None: + e.globals.update(globals) + + if isinstance(target_language, LanguageContext): + lctx = target_language + else: + lctx = LanguageContext(target_language) + + if hasattr(filter, LANGUAGE_FILTER_ATTRIBUTE_NAME): + language_name = getattr(filter, LANGUAGE_FILTER_ATTRIBUTE_NAME) + e.filters[filter_name] = functools.partial(filter, lctx.get_language(language_name)) + else: + e.filters[filter_name] = filter + + rendered = str(e.get_template('test').render()) + if expected != rendered: + msg = 'Unexpected template output\n\texpected : {}\n\twas : {}'.format(expected, rendered) + raise AssertionError(msg) + return rendered + + return _make_filter_test_template + + +def _pytest_integration_that_actually_works() -> typing.Callable: + """ + Sybil matching is pretty broken. We'll have to help it out here. The problem is that + exclude patterns passed into the Sybil object are matched against file name stems such that + files cannot be excluded by path. + """ + + _excludes = [ + '**/markupsafe/*', + '**/jinja2/*', + ] + + _sy = Sybil( + parsers=[ + DocTestParser(optionflags=ELLIPSIS), + CodeBlockParser(), + ], + fixtures=['jinja_filter_tester', 'gen_paths'] + ) + + def pytest_collect_file(parent: typing.Any, path: typing.Any) -> typing.Optional[SybilFile]: + if fnmatch(str(path), '**/nunavut/**/*.py') and not any(fnmatch(str(path), pattern) for pattern in _excludes): + return SybilFile(path, parent, _sy) + else: + return None + + return pytest_collect_file + + +pytest_collect_file = _pytest_integration_that_actually_works() diff --git a/docs/templates.rst b/docs/templates.rst index b56356c7..6cd88708 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -181,19 +181,20 @@ by type if this seems useful for your project. Simply use the Namespace Templates ================================================= -By setting the :code:`generate_namespace_types` parameter of :class:`~nunavut.jinja.Generator` to -true the generator will invoke a template for the root namespace and all nested namespaces allowing -for languages where namespaces are first class objects. For example:: +If the :code:`generate_namespace_types` parameter of :class:`~nunavut.jinja.Generator` is +:code:`YES` then the generator will always invoke a template for the root namespace and all +nested namespaces regardless of language. :code:`NO` suppresses this behavior and :code:`DEFAULT` +will choose the behavior based on the target language. For example:: root_namespace = build_namespace_tree(compound_types, root_ns_folder, out_dir, language_context) - generator = Generator(root_namespace, True, templates_dir) + generator = Generator(root_namespace, YesNoDefault.DEFAULT) -This could be used to generate python :code:`__init__.py` files which would define each namespace -as a python module. +Would generate python :code:`__init__.py` files to define each namespace as a python module but +would not generate any additional headers for C++. The :class:`~nunavut.jinja.Generator` will use the same template name resolution logic as used for pydsdl data types. For namespaces this will resolve first to a template named diff --git a/setup.py b/setup.py index 6501f135..7dbddc44 100755 --- a/setup.py +++ b/setup.py @@ -19,4 +19,4 @@ exec(fp.read(), version) setuptools.setup(version=version['__version__'], - package_data={'': ['*.j2', '*.ini']}) + package_data={'': ['*.j2', '*.ini', '*.hpp']}) diff --git a/src/conftest.py b/src/conftest.py deleted file mode 100644 index 17c1910b..00000000 --- a/src/conftest.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# This software is distributed under the terms of the MIT License. -# -""" -Enable pytest integration of doctests in source and/or in documentation. -""" -import functools -import typing -from doctest import ELLIPSIS -from fnmatch import fnmatch -from unittest.mock import MagicMock - -import pytest -from sybil import Sybil -from sybil.integration.pytest import SybilFile -from sybil.parsers.codeblock import CodeBlockParser -from sybil.parsers.doctest import DocTestParser - -from nunavut.jinja.jinja2 import DictLoader, Environment -from nunavut.lang import LanguageContext -from nunavut.templates import (CONTEXT_FILTER_ATTRIBUTE_NAME, - ENVIRONMENT_FILTER_ATTRIBUTE_NAME, - LANGUAGE_FILTER_ATTRIBUTE_NAME) - - -@pytest.fixture -def jinja_filter_tester(request): # type: ignore - """ - Use to create fluent but testable documentation for Jinja filters. - - Example:: - - .. invisible-code-block: python - - from nunavut.templates import template_environment_filter - - @template_environment_filter - def filter_dummy(env, input): - return input - - .. code-block:: python - - # Given - I = 'foo' - - # and - template = '{{ I | dummy }}' - - # then - rendered = I - - - .. invisible-code-block: python - - jinja_filter_tester(filter_dummy, template, rendered, 'c', I=I) - - """ - def _make_filter_test_template(filter: typing.Callable, - body: str, - expected: str, - target_language: typing.Union[typing.Optional[str], LanguageContext], - **globals: typing.Optional[typing.Dict[str, typing.Any]]) -> str: - e = Environment(loader=DictLoader({'test': body})) - filter_name = filter.__name__[7:] - if hasattr(filter, ENVIRONMENT_FILTER_ATTRIBUTE_NAME) and getattr(filter, ENVIRONMENT_FILTER_ATTRIBUTE_NAME): - e.filters[filter_name] = functools.partial(filter, e) - else: - e.filters[filter_name] = filter - - if hasattr(filter, CONTEXT_FILTER_ATTRIBUTE_NAME) and getattr(filter, CONTEXT_FILTER_ATTRIBUTE_NAME): - context = MagicMock() - e.filters[filter_name] = functools.partial(filter, context) - else: - e.filters[filter_name] = filter - - if globals is not None: - e.globals.update(globals) - - if isinstance(target_language, LanguageContext): - lctx = target_language - else: - lctx = LanguageContext(target_language) - - if hasattr(filter, LANGUAGE_FILTER_ATTRIBUTE_NAME): - language_name = getattr(filter, LANGUAGE_FILTER_ATTRIBUTE_NAME) - e.filters[filter_name] = functools.partial(filter, lctx.get_language(language_name)) - else: - e.filters[filter_name] = filter - - rendered = str(e.get_template('test').render()) - if expected != rendered: - msg = 'Unexpected template output\n\texpected : {}\n\twas : {}'.format(expected, rendered) - raise AssertionError(msg) - return rendered - - return _make_filter_test_template - - -def _pytest_integration_that_actually_works() -> typing.Callable: - """ - Sybil matching is pretty broken. We'll have to help it out here. The problem is that - exclude patterns passed into the Sybil object are matched against file name stems such that - files cannot be excluded by path. - """ - - _excludes = [ - '**/markupsafe/*', - '**/jinja2/*', - ] - - _sy = Sybil( - parsers=[ - DocTestParser(optionflags=ELLIPSIS), - CodeBlockParser(), - ], - fixtures=['jinja_filter_tester'] - ) - - def pytest_collect_file(parent: typing.Any, path: typing.Any) -> typing.Optional[SybilFile]: - if fnmatch(str(path), '**/nunavut/**/*.py') and not any(fnmatch(str(path), pattern) for pattern in _excludes): - return SybilFile(path, parent, _sy) - else: - return None - - return pytest_collect_file - - -pytest_collect_file = _pytest_integration_that_actually_works() diff --git a/src/nunavut/__init__.py b/src/nunavut/__init__.py index 590d593d..eb6eb30a 100644 --- a/src/nunavut/__init__.py +++ b/src/nunavut/__init__.py @@ -56,7 +56,7 @@ language_context) # give the root namespace to the generator and... - generator = Generator(root_namespace, False, language_context, templates_dir) + generator = Generator(root_namespace) # generate all the code! generator.generate_all() @@ -64,6 +64,7 @@ """ import collections +import enum import pathlib import sys import typing @@ -76,6 +77,39 @@ print('A newer version of Python is required', file=sys.stderr) sys.exit(1) + +class YesNoDefault(enum.Enum): + """ + Trinary type for decisions that allow a default behavior to be requested that can + be different based on other contexts. For example: + + .. invisible-code-block: python + + from datetime import datetime + from nunavut import YesNoDefault + + .. code-block:: python + + def should_we_order_pizza(answer: YesNoDefault) -> bool: + if answer == YesNoDefault.YES or ( + answer == YesNoDefault.DEFAULT and + datetime.today().isoweekday() == 5): + # if yes or if we are taking the default action which is to + # order pizza on Friday, and today is Friday, then we order pizza + return True + else: + return False + + .. invisible-code-block: python + + assert should_we_order_pizza(YesNoDefault.YES) + assert not should_we_order_pizza(YesNoDefault.NO) + + """ + NO = 0 + YES = 1 + DEFAULT = 2 + # +---------------------------------------------------------------------------+ @@ -115,6 +149,7 @@ def __init__(self, if output_stem is None: output_stem = self.DefaultOutputStem output_path = self._output_folder / pathlib.PurePath(output_stem) + self._support_output_folder = base_output_path self._output_path = output_path.with_suffix(language_context.get_output_extension()) self._source_folder = pathlib.Path( root_namespace_dir / pathlib.PurePath(*self._namespace_components[1:])).resolve() @@ -124,6 +159,7 @@ def __init__(self, self._short_name = self._namespace_components_stropped[-1] self._data_type_to_outputs = dict() # type: typing.Dict[pydsdl.CompositeType, pathlib.Path] self._nested_namespaces = set() # type: typing.Set[Namespace] + self._language_context = language_context @property def output_folder(self) -> pathlib.Path: @@ -132,6 +168,15 @@ def output_folder(self) -> pathlib.Path: """ return self._output_folder + def get_support_output_folder(self) -> pathlib.PurePath: + return self._support_output_folder + + def get_language_context(self) -> 'lang.LanguageContext': + """ + The generated software language context the namespace is within. + """ + return self._language_context + def get_root_namespace(self) -> 'Namespace': """ Traverses the namespace tree up to the root and returns the root node. @@ -506,3 +551,37 @@ def _extract_dependent_types(cls, inout_dependencies.uses_float = True elif isinstance(dt, pydsdl.BooleanType): inout_dependencies.uses_bool = True + +# +---------------------------------------------------------------------------+ +# | GENERATION HELPERS +# +---------------------------------------------------------------------------+ + + +def generate_types(language_key: str, + root_namespace_dir: pathlib.Path, + out_dir: pathlib.Path, + omit_serialization_support: bool = True, + is_dryrun: bool = False, + allow_overwrite: bool = True, + lookup_directories: typing.Optional[typing.Iterable[str]] = None) -> None: + """ + Helper method that uses default settings and built-in templates to generate types for a given + language. This method is the most direct way to generate code using Nunavut. + """ + from nunavut.generators import create_builtin_source_generator, create_support_generator + + language_context = lang.LanguageContext(language_key, + omit_serialization_support_for_target=omit_serialization_support) + + if lookup_directories is None: + lookup_directories = [] + + type_map = pydsdl.read_namespace(str(root_namespace_dir), lookup_directories) + + namespace = build_namespace_tree(type_map, + str(root_namespace_dir), + str(out_dir), + language_context) + + create_support_generator(namespace).generate_all(is_dryrun, allow_overwrite) + create_builtin_source_generator(namespace).generate_all(is_dryrun, allow_overwrite) diff --git a/src/nunavut/cli.py b/src/nunavut/cli.py index 0b2e110e..e8020dd6 100644 --- a/src/nunavut/cli.py +++ b/src/nunavut/cli.py @@ -68,7 +68,8 @@ def _build_post_processor_list_from_args(args: argparse.Namespace) \ language_context = nunavut.lang.LanguageContext( args.target_language, args.output_extension, - args.namespace_output_stem) + args.namespace_output_stem, + omit_serialization_support_for_target=args.omit_serialization_support) # # nunavut : parse @@ -91,18 +92,21 @@ def _build_post_processor_list_from_args(args: argparse.Namespace) \ sys.stdout.write(';') return 0 + generate_namespace_types = (nunavut.YesNoDefault.YES + if args.generate_namespace_types + else nunavut.YesNoDefault.DEFAULT) generator = nunavut.jinja.Generator(root_namespace, - args.generate_namespace_types, - language_context, + generate_namespace_types, (pathlib.Path(args.templates) if args.templates is not None else None), trim_blocks=args.trim_blocks, - lstrip_blocks=args.lstrip_blocks) + lstrip_blocks=args.lstrip_blocks, + post_processors=_build_post_processor_list_from_args(args)) if args.list_inputs: for input_path in generator.get_templates(): sys.stdout.write(str(input_path.resolve())) sys.stdout.write(';') - if args.generate_namespace_types: + if generator.generate_namespace_types: for output_type, _ in root_namespace.get_all_types(): sys.stdout.write(str(output_type.source_file_path)) sys.stdout.write(';') @@ -112,9 +116,14 @@ def _build_post_processor_list_from_args(args: argparse.Namespace) \ sys.stdout.write(';') return 0 + if args.omit_serialization_support is None or not args.omit_serialization_support: + from nunavut.generators import create_support_generator + support_generator = create_support_generator(root_namespace) + support_generator.generate_all(is_dryrun=args.dry_run, + allow_overwrite=not args.no_overwrite) + return generator.generate_all(is_dryrun=args.dry_run, - allow_overwrite=not args.no_overwrite, - post_processors=_build_post_processor_list_from_args(args)) + allow_overwrite=not args.no_overwrite) class _LazyVersionAction(argparse._VersionAction): @@ -253,6 +262,16 @@ def extension_type(raw_arg: str) -> str: ''').lstrip()) + parser.add_argument('--omit-serialization-support', + '-pod', + action='store_true', + help=textwrap.dedent(''' + If provided then the types generated will be POD datatypes with no additional logic. + By default types generated include serialization routines and additional support libraries, + headers, or methods. + + ''').lstrip()) + parser.add_argument('--namespace-output-stem', default='Namespace', help='The name of the file generated when --generate-namespace-types is provided.') diff --git a/src/nunavut/generators.py b/src/nunavut/generators.py index cf028d0f..4371e2d5 100644 --- a/src/nunavut/generators.py +++ b/src/nunavut/generators.py @@ -13,8 +13,6 @@ import typing import nunavut -import nunavut.postprocessors -import nunavut.lang class AbstractGenerator(metaclass=abc.ABCMeta): @@ -24,17 +22,26 @@ class AbstractGenerator(metaclass=abc.ABCMeta): :param nunavut.Namespace namespace: The top-level namespace to generates types at and from. - :param bool generate_namespace_types: Set to true to emit files - for namespaces. False will only generate files for datatypes. + :param nunavut.YesNoDefault generate_namespace_types: Set to YES + to force generation files for namespaces and NO to suppress. + DEFAULT will generate namespace files based on the language + preference. """ def __init__(self, namespace: nunavut.Namespace, - generate_namespace_types: bool, - language_context: nunavut.lang.LanguageContext): + generate_namespace_types: nunavut.YesNoDefault = nunavut.YesNoDefault.DEFAULT): self._namespace = namespace - self._generate_namespace_types = generate_namespace_types - self._language_context = language_context + if generate_namespace_types == nunavut.YesNoDefault.YES: + self._generate_namespace_types = True + elif generate_namespace_types == nunavut.YesNoDefault.NO: + self._generate_namespace_types = False + else: + target_language = self._namespace.get_language_context().get_target_language() + if target_language is not None and target_language.has_standard_namespace_files: + self._generate_namespace_types = True + else: + self._generate_namespace_types = False @property def namespace(self) -> nunavut.Namespace: @@ -52,15 +59,10 @@ def generate_namespace_types(self) -> bool: """ return self._generate_namespace_types - @property - def language_context(self) -> nunavut.lang.LanguageContext: - return self._language_context - @abc.abstractmethod def generate_all(self, is_dryrun: bool = False, - allow_overwrite: bool = True, - post_processors: typing.Optional[typing.List['nunavut.postprocessors.PostProcessor']] = None) \ + allow_overwrite: bool = True) \ -> int: """ Generates all output for a given :class:`nunavut.Namespace` and using @@ -71,6 +73,40 @@ def generate_all(self, :param bool allow_overwrite: If True then the generator will attempt to overwrite any existing files it encounters. If False then the generator will raise an error if the output file exists and the generation is not a dry-run. - :param post_processors: A list of :class:`nunavut.postprocessors.PostProcessor` + :return: 0 for success. Non-zero for errors. + :raises: PermissionError if :attr:`allow_overwrite` is False and the file exists. """ raise NotImplementedError() + + +def create_builtin_source_generator(namespace: nunavut.Namespace) -> 'AbstractGenerator': + """ + Create a new :class:`Generator ` that uses internal templates + and configuration to generate source code for DSDL messages. + """ + from nunavut.jinja import Generator + + return Generator(namespace) + + +def create_support_generator(namespace: nunavut.Namespace) -> 'AbstractGenerator': + """ + Create a new :class:`Generator ` that uses embedded support + headers, libraries, and other types needed to use generated serialization code for the + :func:`target language `. If no target language + is set or if serialization support has been turned off a no-op generator will be returned instead. + """ + class _NoOpSupportGenerator(AbstractGenerator): + def generate_all(self, + is_dryrun: bool = False, + allow_overwrite: bool = True) \ + -> int: + return 0 + + target_language = namespace.get_language_context().get_target_language() + if target_language is None or target_language.omit_serialization_support: + return _NoOpSupportGenerator(namespace, nunavut.YesNoDefault.DEFAULT) + else: + SupportGenerator = getattr(target_language.module, 'SupportGenerator', _NoOpSupportGenerator) \ + # type: typing.Type[AbstractGenerator] + return SupportGenerator(namespace) diff --git a/src/nunavut/jinja/__init__.py b/src/nunavut/jinja/__init__.py index 9d316bc8..77d394c4 100644 --- a/src/nunavut/jinja/__init__.py +++ b/src/nunavut/jinja/__init__.py @@ -21,6 +21,7 @@ import nunavut.generators import nunavut.lang +import nunavut.postprocessors from nunavut.jinja.jinja2 import (ChoiceLoader, Environment, FileSystemLoader, PackageLoader, StrictUndefined, Template, TemplateAssertionError, nodes, @@ -88,8 +89,9 @@ class Generator(nunavut.generators.AbstractGenerator): :param nunavut.Namespace namespace: The top-level namespace to generates types at and from. - :param bool generate_namespace_types: typing.Set to true to emit files for namespaces. - False will only generate files for datatypes. + :param nunavut.YesNoDefault generate_namespace_types: Set to YES to emit files for namespaces. + NO will suppress namespace file generation and DEFAULT will + use the language's preference. :param templates_dir: Directories containing jinja templates. These will be available along with any built-in templates provided by the target language. The templates at these paths will take precedence masking any built-in templates @@ -111,8 +113,8 @@ class Generator(nunavut.generators.AbstractGenerator): and the callable as the test. :param typing.Dict[str, typing.Any] additional_globals: typing.Optional objects to add to the template environment globals collection. - :param typing.Optional[str] target_language_support: A language to install support for directly into the - global environment. + :param post_processors: A list of :class:`nunavut.postprocessors.PostProcessor` + :type post_processors: typing.Optional[typing.List[nunavut.postprocessors.PostProcessor]] :raises RuntimeError: If any additional filter or test attempts to replace a built-in or otherwise already defined filter or test. """ @@ -347,17 +349,20 @@ def is_padding(value: pydsdl.Field) -> bool: def __init__(self, namespace: nunavut.Namespace, - generate_namespace_types: bool, - language_context: nunavut.lang.LanguageContext, + generate_namespace_types: nunavut.YesNoDefault = nunavut.YesNoDefault.DEFAULT, templates_dir: typing.Optional[typing.Union[pathlib.Path, typing.List[pathlib.Path]]] = None, followlinks: bool = False, trim_blocks: bool = False, lstrip_blocks: bool = False, additional_filters: typing.Optional[typing.Dict[str, typing.Callable]] = None, additional_tests: typing.Optional[typing.Dict[str, typing.Callable]] = None, - additional_globals: typing.Optional[typing.Dict[str, typing.Any]] = None): + additional_globals: typing.Optional[typing.Dict[str, typing.Any]] = None, + post_processors: typing.Optional[typing.List['nunavut.postprocessors.PostProcessor']] = None): - super(Generator, self).__init__(namespace, generate_namespace_types, language_context) + super().__init__(namespace, + generate_namespace_types) + + self._post_processors = post_processors if templates_dir is None: templates_dirs = [] # type: typing.List[pathlib.Path] @@ -384,7 +389,7 @@ def __init__(self, fs_loader = FileSystemLoader((str(d) for d in self._templates_dirs), followlinks=followlinks) - target_language = self._language_context.get_target_language() + target_language = self._namespace.get_language_context().get_target_language() if target_language is not None: template_loader = ChoiceLoader([ @@ -415,7 +420,7 @@ def __init__(self, if additional_globals is not None: self._env.globals.update(additional_globals) - self._add_nunavut_globals() + self._add_nunavut_globals(target_language) self._add_instance_tests_from_root(pydsdl.SerializableType) self._add_filters_and_tests(additional_filters, additional_tests) @@ -435,45 +440,67 @@ def get_templates(self) -> typing.List[pathlib.Path]: def generate_all(self, is_dryrun: bool = False, - allow_overwrite: bool = True, - post_processors: typing.Optional[typing.List['nunavut.postprocessors.PostProcessor']] = None) \ + allow_overwrite: bool = True) \ -> int: if self.generate_namespace_types: for (parsed_type, output_path) in self.namespace.get_all_types(): - self._generate_type(parsed_type, output_path, is_dryrun, allow_overwrite, post_processors) + self._generate_type(parsed_type, output_path, is_dryrun, allow_overwrite, self._post_processors) else: for (parsed_type, output_path) in self.namespace.get_all_datatypes(): - self._generate_type(parsed_type, output_path, is_dryrun, allow_overwrite, post_processors) + self._generate_type(parsed_type, output_path, is_dryrun, allow_overwrite, self._post_processors) return 0 @property def language_context(self) -> nunavut.lang.LanguageContext: - return self._language_context + return self._namespace.get_language_context() # +-----------------------------------------------------------------------+ # | PRIVATE # +-----------------------------------------------------------------------+ - def _add_nunavut_globals(self) -> None: + def _add_nunavut_globals(self, target_language: typing.Optional[nunavut.lang.Language]) -> None: """ Add globals namespaced as 'nunavut'. """ import nunavut.version - self._env.globals['nunavut'] = {'version': nunavut.version.__version__} - pass + + # Helper global so we don't have to futz around with the "omit_serialization_support" + # logic in the templates. The omit_serialization_support property of the Language + # object is read-only so this boolean will remain consistent for the Environment. + if target_language is not None: + omit_serialization_support = target_language.omit_serialization_support + support_namespace = target_language.support_namespace + else: + # If there is no target language then we cannot generate serialization support. + omit_serialization_support = True + support_namespace = [] + + self._env.globals['nunavut'] = { + 'version': nunavut.version.__version__, + 'support': { + 'omit': omit_serialization_support, + 'namespace': support_namespace + } + } def _add_instance_tests_from_root(self, root: typing.Type[object]) -> None: - self._env.tests[root.__name__] = lambda x: isinstance(x, root) + def _field_is_instance(field_or_datatype: typing.Any) -> bool: + if isinstance(field_or_datatype, pydsdl.Field): + return isinstance(field_or_datatype.data_type, root) + else: + return isinstance(field_or_datatype, root) + + self._env.tests[root.__name__] = _field_is_instance for derived in root.__subclasses__(): self._add_instance_tests_from_root(derived) def _add_language_support(self) -> None: - target_language = self._language_context.get_target_language() + target_language = self.language_context.get_target_language() if target_language is not None: for key, value in target_language.get_filters(make_implicit=True).items(): self._add_filter_to_environment(key, value) self._env.globals.update(target_language.get_globals(True)) - for supported_language in self._language_context.get_supported_languages().values(): + for supported_language in self.language_context.get_supported_languages().values(): for key, value in supported_language.get_filters(make_implicit=False).items(): self._add_filter_to_environment(key, value) self._env.globals.update(supported_language.get_globals(False)) @@ -525,7 +552,7 @@ def _generate_type(self, def _add_filter_to_environment(self, filter_name: str, filter: typing.Callable[..., str]) -> None: if hasattr(filter, LANGUAGE_FILTER_ATTRIBUTE_NAME): language_name_or_module_name = getattr(filter, LANGUAGE_FILTER_ATTRIBUTE_NAME) - filter_language = self._language_context.get_language(language_name_or_module_name) + filter_language = self.language_context.get_language(language_name_or_module_name) resolved_filter = functools.partial(filter, filter_language) # type: typing.Callable[..., str] else: resolved_filter = filter diff --git a/src/nunavut/lang/__init__.py b/src/nunavut/lang/__init__.py index 65152bc3..d6f26561 100644 --- a/src/nunavut/lang/__init__.py +++ b/src/nunavut/lang/__init__.py @@ -23,7 +23,10 @@ class Language: """ Facilities for generating source code for a specific language. - :param str language_name: The name of the language used by the :mod:`nunavut.lang` module. + :param str language_name: The name of the language used by the :mod:`nunavut.lang` module. + :param configparser.ConfigParser config: The parser to load language properties from. + :param bool omit_serialization_support: The value to set for the :func:`omit_serialization_support` property + for this language. """ @classmethod @@ -41,11 +44,14 @@ def _find_filters_for_language(cls, language_name: str) -> typing.Mapping[str, t def __init__(self, language_name: str, - config: configparser.ConfigParser): + config: configparser.ConfigParser, + omit_serialization_support: bool): self._language_name = language_name self._section = 'nunavut.lang.{}'.format(language_name) self._filters = self._find_filters_for_language(language_name) self._config = config + self._omit_serialization_support = omit_serialization_support + self._module = None # type: typing.Optional[typing.Any] @property def extension(self) -> str: @@ -82,6 +88,36 @@ def stropping_suffix(self) -> str: """ return self._config.get(self._section, 'stropping_suffix') + @property + def support_namespace(self) -> typing.List[str]: + """ + The hierarchial namespace used by the support headers. The property + is a dot separated string when specified in configuration. This + property returns that value split into namespace components with the + first identifier being the first index in the array, etc. + + .. invisible-code-block: python + + from nunavut.lang import Language + import configparser + + config = configparser.ConfigParser() + config.add_section('nunavut.lang.cpp') + + + lang_cpp = Language('cpp', config, True) + + .. code-block:: python + + config.set('nunavut.lang.cpp', 'support_namespace', 'foo.bar') + assert len(lang_cpp.support_namespace) == 2 + assert lang_cpp.support_namespace[0] == 'foo' + assert lang_cpp.support_namespace[1] == 'bar' + + """ + namespace_str = self._config.get(self._section, 'support_namespace', fallback='') + return namespace_str.split('.') + @property def encoding_prefix(self) -> str: """ @@ -96,6 +132,29 @@ def enable_stropping(self) -> bool: """ return self._config.getboolean(self._section, 'enable_stropping') + @property + def has_standard_namespace_files(self) -> bool: + """ + Whether or not the language defines special namespace files as part of + its core standard (e.g. python's __init__). + """ + return self._config.getboolean(self._section, 'has_standard_namespace_files') + + @property + def omit_serialization_support(self) -> bool: + """ + If True then generators should not include serialization routines, types, + or support libraries for this language. + """ + return self._omit_serialization_support + + @property + def module(self) -> typing.Any: + if self._module is None: + import importlib + self._module = importlib.import_module('nunavut.lang.{}'.format(self._language_name)) + return self._module + def get_config_value(self, key: str, default_value: typing.Union[typing.Mapping[str, typing.Any], str, None] = None)\ @@ -113,6 +172,72 @@ def get_config_value(self, """ return self._config.get(self._section, key, fallback=default_value) + def get_config_value_as_bool(self, key: str, default_value: bool = False) -> bool: + """ + Get an optional language property from the language configuration returning a boolean. The rules + for boolean conversion are as follows: + + .. invisible-code-block: python + + from nunavut.lang import Language + import configparser + + config = configparser.ConfigParser() + config.add_section('nunavut.lang.cpp') + + lang_cpp = Language('cpp', config, True) + + .. code-block:: python + + # "Any string" = True + config.set('nunavut.lang.cpp', 'v', 'Any string') + assert lang_cpp.get_config_value_as_bool('v') + + # "true" = True + config.set('nunavut.lang.cpp', 'v', 'true') + assert lang_cpp.get_config_value_as_bool('v') + + # "TrUe" = True + config.set('nunavut.lang.cpp', 'v', 'TrUe') + assert lang_cpp.get_config_value_as_bool('v') + + # "1" = True + config.set('nunavut.lang.cpp', 'v', '1') + assert lang_cpp.get_config_value_as_bool('v') + + # "false" = False + config.set('nunavut.lang.cpp', 'v', 'false') + assert not lang_cpp.get_config_value_as_bool('v') + + # "FaLse" = False + config.set('nunavut.lang.cpp', 'v', 'FaLse') + assert not lang_cpp.get_config_value_as_bool('v') + + # "0" = False + config.set('nunavut.lang.cpp', 'v', '0') + assert not lang_cpp.get_config_value_as_bool('v') + + # "" = False + config.set('nunavut.lang.cpp', 'v', '') + assert not lang_cpp.get_config_value_as_bool('v') + + # False if not defined + assert not lang_cpp.get_config_value_as_bool('not_a_key') + + # True if not defined but default_value is True + assert lang_cpp.get_config_value_as_bool('not_a_key', True) + + :param str key: The config value to retrieve. + :param bool default_value: The value to use if no value existed. + :return: The config value as either True or False. + :rtype: bool + """ + result = self._config.get(self._section, key, fallback='' if not default_value else '1') + if result.lower() == 'false' or result == '0': + return False + else: + return bool(result) + def get_templates_package_name(self) -> str: """ The name of the nunavut python package containing filters, types, and configuration @@ -182,6 +307,8 @@ class LanguageContext: These will override any values found in the :file:`nunavut.lang.properties.ini` file and files appearing later in this list will override value found in earlier entries. :type additional_config_files: typing.List[pathlib.Path] + :param bool omit_serialization_support_for_target: If True then generators should not include + serialization routines, types, or support libraries for the target language. :raises ValueError: If extension is None and no target language was provided. :raises KeyError: If the target language is not known. """ @@ -219,7 +346,8 @@ def __init__(self, target_language: typing.Optional[str] = None, extension: typing.Optional[str] = None, namespace_output_stem: typing.Optional[str] = None, - additional_config_files: typing.List[pathlib.Path] = []): + additional_config_files: typing.List[pathlib.Path] = [], + omit_serialization_support_for_target: bool = True): self._extension = extension self._namespace_output_stem = namespace_output_stem self._config = self._load_config(*additional_config_files) @@ -229,7 +357,8 @@ def __init__(self, self._target_language = None else: try: - self._target_language = Language(target_language, self._config) + self._target_language = Language(target_language, self._config, + omit_serialization_support_for_target) except ImportError: raise KeyError('{} is not a supported language'.format(target_language)) if namespace_output_stem is not None: @@ -248,7 +377,7 @@ def __init__(self, self._languages[language_name] = self._target_language else: try: - self._languages[language_name] = Language(language_name, self._config) + self._languages[language_name] = Language(language_name, self._config, False) except ImportError: raise KeyError('{} is not a supported language'.format(language_name)) diff --git a/src/nunavut/lang/cpp/__init__.py b/src/nunavut/lang/cpp/__init__.py index 3bfb5daa..ac4aaeb5 100644 --- a/src/nunavut/lang/cpp/__init__.py +++ b/src/nunavut/lang/cpp/__init__.py @@ -10,21 +10,63 @@ import functools import io +import pathlib import re import typing import pydsdl +from ...generators import AbstractGenerator from ...lang import Language -from ...templates import template_language_filter, template_language_list_filter +from ...templates import (template_language_filter, + template_language_list_filter) from ..c import C_RESERVED_PATTERNS, VariableNameEncoder, _CFit -from ._support import IncludeGenerator +from ._utilities import IncludeGenerator CPP_RESERVED_PATTERNS = frozenset([*C_RESERVED_PATTERNS]) CPP_NO_DOUBLE_DASH_RULE = re.compile(r'(__)') +class SupportGenerator(AbstractGenerator): + """ + Copy C++ support types to the :func:`support_output_folder `. + This class name is expected by the :func:`nunavut.generators.create_support_generator()` method. + + .. invisible-code-block: python + + import pathlib + from unittest.mock import MagicMock, NonCallableMagicMock + from nunavut import YesNoDefault + from nunavut.lang import LanguageContext + from nunavut.lang.cpp import SupportGenerator + + language_context = LanguageContext() + namespace = NonCallableMagicMock() + namespace.get_language_context = MagicMock() + namespace.get_language_context.return_value = language_context + namespace.get_support_output_folder = MagicMock() + namespace.get_support_output_folder.return_value = gen_paths.out_dir + + .. code-block:: python + + generator = SupportGenerator(namespace, YesNoDefault.DEFAULT) + assert 0 == generator.generate_all() + + """ + def generate_all(self, + is_dryrun: bool = False, + allow_overwrite: bool = True) \ + -> int: + from .support import copy_support_headers + language = self.namespace.get_language_context().get_language(__package__) + + output_folder = pathlib.Path(self.namespace.get_support_output_folder()) + copy_support_headers(language.support_namespace, output_folder, allow_overwrite) + + return 0 + + @template_language_filter(__name__) def filter_id(language: Language, instance: typing.Any) -> str: @@ -355,29 +397,20 @@ def filter_short_reference_name(language: Language, t: pydsdl.CompositeType) -> @template_language_list_filter(__name__) def filter_includes(language: Language, t: pydsdl.CompositeType, - sort: bool = True, - prefer_system_includes: bool = False, - use_standard_types: typing.Optional[bool] = None) -> typing.List[str]: + sort: bool = True) -> typing.List[str]: """ Returns a list of all include paths for a given type. :param pydsdl.CompositeType t: The type to scan for dependencies. :param bool sort: If true the returned list will be sorted. - :param bool strop: If true the list will contained stropped identifiers. :return: a list of include headers needed for a given type. """ - if use_standard_types is None: - use_standard_types = bool(language.get_config_value('use_standard_types')) - include_gen = IncludeGenerator(language, t, filter_id, - filter_short_reference_name, - use_standard_types) - return include_gen.generate_include_filepart_list(language.extension, - sort, - prefer_system_includes) + filter_short_reference_name) + return include_gen.generate_include_filepart_list(language.extension, sort) @template_language_filter(__name__) @@ -554,3 +587,42 @@ def filter_type_from_primitive(language: Language, if use_standard_types is None: use_standard_types = bool(language.get_config_value('use_standard_types')) return _CFit.get_best_fit(value.bit_length).to_c_type(value, use_standard_types) + + +def filter_to_namespace_qualifier(namespace_list: typing.List[str]) -> str: + """ + Converts a list of namespace names into a qualifer string. For example: + + .. invisible-code-block: python + + from nunavut.lang.cpp import filter_to_namespace_qualifier + import pydsdl + + + .. code-block:: python + + my_namespace = ['foo', 'bar'] + template = '{{ my_namespace | to_namespace_qualifier }}myType()' + expected = 'foo::bar::myType()' + + .. invisible-code-block: python + + jinja_filter_tester(filter_to_namespace_qualifier, template, expected, 'cpp', my_namespace=my_namespace) + + This filter gracefully handles empty namespace lists: + + .. code-block:: python + + my_namespace = [] + template = '{{ my_namespace | to_namespace_qualifier }}myType()' + expected = 'myType()' + + .. invisible-code-block: python + + jinja_filter_tester(filter_to_namespace_qualifier, template, expected, 'cpp', my_namespace=my_namespace) + + """ + if namespace_list is None or len(namespace_list) == 0: + return '' + else: + return '::'.join(namespace_list) + '::' diff --git a/src/nunavut/lang/cpp/_support.py b/src/nunavut/lang/cpp/_utilities.py similarity index 84% rename from src/nunavut/lang/cpp/_support.py rename to src/nunavut/lang/cpp/_utilities.py index 374b81f2..24c791f6 100644 --- a/src/nunavut/lang/cpp/_support.py +++ b/src/nunavut/lang/cpp/_utilities.py @@ -23,22 +23,25 @@ def __init__(self, language: Language, t: pydsdl.CompositeType, id_filter: typing.Callable[[Language, str], str], - short_reference_name_filter: typing.Callable[..., str], - use_standard_types: bool): + short_reference_name_filter: typing.Callable[..., str]): super().__init__(t) self._language = language - self._use_standard_types = use_standard_types + self._use_standard_types = bool(self._language.get_config_value_as_bool('use_standard_types', True)) self._id = id_filter self._short_reference_name_filter = short_reference_name_filter def generate_include_filepart_list(self, output_extension: str, - sort: bool, - prefer_system_includes: bool) -> typing.List[str]: + sort: bool) -> typing.List[str]: dep_types = self.direct() path_list = [self._make_path(dt, output_extension) for dt in dep_types.composite_types] + if not self._language.omit_serialization_support: + from .support import list_support_headers + path_list += [str(p) for p in list_support_headers(self._language.support_namespace)] + + prefer_system_includes = bool(self._language.get_config_value_as_bool('prefer_system_includes', False)) if prefer_system_includes: path_list_with_punctuation = ['<{}>'.format(p) for p in path_list] else: diff --git a/src/nunavut/lang/cpp/support/__init__.py b/src/nunavut/lang/cpp/support/__init__.py new file mode 100644 index 00000000..eed411d8 --- /dev/null +++ b/src/nunavut/lang/cpp/support/__init__.py @@ -0,0 +1,107 @@ +# +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright (C) 2018-2020 UAVCAN Development Team +# This software is distributed under the terms of the MIT License. +# +""" +Contains supporting C++ headers to distribute with generated types. +""" + +import pathlib +import typing + +__version__ = "1.0.0" +"""Version of the c++ support headers.""" + + +def copy_support_headers(target_namespace: typing.List[str], + copy_to_folder: pathlib.Path, + allow_overwrite: bool) -> typing.List[pathlib.Path]: + """ + Copy C++ support headers out of the Nunavut package and into the folder specified. + + .. invisible-code-block: python + + from nunavut.lang.cpp.support import copy_support_headers + import pathlib + import pytest + + .. code-block:: python + + # copied with contain headers like 'test_out_dir/nunavut/support/serialization.hpp' + copied = copy_support_headers(['nunavut', 'support'], gen_paths.out_dir, True) + + # To prevent overwrites set the allow_overwrite parameter to False + with pytest.raises(PermissionError): + copied = copy_support_headers(['nunavut', 'support'], gen_paths.out_dir, False) + + .. invisible-code-block: python + + assert len(copied) > 0 + for header in copied: + assert(header.is_file) + + :param pathlib.Path copy_to_folder: The folder to copy the headers into. + :param bool allow_overwrite: If True then this method will copy the support files over + existing files of the same name. + :return: A list of paths to the copied headers. + :raises: PermissionError if :attr:`allow_overwrite` is False and the file exists. + """ + import pkg_resources + import shutil + copied = [] # type: typing.List[pathlib.Path] + resources = list_support_headers([]) + try: + copy_to_folder.mkdir() + except FileExistsError: + pass + namespaced_path = copy_to_folder + for namespace_part in target_namespace: + namespaced_path = namespaced_path / pathlib.Path(namespace_part) + try: + namespaced_path.mkdir() + except FileExistsError: + pass + for resource in resources: + target = namespaced_path / pathlib.Path(resource) + if not allow_overwrite and target.exists(): + raise PermissionError('{} exists. Refusing to overwrite.'.format(str(target))) + shutil.copy(pkg_resources.resource_filename(__name__, str(resource)), str(namespaced_path)) + copied.append(target) + return copied + + +def list_support_headers(target_namespace: typing.List[str]) -> typing.List[pathlib.Path]: + """ + Get a list of C++ support headers embedded in this package. + + .. invisible-code-block: python + + from nunavut.lang.cpp.support import list_support_headers + import pathlib + + .. code-block:: python + + paths = list_support_headers([]) + for path in paths: + assert pathlib.Path('') == path.parent + + paths = list_support_headers(['nunavut']) + for path in paths: + assert pathlib.Path('nunavut') == path.parent + + paths = list_support_headers(['nunavut', 'support']) + for path in paths: + assert pathlib.Path('nunavut') / pathlib.Path('support') == path.parent + + :return: A list of C++ support header resources. + """ + import pkg_resources + namespace_path = pathlib.Path('') + for namespace_part in target_namespace: + namespace_path = namespace_path / pathlib.Path(namespace_part) + headers = [] # type: typing.List[pathlib.Path] + resources = [r for r in pkg_resources.resource_listdir(__name__, '.') if r.endswith('.hpp')] + for resource in resources: + headers.append(namespace_path / pathlib.Path(resource)) + return headers diff --git a/verification/cpp/include/nunavut/support.hpp b/src/nunavut/lang/cpp/support/serialization.hpp similarity index 56% rename from verification/cpp/include/nunavut/support.hpp rename to src/nunavut/lang/cpp/support/serialization.hpp index 6b96faf7..6fd84ad2 100644 --- a/verification/cpp/include/nunavut/support.hpp +++ b/src/nunavut/lang/cpp/support/serialization.hpp @@ -1,19 +1,29 @@ /* - * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * UAVCAN serialization support routines. +-+ +-+ + * | | | | + * \ - / + * --- + * o + * +------------------------------------------------------------------------------------------------------------------+ */ + #ifndef NUNAVUT_SUPPORT_HPP_INCLUDED #define NUNAVUT_SUPPORT_HPP_INCLUDED #include #include +#include namespace nunavut { +namespace support +{ /** - * Copy aligned bits from a byte array to another byte array using arbitrary alignment. + * Copy aligned bits from a byte buffer to another byte buffer using arbitrary alignment. * - * @param src The byte array to copy from. - * @param dst The byte array to copy data into. + * @param src The byte buffer to copy from. + * @param dst The byte buffer to copy data into. * @param dst_offset_bits The offset, in bits, from the start of the dst array to * start writing to. * @param length_bits The total length of bits to copy. The caller must ensure @@ -22,12 +32,12 @@ namespace nunavut * @return The number of bits copied. */ template -SizeType copyBitsAlignedToUnaligned(const ByteType* const src, - ByteType* const dst, - const SizeType dst_offset_bits, - const SizeType length_bits) +SizeType copyBitsAlignedToUnaligned(const ByteType* const src, + std::vector& dst, + const SizeType dst_offset_bits, + const SizeType length_bits) { - if (nullptr == src || nullptr == dst || length_bits == 0) + if (nullptr == src || length_bits == 0) { return 0; } @@ -58,6 +68,7 @@ SizeType copyBitsAlignedToUnaligned(const ByteType* const src, return bits_copied; } +} // namespace support } // namespace nunavut #endif // NUNAVUT_SUPPORT_HPP_INCLUDED diff --git a/src/nunavut/lang/cpp/templates/Header.j2 b/src/nunavut/lang/cpp/templates/Header.j2 index d8e011d7..b6296121 100644 --- a/src/nunavut/lang/cpp/templates/Header.j2 +++ b/src/nunavut/lang/cpp/templates/Header.j2 @@ -26,8 +26,7 @@ #ifndef {{T.full_name | c.macrofy}} #define {{T.full_name | c.macrofy}} - -{% for n in T | includes(prefer_system_includes=False) -%} +{% for n in T | includes -%} #include {{ n }} {% endfor %} {{T.full_namespace | open_namespace}} diff --git a/src/nunavut/lang/cpp/templates/_composite_type.j2 b/src/nunavut/lang/cpp/templates/_composite_type.j2 index db89aaed..285142c8 100644 --- a/src/nunavut/lang/cpp/templates/_composite_type.j2 +++ b/src/nunavut/lang/cpp/templates/_composite_type.j2 @@ -8,15 +8,24 @@ {{ field.data_type | declaration }} {{ field.name | id }}; {%- endif -%} {%- endfor %} +{%- if not nunavut.support.omit %} static {{ typename_unsigned_length }} serialize( std::vector<{{ typename_byte }}>& inout_byte_buffer, {{ typename_unsigned_length }} bit_offset, {{ composite_type | declaration }}& t) { // Not yet implemented. - (void)inout_byte_buffer; - (void)bit_offset; - (void)t; + const {{ typename_byte_ptr }} bits_in = { reinterpret_cast<{{typename_byte_ptr}}>(&t) }; +{%- for field in composite_type.fields %} +{%- if field is VariableLengthArrayType %} + // var array here +{%- elif field is FixedLengthArrayType %} + // static array here +{%- else %} + bit_offset += {{ nunavut.support.namespace | to_namespace_qualifier }}copyBitsAlignedToUnaligned<{{ typename_unsigned_length }}, {{typename_byte}}>( bits_in, inout_byte_buffer, bit_offset, 4 ); +{%- endif %} +{%- endfor %} return 0; } +{%- endif %} }{{ composite_type | definition_end }} \ No newline at end of file diff --git a/src/nunavut/lang/properties.ini b/src/nunavut/lang/properties.ini index 05b27540..5c809322 100644 --- a/src/nunavut/lang/properties.ini +++ b/src/nunavut/lang/properties.ini @@ -2,6 +2,8 @@ [nunavut.lang.c] extension = .h namespace_file_stem = _namespace_ +has_standard_namespace_files = False +support_namespace = nunavut.support # Taken from https://en.cppreference.com/w/c/keyword reserved_identifiers = asm @@ -74,6 +76,8 @@ named_types = [nunavut.lang.cpp] extension = .hpp namespace_file_stem = _namespace_ +has_standard_namespace_files = False +support_namespace = ${nunavut.lang.c:support_namespace} # Taken from https://en.cppreference.com/w/cpp/keyword reserved_identifiers = ${nunavut.lang.c:reserved_identifiers} @@ -204,9 +208,11 @@ use_standard_types = ${nunavut.lang.c:use_standard_types} named_types = unsigned_length = std::size_t byte = std::uint8_t + byte_ptr = std::uint8_t* [nunavut.lang.py] extension = .py +has_standard_namespace_files = True namespace_file_stem = __init__ enable_stropping = ${nunavut.lang.c:enable_stropping} encoding_prefix = ${nunavut.lang.c:encoding_prefix} @@ -215,6 +221,7 @@ stropping_suffix = _ [nunavut.lang.js] extension = .js +has_standard_namespace_files = False namespace_file_stem = _namespace_ enable_stropping = ${nunavut.lang.c:enable_stropping} encoding_prefix = ${nunavut.lang.c:encoding_prefix} diff --git a/src/nunavut/lang/py.py b/src/nunavut/lang/py.py index bab21683..e62627f6 100644 --- a/src/nunavut/lang/py.py +++ b/src/nunavut/lang/py.py @@ -310,7 +310,7 @@ def filter_imports(language: Language, t: pydsdl.CompositeType, sort: bool = True) -> typing.List[str]: """ - Returns a list of all modules that must be imported to used a given type. + Returns a list of all modules that must be imported to use a given type. :param pydsdl.CompositeType t: The type to scan for dependencies. :param bool sort: If true the returned list will be sorted. diff --git a/src/nunavut/postprocessors.py b/src/nunavut/postprocessors.py index bedf31f9..5fb06f2c 100644 --- a/src/nunavut/postprocessors.py +++ b/src/nunavut/postprocessors.py @@ -50,7 +50,6 @@ def __call__(self, generated: pathlib.Path) -> pathlib.Path: ... - my_generator.generate_all(False, True, [ClangFormat('clang-format')]) """ @abc.abstractmethod diff --git a/src/nunavut/version.py b/src/nunavut/version.py index a68fc405..e24caff9 100644 --- a/src/nunavut/version.py +++ b/src/nunavut/version.py @@ -3,6 +3,6 @@ # This software is distributed under the terms of the MIT License. # -__version__ = "0.1.20" +__version__ = "0.2.0" __license__ = 'MIT' diff --git a/test/gentest_any/test_any.py b/test/gentest_any/test_any.py index 8012ae4b..26b714c6 100644 --- a/test/gentest_any/test_any.py +++ b/test/gentest_any/test_any.py @@ -24,9 +24,9 @@ def test_anygen(gen_paths, lang_key): # type: ignore language_context = LanguageContext(extension='.json') namespace = build_namespace_tree(type_map, root_namespace_dir, - gen_paths.out_dir, + str(gen_paths.out_dir), language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(namespace, templates_dir=gen_paths.templates_dir) generator.generate_all(False) outfile = gen_paths.find_outfile_in_namespace("uavcan.time.SynchronizedTimestamp", namespace) diff --git a/test/gentest_dsdl/test_dsdl.py b/test/gentest_dsdl/test_dsdl.py index bc1197cd..435377f4 100644 --- a/test/gentest_dsdl/test_dsdl.py +++ b/test/gentest_dsdl/test_dsdl.py @@ -6,25 +6,15 @@ from pathlib import Path import pytest -from pydsdl import read_namespace -from nunavut import build_namespace_tree -from nunavut.jinja import Generator -from nunavut.lang import LanguageContext +from nunavut import generate_types -@pytest.mark.parametrize('lang_key', ['cpp']) -def test_realgen(gen_paths, lang_key): # type: ignore +@pytest.mark.parametrize('lang_key,generate_support', [('cpp', False), ('cpp', True)]) +def test_realgen(gen_paths, lang_key, generate_support): # type: ignore """ Sanity test that runs through the entire public, regulated set of UAVCAN types and generates some basic C code. """ root_namespace_dir = gen_paths.root_dir / Path("submodules") / Path("public_regulated_data_types") / Path("uavcan") - type_map = read_namespace(str(root_namespace_dir), '') - language_context = LanguageContext(lang_key) - namespace = build_namespace_tree(type_map, - root_namespace_dir, - gen_paths.out_dir, - language_context) - generator = Generator(namespace, False, language_context) - generator.generate_all(False) + generate_types(lang_key, root_namespace_dir, gen_paths.out_dir, omit_serialization_support=not generate_support) diff --git a/test/gentest_filters/test_filters.py b/test/gentest_filters/test_filters.py index 67589fda..22c30da5 100644 --- a/test/gentest_filters/test_filters.py +++ b/test/gentest_filters/test_filters.py @@ -30,7 +30,7 @@ def test_template_assert(gen_paths): # type: ignore output_path, language_context) template_path = gen_paths.templates_dir / Path('assert') - generator = Generator(namespace, False, language_context, template_path) + generator = Generator(namespace, templates_dir=template_path) try: generator.generate_all() assert False @@ -51,7 +51,7 @@ def test_type_to_include(gen_paths): # type: ignore output_path, language_context) template_path = gen_paths.templates_dir / Path('type_to_include') - generator = Generator(namespace, False, language_context, template_path) + generator = Generator(namespace, templates_dir=template_path) generator.generate_all() outfile = gen_paths.find_outfile_in_namespace("uavcan.time.SynchronizedTimestamp", namespace) @@ -75,9 +75,7 @@ def test_custom_filter_and_test(gen_paths): # type: ignore language_context) template_path = gen_paths.templates_dir / Path('custom_filter_and_test') generator = Generator(namespace, - False, - language_context, - template_path, + templates_dir=template_path, additional_filters={'custom_filter': lambda T: 'hi mum'}, additional_tests={'custom_test': lambda T: True}) @@ -100,17 +98,11 @@ def test_custom_filter_and_test_redefinition(gen_paths): # type: ignore with pytest.raises(RuntimeError): Generator(namespace, - False, - language_context, - Path(), additional_filters={'type_to_include_path': lambda T: ''}, additional_tests={'custom_test': lambda T: False}) with pytest.raises(RuntimeError): Generator(namespace, - False, - language_context, - Path(), additional_filters={'custom_filter': lambda T: ''}, additional_tests={'primitive': lambda T: False}) diff --git a/test/gentest_json/test_json.py b/test/gentest_json/test_json.py index 4d431a09..bb1931f4 100644 --- a/test/gentest_json/test_json.py +++ b/test/gentest_json/test_json.py @@ -6,8 +6,6 @@ import json from pathlib import Path -import pytest - from pydsdl import read_namespace from nunavut import build_namespace_tree from nunavut.lang import LanguageContext @@ -28,7 +26,7 @@ def test_TestType_0_1(gen_paths): # type: ignore root_namespace_dir, gen_paths.out_dir, language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(namespace, templates_dir=gen_paths.templates_dir) generator.generate_all(False) # Now read back in and verify diff --git a/test/gentest_lang/test_lang.py b/test/gentest_lang/test_lang.py index 9eb4c1e5..ef7108b4 100644 --- a/test/gentest_lang/test_lang.py +++ b/test/gentest_lang/test_lang.py @@ -11,7 +11,7 @@ import pytest from pydsdl import read_namespace -from nunavut import build_namespace_tree +from nunavut import build_namespace_tree, YesNoDefault from nunavut.jinja import Generator from nunavut.lang import Language, LanguageContext from nunavut.lang.c import filter_id as c_filter_id @@ -53,9 +53,7 @@ def ptest_lang_c(gen_paths, implicit, unique_name_evaluator): # type: ignore gen_paths.out_dir, language_context) generator = Generator(namespace, - False, - language_context, - templates_dirs) + templates_dir=templates_dirs) generator.generate_all(False) # Now read back in and verify @@ -129,9 +127,7 @@ def ptest_lang_cpp(gen_paths, implicit): # type: ignore language_context) generator = Generator(namespace, - False, - language_context, - templates_dirs) + templates_dir=templates_dirs) generator.generate_all(False) @@ -192,9 +188,8 @@ def ptest_lang_py(gen_paths, implicit, unique_name_evaluator): # type: ignore gen_paths.out_dir, language_context) generator = Generator(namespace, - False, - language_context, - templates_dirs) + generate_namespace_types=YesNoDefault.NO, + templates_dir=templates_dirs) generator.generate_all(False) @@ -347,7 +342,7 @@ def test_language_object() -> None: Verify that the Language module object works as required. """ mock_config = MagicMock() - language = Language('c', mock_config) + language = Language('c', mock_config, True) assert 'c' == language.name @@ -357,6 +352,8 @@ def test_language_object() -> None: explicit_filters = language.get_filters(make_implicit=False) assert 'c.macrofy' in explicit_filters + assert language.omit_serialization_support + def test_language_context() -> None: """ diff --git a/test/gentest_lookup/test_lookup.py b/test/gentest_lookup/test_lookup.py index 35a50efd..0bbbeb30 100644 --- a/test/gentest_lookup/test_lookup.py +++ b/test/gentest_lookup/test_lookup.py @@ -6,7 +6,6 @@ from pathlib import Path -import pytest import json from pydsdl import read_namespace @@ -40,7 +39,7 @@ def test_bfs_of_type_for_template(gen_paths): # type: ignore gen_paths.dsdl_dir, gen_paths.out_dir, language_context) - generator = Generator(empty_namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(empty_namespace, templates_dir=gen_paths.templates_dir) subject = d() template_file = generator.filter_type_to_template(subject) assert str(Path('c').with_suffix(Generator.TEMPLATE_SUFFIX)) == template_file @@ -59,7 +58,7 @@ def test_one_template(gen_paths): # type: ignore root_namespace_dir, gen_paths.out_dir, language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(namespace, templates_dir=gen_paths.templates_dir) generator.generate_all(False) outfile = gen_paths.find_outfile_in_namespace("uavcan.time.TimeSystem", namespace) @@ -84,7 +83,7 @@ def test_get_templates(gen_paths): # type: ignore root_namespace_dir, gen_paths.out_dir, language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(namespace, templates_dir=gen_paths.templates_dir) templates = generator.get_templates() diff --git a/test/gentest_multiple/test_multiple.py b/test/gentest_multiple/test_multiple.py index bfc60187..bedc0d71 100644 --- a/test/gentest_multiple/test_multiple.py +++ b/test/gentest_multiple/test_multiple.py @@ -36,7 +36,7 @@ def test_three_roots(gen_paths): # type: ignore root_namespace, gen_paths.out_dir, language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(namespace, templates_dir=gen_paths.templates_dir) generator.generate_all(False) # Now read back in and verify diff --git a/test/gentest_namespaces/test_namespaces.py b/test/gentest_namespaces/test_namespaces.py index bab719b3..ce291e42 100644 --- a/test/gentest_namespaces/test_namespaces.py +++ b/test/gentest_namespaces/test_namespaces.py @@ -11,7 +11,7 @@ import pytest from pydsdl import Any, CompositeType, read_namespace -from nunavut import Namespace, build_namespace_tree +from nunavut import Namespace, build_namespace_tree, YesNoDefault from nunavut.jinja import Generator from nunavut.lang import LanguageContext @@ -106,7 +106,8 @@ def test_empty_namespace(gen_paths): # type: ignore def parameterized_test_namespace_(gen_paths, templates_subdir): # type: ignore language_context = LanguageContext(extension='.json') namespace, root_namespace_path, _ = gen_test_namespace(gen_paths, language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir / Path(templates_subdir)) + generator = Generator(namespace, YesNoDefault.NO, + templates_dir=gen_paths.templates_dir / Path(templates_subdir)) generator.generate_all() assert namespace.source_file_path == root_namespace_path assert namespace.full_name == 'scotec' @@ -130,7 +131,8 @@ def test_namespace_generation(gen_paths): # type: ignore language_context = LanguageContext(extension='.json', namespace_output_stem='__module__') namespace, root_namespace_path, compound_types = gen_test_namespace(gen_paths, language_context) assert len(compound_types) == 2 - generator = Generator(namespace, True, language_context, gen_paths.templates_dir / Path('default')) + generator = Generator(namespace, YesNoDefault.YES, + templates_dir=gen_paths.templates_dir / Path('default')) generator.generate_all() for nested_namespace in namespace.get_nested_namespaces(): nested_namespace_path = Path(root_namespace_path) / Path(*nested_namespace.full_name.split('.')[1:]) @@ -167,7 +169,8 @@ def test_namespace_stropping(gen_paths, language_context = LanguageContext(language_key) namespace, root_namespace_path, compound_types = gen_test_namespace(gen_paths, language_context) assert len(compound_types) == 2 - generator = Generator(namespace, True, language_context, gen_paths.templates_dir / Path('default')) + generator = Generator(namespace, YesNoDefault.YES, + templates_dir=gen_paths.templates_dir / Path('default')) generator.generate_all() expected_stropped_ns = 'scotec.{}.{}'.format(expected_stropp_part_0, expected_stropp_part_1) diff --git a/test/gentest_postprocessors/test_postprocessors.py b/test/gentest_postprocessors/test_postprocessors.py index cebbf4d6..01972523 100644 --- a/test/gentest_postprocessors/test_postprocessors.py +++ b/test/gentest_postprocessors/test_postprocessors.py @@ -18,13 +18,13 @@ from nunavut.lang import LanguageContext -def _test_common_namespace(gen_paths): # type: ignore +def _test_common_namespace(gen_paths, target_language: str = 'js', extension: str = '.json'): # type: ignore root_namespace_dir = gen_paths.dsdl_dir / pathlib.Path("uavcan") root_namespace = str(root_namespace_dir) return nunavut.build_namespace_tree(pydsdl.read_namespace(root_namespace, ''), root_namespace_dir, gen_paths.out_dir, - LanguageContext('js')) + LanguageContext(target_language, extension=extension)) def _test_common_post_condition(gen_paths, namespace): # type: ignore @@ -65,17 +65,21 @@ def __call__(self, generated: pathlib.Path) -> pathlib.Path: return generated namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[InvalidType()]) with pytest.raises(ValueError): - generator.generate_all(False, True, [InvalidType()]) + generator.generate_all(False, True) def test_empty_pp_array(gen_paths): # type: ignore """ Verifies the behavior of a zero length post_processors argument. """ namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, []) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[]) + generator.generate_all(False, True) _test_common_post_condition(gen_paths, namespace) @@ -84,8 +88,10 @@ def test_chmod(gen_paths): # type: ignore """ Generates a file using a SetFileMode post-processor. """ namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [nunavut.postprocessors.SetFileMode(0o444)]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[nunavut.postprocessors.SetFileMode(0o444)]) + generator.generate_all(False, True) outfile = _test_common_post_condition(gen_paths, namespace) @@ -96,8 +102,10 @@ def test_overwrite(gen_paths): # type: ignore """ Verifies the allow_overwrite flag contracts. """ namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [nunavut.postprocessors.SetFileMode(0o444)]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[nunavut.postprocessors.SetFileMode(0o444)]) + generator.generate_all(False, True) with pytest.raises(PermissionError): generator.generate_all(False, False) @@ -111,8 +119,10 @@ def test_overwrite_dryrun(gen_paths): # type: ignore """ Verifies the allow_overwrite flag contracts. """ namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [nunavut.postprocessors.SetFileMode(0o444)]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[nunavut.postprocessors.SetFileMode(0o444)]) + generator.generate_all(False, True) with pytest.raises(PermissionError): generator.generate_all(False, False) @@ -186,8 +196,10 @@ def __call__(self, generated: pathlib.Path) -> pathlib.Path: verifier = Verifier() namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [mover, verifier]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[mover, verifier]) + generator.generate_all(False, True) assert mover.called assert mover.generated_path != mover.target_path @@ -218,8 +230,10 @@ def __call__(self, line_and_lineend: typing.Tuple[str, str]) -> typing.Tuple[str line_pp0 = TestLinePostProcessor0() line_pp1 = TestLinePostProcessor1() namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [line_pp0, line_pp1]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[line_pp0, line_pp1]) + generator.generate_all(False, True) assert len(line_pp1._lines) > 0 _test_common_post_condition(gen_paths, namespace) @@ -231,15 +245,19 @@ def __call__(self, line_and_lineend: typing.Tuple[str, str]) -> typing.Tuple[str return None # type: ignore namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[TestBadLinePostProcessor()]) with pytest.raises(ValueError): - generator.generate_all(False, True, [TestBadLinePostProcessor()]) + generator.generate_all(False, True) def test_trim_trailing_ws(gen_paths): # type: ignore namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [nunavut.postprocessors.TrimTrailingWhitespace()]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[nunavut.postprocessors.TrimTrailingWhitespace()]) + generator.generate_all(False, True) outfile = _test_common_post_condition(gen_paths, namespace) with open(str(outfile), 'r') as json_file: @@ -249,8 +267,10 @@ def test_trim_trailing_ws(gen_paths): # type: ignore def test_limit_empty_lines(gen_paths): # type: ignore namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) - generator.generate_all(False, True, [nunavut.postprocessors.LimitEmptyLines(0)]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[nunavut.postprocessors.LimitEmptyLines(0)]) + generator.generate_all(False, True) outfile = _test_common_post_condition(gen_paths, namespace) with open(str(outfile), 'r') as json_file: @@ -349,10 +369,12 @@ def test_external_edit_in_place(gen_paths): # type: ignore Test that ExternalProgramEditInPlace is invoked as expected """ namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) ext_program = gen_paths.test_dir / pathlib.Path('ext_program.py') edit_in_place = nunavut.postprocessors.ExternalProgramEditInPlace([str(ext_program)]) - generator.generate_all(False, True, [edit_in_place]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[edit_in_place]) + generator.generate_all(False, True) outfile = gen_paths.find_outfile_in_namespace("uavcan.test.TestType", namespace) assert outfile is not None @@ -369,14 +391,19 @@ def test_external_edit_in_place_fail(gen_paths): # type: ignore Test that ExternalProgramEditInPlace handles error as expected. """ namespace = _test_common_namespace(gen_paths) - generator = nunavut.jinja.Generator(namespace, False, LanguageContext(extension='.json'), gen_paths.templates_dir) ext_program = gen_paths.test_dir / pathlib.Path('ext_program.py') simulated_error_args = [str(ext_program), '--simulate-error'] edit_in_place_checking = nunavut.postprocessors.ExternalProgramEditInPlace(simulated_error_args) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[edit_in_place_checking]) with pytest.raises(subprocess.CalledProcessError): - generator.generate_all(False, True, [edit_in_place_checking]) + generator.generate_all(False, True) edit_in_place_not_checking = nunavut.postprocessors.ExternalProgramEditInPlace(simulated_error_args, check=False) - generator.generate_all(False, True, [edit_in_place_not_checking]) + generator = nunavut.jinja.Generator(namespace, + templates_dir=gen_paths.templates_dir, + post_processors=[edit_in_place_not_checking]) + generator.generate_all(False, True) def test_pp_run_program(gen_paths, run_nnvg): # type: ignore diff --git a/test/gentest_serialization/test_serialization.py b/test/gentest_serialization/test_serialization.py new file mode 100644 index 00000000..4289b763 --- /dev/null +++ b/test/gentest_serialization/test_serialization.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright (C) 2018-2019 UAVCAN Development Team +# This software is distributed under the terms of the MIT License. +# +""" +Test the generation of serialization support generically and for python. +""" +from pathlib import Path + +from nunavut import generate_types + + +def test_no_serialization_cpp(gen_paths): # type: ignore + root_namespace_dir = gen_paths.root_dir / Path("submodules") / Path("public_regulated_data_types") / Path("uavcan") + generate_types('cpp', + root_namespace_dir, + gen_paths.out_dir, + omit_serialization_support=True) diff --git a/test/gentest_tests/templates/Any.j2 b/test/gentest_tests/templates/Any.j2 index 7924a353..e61d3bcd 100644 --- a/test/gentest_tests/templates/Any.j2 +++ b/test/gentest_tests/templates/Any.j2 @@ -5,7 +5,8 @@ "type": "{{ field.data_type }}", "isSerializableType":{% if field.data_type is SerializableType %}true{% else %}false{% endif %}, "isFloatType":{% if field.data_type is FloatType %}true{% else %}false{% endif %}, - "isIntegerType":{% if field.data_type is IntegerType %}true{% else %}false{% endif %} + "isIntegerType":{% if field.data_type is IntegerType %}true{% else %}false{% endif %}, + "isIntegerType_field":{% if field is IntegerType %}true{% else %}false{% endif %} }{% if not loop.last %},{% endif %} {% endfor %} } diff --git a/test/gentest_tests/test_tests.py b/test/gentest_tests/test_tests.py index a4ce1395..35d076b8 100644 --- a/test/gentest_tests/test_tests.py +++ b/test/gentest_tests/test_tests.py @@ -25,7 +25,7 @@ def test_instance_tests(gen_paths): # type: ignore root_namespace_dir, gen_paths.out_dir, language_context) - generator = Generator(namespace, False, language_context, gen_paths.templates_dir) + generator = Generator(namespace, templates_dir=gen_paths.templates_dir) generator.generate_all(False) outfile = gen_paths.find_outfile_in_namespace("buncho.serializables", namespace) @@ -39,7 +39,9 @@ def test_instance_tests(gen_paths): # type: ignore assert json_blob["this_field_is_an_int32"]["isSerializableType"] is True assert json_blob["this_field_is_an_int32"]["isIntegerType"] is True assert json_blob["this_field_is_an_int32"]["isFloatType"] is False + assert json_blob["this_field_is_an_int32"]["isIntegerType_field"] is True assert json_blob["this_field_is_a_float"]["isSerializableType"] is True assert json_blob["this_field_is_a_float"]["isIntegerType"] is False + assert json_blob["this_field_is_a_float"]["isIntegerType_field"] is False assert json_blob["this_field_is_a_float"]["isFloatType"] is True diff --git a/tox.ini b/tox.ini index e2acb408..843847ee 100644 --- a/tox.ini +++ b/tox.ini @@ -283,7 +283,7 @@ commands = [testenv:local] -basepython = python3.7 +usedevelop = true deps = {[base]deps} {[dev]deps} diff --git a/verification/cpp/CMakeLists.txt b/verification/cpp/CMakeLists.txt index eda0f84b..aa89ef8a 100644 --- a/verification/cpp/CMakeLists.txt +++ b/verification/cpp/CMakeLists.txt @@ -77,21 +77,9 @@ file(WRITE ${NUNAVUT_VERIFICATIONS_BINARY_DIR}/README.txt find_package(lcov REQUIRED) # -# We need python, of course. +# We need tox to enable a reproducable python environment. # -find_package(python3 REQUIRED) - -# -# We need virtualenv to enable a reproducable python environment. -# -find_package(virtualenv REQUIRED) - -# -# Install Nunavut into our local environment. -# -execute_process(COMMAND ${PIP} --isolated --disable-pip-version-check install --editable ${NUNAVUT_PROJECT_ROOT} - WORKING_DIRECTORY ${NUNAVUT_PROJECT_ROOT}) - +find_package(tox REQUIRED) # +---------------------------------------------------------------------------+ # | SOURCE GENERATION @@ -114,9 +102,10 @@ create_dsdl_target(dsdl-regulated ${NUNAVUT_SUBMODULES_ROOT}/public_regulated_data_types/uavcan ON) -target_include_directories(dsdl-regulated INTERFACE - ${CMAKE_CURRENT_SOURCE_DIR}/include -) +# Bring this back if we end up with test-specific headers. +# target_include_directories(dsdl-regulated INTERFACE +# ${CMAKE_CURRENT_SOURCE_DIR}/include +# ) # +---------------------------------------------------------------------------+ # | FLAG SETS diff --git a/verification/cpp/cmake/modules/Findnnvg.cmake b/verification/cpp/cmake/modules/Findnnvg.cmake index 206a5b50..0a045dae 100644 --- a/verification/cpp/cmake/modules/Findnnvg.cmake +++ b/verification/cpp/cmake/modules/Findnnvg.cmake @@ -103,16 +103,16 @@ endfunction(create_dsdl_target) # | CONFIGURE: PYTHON ENVIRONMENT # +---------------------------------------------------------------------------+ -if(NOT VIRTUALENV) +if(NOT TOX) - message(STATUS "virtualenv was not found. You must have nunavut and its" + message(STATUS "tox was not found. You must have nunavut and its" " dependencies available in the global python environment.") find_program(NNVG nnvg) else() - find_program(NNVG nnvg HINTS ${VIRTUALENV_PYTHON_BIN}) + find_program(NNVG nnvg HINTS ${TOX_LOCAL_PYTHON_BIN}) if (NOT NNVG) message(WARNING "nnvg program was not found. The build will probably fail. (${NNVG})") diff --git a/verification/cpp/cmake/modules/Findpython3.cmake b/verification/cpp/cmake/modules/Findtox.cmake similarity index 51% rename from verification/cpp/cmake/modules/Findpython3.cmake rename to verification/cpp/cmake/modules/Findtox.cmake index b6d16e4c..e00449e0 100644 --- a/verification/cpp/cmake/modules/Findpython3.cmake +++ b/verification/cpp/cmake/modules/Findtox.cmake @@ -1,53 +1,51 @@ # -# Find the newest python3 version available. +# Find tox. # Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # set(PYTHON3_MINIMUM_VERSION 3.5) -find_program(PYTHON3 python3.8) +find_program(TOX tox) -if (NOT PYTHON3) - find_program(PYTHON3 python3.7) -endif() +if(TOX) -if (NOT PYTHON3) - find_program(PYTHON3 python3.6) -endif() + set(TOX_LOCAL_OUTPUT ${NUNAVUT_PROJECT_ROOT}/.tox/local) -if (NOT PYTHON3) - find_program(PYTHON3 python3.5) -endif() + set(TOX_LOCAL_PYTHON_BIN ${TOX_LOCAL_OUTPUT}/bin) + set(PYTHON ${TOX_LOCAL_PYTHON_BIN}/python) + + + execute_process(COMMAND ${TOX} -e local + WORKING_DIRECTORY ${NUNAVUT_PROJECT_ROOT} + RESULT_VARIABLE TOX_LOCAL_RESULT) + + if(NOT TOX_LOCAL_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to run tox local (${TOX_LOCAL_RESULT})") + endif() -if (NOT PYTHON3) - find_program(PYTHON3 python3) -endif() -if (NOT PYTHON3) - message(FATAL_ERROR "Could not find python3.") endif() # +---------------------------------------------------------------------------+ -# | CONFIGURE: VALIDATE NNVG +# | CONFIGURE: VALIDATE TOX AND PYTHON # +---------------------------------------------------------------------------+ -execute_process(COMMAND ${PYTHON3} --version +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args(tox + REQUIRED_VARS TOX TOX_LOCAL_PYTHON_BIN +) + +execute_process(COMMAND ${PYTHON} --version OUTPUT_VARIABLE PYTHON3_VERSION RESULT_VARIABLE PYTHON3_VERSION_RESULT) if(PYTHON3_VERSION_RESULT EQUAL 0) string(REPLACE "Python" "" PYTHON3_VERSION ${PYTHON3_VERSION}) string(STRIP ${PYTHON3_VERSION} PYTHON3_VERSION) - message(STATUS "${PYTHON3} --version: ${PYTHON3_VERSION}") + message(STATUS "${TOX_LOCAL_PYTHON_BIN} --version: ${PYTHON3_VERSION}") endif() - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(nnvg - REQUIRED_VARS PYTHON3_VERSION -) - if(PYTHON3_VERSION VERSION_LESS ${PYTHON3_MINIMUM_VERSION}) message(FATAL_ERROR "Nunavut requires Python ${PYTHON3_MINIMUM_VERSION} or greater. (found ${PYTHON3_VERSION})") endif() diff --git a/verification/cpp/cmake/modules/Findvirtualenv.cmake b/verification/cpp/cmake/modules/Findvirtualenv.cmake deleted file mode 100644 index 4e0a9c9f..00000000 --- a/verification/cpp/cmake/modules/Findvirtualenv.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# -# Find virtualenv. If found provide a way to setup a virtualenv for the build. -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -find_program(VIRTUALENV virtualenv) - -if(VIRTUALENV) - - if (NOT DEFINED VIRTUALENV_OUTPUT) - set(VIRTUALENV_OUTPUT ${NUNAVUT_PROJECT_ROOT}/.pyenv) - else() - message(STATUS "Using predefined VIRTUALENV_OUTPUT=${VIRTUALENV_OUTPUT}") - endif() - - set(VIRTUALENV_PYTHON_BIN ${VIRTUALENV_OUTPUT}/bin) - set(PYTHON ${VIRTUALENV_PYTHON_BIN}/python) - set(PIP ${VIRTUALENV_PYTHON_BIN}/pip) - - if(NOT EXISTS ${VIRTUALENV_OUTPUT}) - message(STATUS "virtualenv found. Creating a virtual environment and installing core requirements.") - - execute_process(COMMAND ${VIRTUALENV} -p ${PYTHON3} ${VIRTUALENV_OUTPUT} - WORKING_DIRECTORY ${NUNAVUT_PROJECT_ROOT} - RESULT_VARIABLE VIRTUALENV_CREATE_RESULT) - - if(NOT VIRTUALENV_CREATE_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to create a virtualenv (${VIRTUALENV_CREATE_RESULT})") - endif() - - else() - message(STATUS "virtualenv ${VIRTUALENV_OUTPUT} exists. Not recreating (delete this directory to re-create).") - endif() - -endif() - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(nnvg - REQUIRED_VARS VIRTUALENV VIRTUALENV_PYTHON_BIN -) diff --git a/verification/cpp/suite/test_support.cpp b/verification/cpp/suite/test_support.cpp index 7bf80e6a..908d261e 100644 --- a/verification/cpp/suite/test_support.cpp +++ b/verification/cpp/suite/test_support.cpp @@ -4,7 +4,7 @@ * Tests of the Nunavut support header. */ #include "gmock/gmock.h" -#include "nunavut/support.hpp" +#include "nunavut/support/serialization.hpp" #include template @@ -40,11 +40,12 @@ TYPED_TEST(SupportTest, UnalignedCopy) memset(test_pattern.data(), 0xAA, test_pattern.capacity()); std::vector test_buffer; test_buffer.reserve(test_pattern.capacity() + 1); - auto copied_bits = nunavut::copyBitsAlignedToUnaligned(test_pattern.data(), - test_buffer.data(), - 0, - test_pattern.capacity()); + auto copied_bits = + nunavut::support::copyBitsAlignedToUnaligned(test_pattern.data(), + test_buffer, + 0, + test_pattern.capacity()); ASSERT_EQ(test_pattern.capacity(), copied_bits); ASSERT_EQ(static_cast(test_buffer[0] & 0xFF), 0xAAU); }