From 59a0f62e2033d6056c0278c13695af4dff408e10 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Wed, 18 Jan 2023 11:58:10 -0500 Subject: [PATCH 1/2] yaml_preprocessor: add support for $include This commit adds support for a special `$include` keyword in the YAML preprocessor. This new keyword is similar to the existing `$eval` keyword and allows building the `robot.model.yaml` up from multiple separate files. $include can process both absolute filenames as well as relative filenames, which are interpreted relative to the file currently being processed. The included document is also processed by the YAML preprocessor, and so can include nested `$include`s and `$eval`s. This commit also updates the documentation and tests. --- docs/core_functions/yaml_preprocessor.rst | 24 +++++ flatland_server/CMakeLists.txt | 2 +- .../flatland_server/yaml_preprocessor.h | 42 ++++++-- flatland_server/src/yaml_preprocessor.cpp | 97 ++++++++++++++++--- flatland_server/src/yaml_reader.cpp | 6 +- .../yaml_preprocessor/yaml/include.child.yaml | 3 + .../yaml/include.parent.out.yaml | 8 ++ .../yaml/include.parent.yaml | 5 + .../yaml/include.string.yaml | 1 + .../yaml_preprocessor_test.cpp | 27 ++++++ 10 files changed, 189 insertions(+), 26 deletions(-) create mode 100644 flatland_server/test/yaml_preprocessor/yaml/include.child.yaml create mode 100644 flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml create mode 100644 flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml create mode 100644 flatland_server/test/yaml_preprocessor/yaml/include.string.yaml diff --git a/docs/core_functions/yaml_preprocessor.rst b/docs/core_functions/yaml_preprocessor.rst index c125bec7..1a445779 100644 --- a/docs/core_functions/yaml_preprocessor.rst +++ b/docs/core_functions/yaml_preprocessor.rst @@ -9,6 +9,30 @@ Yaml Preprocessor Flatland Server has a lua preprocessor for YAML with simple bindings for the environment variables and rosparam. The intent is to be able to build parametric models that are defined by dimensions and flags found in either environment variables or rosparam, in a similar way to xacro+urdf. Because this is parsed at model load time, any roslaunch rosparam loading will have completed and those parameters will be available. +Including additional YAML files +------------------------------- +The YAML preprocessor allows for including of other YAML files using the `$include` keyword. When this string is encountered, the following filename is parsed as a YAML file and substituted for the `$include`-string. The file path bay be absolute or relative. Relative file paths are relative to the YAML file currently being processed. + +.. code-block:: yaml + + # project/param/parent.yaml + foo: 123 + bar: $include child.yaml # Looks in current directory (project/param/) for relative filenames + baz: $include /absolute/path/to/project/param/child.yaml # or an absolute path can be specified + + # project/param/child.yaml + a: 1 + b: 2 + + # result of YAML preprocessing: + foo: 123 + bar: + a: 1 + b: 2 + baz: + a: 1 + b: 2 + body/joint/plugin enabled flag ------------------------------ Model bodies, plugins and joints now have a new flag `enabled` which can be set to true or false either directly in the yaml, or based on more complex logic from a lua `$eval` string that returns "true" or "false". Disabled bodies, plugins and joints are skipped during yaml loading, and as a result are never instantiated. From Flatland's perspective `enabled: false` causes the affected body/plugin/joint to be deleted. diff --git a/flatland_server/CMakeLists.txt b/flatland_server/CMakeLists.txt index 9978aa9a..0c958b5c 100644 --- a/flatland_server/CMakeLists.txt +++ b/flatland_server/CMakeLists.txt @@ -42,7 +42,7 @@ find_package(Lua 5.1 QUIET) find_package(OpenCV REQUIRED) # Boost -find_package(Boost REQUIRED COMPONENTS date_time system thread) +find_package(Boost REQUIRED COMPONENTS date_time system thread filesystem) find_package(Threads) ############## diff --git a/flatland_server/include/flatland_server/yaml_preprocessor.h b/flatland_server/include/flatland_server/yaml_preprocessor.h index 9bcf3e32..5988ca62 100644 --- a/flatland_server/include/flatland_server/yaml_preprocessor.h +++ b/flatland_server/include/flatland_server/yaml_preprocessor.h @@ -65,9 +65,11 @@ namespace YamlPreprocessor { /** * @brief Preprocess with a given node * @param[in/out] node A Yaml node to parse + * @param[in] ref_path The path the file was loaded from; used to locate + * include files with relative filenames. * @return The parsed YAML::Node */ -void Parse(YAML::Node &node); +void Parse(YAML::Node &node, const std::string &ref_path); /** * @brief Constructor with a given path to a yaml file, throws exception on @@ -80,25 +82,47 @@ YAML::Node LoadParse(const std::string &path); /** * @brief Find and run any $eval nodes * @param[in/out] node A Yaml node to recursively parse + * @param[in] ref_path The path the file was loaded from; used to locate + * include files with relative filenames. */ -void ProcessNodes(YAML::Node &node); +void ProcessNodes(YAML::Node &node, const std::string &ref_path); /** * @brief Find and run any $eval expressions * @param[in/out] node A Yaml string node to parse */ -void ProcessScalarNode(YAML::Node &node); +void ProcessEvalNode(YAML::Node &node); /** - * @brief Get an environment variable with an optional default value - * @param[in/out] lua_State The lua state/stack to read/write to/from - */ + * @brief Resolve a given filename to an absolute path, resolving relative + * filenames relative to ref_path + * @param filename The filename to find. + * @param ref_path Filename of the YAML file containing the `$include`, used to + * locate relative filenames + * @return The absolute path of the file to be included. + */ +std::string ResolveIncludeFilePath(const std::string &filename, + const std::string &ref_path); + +/** + * @brief Find and process any $include expressions + * @param[in/out] node A Yaml string node to parse. If an include is processed, + * the node is replaced with the contents of the specified file. + * @param[in] ref_path The path the file was loaded from; used to locate + * include files with relative filenames. + */ +void ProcessIncludeNode(YAML::Node &node, const std::string &ref_path); + +/** + * @brief Get an environment variable with an optional default value + * @param[in/out] lua_State The lua state/stack to read/write to/from + */ int LuaGetEnv(lua_State *L); /** - * @brief Get a rosparam with an optional default value - * @param[in/out] lua_State The lua state/stack to read/write to/from - */ + * @brief Get a rosparam with an optional default value + * @param[in/out] lua_State The lua state/stack to read/write to/from + */ int LuaGetParam(lua_State *L); }; } diff --git a/flatland_server/src/yaml_preprocessor.cpp b/flatland_server/src/yaml_preprocessor.cpp index b35f99f9..292396f6 100644 --- a/flatland_server/src/yaml_preprocessor.cpp +++ b/flatland_server/src/yaml_preprocessor.cpp @@ -45,36 +45,50 @@ */ #include "flatland_server/yaml_preprocessor.h" + #include +#include +#include + #include +#include #include - #include #include +#include "flatland_server/exceptions.h" + namespace flatland_server { -void YamlPreprocessor::Parse(YAML::Node &node) { - YamlPreprocessor::ProcessNodes(node); +static const char *kEvalMarker = "$eval"; +static const char *kIncludeMarker = "$include"; + +void YamlPreprocessor::Parse(YAML::Node &node, const std::string &ref_path) { + YamlPreprocessor::ProcessNodes(node, ref_path); } -void YamlPreprocessor::ProcessNodes(YAML::Node &node) { +void YamlPreprocessor::ProcessNodes(YAML::Node &node, + const std::string &ref_path) { switch (node.Type()) { case YAML::NodeType::Sequence: for (YAML::Node child : node) { - YamlPreprocessor::ProcessNodes(child); + YamlPreprocessor::ProcessNodes(child, ref_path); } break; case YAML::NodeType::Map: for (YAML::iterator it = node.begin(); it != node.end(); ++it) { - YamlPreprocessor::ProcessNodes(it->second); + YamlPreprocessor::ProcessNodes(it->second, ref_path); } break; - case YAML::NodeType::Scalar: - if (node.as().compare(0, 5, "$eval") == 0) { - ProcessScalarNode(node); + case YAML::NodeType::Scalar: { + auto s = node.as(); + if (s.compare(0, strlen(kEvalMarker), kEvalMarker) == 0) { + ProcessEvalNode(node); + } else if (s.compare(0, strlen(kIncludeMarker), kIncludeMarker) == 0) { + ProcessIncludeNode(node, ref_path); } break; + } default: ROS_DEBUG_STREAM( "Yaml Preprocessor found an unexpected type: " << node.Type()); @@ -82,9 +96,10 @@ void YamlPreprocessor::ProcessNodes(YAML::Node &node) { } } -void YamlPreprocessor::ProcessScalarNode(YAML::Node &node) { - std::string value = node.as().substr(5); // omit the $parse - boost::algorithm::trim(value); // trim whitespace +void YamlPreprocessor::ProcessEvalNode(YAML::Node &node) { + std::string value = + node.as().substr(strlen(kEvalMarker)); // omit the $parse + boost::algorithm::trim(value); // trim whitespace ROS_INFO_STREAM("Attempting to parse lua " << value); if (value.find("return ") == std::string::npos) { // Has no return statement @@ -127,6 +142,60 @@ void YamlPreprocessor::ProcessScalarNode(YAML::Node &node) { ROS_ERROR_STREAM("Lua error in: " << value); } } +std::string YamlPreprocessor::ResolveIncludeFilePath( + const std::string &filename, const std::string &ref_path) { + namespace fs = boost::filesystem; + fs::path f(filename); + // already an absolute path, return as-is. + if (f.is_absolute()) { + ROS_DEBUG_STREAM("Path is already absolute."); + return filename; + } + // if we're not loading from a file, then we can't resolve relative paths. + // just pass along and hope the caller is requesting something in the CWD. + if (ref_path.empty()) { + ROS_WARN_STREAM( + "$include specified a relative path but no original filename " + "specified"); + return filename; + } + + fs::path rel(ref_path); + if (fs::is_regular_file(rel)) { + rel = rel.parent_path(); + } + + fs::path result = rel / f; + return result.string(); +} +void YamlPreprocessor::ProcessIncludeNode(YAML::Node &node, + const std::string &ref_path) { + std::string value = node.as().substr( + strlen(kIncludeMarker)); // omit the $include + ROS_INFO_STREAM("Attempting to parse include: " << value); + boost::algorithm::trim(value); // remove whitespace + + // format the common file & include info for any thrown exceptions. + const auto format_error_info = [&]() { + return "path=" + ref_path + ", include=" + value; + }; + + try { + auto path = ResolveIncludeFilePath(value, ref_path); + // replace string node with parsed contents of file + node = YAML::LoadFile(path); + ProcessNodes(node, path); // recursively process the included file, too + ROS_INFO_STREAM("Successfully loaded include file " + path); + } catch (const YAML::BadFile &) { + throw YAMLException("File specified in $include does not exist," + + format_error_info()); + } catch (const YAML::ParserException &e) { + throw YAMLException( + "Malformatted file specified as include, " + format_error_info(), e); + } catch (const YAML::Exception &e) { + throw YAMLException("Error loading include file, " + format_error_info(), e); + } +} YAML::Node YamlPreprocessor::LoadParse(const std::string &path) { YAML::Node node; @@ -141,7 +210,7 @@ YAML::Node YamlPreprocessor::LoadParse(const std::string &path) { throw YAMLException("Error loading file, path=" + path, e); } - YamlPreprocessor::Parse(node); + YamlPreprocessor::Parse(node, path); return node; } @@ -214,4 +283,4 @@ int YamlPreprocessor::LuaGetParam(lua_State *L) { return 1; // 1 return value } -} \ No newline at end of file +} // namespace flatland_server \ No newline at end of file diff --git a/flatland_server/src/yaml_reader.cpp b/flatland_server/src/yaml_reader.cpp index 6c3df6f1..55f6d151 100644 --- a/flatland_server/src/yaml_reader.cpp +++ b/flatland_server/src/yaml_reader.cpp @@ -55,14 +55,16 @@ YamlReader::YamlReader() : node_(YAML::Node()) { } YamlReader::YamlReader(const YAML::Node &node) : node_(node) { - YamlPreprocessor::Parse(node_); + // note: we don't have a filename here so $include parsing will not work + // with relative filenames. + YamlPreprocessor::Parse(node_, ""); SetErrorInfo("_NONE_", "_NONE_"); } YamlReader::YamlReader(const std::string &path) { try { node_ = YAML::LoadFile(path); - YamlPreprocessor::Parse(node_); + YamlPreprocessor::Parse(node_, path); } catch (const YAML::BadFile &e) { throw YAMLException("File does not exist, path=" + Q(path)); diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml new file mode 100644 index 00000000..fa0924d3 --- /dev/null +++ b/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml @@ -0,0 +1,3 @@ +foo: bar +spam: 2 +eggs: $include include.string.yaml \ No newline at end of file diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml new file mode 100644 index 00000000..67811e9f --- /dev/null +++ b/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml @@ -0,0 +1,8 @@ +a: this is a +b: 52 +c: this is a technically a YAML string. +d: contents of absolute file +e: + foo: bar + spam: 2 + eggs: this is a technically a YAML string. diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml new file mode 100644 index 00000000..677bcf31 --- /dev/null +++ b/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml @@ -0,0 +1,5 @@ +a: this is a +b: 52 +c: $include include.string.yaml +d: $include /tmp/9aaafd40-2083-49d9-a300-9d01f94d6671.yaml +e: $include include.child.yaml \ No newline at end of file diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml new file mode 100644 index 00000000..e34d652c --- /dev/null +++ b/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml @@ -0,0 +1 @@ +this is a technically a YAML string. \ No newline at end of file diff --git a/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp b/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp index 50bfd619..3b46d413 100644 --- a/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp +++ b/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp @@ -47,6 +47,7 @@ #include "flatland_server/yaml_preprocessor.h" #include #include +#include #include namespace fs = boost::filesystem; @@ -127,6 +128,32 @@ TEST(YamlPreprocTest, testEvalStrings) { compareNodes("testParam", "param9", in, out); } +TEST(YamlPreprocTest, testIncludeStrings) { + boost::filesystem::path cwd = fs::path(__FILE__).parent_path(); + // randomly-generated UUID + const std::string kAbsoluteFileName = "/tmp/9aaafd40-2083-49d9-a300-9d01f94d6671.yaml"; + + auto parent_yaml_filename = (cwd / "yaml/include.parent.yaml").string(); + auto result_yaml_filename = (cwd / "yaml/include.parent.out.yaml").string(); + + std::ofstream f(kAbsoluteFileName); + f << "contents of absolute file\n"; + ASSERT_TRUE(f.good()) << "Failure writing temp file " << kAbsoluteFileName; + f.close(); + + YAML::Node in = YamlPreprocessor::LoadParse(parent_yaml_filename); + YAML::Node out = YamlPreprocessor::LoadParse(result_yaml_filename); + + // make sure we get the expected node structure. + compareNodes("a", in, out); + compareNodes("b", in, out); + compareNodes("c", in, out); + compareNodes("d", in, out); + compareNodes("e", "foo", in, out); + compareNodes("e", "spam", in, out); + compareNodes("e", "eggs", in, out); +} + // Run all the tests that were declared with TEST() int main(int argc, char **argv) { ros::init(argc, argv, "yaml_preprocessor_test"); From deaed5609f8571adf0de8e6a136c7e3bc26f3216 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Wed, 18 Jan 2023 14:07:46 -0500 Subject: [PATCH 2/2] yaml_preprocessor: Add sequence include $[include] * Add sequence include keyword $[include], which generates into one or more sequence elements * One element is generated for each separate YAML document in the included file (separated by '---') * Update tests * Update docs --- docs/core_functions/yaml_preprocessor.rst | 33 ++++++- .../flatland_server/yaml_preprocessor.h | 24 ++++- flatland_server/src/yaml_preprocessor.cpp | 87 +++++++++++++++++-- .../yaml_preprocessor/yaml/include.child.yaml | 2 +- .../yaml/include.parent.out.yaml | 6 ++ .../yaml/include.parent.yaml | 6 +- .../yaml/include.sequence.yaml | 5 ++ .../yaml/include.string.yaml | 2 +- .../yaml_preprocessor_test.cpp | 6 ++ 9 files changed, 154 insertions(+), 17 deletions(-) create mode 100644 flatland_server/test/yaml_preprocessor/yaml/include.sequence.yaml diff --git a/docs/core_functions/yaml_preprocessor.rst b/docs/core_functions/yaml_preprocessor.rst index 1a445779..09910f5b 100644 --- a/docs/core_functions/yaml_preprocessor.rst +++ b/docs/core_functions/yaml_preprocessor.rst @@ -11,7 +11,7 @@ The intent is to be able to build parametric models that are defined by dimensio Including additional YAML files ------------------------------- -The YAML preprocessor allows for including of other YAML files using the `$include` keyword. When this string is encountered, the following filename is parsed as a YAML file and substituted for the `$include`-string. The file path bay be absolute or relative. Relative file paths are relative to the YAML file currently being processed. +The YAML preprocessor allows for including of other YAML files using the `$include` keyword. When this string is encountered, the following filename is parsed as a YAML file and substituted for the `$include`-string. The file path bay be absolute or relative. Relative file paths are relative to the YAML file currently being processed. .. code-block:: yaml @@ -24,7 +24,7 @@ The YAML preprocessor allows for including of other YAML files using the `$inclu a: 1 b: 2 - # result of YAML preprocessing: + # result of YAML preprocessing parent.yaml: foo: 123 bar: a: 1 @@ -33,9 +33,36 @@ The YAML preprocessor allows for including of other YAML files using the `$inclu a: 1 b: 2 +There is also a special form of `$include`, `$[include]`, that can be used to populate a sequence with an arbitrary number of items. Each individual document in the specified YAML file (separated by `---` as is standard for multi-document YAML files) becomes its own element in the sequence. For example: + +.. code-block:: yaml + + # parent.yaml + foo: + - first + - $[include] child.yaml + - last + + #child.yaml + one + --- + a: foo + b: baz + --- + three + + # result of YAML preprocessing of parent.yaml: + foo: + - first + - one + - a: foo + b: baz + - three + - last + body/joint/plugin enabled flag ------------------------------ -Model bodies, plugins and joints now have a new flag `enabled` which can be set to true or false either directly in the yaml, or based on more complex logic from a lua `$eval` string that returns "true" or "false". Disabled bodies, plugins and joints are skipped during yaml loading, and as a result are never instantiated. From Flatland's perspective `enabled: false` causes the affected body/plugin/joint to be deleted. +Model bodies, plugins and joints now have a new flag `enabled` which can be set to true or false either directly in the yaml, or based on more complex logic from a lua `$eval` string that returns "true" or "false". Disabled bodies, plugins and joints are skipped during yaml loading, and as a result are never instantiated. From Flatland's perspective `enabled: false` causes the affected body/plugin/joint to be deleted. bindings for env and param ------------------------------- diff --git a/flatland_server/include/flatland_server/yaml_preprocessor.h b/flatland_server/include/flatland_server/yaml_preprocessor.h index 5988ca62..3fac6a7d 100644 --- a/flatland_server/include/flatland_server/yaml_preprocessor.h +++ b/flatland_server/include/flatland_server/yaml_preprocessor.h @@ -96,16 +96,16 @@ void ProcessEvalNode(YAML::Node &node); /** * @brief Resolve a given filename to an absolute path, resolving relative * filenames relative to ref_path - * @param filename The filename to find. - * @param ref_path Filename of the YAML file containing the `$include`, used to - * locate relative filenames + * @param[in] filename The filename to find. + * @param[in] ref_path Filename of the YAML file containing the `$include`, used + * to locate relative filenames * @return The absolute path of the file to be included. */ std::string ResolveIncludeFilePath(const std::string &filename, const std::string &ref_path); /** - * @brief Find and process any $include expressions + * @brief Potentially process an $include expression. * @param[in/out] node A Yaml string node to parse. If an include is processed, * the node is replaced with the contents of the specified file. * @param[in] ref_path The path the file was loaded from; used to locate @@ -113,6 +113,22 @@ std::string ResolveIncludeFilePath(const std::string &filename, */ void ProcessIncludeNode(YAML::Node &node, const std::string &ref_path); +/** + * @brief Process a node, converting sequence include expression + * ('$[include]')to a series of nodes. + * @param[in/out] out_elems Vector that will be populated with parsed nodes + * from the included file. + * @param[in] node Node to parse. Should be a scalar node. + * @param ref_path Reference path used for locating relative filenames. + * @return If the node is an include expression, returns True. Otherwise, + * False. + * + * If the function returns true, then `node` should be replaced by out_elems in + * its sequence. + */ +bool ProcessSequenceIncludeNode(std::vector &out_elems, + YAML::Node &node, const std::string &ref_path); + /** * @brief Get an environment variable with an optional default value * @param[in/out] lua_State The lua state/stack to read/write to/from diff --git a/flatland_server/src/yaml_preprocessor.cpp b/flatland_server/src/yaml_preprocessor.cpp index 292396f6..eacca01d 100644 --- a/flatland_server/src/yaml_preprocessor.cpp +++ b/flatland_server/src/yaml_preprocessor.cpp @@ -49,6 +49,8 @@ #include #include #include +#include +#include #include #include @@ -62,6 +64,7 @@ namespace flatland_server { static const char *kEvalMarker = "$eval"; static const char *kIncludeMarker = "$include"; +static const char *kSequenceIncludeMarker = "$[include]"; void YamlPreprocessor::Parse(YAML::Node &node, const std::string &ref_path) { YamlPreprocessor::ProcessNodes(node, ref_path); @@ -70,11 +73,31 @@ void YamlPreprocessor::Parse(YAML::Node &node, const std::string &ref_path) { void YamlPreprocessor::ProcessNodes(YAML::Node &node, const std::string &ref_path) { switch (node.Type()) { - case YAML::NodeType::Sequence: + case YAML::NodeType::Sequence: { + // copy the elements to a new sequence, checking for $[include] expressions + // as we go. If an include is found, replace it with one or more nodes + // parsed from the included file. + YAML::Node new_sequence; for (YAML::Node child : node) { - YamlPreprocessor::ProcessNodes(child, ref_path); + std::vector included_nodes = {}; + if (ProcessSequenceIncludeNode(included_nodes, child, ref_path)) { + ROS_INFO_STREAM("Sequence include yielded " << included_nodes.size() + << " nodes"); + // node was an $include + for (auto &include_child : included_nodes) { + // ProcessSequenceIncludeNode handles processing of children itself. + new_sequence.push_back(include_child); + } + } else { + // not an include, just process and copy over normally. + // the sequence itself is the parent. + YamlPreprocessor::ProcessNodes(child, ref_path); + new_sequence.push_back(child); + } } + node = new_sequence; break; + } case YAML::NodeType::Map: for (YAML::iterator it = node.begin(); it != node.end(); ++it) { YamlPreprocessor::ProcessNodes(it->second, ref_path); @@ -170,8 +193,8 @@ std::string YamlPreprocessor::ResolveIncludeFilePath( } void YamlPreprocessor::ProcessIncludeNode(YAML::Node &node, const std::string &ref_path) { - std::string value = node.as().substr( - strlen(kIncludeMarker)); // omit the $include + // omit the $include + std::string value = node.as().substr(strlen(kIncludeMarker)); ROS_INFO_STREAM("Attempting to parse include: " << value); boost::algorithm::trim(value); // remove whitespace @@ -182,9 +205,9 @@ void YamlPreprocessor::ProcessIncludeNode(YAML::Node &node, try { auto path = ResolveIncludeFilePath(value, ref_path); - // replace string node with parsed contents of file node = YAML::LoadFile(path); - ProcessNodes(node, path); // recursively process the included file, too + // recursively process the included file, too + ProcessNodes(node, path); ROS_INFO_STREAM("Successfully loaded include file " + path); } catch (const YAML::BadFile &) { throw YAMLException("File specified in $include does not exist," + @@ -193,8 +216,58 @@ void YamlPreprocessor::ProcessIncludeNode(YAML::Node &node, throw YAMLException( "Malformatted file specified as include, " + format_error_info(), e); } catch (const YAML::Exception &e) { - throw YAMLException("Error loading include file, " + format_error_info(), e); + throw YAMLException("Error loading include file, " + format_error_info(), + e); + } +} + +bool YamlPreprocessor::ProcessSequenceIncludeNode( + std::vector &out_elems, YAML::Node &node, + const std::string &ref_path) { + if (node.Type() != YAML::NodeType::Scalar) { + return false; } + auto node_string = node.as(); + // check for the actual sequence include marker + if (node_string.compare(0, strlen(kSequenceIncludeMarker), + kSequenceIncludeMarker) != 0) { + return false; + } + // omit the $include + std::string value = node_string.substr(strlen(kSequenceIncludeMarker)); + boost::algorithm::trim(value); + out_elems.clear(); + + ROS_INFO_STREAM("Attempting to parse sequence include: " << value); + + // format the common file & include info for any thrown exceptions. + const auto format_error_info = [&]() { + return "path=" + ref_path + ", include=" + value; + }; + + try { + auto path = ResolveIncludeFilePath(value, ref_path); + out_elems = YAML::LoadAllFromFile(path); + + // recursively process the included nodes, too + for (auto &included_node : out_elems) { + ProcessNodes(included_node, path); + } + ROS_INFO_STREAM("Successfully loaded sequence include file " + path); + + } catch (const YAML::BadFile &) { + throw YAMLException("File specified in $[include] does not exist," + + format_error_info()); + } catch (const YAML::ParserException &e) { + throw YAMLException( + "Malformatted file specified as include, " + format_error_info(), e); + } catch (const YAML::Exception &e) { + throw YAMLException("Error loading include file, " + format_error_info(), + e); + } + + // parsed include successfully + return true; } YAML::Node YamlPreprocessor::LoadParse(const std::string &path) { diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml index fa0924d3..f9277602 100644 --- a/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml +++ b/flatland_server/test/yaml_preprocessor/yaml/include.child.yaml @@ -1,3 +1,3 @@ foo: bar spam: 2 -eggs: $include include.string.yaml \ No newline at end of file +eggs: $include include.string.yaml diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml index 67811e9f..c810a0ba 100644 --- a/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml +++ b/flatland_server/test/yaml_preprocessor/yaml/include.parent.out.yaml @@ -6,3 +6,9 @@ e: foo: bar spam: 2 eggs: this is a technically a YAML string. +f: + - parent first element + - first element + - second element + - third element + - parent last element diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml index 677bcf31..bd974a0d 100644 --- a/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml +++ b/flatland_server/test/yaml_preprocessor/yaml/include.parent.yaml @@ -2,4 +2,8 @@ a: this is a b: 52 c: $include include.string.yaml d: $include /tmp/9aaafd40-2083-49d9-a300-9d01f94d6671.yaml -e: $include include.child.yaml \ No newline at end of file +e: $include include.child.yaml +f: + - parent first element + - $[include] include.sequence.yaml + - parent last element diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.sequence.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.sequence.yaml new file mode 100644 index 00000000..56cd4273 --- /dev/null +++ b/flatland_server/test/yaml_preprocessor/yaml/include.sequence.yaml @@ -0,0 +1,5 @@ +first element +--- +second element +--- +third element diff --git a/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml b/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml index e34d652c..396ca4e0 100644 --- a/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml +++ b/flatland_server/test/yaml_preprocessor/yaml/include.string.yaml @@ -1 +1 @@ -this is a technically a YAML string. \ No newline at end of file +this is a technically a YAML string. diff --git a/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp b/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp index 3b46d413..40b00ea1 100644 --- a/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp +++ b/flatland_server/test/yaml_preprocessor/yaml_preprocessor_test.cpp @@ -152,6 +152,12 @@ TEST(YamlPreprocTest, testIncludeStrings) { compareNodes("e", "foo", in, out); compareNodes("e", "spam", in, out); compareNodes("e", "eggs", in, out); + + compareNodes("f", 0, in, out); + compareNodes("f", 1, in, out); + compareNodes("f", 2, in, out); + compareNodes("f", 3, in, out); + compareNodes("f", 4, in, out); } // Run all the tests that were declared with TEST()