Static test cases definition + supporting multi-model cases in MemLeaks tests

tests/stress_tests/.automation/memleaks_tests/nightly_configs/desktop_test_config.xml

@@ -1,21 +1,15 @@
-<attributes>
-    <processes>
-        <value>1</value>
-    </processes>
-    <threads>
-        <value>1</value>
-    </threads>
-    <iterations>
-        <value>1000</value>
-    </iterations>
-    <devices>
-        <value>CPU</value>
-        <value>GPU</value>
-    </devices>
-    <models>
-        <!-- Alexnet model contains MatMul node which is needed to cover *-52623 -->
+<cases>
+
+    <device name="CPU" iterations="1000" processes="1" threads="1">
         <model name="alexnet" precision="FP32" source="omz" />
         <model name="mobilenet-ssd" precision="FP32" source="omz" />
         <model name="mtcnn-r" precision="FP32" source="omz" />
-    </models>
-</attributes>
+    </device>
+
+    <device name="GPU" iterations="1000" processes="1" threads="1">
+        <model name="alexnet" precision="FP32" source="omz" />
+        <model name="mobilenet-ssd" precision="FP32" source="omz" />
+        <model name="mtcnn-r" precision="FP32" source="omz" />
+    </device>
+
+</cases>

tests/stress_tests/.automation/memleaks_tests/nightly_configs/myriad_test_config.xml

@@ -1,19 +1,9 @@
-<attributes>
-    <processes>
-        <value>1</value>
-    </processes>
-    <threads>
-        <value>1</value>
-    </threads>
-    <iterations>
-        <value>1000</value>
-    </iterations>
-    <devices>
-        <value>MYRIAD</value>
-    </devices>
-    <models>
+<cases>
+
+    <device name="MYRIAD" iterations="1000" processes="1" threads="1">
         <model name="alexnet" precision="FP32" source="omz" />
         <model name="mobilenet-ssd" precision="FP32" source="omz" />
         <model name="mtcnn-r" precision="FP32" source="omz" />
-    </models>
-</attributes>
+    </device>
+
+</cases>

tests/stress_tests/.automation/memleaks_tests/precommit_configs/desktop_test_config.xml

@@ -1,18 +1,11 @@
-<attributes>
-    <processes>
-        <value>1</value>
-    </processes>
-    <threads>
-        <value>1</value>
-    </threads>
-    <iterations>
-        <value>30</value>
-    </iterations>
-    <devices>
-        <value>CPU</value>
-        <value>GPU</value>
-    </devices>
-    <models>
+<cases>
+
+    <device name="CPU" iterations="30" processes="1" threads="1">
         <model name="mobilenet-ssd" precision="FP32" source="omz" />
-    </models>
-</attributes>
+    </device>
+
+    <device name="GPU" iterations="30" processes="1" threads="1">
+        <model name="mobilenet-ssd" precision="FP32" source="omz" />
+    </device>
+
+</cases>

tests/stress_tests/.automation/memleaks_tests/weekly_configs/desktop_test_config.xml

@@ -1,21 +1,15 @@
-<attributes>
-    <processes>
-        <value>1</value>
-    </processes>
-    <threads>
-        <value>1</value>
-    </threads>
-    <iterations>
-        <value>5000</value>
-    </iterations>
-    <devices>
-        <value>CPU</value>
-        <value>GPU</value>
-    </devices>
-    <models>
-        <!-- Alexnet model contains MatMul node which is needed to cover *-52623 -->
+<cases>
+
+    <device name="CPU" iterations="5000" processes="1" threads="1">
         <model name="alexnet" precision="FP32" source="omz" />
         <model name="mobilenet-ssd" precision="FP32" source="omz" />
         <model name="mtcnn-r" precision="FP32" source="omz" />
-    </models>
-</attributes>
+    </device>
+
+    <device name="GPU" iterations="5000" processes="1" threads="1">
+        <model name="alexnet" precision="FP32" source="omz" />
+        <model name="mobilenet-ssd" precision="FP32" source="omz" />
+        <model name="mtcnn-r" precision="FP32" source="omz" />
+    </device>
+
+</cases>

tests/stress_tests/common/tests_utils.cpp

@@ -5,22 +5,19 @@
 #include "tests_utils.h"
 
 #include <gtest/gtest.h>
+#include <map>
 #include <pugixml.hpp>
 #include <string>
 
 #define DEBUG_MODE false
 
-const pugi::xml_document & Environment::getTestConfig() {
-    return _test_config;
-}
+const pugi::xml_document &Environment::getTestConfig() { return _test_config; }
 
-void Environment::setTestConfig(const pugi::xml_document &test_config) {
-    _test_config.reset(test_config);
-}
+void Environment::setTestConfig(const pugi::xml_document &test_config) { _test_config.reset(test_config); }
 
 std::vector<TestCase> generateTestsParams(std::initializer_list<std::string> fields) {
     std::vector<TestCase> tests_cases;
-    const pugi::xml_document & test_config = Environment::Instance().getTestConfig();
+    const pugi::xml_document &test_config = Environment::Instance().getTestConfig();
 
     std::vector<int> processes, threads, iterations;
     std::vector<std::string> devices, models, models_names, precisions;
@@ -49,7 +46,8 @@ std::vector<TestCase> generateTestsParams(std::initializer_list<std::string> fie
                 std::string full_path = val.attribute("full_path").as_string();
                 std::string path = val.attribute("path").as_string();
                 if (full_path.empty() || path.empty())
-                    throw std::logic_error("One of the 'model' records from test config doesn't contain 'full_path' or 'path' attributes");
+                    throw std::logic_error("One of the 'model' records from test config doesn't contain 'full_path' or "
+                                           "'path' attributes");
                 else {
                     models.push_back(full_path);
                     models_names.push_back(path);
@@ -61,9 +59,9 @@ std::vector<TestCase> generateTestsParams(std::initializer_list<std::string> fie
     }
 
     // Initialize variables with default value if it weren't filled
-    processes = !processes.empty() ? processes: std::vector<int>{1};
-    threads = !threads.empty() ? threads: std::vector<int>{1};
-    iterations = !iterations.empty() ? iterations: std::vector<int>{1};
+    processes = !processes.empty() ? processes : std::vector<int>{1};
+    threads = !threads.empty() ? threads : std::vector<int>{1};
+    iterations = !iterations.empty() ? iterations : std::vector<int>{1};
     devices = !devices.empty() ? devices : std::vector<std::string>{"NULL"};
     models = !models.empty() ? models : std::vector<std::string>{"NULL"};
     precisions = !precisions.empty() ? precisions : std::vector<std::string>{"NULL"};
@@ -74,14 +72,58 @@ std::vector<TestCase> generateTestsParams(std::initializer_list<std::string> fie
             for (auto &numiters : iterations)
                 for (auto &device : devices)
                     for (int i = 0; i < models.size(); i++)
-                        tests_cases.push_back(TestCase(numprocesses, numthreads, numiters, device, models[i], models_names[i], precisions[i]));
+                        tests_cases.push_back(TestCase(numprocesses, numthreads, numiters, device, models[i],
+                                                       models_names[i], precisions[i]));
+    return tests_cases;
+}
+
+// Generate multi-model test cases from config file with static test definition.
+std::vector<MemLeaksTestCase> generateTestsParamsMemLeaks() {
+    std::vector<MemLeaksTestCase> tests_cases;
+    const pugi::xml_document &test_config = Environment::Instance().getTestConfig();
+
+    int numprocesses, numthreads, numiterations;
+    std::string device_name;
+
+    pugi::xml_node cases;
+    cases = test_config.child("cases");
+
+    for (pugi::xml_node device = cases.first_child(); device; device = device.next_sibling()) {
+        device_name = device.attribute("name").as_string("NULL");
+        numprocesses = device.attribute("processes").as_int(1);
+        numthreads = device.attribute("threads").as_int(1);
+        numiterations = device.attribute("iterations").as_int(1);
+
+        std::vector<std::map<std::string, std::string>> models;
+
+        for (pugi::xml_node model = device.first_child(); model; model = model.next_sibling()) {
+            std::string full_path = model.attribute("full_path").as_string();
+            std::string path = model.attribute("path").as_string();
+            if (full_path.empty() || path.empty())
+                throw std::logic_error(
+                        "One of the 'model' records from test config doesn't contain 'full_path' or 'path' attributes");
+            std::string name = model.attribute("name").as_string();
+            std::string precision = model.attribute("precision").as_string();
+            std::map<std::string, std::string> model_map{{"name", name},
+                                                         {"path", path},
+                                                         {"full_path", full_path},
+                                                         {"precision", precision}};
+            models.push_back(model_map);
+        }
+        tests_cases.push_back(MemLeaksTestCase(numprocesses, numthreads, numiterations, device_name, models));
+    }
+
     return tests_cases;
 }
 
 std::string getTestCaseName(const testing::TestParamInfo<TestCase> &obj) {
     return obj.param.test_case_name;
 }
 
+std::string getTestCaseNameMemLeaks(const testing::TestParamInfo<MemLeaksTestCase> &obj) {
+    return obj.param.test_case_name;
+}
+
 void test_wrapper(const std::function<void(std::string, std::string, int)> &tests_pipeline, const TestCase &params) {
     tests_pipeline(params.model, params.device, params.numiters);
 }
@@ -94,8 +136,7 @@ void runTest(const std::function<void(std::string, std::string, int)> &tests_pip
 #if DEBUG_MODE
     tests_pipeline(params.model, params.device, params.numiters);
 #else
-    int status = run_in_processes(params.numprocesses, [&](){ _runTest(tests_pipeline, params); });
+    int status = run_in_processes(params.numprocesses, [&]() { _runTest(tests_pipeline, params); });
     ASSERT_EQ(status, 0) << "Test failed with exitcode " << std::to_string(status);
 #endif
 }
-

tests/stress_tests/common/tests_utils.h

@@ -9,69 +9,90 @@
 #include <gtest/gtest.h>
 #include <pugixml.hpp>
 #include <string>
-#include <vector>
 #include <thread>
+#include <vector>
 
 
-enum TestStatus
-{
-    TEST_NOT_STARTED = 0,
-    TEST_FAILED,
-    TEST_OK
-};
+enum TestStatus { TEST_NOT_STARTED = 0, TEST_FAILED, TEST_OK };
 
 using TestResult = std::pair<TestStatus, std::string>;
 
-class TestCase {
+class TestCaseBase {
 public:
     int numprocesses;
     int numthreads;
     int numiters;
-    std::string device;
-    std::string model_name;
-    std::string model;
     std::string precision;
     std::string test_case_name;
+    std::string model_name;
+    std::string device;
 
-    TestCase(int _numprocesses, int _numthreads, int _numiters, std::string _device, const std::string& _model, const std::string& _model_name, const std::string& _precision) {
-        numprocesses = _numprocesses, numthreads = _numthreads, numiters = _numiters, device = _device, model = _model, model_name = _model_name, precision = _precision;
-        test_case_name =
-                "Numprocesses_" + std::to_string(numprocesses) + "_Numthreads_" + std::to_string(numthreads) +
-                "_Numiters_" + std::to_string(numiters) + "_Device_" + update_item_for_name(device) + "_Precision_" +
-                update_item_for_name(precision) + "_Model_" + update_item_for_name(model_name);
-    }
-
-private:
+protected:
+    // Replace non-alphabetic/numeric symbols with "_" to prevent logging errors
     std::string update_item_for_name(const std::string &item) {
         std::string _item(item);
         for (std::string::size_type index = 0; index < _item.size(); ++index) {
-            if (!isalnum(_item[index]) && _item[index] != '_')
-                _item[index] = '_';
+            if (!isalnum(_item[index]) && _item[index] != '_') _item[index] = '_';
         }
         return _item;
     }
 };
 
+class TestCase : public TestCaseBase {
+public:
+    std::string model;
+
+    TestCase(int _numprocesses, int _numthreads, int _numiters, std::string _device, const std::string &_model,
+             const std::string &_model_name, const std::string &_precision) {
+        numprocesses = _numprocesses, numthreads = _numthreads, numiters = _numiters, device = _device, model = _model,
+        model_name = _model_name, precision = _precision;
+        test_case_name = "Numprocesses_" + std::to_string(numprocesses) + "_Numthreads_" + std::to_string(numthreads) +
+                         "_Numiters_" + std::to_string(numiters) + "_Device_" + update_item_for_name(device) +
+                         "_Precision_" + update_item_for_name(precision) + "_Model_" + update_item_for_name(model_name);
+    }
+};
+
+class MemLeaksTestCase : public TestCaseBase {
+public:
+    std::vector<std::map<std::string, std::string>> models;
+
+    MemLeaksTestCase(int _numprocesses, int _numthreads, int _numiters, std::string _device,
+                     std::vector<std::map<std::string, std::string>> _models) {
+        numprocesses = _numprocesses, numthreads = _numthreads, numiters = _numiters, device = _device,
+        models = _models;
+        test_case_name = "Numprocesses_" + std::to_string(numprocesses) + "_Numthreads_" + std::to_string(numthreads) +
+                         "_Numiters_" + std::to_string(numiters) + "_Device_" + update_item_for_name(device);
+        for (int i = 0; i < models.size(); i++) {
+            test_case_name += "_Model" + std::to_string(i + 1) + "_" + update_item_for_name(models[i]["name"]) + "_" +
+                              update_item_for_name(models[i]["precision"]);
+            model_name += "\"" + models[i]["path"] + "\"" + (i < models.size() - 1 ? ", " : "");
+        }
+    }
+};
+
 class Environment {
 private:
     pugi::xml_document _test_config;
     bool _collect_results_only = false;
 
     Environment() = default;
-    Environment(const Environment&) = delete;
-    Environment& operator=(const Environment&) = delete;
+    Environment(const Environment &) = delete;
+    Environment &operator=(const Environment &) = delete;
+
 public:
-    static Environment& Instance(){
+    static Environment &Instance() {
         static Environment env;
         return env;
     }
 
-    const pugi::xml_document & getTestConfig();
+    const pugi::xml_document &getTestConfig();
     void setTestConfig(const pugi::xml_document &test_config);
 };
 
 std::vector<TestCase> generateTestsParams(std::initializer_list<std::string> items);
+std::vector<MemLeaksTestCase> generateTestsParamsMemLeaks();
 std::string getTestCaseName(const testing::TestParamInfo<TestCase> &obj);
+std::string getTestCaseNameMemLeaks(const testing::TestParamInfo<MemLeaksTestCase> &obj);
 
 void runTest(const std::function<void(std::string, std::string, int)> &tests_pipeline, const TestCase &params);
 void _runTest(const std::function<void(std::string, std::string, int)> &tests_pipeline, const TestCase &params);