Migrate test suite to GoogleTest framework

CMakeLists.txt

@@ -6,12 +6,13 @@ add_subdirectory(RTNeural)
 
 include(cmake/SIMDExtensions.cmake)
 include(cmake/ChooseBackend.cmake)
+include(cmake/CPM.cmake)
+include(cmake/Testing.cmake)
 
 option(BUILD_TESTS "Build RTNeural accuracy tests" OFF)
 if(BUILD_TESTS)
     message(STATUS "RTNeural -- Configuring tests...")
-    include(CTest)
-    enable_testing()
+    rtneural_setup_testing()
     add_subdirectory(tests)
 endif()
 

README.md

@@ -18,7 +18,7 @@ This library was designed with the intention of being used in
 real-time systems, specifically real-time audio processing.
 
 Currently supported layers:
-  
+
   - [x] Dense
   - [x] GRU
   - [x] LSTM
@@ -49,7 +49,7 @@ Additional resources:
 If you are using RTNeural as part of an academic work, please cite the library as follows:
 ```
 @article{chowdhury2021rtneural,
-        title={RTNeural: Fast Neural Inferencing for Real-Time Systems}, 
+        title={RTNeural: Fast Neural Inferencing for Real-Time Systems},
         author={Jatin Chowdhury},
         year={2021},
         journal={arXiv preprint arXiv:2106.03037}
@@ -187,17 +187,17 @@ or the C++ STL. You can choose your backend by passing
 either `-DRTNEURAL_EIGEN=ON`, `-DRTNEURAL_XSIMD=ON`,
 or `-DRTNEURAL_STL=ON` to your CMake configuration. By
 default, the `Eigen` backend will be used. Alternatively,
-you may select your choice of backends in your CMake 
+you may select your choice of backends in your CMake
 configuration as follows:
 ```cmake
 set(RTNEURAL_XSIMD ON CACHE BOOL "Use RTNeural with this backend" FORCE)
 add_subdirectory(modules/RTNeural)
 ```
 
-In general, the `Eigen` backend typically has the best 
+In general, the `Eigen` backend typically has the best
 performance for larger networks, while smaller networks
 may perform better with XSIMD. However, it is recommended
-to measure the performance of your network with all the 
+to measure the performance of your network with all the
 backends that are available on your target platform
 to ensure optimal performance. For more information see the
 [benchmark results](https://github.com/jatinchowdhury18/RTNeural/actions?query=workflow%3ABench).
@@ -211,13 +211,11 @@ you may run CMake with the `-DRTNEURAL_USE_AVX=ON`. Note that
 this flag will have no effect when compiling for platforms that
 do not support AVX instructions.
 
-### Building the Unit Tests
+### Building the test suite
 
-To build RTNeural's unit tests, run
-`cmake -Bbuild -DBUILD_TESTS=ON`, followed by
-`cmake --build build`. To run the full testing suite,
-run `./build/rtneural_tests all`. For more information,
-run `./build/rtneural_tests --help`.
+To build RTNeural's test suite, run `cmake -Bbuild -DBUILD_TESTS=ON`, followed
+by `cmake --build build`. To run the full testing suite, run `ctest` from the
+`build` folder. For more information, see `tests/README.md`.
 
 ### Building the Performance Benchmarks
 

RTNeural/CMakeLists.txt

@@ -44,7 +44,7 @@ target_include_directories(RTNeural
     PUBLIC
         ../modules/json
     INTERFACE
-        ..
+        ./
 )
 set(RTNEURAL_NAMESPACE "RTNeural" CACHE STRING "Namespace to use for RTNeural code")
 target_compile_definitions(RTNeural

cmake/CPM.cmake

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: MIT
+#
+# SPDX-FileCopyrightText: Copyright (c) 2019-2023 Lars Melchior and contributors
+
+set(CPM_DOWNLOAD_VERSION 0.38.6)
+set(CPM_HASH_SUM "11c3fa5f1ba14f15d31c2fb63dbc8628ee133d81c8d764caad9a8db9e0bacb07")
+
+if(CPM_SOURCE_CACHE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+# Expand relative path. This is important if the provided path contains a tilde (~)
+get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE)
+
+file(DOWNLOAD
+     https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+     ${CPM_DOWNLOAD_LOCATION} EXPECTED_HASH SHA256=${CPM_HASH_SUM}
+)
+
+include(${CPM_DOWNLOAD_LOCATION})

cmake/Testing.cmake

@@ -0,0 +1,31 @@
+option(RTNEURAL_TEST_REPORTS "Output test reports to XML files" OFF)
+
+macro(rtneural_setup_testing)
+    include(CTest)
+    enable_testing()
+    add_custom_target(rtneural_test COMMAND ctest -C ${Configuration} --output-on-failure)
+    CPMAddPackage("gh:google/[email protected]")
+endmacro()
+
+function(rtneural_add_test)
+
+    set(one_val_args TARGET)
+    set(multi_val_args SOURCES DEPENDENCIES)
+    cmake_parse_arguments(arg "" "${one_val_args}" "${multi_val_args}" ${ARGN})
+
+    add_executable(${arg_TARGET} ${arg_SOURCES})
+    target_link_libraries(${arg_TARGET} PUBLIC gtest_main gmock ${arg_DEPENDENCIES})
+    target_compile_definitions(${arg_TARGET} PRIVATE RTNEURAL_ROOT_DIR="${CMAKE_SOURCE_DIR}/")
+
+    if(RTNEURAL_TEST_REPORTS)
+        set(test_cmd_args --gtest_output=xml:${arg_TARGET}_report.xml)
+    endif()
+
+    add_test(
+        NAME ${arg_TARGET}
+        WORKING_DIRECTORY ./
+        COMMAND ${arg_TARGET} ${test_cmd_args})
+
+    add_dependencies(rtneural_test ${arg_TARGET})
+
+endfunction()

tests/CMakeLists.txt

@@ -1,24 +1,9 @@
-include_directories(../RTNeural)
-
-add_executable(rtneural_tests tests.cpp)
-target_link_libraries(rtneural_tests LINK_PUBLIC RTNeural)
-target_compile_definitions(rtneural_tests PRIVATE RTNEURAL_ROOT_DIR="${CMAKE_SOURCE_DIR}/")
-
-add_test(NAME "RTNeural_Bad_Model_Test" COMMAND $<TARGET_FILE:rtneural_tests> bad_model)
-add_test(NAME "RTNeural_Model_Test" COMMAND $<TARGET_FILE:rtneural_tests> model)
-add_test(NAME "RTNeural_Sample_Rate_RNN_Test" COMMAND $<TARGET_FILE:rtneural_tests> sample_rate_rnn)
-add_test(NAME "RTNeural_Torch_Test" COMMAND $<TARGET_FILE:rtneural_tests> torch)
-add_test(NAME "RTNeural_Util_Test" COMMAND $<TARGET_FILE:rtneural_tests> util)
-add_test(NAME "RTNeural_Conv1D_Test" COMMAND $<TARGET_FILE:rtneural_tests> conv1d)
-add_test(NAME "RTNeural_Conv2D_Test" COMMAND $<TARGET_FILE:rtneural_tests> conv2d)
-add_test(NAME "RTNeural_Dense_Test" COMMAND $<TARGET_FILE:rtneural_tests> dense)
-add_test(NAME "RTNeural_GRU_Test" COMMAND $<TARGET_FILE:rtneural_tests> gru)
-add_test(NAME "RTNeural_GRU_1D_Test" COMMAND $<TARGET_FILE:rtneural_tests> gru_1d)
-add_test(NAME "RTNeural_LSTM_Test" COMMAND $<TARGET_FILE:rtneural_tests> lstm)
-add_test(NAME "RTNeural_LSTM_1D_Test" COMMAND $<TARGET_FILE:rtneural_tests> lstm_1d)
+add_subdirectory(unit)
+add_subdirectory(functional)
 
 option(RTNEURAL_CODE_COVERAGE "Build RTNeural tests with code coverage flags" OFF)
 if(RTNEURAL_CODE_COVERAGE)
     include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/EnableCoverageFlags.cmake)
-    enable_coverage_flags(rtneural_tests)
+    enable_coverage_flags(rtneural_test_unit)
+    enable_coverage_flags(rtneural_test_functional)
 endif()

tests/README.md

@@ -0,0 +1,55 @@
+# RTNeural Testing
+
+RTNeural tests are configured with CMake's [CTest](https://cmake.org/cmake/help/book/mastering-cmake/chapter/Testing%20With%20CMake%20and%20CTest.html) and developed with the [GoogleTest framework](https://github.com/google/googletest). Tests are split into two categories:
+
+1. unit tests, and
+2. functional tests.
+
+A unit test should (ideally) be small, and will usually verify that one piece of
+logic in an individual component is working correctly. Functional tests operate
+at a higher level, and should check that the combination of multiple components
+(or multiple pieces of logic) are operating together as expected.
+
+GoogleTest has excellent [documentation](https://google.github.io/googletest/),
+which is highly recommended reading if you're new to testing or new to the
+framework.
+
+Happy testing!
+
+## Running tests
+
+### Full test suite
+
+To build and run all tests, make sure that you have passed `-DBUILD_TESTS=ON`
+to your `cmake` command. You then have two options:
+
+1. build all targets as normal, then run the test suite by invoking `ctest` in your build folder, or
+2. build the `rtneural_test` target, which should also run the test suite.
+
+### All unit tests
+
+To build and run only the unit tests,
+
+1. build the `rtneural_test_unit` target, and
+2. run the executable in your build folder at `./tests/unit/rtneural_test_unit`
+
+### All functional tests
+
+To build and run only the functional tests,
+
+1. build the `rtneural_test_functional` target, and
+2. run the executable in your build folder at `./tests/functional/rtneural_test_functional`
+
+### A subset of the tests
+
+Running a sub-set of the tests within a test target like `rtneural_test_unit` can be done by passing
+a filter string to the `--gtest_filter` command-line argument, e.g.
+
+```sh
+./tests/unit/rtneural_test_unit --gtest_filter="*Activation*"
+```
+
+There are a wealth of other useful command-line options that can be found by passing the `--help`
+argument to the test executable.
+
+