From 2221ecb128b2fed8ebd225a4476568a398db4d14 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 9 May 2023 17:14:21 -0500 Subject: [PATCH 01/50] WIP --- Makefile | 12 ++++++++++-- doc/changelog.rst | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c9561703e..389f69e5d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,14 @@ MAKEFLAGS += --no-print-directory COV_FLAGS := SHELL:=/bin/bash + +# Build variables +SR_BUILD := Release +SR_LINK := Shared +SR_TEST_REDIS_MODE := Clustered +SR_TEST_RAI_VER := 1.2.7 +SR_WLM := Local +SR_WLM_FLAGS := SR_DEVICE := cpu # Params for third-party software @@ -76,8 +84,8 @@ test-deps: lcov # help: test-deps-gpu - Make SmartRedis GPU testing dependencies .PHONY: test-deps-gpu -test-deps-gpu: - $(error To build test-deps for GPU, please execute "make test-deps SR_DEVICE=gpu") +test-deps-gpu: SR_DEVICE=gpu +test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) .PHONY: build-tests diff --git a/doc/changelog.rst b/doc/changelog.rst index 08e7f9f2d..bb62ffd05 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Revamp build and test systems for SmartRedis - Fix the spelling of the Dataset destructor's C interface (now DeallocateDataSet) - Update Redis++ version to 1.3.8 - Refactor third-party software dependency installation @@ -16,13 +17,15 @@ Description Detailed Notes +- Rework the build and test system to improve maintainability of the library - Correct the spelling of the C DataSet destruction interface from DeallocateeDataSet to DeallocateDataSet (PR338_) - Updated the version of Redis++ to v1.3.8 to pull in a change that ensures the redis++.pc file properly points to the generated libraries (PR334_) - Third-party software dependency installation is now handled in the Makefile instead of separate scripts - New pip-install target in Makefile will be a dependency of the lib target going forward so that users don't have to manually pip install SmartRedis in the future (PR330_) - Added ConfigOptions class and API, which will form the backbone of multiDB support (PR303_) -.. _PR335: https://github.com/CrayLabs/SmartRedis/pull/338 +.. _PR340: https://github.com/CrayLabs/SmartRedis/pull/340 +.. _PR338: https://github.com/CrayLabs/SmartRedis/pull/338 .. _PR334: https://github.com/CrayLabs/SmartRedis/pull/334 .. _PR331: https://github.com/CrayLabs/SmartRedis/pull/331 .. _PR330: https://github.com/CrayLabs/SmartRedis/pull/330 From e9fef6f50e896f42469098c8a749123cd30beb07 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 12 May 2023 13:49:06 -0500 Subject: [PATCH 02/50] Examples work --- Makefile | 54 ++++++++++--------- conftest.py | 9 ++++ examples/CMakeLists.txt | 37 +++++++++++++ examples/parallel/CMakeLists.txt | 34 ++++++++++++ examples/parallel/cpp/CMakeLists.txt | 38 ++++++------- examples/parallel/cpp/smartredis_mnist.cpp | 6 +-- examples/parallel/fortran/CMakeLists.txt | 12 ++--- .../parallel/fortran/smartredis_mnist.F90 | 4 +- examples/serial/CMakeLists.txt | 36 +++++++++++++ examples/serial/c/CMakeLists.txt | 33 ++++++------ examples/serial/cpp/CMakeLists.txt | 52 ++++++++---------- examples/serial/cpp/smartredis_mnist.cpp | 11 ++-- examples/serial/cpp/smartredis_model.cpp | 6 +-- examples/serial/fortran/CMakeLists.txt | 13 ++--- examples/test_examples.py | 21 +++++--- 15 files changed, 241 insertions(+), 125 deletions(-) create mode 100644 examples/CMakeLists.txt create mode 100644 examples/parallel/CMakeLists.txt create mode 100644 examples/serial/CMakeLists.txt diff --git a/Makefile b/Makefile index 389f69e5d..32e5c5303 100644 --- a/Makefile +++ b/Makefile @@ -90,51 +90,55 @@ test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) .PHONY: build-tests build-tests: test-lib-with-fortran - ./build-scripts/build_cpp_tests.sh $(CMAKE_ARGS) - ./build-scripts/build_cpp_unit_tests.sh $(CMAKE_ARGS) - ./build-scripts/build_c_tests.sh $(CMAKE_ARGS) - ./build-scripts/build_fortran_tests.sh $(CMAKE_ARGS) + cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/tests # help: build-test-cpp - build the C++ tests .PHONY: build-test-cpp build-test-cpp: test-lib - ./build-scripts/build_cpp_tests.sh - ./build-scripts/build_cpp_unit_tests.sh + cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/tests/cpp # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib - ./build-scripts/build_cpp_unit_tests.sh + cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib - ./build-scripts/build_c_tests.sh + cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/tests/c # help: build-test-fortran - build the Fortran tests .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran - ./build-scripts/build_fortran_tests.sh + cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/tests/fortran # help: build-examples - build all examples (serial, parallel) .PHONY: build-examples build-examples: lib-with-fortran - ./build-scripts/build_serial_examples.sh - ./build-scripts/build_parallel_examples.sh + cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/examples + # help: build-example-serial - buld serial examples .PHONY: build-example-serial -build-example-serial: lib - ./build-scripts/build_serial_examples.sh +build-example-serial: lib-with-fortran + cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/examples/serial # help: build-example-parallel - build parallel examples (requires MPI) .PHONY: build-example-parallel -build-example-parallel: lib - ./build-scripts/build_parallel_examples.sh +build-example-parallel: lib-with-fortran + cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + cmake --build build/$(SR_BUILD)/examples/parellel # help: clean-deps - remove third-party deps @@ -222,7 +226,7 @@ cov: test: test-deps test: build-tests test: - @PYTHONFAULTHANDLER=1 python -m pytest --ignore ./tests/docker -vv ./tests + @PYTHONFAULTHANDLER=1 python -m pytest --ignore ./tests/docker -vv ./tests --build $(SR_BUILD) # help: test-verbose - Build and run all tests [verbosely] @@ -230,7 +234,7 @@ test: test-verbose: test-deps test-verbose: build-tests test-verbose: - PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests + PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) # help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] .PHONY: test-verbose-with-coverage @@ -238,14 +242,14 @@ test-verbose-with-coverage: test-deps test-verbose-with-coverage: build-tests test-verbose-with-coverage: CMAKE_ARGS="-DCOVERAGE=on" test-verbose-with-coverage: - PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests + PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) # help: test-c - Build and run all C tests .PHONY: test-c test-c: test-deps test-c: build-test-c test-c: - @python -m pytest -vv -s ./tests/c/ + @python -m pytest -vv -s ./tests/c/ --build $(SR_BUILD) # help: test-cpp - Build and run all C++ tests .PHONY: test-cpp @@ -253,39 +257,39 @@ test-cpp: test-deps test-cpp: build-test-cpp test-cpp: build-unit-test-cpp test-cpp: - @python -m pytest -vv -s ./tests/cpp/ + @python -m pytest -vv -s ./tests/cpp/ --build $(SR_BUILD) # help: unit-test-cpp - Build and run unit tests for C++ .PHONY: unit-test-cpp unit-test-cpp: test-deps unit-test-cpp: build-unit-test-cpp unit-test-cpp: - @python -m pytest -vv -s ./tests/cpp/unit-tests/ + @python -m pytest -vv -s ./tests/cpp/unit-tests/ --build $(SR_BUILD) # help: test-py - run python tests .PHONY: test-py test-py: test-deps test-py: - @PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/ + @PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/ --build $(SR_BUILD) # help: test-fortran - run fortran tests .PHONY: test-fortran test-fortran: test-deps test-fortran: build-test-fortran - @python -m pytest -vv ./tests/fortran/ + @python -m pytest -vv ./tests/fortran/ --build $(SR_BUILD) # help: testpy-cov - run python tests with coverage .PHONY: testpy-cov testpy-cov: test-deps testpy-cov: - @PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ -vv ./tests/python/ + @PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ -vv ./tests/python/ --build $(SR_BUILD) # help: test-examples - Build and run all examples .PHONY: test-examples test-examples: test-deps test-examples: build-examples test-examples: - @python -m pytest -vv -s ./examples + @python -m pytest -vv -s ./examples --build $(SR_BUILD) ############################################################################ diff --git a/conftest.py b/conftest.py index 8b4eca3a6..b272145e9 100644 --- a/conftest.py +++ b/conftest.py @@ -136,3 +136,12 @@ def create_torch_cnn(filepath=None): torch.jit.save(module, buffer) str_model = buffer.getvalue() return str_model + +# Add a build type option to pytest command lines +def pytest_addoption(parser): + parser.addoption("--build", action="store", default="Release") + +# Fixture to retrieve the build type setting +@pytest.fixture(scope="session") +def build(request): + return request.config.getoption("--build") diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 000000000..59e294ca5 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,37 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +project(Examples) +cmake_minimum_required(VERSION 3.13) + +enable_language(C) +enable_language(CXX) +enable_language(Fortran) + +set(ALLOW_DUPLICATE_CUSTOM_TARGETS true) + +add_subdirectory(parallel) +add_subdirectory(serial) diff --git a/examples/parallel/CMakeLists.txt b/examples/parallel/CMakeLists.txt new file mode 100644 index 000000000..600ceeeff --- /dev/null +++ b/examples/parallel/CMakeLists.txt @@ -0,0 +1,34 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +project(ParallelExamples) +cmake_minimum_required(VERSION 3.13) + +enable_language(CXX) +enable_language(Fortran) + +add_subdirectory(cpp) +add_subdirectory(fortran) diff --git a/examples/parallel/cpp/CMakeLists.txt b/examples/parallel/cpp/CMakeLists.txt index 2dba75e3a..00db4d5c8 100644 --- a/examples/parallel/cpp/CMakeLists.txt +++ b/examples/parallel/cpp/CMakeLists.txt @@ -24,9 +24,9 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientExamples) - +project(CppClientParallelExamples) cmake_minimum_required(VERSION 3.13) +enable_language(CXX) set(CMAKE_BUILD_TYPE Release) set(CMAKE_CXX_STANDARD 17) @@ -47,22 +47,22 @@ set(ftn_client_src ../../../src/fortran/client.F90 ) -# Build executables - -add_executable(smartredis_put_get_3D - smartredis_put_get_3D.cpp - ${ftn_client_src} -) -target_link_libraries(smartredis_put_get_3D - MPI::MPI_CXX - ${SR_LIB} +# Define all the tests to be built +list(APPEND EXECUTABLES + smartredis_put_get_3D + smartredis_mnist ) -add_executable(smartredis_mnist - smartredis_mnist.cpp - ${ftn_client_src} -) -target_link_libraries(smartredis_mnist - MPI::MPI_CXX - ${SR_LIB} -) +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_cpp_parallel + ${EXECUTABLE}.cpp + ${ftn_client_src} + ) + set_target_properties(${EXECUTABLE}_cpp_parallel PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_cpp_parallel + MPI::MPI_CXX + ${SR_LIB} + ) +endforeach() diff --git a/examples/parallel/cpp/smartredis_mnist.cpp b/examples/parallel/cpp/smartredis_mnist.cpp index 05266e006..974eeca01 100644 --- a/examples/parallel/cpp/smartredis_mnist.cpp +++ b/examples/parallel/cpp/smartredis_mnist.cpp @@ -43,7 +43,7 @@ void run_mnist(const std::string& model_name, // Load the mnist image from a file using MPI rank 0 if (rank == 0) { - std::string image_file = "../../../common/mnist_data/one.raw"; + std::string image_file = "../../../../../examples/common/mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -113,7 +113,7 @@ int main(int argc, char* argv[]) { // Build model key, file name, and then set model // from file using client API std::string model_key = "mnist_model"; - std::string model_file = "../../../"\ + std::string model_file = "../../../../../examples/"\ "common/mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU", 20); @@ -121,7 +121,7 @@ int main(int argc, char* argv[]) { // Build script key, file name, and then set script // from file using client API std::string script_key = "mnist_script"; - std::string script_file = "../../../common/mnist_data/" + std::string script_file = "../../../../../examples/common/mnist_data/" "data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index 8cf05c2bd..dbe44957c 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -24,10 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientExamples) - +project(FortranClientParallelExamples) cmake_minimum_required(VERSION 3.13) - enable_language(Fortran) set(CMAKE_VERBOSE_MAKEFILE ON) @@ -69,12 +67,14 @@ list(APPEND EXECUTABLES ) foreach(EXECUTABLE ${EXECUTABLES}) - - add_executable(${EXECUTABLE} + add_executable(${EXECUTABLE}_fortran_parallel ${EXECUTABLE}.F90 example_utils.F90 ) - target_link_libraries(${EXECUTABLE} + set_target_properties(${EXECUTABLE}_fortran_parallel PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_fortran_parallel ${SMARTREDIS_LIBRARIES} MPI::MPI_Fortran ) diff --git a/examples/parallel/fortran/smartredis_mnist.F90 b/examples/parallel/fortran/smartredis_mnist.F90 index 780c24159..940cd23fb 100644 --- a/examples/parallel/fortran/smartredis_mnist.F90 +++ b/examples/parallel/fortran/smartredis_mnist.F90 @@ -35,9 +35,9 @@ program mnist_example #include "enum_fortran.inc" character(len=*), parameter :: model_key = "mnist_model" - character(len=*), parameter :: model_file = "../../../common/mnist_data/mnist_cnn.pt" + character(len=*), parameter :: model_file = "../../../../../examples/common/mnist_data/mnist_cnn.pt" character(len=*), parameter :: script_key = "mnist_script" - character(len=*), parameter :: script_file = "../../../common/mnist_data/data_processing_script.txt" + character(len=*), parameter :: script_file = "../../../../../examples/common/mnist_data/data_processing_script.txt" type(client_type) :: client integer :: err_code, pe_id, result diff --git a/examples/serial/CMakeLists.txt b/examples/serial/CMakeLists.txt new file mode 100644 index 000000000..b7bea33b5 --- /dev/null +++ b/examples/serial/CMakeLists.txt @@ -0,0 +1,36 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +project(SerialExamples) +cmake_minimum_required(VERSION 3.13) + +enable_language(C) +enable_language(CXX) +enable_language(Fortran) + +add_subdirectory(c) +add_subdirectory(cpp) +add_subdirectory(fortran) diff --git a/examples/serial/c/CMakeLists.txt b/examples/serial/c/CMakeLists.txt index 4184cfecd..e2dd09d98 100644 --- a/examples/serial/c/CMakeLists.txt +++ b/examples/serial/c/CMakeLists.txt @@ -24,35 +24,36 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CClientExamples) - +project(CClientSerialExamples) cmake_minimum_required(VERSION 3.13) +enable_language(C) set(CMAKE_BUILD_TYPE RELEASE) set(CMAKE_CXX_STANDARD 17) SET(CMAKE_C_STANDARD 99) find_library(SR_LIB smartredis PATHS ../../../install/lib - NO_DEFAULT_PATH REQUIRED) + NO_DEFAULT_PATH REQUIRED) include_directories(SYSTEM /usr/local/include ../../../install/include ) -add_executable(example_put_unpack_1D - example_put_unpack_1D.c -) - -target_link_libraries(example_put_unpack_1D - ${SR_LIB} +# Define all the tests to be built +list(APPEND EXECUTABLES + example_put_unpack_1D + example_put_get_3D ) -add_executable(example_put_get_3D - example_put_get_3D.c -) - -target_link_libraries(example_put_get_3D +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_c_serial + ${EXECUTABLE}.c + ) + set_target_properties(${EXECUTABLE}_c_serial PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_c_serial ${SR_LIB} -) - + ) +endforeach() diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index d3f436973..a22c1b73a 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -24,9 +24,9 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientExamples) - +project(CppClientSerialExamples) cmake_minimum_required(VERSION 3.13) +enable_language(CXX) set(CMAKE_BUILD_TYPE RELEASE) set(CMAKE_CXX_STANDARD 17) @@ -39,32 +39,22 @@ include_directories(SYSTEM ../../../install/include ) -# Build executables - -add_executable(smartredis_put_get_3D - smartredis_put_get_3D.cpp -) -target_link_libraries(smartredis_put_get_3D - ${SR_LIB} -) - -add_executable(smartredis_dataset - smartredis_dataset.cpp -) -target_link_libraries(smartredis_dataset - ${SR_LIB} -) - -add_executable(smartredis_model - smartredis_model.cpp -) -target_link_libraries(smartredis_model - ${SR_LIB} -) - -add_executable(smartredis_mnist - smartredis_mnist.cpp -) -target_link_libraries(smartredis_mnist - ${SR_LIB} -) +# Define all the tests to be built +list(APPEND EXECUTABLES + smartredis_put_get_3D + smartredis_dataset + smartredis_model + smartredis_mnist +) + +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_cpp_serial + ${EXECUTABLE}.cpp + ) + set_target_properties(${EXECUTABLE}_cpp_serial PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_cpp_serial + ${SR_LIB} + ) +endforeach() diff --git a/examples/serial/cpp/smartredis_mnist.cpp b/examples/serial/cpp/smartredis_mnist.cpp index 0a3b05296..0c85e4e9d 100644 --- a/examples/serial/cpp/smartredis_mnist.cpp +++ b/examples/serial/cpp/smartredis_mnist.cpp @@ -37,7 +37,7 @@ void run_mnist(const std::string& model_name, std::vector img(n_values, 0); // Load the MNIST image from a file - std::string image_file = "../../../common/mnist_data/one.raw"; + std::string image_file = "../../../../../examples/common/mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -83,15 +83,14 @@ int main(int argc, char* argv[]) { // Build model key, file name, and then set model // from file using client API std::string model_key = "mnist_model"; - std::string model_file = "../../../"\ - "common/mnist_data/mnist_cnn.pt"; - client.set_model_from_file(model_key, model_file, - "TORCH", "CPU", 20); + std::string model_file = "../../../../../examples/common/mnist_data/mnist_cnn.pt"; + client.set_model_from_file( + model_key, model_file, "TORCH", "CPU", 20); // Build script key, file name, and then set script // from file using client API std::string script_key = "mnist_script"; - std::string script_file = "../../../common/mnist_data/" + std::string script_file = "../../../../../examples/common/mnist_data/" "data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); diff --git a/examples/serial/cpp/smartredis_model.cpp b/examples/serial/cpp/smartredis_model.cpp index 0c520f21d..1fab8ad59 100644 --- a/examples/serial/cpp/smartredis_model.cpp +++ b/examples/serial/cpp/smartredis_model.cpp @@ -37,7 +37,7 @@ int main(int argc, char* argv[]) { std::vector img(n_values, 0); // Load the mnist image from a file - std::string image_file = "../../../common/mnist_data/one.raw"; + std::string image_file = "../../../../../examples/common/mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -52,12 +52,12 @@ int main(int argc, char* argv[]) { // Use the client to set a model in the database from a file std::string model_key = "mnist_model"; - std::string model_file = "../../../common/mnist_data/mnist_cnn.pt"; + std::string model_file = "../../../../../examples/common/mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU", 20); // Use the client to set a script from the database form a file std::string script_key = "mnist_script"; - std::string script_file = "../../../common/mnist_data/data_processing_script.txt"; + std::string script_file = "../../../../../examples/common/mnist_data/data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); // Declare keys that we will use in forthcoming client commands diff --git a/examples/serial/fortran/CMakeLists.txt b/examples/serial/fortran/CMakeLists.txt index 59008918d..abdef7177 100644 --- a/examples/serial/fortran/CMakeLists.txt +++ b/examples/serial/fortran/CMakeLists.txt @@ -24,10 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientExamples) - +project(FortranClientSerialExamples) cmake_minimum_required(VERSION 3.13) - enable_language(Fortran) set(CMAKE_VERBOSE_MAKEFILE ON) @@ -64,10 +62,13 @@ list(APPEND EXECUTABLES foreach(EXECUTABLE ${EXECUTABLES}) - add_executable(${EXECUTABLE} + add_executable(${EXECUTABLE}_fortran_serial ${EXECUTABLE}.F90 ) - target_link_libraries(${EXECUTABLE} + set_target_properties(${EXECUTABLE}_fortran_serial PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_fortran_serial ${SMARTREDIS_LIBRARIES} ) -endforeach() \ No newline at end of file +endforeach() diff --git a/examples/test_examples.py b/examples/test_examples.py index ef0ebe818..e6f758472 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -26,8 +26,8 @@ import pytest from os import path as osp +from os import getcwd from glob import glob -from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import time @@ -38,18 +38,25 @@ def get_test_names(): """Obtain test names by globbing for client_test Add tests manually if necessary """ - glob_path_1 = osp.join(TEST_PATH, "*/*/build/example*") - glob_path_2 = osp.join(TEST_PATH, "*/*/build/smartredis*") + glob_path_1 = osp.join(TEST_PATH, "*/*/example*") + glob_path_2 = osp.join(TEST_PATH, "*/*/smartredis*") test_names = glob(glob_path_1) + glob(glob_path_2) - test_names = list(filter(lambda test: test.find('.mod') == -1, test_names)) + test_names = list(filter(lambda test: test.find('example_utils') == -1, test_names)) + test_names = list(filter(lambda test: test.find('.py') == -1, test_names)) return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_cpp_client(test, use_cluster): +def test_example(test, use_cluster, build): + # Build the path to the test executable from the source file name + # . keep only the last three parts of the path (parallel/serial, language, basename) + test = "/".join(test.split("/")[-3:]) + # . drop the file extension + test = ".".join(test.split(".")[:-1]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/examples/{test}" cmd = [] cmd.append(test) print(f"Running test: {osp.basename(test)}") - print(f"Test command {' '.join(cmd)}") print(f"Using cluster: {use_cluster}") execute_cmd(cmd) time.sleep(1) @@ -71,8 +78,6 @@ def execute_cmd(cmd_list): # spawning the subprocess and connecting to its output run_path = find_path(cmd_list[0]) - print("cmd_list", cmd_list) - print("cwd", run_path) proc = Popen( cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=run_path) try: From 6d6933f38165993e646d4bcdf1488d93b9096e5f Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 12 May 2023 17:35:06 -0500 Subject: [PATCH 03/50] Unit tests buildable --- examples/serial/cpp/CMakeLists.txt | 6 +- tests/CMakeLists.txt | 42 ++++++ tests/c/CMakeLists.txt | 73 +++------- tests/cpp/CMakeLists.txt | 213 ++++++---------------------- tests/docker/CMakeLists.txt | 38 +++++ tests/docker/c/CMakeLists.txt | 12 +- tests/docker/cpp/CMakeLists.txt | 11 +- tests/docker/fortran/CMakeLists.txt | 12 +- tests/fortran/CMakeLists.txt | 12 +- 9 files changed, 174 insertions(+), 245 deletions(-) create mode 100644 tests/CMakeLists.txt create mode 100644 tests/docker/CMakeLists.txt diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index a22c1b73a..c1dbfd0c6 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -41,10 +41,10 @@ include_directories(SYSTEM # Define all the tests to be built list(APPEND EXECUTABLES - smartredis_put_get_3D + smartredis_put_get_3D smartredis_dataset - smartredis_model - smartredis_mnist + smartredis_model + smartredis_mnist ) foreach(EXECUTABLE ${EXECUTABLES}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..a372b8821 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,42 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +project(SmartSim-Tests) +cmake_minimum_required(VERSION 3.13) + +enable_language(C) +enable_language(CXX) +enable_language(Fortran) + +set(ALLOW_DUPLICATE_CUSTOM_TARGETS true) + +add_subdirectory(c) +add_subdirectory(cpp) +add_subdirectory(fortran) + +# The docker subdirectory is designed to be built within a container, not as part +# of the normal test build. We therefore do not include it in a test build +#add_subdirectory(docker) diff --git a/tests/c/CMakeLists.txt b/tests/c/CMakeLists.txt index 5926beb68..e2d3ce640 100644 --- a/tests/c/CMakeLists.txt +++ b/tests/c/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project(CClientTester) - cmake_minimum_required(VERSION 3.13) +enable_language(C) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_BUILD_TYPE DEBUG) @@ -47,58 +47,25 @@ include_directories(SYSTEM ../../install/include ) -add_executable(client_test_dataset_aggregation -client_test_dataset_aggregation.c +# Define all the tests to be built +list(APPEND EXECUTABLES + client_test_dataset_aggregation + client_test_dataset_exists + client_test_logging + client_test_put_unpack_1D + client_test_put_get_1D + client_test_put_get_2D + client_test_put_get_3D ) -target_link_libraries(client_test_dataset_aggregation +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_c_test + ${EXECUTABLE}.c + ) + set_target_properties(${EXECUTABLE}_c_test PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_c_test ${SR_LIB} -) - -add_executable(client_test_dataset_exists - client_test_dataset_exists.c -) - -target_link_libraries(client_test_dataset_exists - ${SR_LIB} -) - -add_executable(client_test_logging - client_test_logging.c -) - -target_link_libraries(client_test_logging - ${SR_LIB} -) - -add_executable(client_test_put_unpack_1D - client_test_put_unpack_1D.c -) - -target_link_libraries(client_test_put_unpack_1D - ${SR_LIB} -) - -add_executable(client_test_put_get_1D - client_test_put_get_1D.c -) - -target_link_libraries(client_test_put_get_1D - ${SR_LIB} -) - -add_executable(client_test_put_get_2D - client_test_put_get_2D.c -) - -target_link_libraries(client_test_put_get_2D - ${SR_LIB} -) - -add_executable(client_test_put_get_3D - client_test_put_get_3D.c -) - -target_link_libraries(client_test_put_get_3D - ${SR_LIB} -) + ) +endforeach() diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index f320fecac..a47c0592d 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -25,18 +25,20 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project(CppClientTester) - cmake_minimum_required(VERSION 3.13) +enable_language(CXX) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_BUILD_TYPE DEBUG) set(CMAKE_CXX_STANDARD 17) +add_subdirectory(unit-tests) + if(COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - set(CMAKE_CXX_FLAGS --coverage) - add_link_options(--coverage) - endif() + if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) + set(CMAKE_CXX_FLAGS --coverage) + add_link_options(--coverage) + endif() endif() find_library(SR_LIB smartredis PATHS ../../install/lib NO_DEFAULT_PATH REQUIRED) @@ -46,166 +48,41 @@ include_directories(SYSTEM ../../install/include ) -# Build executables - -add_executable(client_test_dataset - client_test_dataset.cpp -) -target_link_libraries(client_test_dataset - ${SR_LIB} -) - -add_executable(client_test_dataset_aggregation - client_test_dataset_aggregation.cpp -) -target_link_libraries(client_test_dataset_aggregation - ${SR_LIB} -) - -add_executable(client_test_dataset_no_tensors - client_test_dataset_no_tensors.cpp -) -target_link_libraries(client_test_dataset_no_tensors - ${SR_LIB} -) - -add_executable(client_test_dataset_no_meta - client_test_dataset_no_meta.cpp -) -target_link_libraries(client_test_dataset_no_meta - ${SR_LIB} -) - -add_executable(client_test_dataset_empty - client_test_dataset_empty.cpp -) -target_link_libraries(client_test_dataset_empty - ${SR_LIB} -) - -add_executable(client_test_dataset_multiple_get_tensor - client_test_dataset_multiple_get_tensor.cpp -) -target_link_libraries(client_test_dataset_multiple_get_tensor - ${SR_LIB} -) - -add_executable(client_test_delete_dataset - client_test_delete_dataset.cpp -) -target_link_libraries(client_test_delete_dataset - ${SR_LIB} -) - -add_executable(client_test_rename_dataset - client_test_rename_dataset.cpp -) -target_link_libraries(client_test_rename_dataset - ${SR_LIB} -) - -add_executable(client_test_copy_dataset - client_test_copy_dataset.cpp -) -target_link_libraries(client_test_copy_dataset - ${SR_LIB} -) - -add_executable(client_test_dataset_copy_constructor - client_test_dataset_copy_constructor.cpp -) -target_link_libraries(client_test_dataset_copy_constructor - ${SR_LIB} -) - -add_executable(client_test_dataset_move_constructor - client_test_dataset_move_constructor.cpp -) -target_link_libraries(client_test_dataset_move_constructor - ${SR_LIB} -) - -add_executable(client_test_dataset_copy_assignment - client_test_dataset_copy_assignment.cpp -) -target_link_libraries(client_test_dataset_copy_assignment - ${SR_LIB} -) - -add_executable(client_test_dataset_move_assignment - client_test_dataset_move_assignment.cpp -) -target_link_libraries(client_test_dataset_move_assignment - ${SR_LIB} -) - -add_executable(client_test_put_get_3D - client_test_put_get_3D.cpp -) -target_link_libraries(client_test_put_get_3D - ${SR_LIB} -) - -add_executable(client_test_put_get_3D_static_values - client_test_put_get_3D_static_values.cpp -) -target_link_libraries(client_test_put_get_3D_static_values - ${SR_LIB} -) - -add_executable(client_test_put_get_contiguous_3D - client_test_put_get_contiguous_3D.cpp -) -target_link_libraries(client_test_put_get_contiguous_3D - ${SR_LIB} -) - -add_executable(client_test_put_get_transpose_3D - client_test_put_get_transpose_3D.cpp -) -target_link_libraries(client_test_put_get_transpose_3D - ${SR_LIB} -) - -add_executable(client_test_put_get_2D - client_test_put_get_2D.cpp -) -target_link_libraries(client_test_put_get_2D - ${SR_LIB} -) - -add_executable(client_test_put_get_1D - client_test_put_get_1D.cpp -) -target_link_libraries(client_test_put_get_1D - ${SR_LIB} -) - -add_executable(client_test_mnist - client_test_mnist.cpp -) -target_link_libraries(client_test_mnist - ${SR_LIB} -) - -add_executable(client_test_mnist_dataset - client_test_mnist_dataset.cpp -) -target_link_libraries(client_test_mnist_dataset - ${SR_LIB} -) - -add_executable(client_test_ensemble - client_test_ensemble.cpp -) -target_link_libraries(client_test_ensemble - ${SR_LIB} -) - -add_executable(client_test_ensemble_dataset - client_test_ensemble_dataset.cpp -) -target_link_libraries(client_test_ensemble_dataset - ${SR_LIB} -) - +# Define all the tests to be built +list(APPEND EXECUTABLES + client_test_dataset + client_test_dataset_aggregation + client_test_dataset_no_tensors + client_test_dataset_no_meta + client_test_dataset_empty + client_test_dataset_multiple_get_tensor + client_test_delete_dataset + client_test_rename_dataset + client_test_copy_dataset + client_test_dataset_copy_constructor + client_test_dataset_move_constructor + client_test_dataset_copy_assignment + client_test_dataset_move_assignment + client_test_put_get_3D + client_test_put_get_3D_static_values + client_test_put_get_contiguous_3D + client_test_put_get_transpose_3D + client_test_put_get_2D + client_test_put_get_1D + client_test_mnist + client_test_mnist_dataset + client_test_ensemble + client_test_ensemble_dataset +) + +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_cpp_test + ${EXECUTABLE}.cpp + ) + set_target_properties(${EXECUTABLE}_cpp_test PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_cpp_test + ${SR_LIB} + ) +endforeach() diff --git a/tests/docker/CMakeLists.txt b/tests/docker/CMakeLists.txt new file mode 100644 index 000000000..7e5a8174d --- /dev/null +++ b/tests/docker/CMakeLists.txt @@ -0,0 +1,38 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +project(SmartSim-Tests-Docker) +cmake_minimum_required(VERSION 3.13) + +enable_language(C) +enable_language(CXX) +enable_language(Fortran) + +set(ALLOW_DUPLICATE_CUSTOM_TARGETS true) + +add_subdirectory(c) +add_subdirectory(cpp) +add_subdirectory(fortran) diff --git a/tests/docker/c/CMakeLists.txt b/tests/docker/c/CMakeLists.txt index 3b09350fd..a44364b57 100644 --- a/tests/docker/c/CMakeLists.txt +++ b/tests/docker/c/CMakeLists.txt @@ -26,9 +26,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -project(DockerTester) - +project(DockerTesterC) cmake_minimum_required(VERSION 3.13) +enable_language(C) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_BUILD_TYPE DEBUG) @@ -41,8 +41,10 @@ include_directories(SYSTEM ) # Build executables - -add_executable(docker_test +add_executable(docker_test_c test_docker.c ) -target_link_libraries(docker_test ${SR_LIB} pthread) +set_target_properties(docker_test_c PROPERTIES + OUTPUT_NAME docker_test +) +target_link_libraries(docker_test_c ${SR_LIB} pthread) diff --git a/tests/docker/cpp/CMakeLists.txt b/tests/docker/cpp/CMakeLists.txt index 334140440..19b3e9e3c 100644 --- a/tests/docker/cpp/CMakeLists.txt +++ b/tests/docker/cpp/CMakeLists.txt @@ -26,9 +26,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -project(DockerTester) - +project(DockerTesterCpp) cmake_minimum_required(VERSION 3.13) +enable_language(CXX) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_BUILD_TYPE DEBUG) @@ -42,7 +42,10 @@ include_directories(SYSTEM # Build executables -add_executable(docker_test +add_executable(docker_test_cpp docker_test.cpp ) -target_link_libraries(docker_test ${SR_LIB} pthread) +set_target_properties(docker_test_cpp PROPERTIES + OUTPUT_NAME docker_test +) +target_link_libraries(docker_test_cpp ${SR_LIB} pthread) diff --git a/tests/docker/fortran/CMakeLists.txt b/tests/docker/fortran/CMakeLists.txt index 0c983d9f9..86e28588f 100644 --- a/tests/docker/fortran/CMakeLists.txt +++ b/tests/docker/fortran/CMakeLists.txt @@ -26,10 +26,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -project(DockerTester) - +project(DockerTesterFortran) cmake_minimum_required(VERSION 3.13) - enable_language(Fortran) set(CMAKE_VERBOSE_MAKEFILE ON) @@ -49,9 +47,11 @@ include_directories(SYSTEM /usr/local/include/smartredis ) -add_executable(docker_test +add_executable(docker_test_fortran test_docker.F90 ${ftn_client_src} ) - -target_link_libraries(docker_test ${SR_LIB} pthread) +set_target_properties(docker_test_fortran PROPERTIES + OUTPUT_NAME docker_test +) +target_link_libraries(docker_test_fortran ${SR_LIB} pthread) diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 0ada74d4c..3a06941d7 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -25,9 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project(FortranClientTester) - cmake_minimum_required(VERSION 3.13) - enable_language(Fortran) set(CMAKE_VERBOSE_MAKEFILE ON) @@ -46,7 +44,7 @@ endif() # Assume by default that users should link against the install directory in this repository if(NOT DEFINED SMARTREDIS_INSTALL_PATH) - set(SMARTREDIS_INSTALL_PATH "../../install/") + set(SMARTREDIS_INSTALL_PATH "../../install/") endif() # Specify all pre-processor and library dependencies @@ -85,12 +83,14 @@ list(APPEND EXECUTABLES ) foreach(EXECUTABLE ${EXECUTABLES}) - - add_executable(${EXECUTABLE} + add_executable(${EXECUTABLE}_fortran_test ${EXECUTABLE}.F90 test_utils.F90 ) - target_link_libraries(${EXECUTABLE} + set_target_properties(${EXECUTABLE}_fortran_test PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_fortran_test ${SMARTREDIS_LIBRARIES} ) endforeach() From 5e76d5d2b66cf1c5c5004d9aa5f4cc6f60cfc552 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 15 May 2023 16:16:59 -0500 Subject: [PATCH 04/50] Unit tests all pass again --- examples/test_examples.py | 18 ++++-------- tests/c/test_c_client.py | 16 +++++++---- tests/cpp/test_cpp_client.py | 17 ++++++++---- tests/cpp/unit-tests/test_unit_cpp_client.py | 17 ++++++------ tests/fortran/test_fortran_client.py | 29 ++++++++++++-------- 5 files changed, 55 insertions(+), 42 deletions(-) diff --git a/examples/test_examples.py b/examples/test_examples.py index e6f758472..b155a8265 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -43,35 +43,29 @@ def get_test_names(): test_names = glob(glob_path_1) + glob(glob_path_2) test_names = list(filter(lambda test: test.find('example_utils') == -1, test_names)) test_names = list(filter(lambda test: test.find('.py') == -1, test_names)) + test_names = [(pytest.param(test, + id=osp.basename(test))) for test in test_names] return test_names @pytest.mark.parametrize("test", get_test_names()) def test_example(test, use_cluster, build): # Build the path to the test executable from the source file name - # . keep only the last three parts of the path (parallel/serial, language, basename) + # . keep only the last three parts of the path: (parallel/serial, language, basename) test = "/".join(test.split("/")[-3:]) # . drop the file extension test = ".".join(test.split(".")[:-1]) # . prepend the path to the built test executable test = f"{getcwd()}/build/{build}/examples/{test}" - cmd = [] - cmd.append(test) + cmd = [test] print(f"Running test: {osp.basename(test)}") print(f"Using cluster: {use_cluster}") execute_cmd(cmd) time.sleep(1) def find_path(executable_path): - if(executable_path.find('/') == -1): + if executable_path.find('/') == -1: return '.' - - start = 0 - while True: - slash = executable_path.find('/', start) - if (slash == -1): - return executable_path[0:start] - else: - start = slash + 1 + return "/".join(executable_path.split("/")[:-1]) def execute_cmd(cmd_list): """Execute a command """ diff --git a/tests/c/test_c_client.py b/tests/c/test_c_client.py index ebe6a8a3b..5ae827ac0 100644 --- a/tests/c/test_c_client.py +++ b/tests/c/test_c_client.py @@ -26,8 +26,8 @@ import pytest from os import path as osp +from os import getcwd from glob import glob -from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import time @@ -38,7 +38,7 @@ def get_test_names(): """Obtain test names by globbing for client_test Add tests manually if necessary """ - glob_path = osp.join(TEST_PATH, "build/client_test*") + glob_path = osp.join(TEST_PATH, "client_test*") test_names = glob(glob_path) test_names = [(pytest.param(test, id=osp.basename(test))) for test in test_names] @@ -46,15 +46,21 @@ def get_test_names(): @pytest.mark.parametrize("test", get_test_names()) -def test_c_client(test, use_cluster): +def test_c_client(test, use_cluster, build): """This function actually runs the tests using the parameterization function provided in Pytest :param test: a path to a test to run :type test: str """ - cmd = [] - cmd.append(test) + # Build the path to the test executable from the source file name + # . keep only the last three parts of the path: (tests, language, basename) + test = "/".join(test.split("/")[-3:]) + # . drop the file extension + test = ".".join(test.split(".")[:-1]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/{test}" + cmd = [test] print(f"Running test: {osp.basename(test)}") print(f"Test command {' '.join(cmd)}") print(f"Using cluster: {use_cluster}") diff --git a/tests/cpp/test_cpp_client.py b/tests/cpp/test_cpp_client.py index 6eeef3bd0..f5a5a2cd7 100644 --- a/tests/cpp/test_cpp_client.py +++ b/tests/cpp/test_cpp_client.py @@ -26,8 +26,8 @@ import pytest from os import path as osp +from os import getcwd from glob import glob -from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import time @@ -38,16 +38,23 @@ def get_test_names(): """Obtain test names by globbing for client_test Add tests manually if necessary """ - glob_path = osp.join(TEST_PATH, "build/client_test*") + glob_path = osp.join(TEST_PATH, "client_test*") test_names = glob(glob_path) + test_names = list(filter(lambda test: test.find('.h') == -1, test_names)) test_names = [(pytest.param(test, id=osp.basename(test))) for test in test_names] return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_cpp_client(test, use_cluster): - cmd = [] - cmd.append(test) +def test_cpp_client(test, use_cluster, build): + # Build the path to the test executable from the source file name + # . keep only the last three parts of the path: (tests, language, basename) + test = "/".join(test.split("/")[-3:]) + # . drop the file extension + test = ".".join(test.split(".")[:-1]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/{test}" + cmd = [test] print(f"Running test: {osp.basename(test)}") print(f"Test command {' '.join(cmd)}") print(f"Using cluster: {use_cluster}") diff --git a/tests/cpp/unit-tests/test_unit_cpp_client.py b/tests/cpp/unit-tests/test_unit_cpp_client.py index dffe12ec7..30deec19d 100644 --- a/tests/cpp/unit-tests/test_unit_cpp_client.py +++ b/tests/cpp/unit-tests/test_unit_cpp_client.py @@ -26,8 +26,8 @@ import pytest from os import path as osp +from os import getcwd from glob import glob -from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import time @@ -39,19 +39,20 @@ def get_test_names(): """Obtain test names by globbing for client_test Add tests manually if necessary """ - glob_path = osp.join(TEST_PATH, "build/cpp_unit_tests") - test_names = glob(glob_path) + test_names = [osp.join(TEST_PATH, "cpp_unit_tests")] test_names = [(pytest.param(test, id=osp.basename(test))) for test in test_names] - print(test_names) - #test_names = [("build/test", "unit_tests")] return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_unit_cpp_client(test, use_cluster): - cmd = [] - cmd.append(test) +def test_unit_cpp_client(test, use_cluster, build): + # Build the path to the test executable from the source file name + # . keep only the last four parts of the path: (tests, language, unit-tests, basename) + test = "/".join(test.split("/")[-4:]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/{test}" + cmd = [test] print(f"Running test: {osp.basename(test)}") print(f"Test command {' '.join(cmd)}") print(f"Using cluster: {use_cluster}") diff --git a/tests/fortran/test_fortran_client.py b/tests/fortran/test_fortran_client.py index 0a50342b4..d11f4b959 100644 --- a/tests/fortran/test_fortran_client.py +++ b/tests/fortran/test_fortran_client.py @@ -26,13 +26,12 @@ import pytest from os import path as osp -import os +from os import getcwd, environ from glob import glob -from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import time -test_gpu = os.environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" +test_gpu = environ.get("SMARTREDIS_TEST_DEVICE","cpu").lower() == "gpu" RANKS = 1 TEST_PATH = osp.dirname(osp.abspath(__file__)) @@ -41,23 +40,30 @@ def get_test_names(): """Obtain test names by globbing for client_test Add tests manually if necessary """ - glob_path = osp.join(TEST_PATH, "build/client_test*") + glob_path = osp.join(TEST_PATH, "client_test*") test_names = glob(glob_path) - test_names = [(pytest.param(test, id=osp.basename(test))) - for test in test_names if not "gpu" in test] + test_names = list(filter(lambda test: test.find('gpu') == -1, test_names)) + test_names = [(pytest.param(test, + id=osp.basename(test))) for test in test_names] return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_fortran_client(test): +def test_fortran_client(test, build): """This function actually runs the tests using the parameterization function provided in Pytest :param test: a path to a test to run :type test: str """ - cmd = [] - cmd.append(test) + # Build the path to the test executable from the source file name + # . keep only the last three parts of the path: (tests, language, basename) + test = "/".join(test.split("/")[-3:]) + # . drop the file extension + test = ".".join(test.split(".")[:-1]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/{test}" + cmd = [test] print(f"Running test: {osp.basename(test)}") print(f"Test command {' '.join(cmd)}") execute_cmd(cmd) @@ -104,7 +110,6 @@ def test_client_multigpu_mnist(): Test setting and running a machine learning model via the Fortran client on an orchestrator with multiple GPUs """ - - tester_path = osp.join(TEST_PATH, "build/client_test_mnist_multigpu") + tester_path = osp.join(TEST_PATH, "client_test_mnist_multigpu.F90") test_fortran_client(tester_path) - + From 431880185f943d087797ffb7cc535054243894fb Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 15 May 2023 17:25:19 -0500 Subject: [PATCH 05/50] Adjust tests to run from the source directory to simplify creation of relative paths to MNIST resources --- tests/c/test_c_client.py | 3 +-- tests/cpp/client_test_ensemble.cpp | 6 +++--- tests/cpp/client_test_mnist.cpp | 6 +++--- tests/cpp/client_test_mnist_dataset.cpp | 6 +++--- tests/cpp/test_cpp_client.py | 3 +-- tests/cpp/unit-tests/test_client.cpp | 6 +++--- tests/cpp/unit-tests/test_client_ensemble.cpp | 6 +++--- tests/cpp/unit-tests/test_unit_cpp_client.py | 3 +-- tests/fortran/client_test_ensemble.F90 | 4 ++-- tests/fortran/client_test_mnist.F90 | 4 ++-- tests/fortran/client_test_mnist_multigpu.F90 | 4 ++-- tests/fortran/test_fortran_client.py | 3 +-- 12 files changed, 25 insertions(+), 29 deletions(-) diff --git a/tests/c/test_c_client.py b/tests/c/test_c_client.py index 5ae827ac0..085f4d0fc 100644 --- a/tests/c/test_c_client.py +++ b/tests/c/test_c_client.py @@ -71,9 +71,8 @@ def execute_cmd(cmd_list): """Execute a command """ # spawning the subprocess and connecting to its output - run_path = osp.join(TEST_PATH, "build/") proc = Popen( - cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=run_path) + cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=TEST_PATH) try: out, err = proc.communicate(timeout=120) if out: diff --git a/tests/cpp/client_test_ensemble.cpp b/tests/cpp/client_test_ensemble.cpp index 95c33a7a9..d7b978b02 100644 --- a/tests/cpp/client_test_ensemble.cpp +++ b/tests/cpp/client_test_ensemble.cpp @@ -35,7 +35,7 @@ void load_mnist_image_to_array(float**** img) { - std::string image_file = "../mnist_data/one.raw"; + std::string image_file = "mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -77,7 +77,7 @@ void produce( // Models std::string model_key = "mnist_model"; - std::string model_file = "./../mnist_data/mnist_cnn.pt"; + std::string model_file = "mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU"); @@ -86,7 +86,7 @@ void produce( // Scripts std::string script_key = "mnist_script"; - std::string script_file = "./../mnist_data/data_processing_script.txt"; + std::string script_file = "mnist_data/data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); if(!client.model_exists(script_key)) diff --git a/tests/cpp/client_test_mnist.cpp b/tests/cpp/client_test_mnist.cpp index a07d4d9da..7fd5f1d78 100644 --- a/tests/cpp/client_test_mnist.cpp +++ b/tests/cpp/client_test_mnist.cpp @@ -31,7 +31,7 @@ void load_mnist_image_to_array(float**** img) { - std::string image_file = "../mnist_data/one.raw"; + std::string image_file = "mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -80,11 +80,11 @@ int main(int argc, char* argv[]) { SmartRedis::Client client(use_cluster(), "client_test_mnist"); std::string model_key = "mnist_model"; - std::string model_file = "./../mnist_data/mnist_cnn.pt"; + std::string model_file = "mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU"); std::string script_key = "mnist_script"; - std::string script_file = "./../mnist_data/data_processing_script.txt"; + std::string script_file = "mnist_data/data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); std::string_view model = client.get_model(model_key); diff --git a/tests/cpp/client_test_mnist_dataset.cpp b/tests/cpp/client_test_mnist_dataset.cpp index 402adf0d0..d732b27ea 100644 --- a/tests/cpp/client_test_mnist_dataset.cpp +++ b/tests/cpp/client_test_mnist_dataset.cpp @@ -31,7 +31,7 @@ void load_mnist_image_to_array(float**** img) { - std::string image_file = "../mnist_data/one.raw"; + std::string image_file = "mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -84,11 +84,11 @@ int main(int argc, char* argv[]) { SmartRedis::Client client(use_cluster(), "client_test_mnist_dataset"); std::string model_key = "mnist_model"; - std::string model_file = "./../mnist_data/mnist_cnn.pt"; + std::string model_file = "mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU"); std::string script_key = "mnist_script"; - std::string script_file = "./../mnist_data/data_processing_script.txt"; + std::string script_file = "mnist_data/data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); std::string_view model = client.get_model(model_key); diff --git a/tests/cpp/test_cpp_client.py b/tests/cpp/test_cpp_client.py index f5a5a2cd7..8b199bdfa 100644 --- a/tests/cpp/test_cpp_client.py +++ b/tests/cpp/test_cpp_client.py @@ -65,9 +65,8 @@ def execute_cmd(cmd_list): """Execute a command """ # spawning the subprocess and connecting to its output - run_path = osp.join(TEST_PATH, "build/") proc = Popen( - cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=run_path) + cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=TEST_PATH) try: out, err = proc.communicate(timeout=180) if out: diff --git a/tests/cpp/unit-tests/test_client.cpp b/tests/cpp/unit-tests/test_client.cpp index df053a9f6..dd637b383 100644 --- a/tests/cpp/unit-tests/test_client.cpp +++ b/tests/cpp/unit-tests/test_client.cpp @@ -572,7 +572,7 @@ SCENARIO("Testing AI.INFO Functions on Client Object", "[Client]") { std::string db_address = parse_SSDB(std::getenv("SSDB")); std::string model_key = "ai_info_model"; - std::string model_file = "./../../mnist_data/mnist_cnn.pt"; + std::string model_file = "../mnist_data/mnist_cnn.pt"; std::string backend = "TORCH"; std::string device = "CPU"; parsed_reply_map reply; @@ -867,9 +867,9 @@ SCENARIO("Testing Multi-GPU Function error cases", "[Client]") { Client client(use_cluster(), "test_client"); std::string model_key = "a_model"; - std::string model_file = "./../../mnist_data/mnist_cnn.pt"; + std::string model_file = "../mnist_data/mnist_cnn.pt"; std::string script_key = "a_script"; - std::string script_file = "./../../mnist_data/data_processing_script.txt"; + std::string script_file = "../mnist_data/data_processing_script.txt"; std::string backend = "TORCH"; WHEN("set_model_multigpu() called with invalid first gpu") diff --git a/tests/cpp/unit-tests/test_client_ensemble.cpp b/tests/cpp/unit-tests/test_client_ensemble.cpp index 737cc2c72..6dc105e7c 100644 --- a/tests/cpp/unit-tests/test_client_ensemble.cpp +++ b/tests/cpp/unit-tests/test_client_ensemble.cpp @@ -57,7 +57,7 @@ void reset_env_vars(const char* old_keyin, const char* old_keyout) // helper function for loading mnist void load_mnist_image_to_array(float**** img) { - std::string image_file = "../../mnist_data/one.raw"; + std::string image_file = "../mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -100,11 +100,11 @@ SCENARIO("Testing Client ensemble using a producer/consumer paradigm") std::string tensor_key = "ensemble_test"; // for model std::string model_key = "mnist_model"; - std::string model_file = "./../../mnist_data/mnist_cnn.pt"; + std::string model_file = "./../mnist_data/mnist_cnn.pt"; // for script std::string script_key = "mnist_script"; std::string script_file = - "./../../mnist_data/data_processing_script.txt"; + "./../mnist_data/data_processing_script.txt"; // for setup mnist std::string in_key = "mnist_input"; std::string out_key = "mnist_output"; diff --git a/tests/cpp/unit-tests/test_unit_cpp_client.py b/tests/cpp/unit-tests/test_unit_cpp_client.py index 30deec19d..b213e394c 100644 --- a/tests/cpp/unit-tests/test_unit_cpp_client.py +++ b/tests/cpp/unit-tests/test_unit_cpp_client.py @@ -63,9 +63,8 @@ def execute_cmd(cmd_list): """Execute a command """ # spawning the subprocess and connecting to its output - run_path = osp.join(TEST_PATH, "build/") proc = Popen( - cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=run_path) + cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=TEST_PATH) try: out, err = proc.communicate(timeout=timeout_limit) print("OUTPUT:", out.decode("utf-8") if out else "None") diff --git a/tests/fortran/client_test_ensemble.F90 b/tests/fortran/client_test_ensemble.F90 index 6cb520b01..768165684 100644 --- a/tests/fortran/client_test_ensemble.F90 +++ b/tests/fortran/client_test_ensemble.F90 @@ -74,7 +74,7 @@ program main endif script_key = "ensemble_script" -script_file = "../../cpp/mnist_data/data_processing_script.txt" +script_file = "../cpp/mnist_data/data_processing_script.txt" result = client%set_script_from_file(script_key, "CPU", script_file) if (result .ne. SRNoError) error stop result = client%model_exists(script_key, exists) @@ -85,7 +85,7 @@ program main endif model_key = "ensemble_model" -model_file = "../../cpp/mnist_data/mnist_cnn.pt" +model_file = "../cpp/mnist_data/mnist_cnn.pt" result = client%set_model_from_file(model_key, model_file, "TORCH", "CPU") if (result .ne. SRNoError) error stop 'set_model_from_file failed' result = client%model_exists(model_key, exists) diff --git a/tests/fortran/client_test_mnist.F90 b/tests/fortran/client_test_mnist.F90 index d3aae8611..52e126dba 100644 --- a/tests/fortran/client_test_mnist.F90 +++ b/tests/fortran/client_test_mnist.F90 @@ -35,9 +35,9 @@ program mnist_test #include "enum_fortran.inc" character(len=*), parameter :: model_key = "mnist_model" - character(len=*), parameter :: model_file = "../../cpp/mnist_data/mnist_cnn.pt" + character(len=*), parameter :: model_file = "../cpp/mnist_data/mnist_cnn.pt" character(len=*), parameter :: script_key = "mnist_script" - character(len=*), parameter :: script_file = "../../cpp/mnist_data/data_processing_script.txt" + character(len=*), parameter :: script_file = "../cpp/mnist_data/data_processing_script.txt" type(client_type) :: client integer :: err_code diff --git a/tests/fortran/client_test_mnist_multigpu.F90 b/tests/fortran/client_test_mnist_multigpu.F90 index de83efdc6..e8d5b7c47 100644 --- a/tests/fortran/client_test_mnist_multigpu.F90 +++ b/tests/fortran/client_test_mnist_multigpu.F90 @@ -35,9 +35,9 @@ program mnist_test #include "enum_fortran.inc" character(len=*), parameter :: model_key = "mnist_model" - character(len=*), parameter :: model_file = "../../cpp/mnist_data/mnist_cnn.pt" + character(len=*), parameter :: model_file = "../cpp/mnist_data/mnist_cnn.pt" character(len=*), parameter :: script_key = "mnist_script" - character(len=*), parameter :: script_file = "../../cpp/mnist_data/data_processing_script.txt" + character(len=*), parameter :: script_file = "../cpp/mnist_data/data_processing_script.txt" integer, parameter :: first_gpu = 0 integer, parameter :: num_gpus = 1 integer, parameter :: offset = 0 diff --git a/tests/fortran/test_fortran_client.py b/tests/fortran/test_fortran_client.py index d11f4b959..6af947f06 100644 --- a/tests/fortran/test_fortran_client.py +++ b/tests/fortran/test_fortran_client.py @@ -73,9 +73,8 @@ def execute_cmd(cmd_list): """Execute a command """ # spawning the subprocess and connecting to its output - run_path = osp.join(TEST_PATH, "build/") proc = Popen( - cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=run_path) + cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=TEST_PATH) try: out, err = proc.communicate(timeout=120) if out: From dea8a240f8eb296296663c8c05d436a87f2d0999 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 16 May 2023 17:54:13 -0500 Subject: [PATCH 06/50] Lib building now via cmake --- CMakeLists.txt | 27 ++++++++---- Makefile | 18 ++++---- examples/parallel/fortran/CMakeLists.txt | 4 -- examples/serial/fortran/CMakeLists.txt | 4 -- smartredis_defs.cmake | 52 ++++++++++++++++++++++++ tests/fortran/CMakeLists.txt | 4 -- 6 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 smartredis_defs.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 039dd1452..b29635d67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,29 +24,37 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(SmartRedis) +# Enable setting version in the project statement +if (POLICY CMP0048) + cmake_policy(SET CMP0048 NEW) +endif (POLICY CMP0048) + +project(SmartRedis VERSION "0.4.0") cmake_minimum_required(VERSION 3.13) option(BUILD_PYTHON "Build the python module" ON) -option(BUILD_FORTRAN "Build the fortran client library" OFF) +option(SR_FORTRAN "Build the fortran client library" OFF) option(COVERAGE "Build the fortran client library" OFF) +option(SR_PEDANTIC "Build with pickiest compiler settings" OFF) -set(CMAKE_BUILD_TYPE RELEASE) set(CMAKE_CXX_STANDARD 17) set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install) set(CMAKE_CXX_VISIBILITY_PRESET default) set(THREADS_PREFER_PTHREAD_FLAG ON) -if (BUILD_FORTRAN) +#include(smartredis_defs) +set(CMAKE_BUILD_TYPE RELEASE) + +if (SR_FORTRAN) enable_language(Fortran) endif() -if (WERROR) +if (SR_PEDANTIC) if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) add_compile_options(-Wall -Werror) else() - message(WARNING "WERROR was specified, but the CMAKE compiler is not GCC") + message(WARNING "SR_PEDANTIC was specified, but the CMAKE compiler is not GCC") endif() endif() @@ -111,7 +119,8 @@ include_directories(SYSTEM install/include ) -if (BUILD_FORTRAN) +# Build the Fortran library +if (SR_FORTRAN) set(FORTRAN_SRC src/fortran/errors.F90 src/fortran/client.F90 @@ -166,8 +175,8 @@ if(BUILD_PYTHON) src/python/bindings/bind.cpp) target_link_libraries(smartredisPy PUBLIC ${EXT_CLIENT_LIBRARIES}) - install(TARGETS smartredisPy - LIBRARY DESTINATION lib) + install(TARGETS smartredisPy LIBRARY DESTINATION lib) + install(TARGETS smartredisPy LIBRARY DESTINATION ../src/python/module/smartredis) else() message("-- Skipping Python client build") endif() diff --git a/Makefile b/Makefile index 32e5c5303..802180a0a 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,8 @@ SR_TEST_RAI_VER := 1.2.7 SR_WLM := Local SR_WLM_FLAGS := SR_DEVICE := cpu +SR_PEDANTIC := OFF +SR_FORTRAN := OFF # Params for third-party software HIREDIS_URL := https://github.com/redis/hiredis.git @@ -52,27 +54,29 @@ deps: # help: pip-install - Register the SmartRedis library with pip .PHONY: pip-install pip-install: - @if ! python -c "import smartredis" >& /dev/null; then pip install -e.; fi + @python -c "import smartredis" >& /dev/null || pip install -e. # help: lib - Build SmartRedis C/C++/Python clients into a dynamic library .PHONY: lib lib: pip-install lib: deps - @bash ./build-scripts/build_lib.sh $(LIB_BUILD_ARGS) + cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) + cmake --build build/$(SR_BUILD) + cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran -lib-with-fortran: deps - @bash ./build-scripts/build_lib.sh $(LIB_BUILD_ARGS) $(CMAKE_ARGS) -DBUILD_FORTRAN=ON +lib-with-fortran: SR_FORTRAN=ON +lib-with-fortran: lib # help: test-lib - Build SmartRedis clients into a dynamic library with least permissive compiler settings .PHONY: test-lib -test-lib: LIB_BUILD_ARGS="-DWERROR=ON" +test-lib: SR_PEDANTIC=ON test-lib: lib # help: test-lib-with-fortran - Build SmartRedis clients into a dynamic library with least permissive compiler settings .PHONY: test-lib-with-fortran -test-lib-with-fortran: LIB_BUILD_ARGS="-DWERROR=ON" +test-lib-with-fortran: SR_PEDANTIC=ON test-lib-with-fortran: lib-with-fortran # help: test-deps - Make SmartRedis testing dependencies @@ -238,9 +242,9 @@ test-verbose: # help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] .PHONY: test-verbose-with-coverage +test-verbose-with-coverage: SR_BUILD=Coverage test-verbose-with-coverage: test-deps test-verbose-with-coverage: build-tests -test-verbose-with-coverage: CMAKE_ARGS="-DCOVERAGE=on" test-verbose-with-coverage: PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index dbe44957c..5399c7be1 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -45,13 +45,9 @@ IF(NOT MPI_Fortran_FOUND) endif() find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(REDISPP redis++ PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) set(SMARTREDIS_LIBRARIES ${SMARTREDIS_LIBRARY} ${SMARTREDIS_FORTRAN_LIBRARY} - ${HIREDIS} - ${REDISPP} ) include_directories(SYSTEM /usr/local/include diff --git a/examples/serial/fortran/CMakeLists.txt b/examples/serial/fortran/CMakeLists.txt index abdef7177..2502539ad 100644 --- a/examples/serial/fortran/CMakeLists.txt +++ b/examples/serial/fortran/CMakeLists.txt @@ -41,13 +41,9 @@ endif() # Specify all pre-processor and library dependencies find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(REDISPP redis++ PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) set(SMARTREDIS_LIBRARIES ${SMARTREDIS_LIBRARY} ${SMARTREDIS_FORTRAN_LIBRARY} - ${HIREDIS} - ${REDISPP} ) include_directories(SYSTEM /usr/local/include diff --git a/smartredis_defs.cmake b/smartredis_defs.cmake new file mode 100644 index 000000000..4b2bad046 --- /dev/null +++ b/smartredis_defs.cmake @@ -0,0 +1,52 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Select CMake build type based on SR_BUILD variable +if(SR_BUILD == Release) + set(CMAKE_BUILD_TYPE RELEASE) + set(SRLIB_NAME_SUFFIX "") +elseif(SR_BUILD == Debug) + set(CMAKE_BUILD_TYPE DEBUG) + set(SRLIB_NAME_SUFFIX "-debug") +elseif(SR_BUILD == Coverage) + set(CMAKE_BUILD_TYPE DEBUG) + set(SRLIB_NAME_SUFFIX "-coverage") + set(COVERAGE ON) +else() + message(ERROR "Unrecognized build type specified in SR_BUILD") +endif() + +# Select CMake linkage based of SR_LINK variable +if(SR_LINK == Static) + set(CMAKE_LINK_LIBRARY_SUFFIX .a) +elseif(SR_LINK == Shared) + set(CMAKE_LINK_LIBRARY_SUFFIX .so) +else() + message(ERROR "Unrecognized link type specified in SR_LINK") +endif() + +# Identify the SmartRedis library name based on the build and link +set(SMARTREDIS_LIB smartredis${SRLIB_NAME_SUFFIX}) diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 3a06941d7..d246f4652 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -50,13 +50,9 @@ endif() # Specify all pre-processor and library dependencies find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(REDISPP redis++ PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) set(SMARTREDIS_LIBRARIES ${SMARTREDIS_LIBRARY} ${SMARTREDIS_FORTRAN_LIBRARY} - ${HIREDIS} - ${REDISPP} ) include_directories(SYSTEM /usr/local/include From 609ea05189be0c5d9c23a9f31586d23d49e78d5f Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 16 May 2023 19:19:32 -0500 Subject: [PATCH 07/50] EOD checkin --- CMakeLists.txt | 10 ++++++++-- examples/parallel/cpp/CMakeLists.txt | 7 ------- smartredis_defs.cmake | 24 ++++++++++++++++-------- tests/cpp/unit-tests/CMakeLists.txt | 6 +----- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b29635d67..190da4f18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,12 +30,10 @@ if (POLICY CMP0048) endif (POLICY CMP0048) project(SmartRedis VERSION "0.4.0") - cmake_minimum_required(VERSION 3.13) option(BUILD_PYTHON "Build the python module" ON) option(SR_FORTRAN "Build the fortran client library" OFF) -option(COVERAGE "Build the fortran client library" OFF) option(SR_PEDANTIC "Build with pickiest compiler settings" OFF) set(CMAKE_CXX_STANDARD 17) @@ -67,6 +65,7 @@ if (COVERAGE) endif() endif() +# Bring in third-party libaries find_library(REDISPP redis++ PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) find_library(HIREDIS hiredis PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) find_package(Threads REQUIRED) @@ -117,6 +116,7 @@ set(CLIENT_SRC include_directories(SYSTEM include install/include + third-party/install/include ) # Build the Fortran library @@ -136,6 +136,9 @@ if (SR_FORTRAN) # Fortran library add_library(smartredis-fortran SHARED ${FORTRAN_SRC}) set_target_properties(smartredis-fortran PROPERTIES SUFFIX ".so") +# set_target_properties(smartredis-fortran PROPERTIES +# OUTPUT_NAME ${SMARTREDIS_FORTRAN_LIB} +# ) target_link_libraries(smartredis-fortran PUBLIC smartredis ${EXT_CLIENT_LIBRARIES}) # Install dynamic library and headers install(TARGETS smartredis-fortran @@ -146,6 +149,9 @@ endif() # Build dynamic library add_library(smartredis SHARED ${CLIENT_SRC}) set_target_properties(smartredis PROPERTIES SUFFIX ".so") +# set_target_properties(smartredis-fortran PROPERTIES +# OUTPUT_NAME ${SMARTREDIS_LIB} +# ) target_link_libraries(smartredis PUBLIC ${EXT_CLIENT_LIBRARIES} PRIVATE Threads::Threads) install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" diff --git a/examples/parallel/cpp/CMakeLists.txt b/examples/parallel/cpp/CMakeLists.txt index 00db4d5c8..07a847401 100644 --- a/examples/parallel/cpp/CMakeLists.txt +++ b/examples/parallel/cpp/CMakeLists.txt @@ -41,12 +41,6 @@ include_directories(SYSTEM ../../../install/include ) -set(ftn_client_src - ../../../src/fortran/fortran_c_interop.F90 - ../../../src/fortran/dataset.F90 - ../../../src/fortran/client.F90 -) - # Define all the tests to be built list(APPEND EXECUTABLES smartredis_put_get_3D @@ -56,7 +50,6 @@ list(APPEND EXECUTABLES foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_cpp_parallel ${EXECUTABLE}.cpp - ${ftn_client_src} ) set_target_properties(${EXECUTABLE}_cpp_parallel PROPERTIES OUTPUT_NAME ${EXECUTABLE} diff --git a/smartredis_defs.cmake b/smartredis_defs.cmake index 4b2bad046..553e69686 100644 --- a/smartredis_defs.cmake +++ b/smartredis_defs.cmake @@ -24,7 +24,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Select CMake build type based on SR_BUILD variable +# Configure the CMake build based on the SR_BUILD selection if(SR_BUILD == Release) set(CMAKE_BUILD_TYPE RELEASE) set(SRLIB_NAME_SUFFIX "") @@ -34,19 +34,27 @@ elseif(SR_BUILD == Debug) elseif(SR_BUILD == Coverage) set(CMAKE_BUILD_TYPE DEBUG) set(SRLIB_NAME_SUFFIX "-coverage") - set(COVERAGE ON) + if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) + add_compile_options(--coverage) + add_link_options(--coverage) + else() + message(WARNING "A coverage build was specified, but the CMAKE compiler is not GCC") + endif() else() - message(ERROR "Unrecognized build type specified in SR_BUILD") + message(FATAL_ERROR "Unrecognized build type (${SR_BUILD}) specified in SR_BUILD") endif() -# Select CMake linkage based of SR_LINK variable +# Configure CMake linkage on the SR_LINK selection if(SR_LINK == Static) - set(CMAKE_LINK_LIBRARY_SUFFIX .a) +set(SMARTREDIS_LINK_MODE STATIC) +set(SMARTREDIS_LINK_LIBRARY_SUFFIX .a) elseif(SR_LINK == Shared) - set(CMAKE_LINK_LIBRARY_SUFFIX .so) + set(SMARTREDIS_LINK_MODE SHARED) + set(SMARTREDIS_LINK_LIBRARY_SUFFIX .so) else() - message(ERROR "Unrecognized link type specified in SR_LINK") + message(FATAL_ERROR "Unrecognized link type (${SR_LINK}) specified in SR_LINK") endif() -# Identify the SmartRedis library name based on the build and link +# Identify the SmartRedis library names based on the build and link set(SMARTREDIS_LIB smartredis${SRLIB_NAME_SUFFIX}) +set(SMARTREDIS_FORTRAN_LIB smartredis-fortran${SRLIB_NAME_SUFFIX}) diff --git a/tests/cpp/unit-tests/CMakeLists.txt b/tests/cpp/unit-tests/CMakeLists.txt index 9a81be40f..8557e9fba 100644 --- a/tests/cpp/unit-tests/CMakeLists.txt +++ b/tests/cpp/unit-tests/CMakeLists.txt @@ -40,12 +40,8 @@ if(COVERAGE) endif() set(THREADS_PREFER_PTHREAD_FLAG ON) -find_library(REDISPP redis++ PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) find_library(SRLIB smartredis PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) -find_library(SRLIB_FORTRAN smartredis-fortran PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) find_package(Threads REQUIRED) -set(EXT_LIBRARIES ${REDISPP} ${HIREDIS} ${SRLIB} ${SRLIB_FORTRAN}) include_directories(SYSTEM /usr/local/include @@ -57,4 +53,4 @@ file(GLOB UNIT_TESTS CONFIGURE_DEPENDS ./*.cpp) add_executable(cpp_unit_tests ${UNIT_TESTS}) -target_link_libraries(cpp_unit_tests PUBLIC ${EXT_LIBRARIES} PRIVATE Threads::Threads) +target_link_libraries(cpp_unit_tests PUBLIC ${SRLIB} PRIVATE Threads::Threads) From 8afe5ab266f775d2481273c214028120f9120384 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 17 May 2023 15:54:31 -0500 Subject: [PATCH 08/50] build mode and link type support in place for SmartRedis and SmartRedis-Fortran libraries --- CMakeLists.txt | 96 +++++++++++++++++++++-------------------- examples/CMakeLists.txt | 7 +-- smartredis_defs.cmake | 15 ++++--- 3 files changed, 62 insertions(+), 56 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 190da4f18..e1537dd00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,9 +29,11 @@ if (POLICY CMP0048) cmake_policy(SET CMP0048 NEW) endif (POLICY CMP0048) +# Project definition for the SmartRedis project project(SmartRedis VERSION "0.4.0") cmake_minimum_required(VERSION 3.13) +# Configure options for the SmartRedis project option(BUILD_PYTHON "Build the python module" ON) option(SR_FORTRAN "Build the fortran client library" OFF) option(SR_PEDANTIC "Build with pickiest compiler settings" OFF) @@ -40,14 +42,17 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install) set(CMAKE_CXX_VISIBILITY_PRESET default) set(THREADS_PREFER_PTHREAD_FLAG ON) +set(CMAKE_MODULE_PATH .) +include(smartredis_defs) -#include(smartredis_defs) -set(CMAKE_BUILD_TYPE RELEASE) - +# If we want to use Fortran, we have to tell CMake to use it if (SR_FORTRAN) enable_language(Fortran) endif() +# For now, we only support Pedantic on the main library build. +# If/when we fine-tune the examples and test cases, move this block +# to smartredis_defs.cmake if (SR_PEDANTIC) if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) add_compile_options(-Wall -Werror) @@ -56,22 +61,13 @@ if (SR_PEDANTIC) endif() endif() -if (COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - add_compile_options(--coverage) - add_link_options(--coverage) - else() - message(WARNING "COVERAGE was specified, but the CMAKE compiler is not GCC") - endif() -endif() - -# Bring in third-party libaries +# Bring in third-party libaries needed for the SmartRedis library find_library(REDISPP redis++ PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) find_library(HIREDIS hiredis PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) find_package(Threads REQUIRED) - set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS}) +# Define source code that goes into the SmartRedis library set(CLIENT_SRC src/c/c_client.cpp src/c/c_configoptions.cpp @@ -80,39 +76,40 @@ set(CLIENT_SRC src/c/c_logcontext.cpp src/c/c_logger.cpp src/cpp/address.cpp + src/cpp/addressallcommand.cpp + src/cpp/addressanycommand.cpp + src/cpp/addressatcommand.cpp src/cpp/client.cpp - src/cpp/dataset.cpp + src/cpp/clusterinfocommand.cpp src/cpp/command.cpp - src/cpp/keyedcommand.cpp - src/cpp/nonkeyedcommand.cpp - src/cpp/multikeycommand.cpp - src/cpp/singlekeycommand.cpp + src/cpp/commandlist.cpp + src/cpp/commandreply.cpp src/cpp/compoundcommand.cpp - src/cpp/addressatcommand.cpp - src/cpp/addressanycommand.cpp - src/cpp/addressallcommand.cpp - src/cpp/clusterinfocommand.cpp + src/cpp/configoptions.cpp + src/cpp/dataset.cpp src/cpp/dbinfocommand.cpp + src/cpp/dbnode.cpp src/cpp/gettensorcommand.cpp - src/cpp/commandlist.cpp + src/cpp/keyedcommand.cpp + src/cpp/logger.cpp src/cpp/metadata.cpp - src/cpp/tensorbase.cpp - src/cpp/tensorpack.cpp - src/cpp/dbnode.cpp - src/cpp/commandreply.cpp - src/cpp/redisserver.cpp - src/cpp/rediscluster.cpp - src/cpp/redis.cpp src/cpp/metadatafield.cpp - src/cpp/stringfield.cpp + src/cpp/multikeycommand.cpp + src/cpp/nonkeyedcommand.cpp src/cpp/pipelinereply.cpp + src/cpp/redis.cpp + src/cpp/rediscluster.cpp + src/cpp/redisserver.cpp + src/cpp/singlekeycommand.cpp + src/cpp/srobject.cpp + src/cpp/stringfield.cpp + src/cpp/tensorbase.cpp + src/cpp/tensorpack.cpp src/cpp/threadpool.cpp src/cpp/utility.cpp - src/cpp/logger.cpp - src/cpp/srobject.cpp - src/cpp/configoptions.cpp ) +# Define include directories for header files include_directories(SYSTEM include install/include @@ -134,11 +131,13 @@ if (SR_FORTRAN) # Note the following has to be before ANY add_library command) set(CMAKE_Fortran_MODULE_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include") # Fortran library - add_library(smartredis-fortran SHARED ${FORTRAN_SRC}) - set_target_properties(smartredis-fortran PROPERTIES SUFFIX ".so") -# set_target_properties(smartredis-fortran PROPERTIES -# OUTPUT_NAME ${SMARTREDIS_FORTRAN_LIB} -# ) + add_library(smartredis-fortran ${SMARTREDIS_LINK_MODE} ${FORTRAN_SRC}) + set_target_properties(smartredis-fortran PROPERTIES + SUFFIX ${SMARTREDIS_LINK_LIBRARY_SUFFIX} + ) + set_target_properties(smartredis-fortran PROPERTIES + OUTPUT_NAME ${SMARTREDIS_FORTRAN_LIB} + ) target_link_libraries(smartredis-fortran PUBLIC smartredis ${EXT_CLIENT_LIBRARIES}) # Install dynamic library and headers install(TARGETS smartredis-fortran @@ -146,14 +145,17 @@ if (SR_FORTRAN) endif() -# Build dynamic library -add_library(smartredis SHARED ${CLIENT_SRC}) -set_target_properties(smartredis PROPERTIES SUFFIX ".so") -# set_target_properties(smartredis-fortran PROPERTIES -# OUTPUT_NAME ${SMARTREDIS_LIB} -# ) +# Build the main SmartRedis library +add_library(smartredis ${SMARTREDIS_LINK_MODE} ${CLIENT_SRC}) +set_target_properties(smartredis PROPERTIES + SUFFIX ${SMARTREDIS_LINK_LIBRARY_SUFFIX} +) +set_target_properties(smartredis PROPERTIES + OUTPUT_NAME ${SMARTREDIS_LIB} +) target_link_libraries(smartredis PUBLIC ${EXT_CLIENT_LIBRARIES} PRIVATE Threads::Threads) +# Install SmartRedis header files install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION "include" FILES_MATCHING @@ -164,11 +166,13 @@ install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" install(TARGETS smartredis LIBRARY DESTINATION lib) +# Build the Python library for SmartRedis if(BUILD_PYTHON) message("-- Python client build enabled") add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind ${CMAKE_SOURCE_DIR}/third-party/pybind/build) + # Note that this library is always static add_library(smartredis_static STATIC ${CLIENT_SRC}) pybind11_add_module(smartredisPy diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 59e294ca5..6fbbb21ef 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -24,14 +24,15 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(Examples) +# Project definition for the SmartRedis-Examples project +project(SmartRedis-Examples) cmake_minimum_required(VERSION 3.13) +# Enable language support for the examples enable_language(C) enable_language(CXX) enable_language(Fortran) -set(ALLOW_DUPLICATE_CUSTOM_TARGETS true) - +# Bring in subdirectories add_subdirectory(parallel) add_subdirectory(serial) diff --git a/smartredis_defs.cmake b/smartredis_defs.cmake index 553e69686..15ca49e39 100644 --- a/smartredis_defs.cmake +++ b/smartredis_defs.cmake @@ -25,13 +25,13 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Configure the CMake build based on the SR_BUILD selection -if(SR_BUILD == Release) +if(SR_BUILD STREQUAL "Release") set(CMAKE_BUILD_TYPE RELEASE) set(SRLIB_NAME_SUFFIX "") -elseif(SR_BUILD == Debug) +elseif(SR_BUILD STREQUAL "Debug") set(CMAKE_BUILD_TYPE DEBUG) set(SRLIB_NAME_SUFFIX "-debug") -elseif(SR_BUILD == Coverage) +elseif(SR_BUILD STREQUAL "Coverage") set(CMAKE_BUILD_TYPE DEBUG) set(SRLIB_NAME_SUFFIX "-coverage") if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) @@ -45,10 +45,11 @@ else() endif() # Configure CMake linkage on the SR_LINK selection -if(SR_LINK == Static) -set(SMARTREDIS_LINK_MODE STATIC) -set(SMARTREDIS_LINK_LIBRARY_SUFFIX .a) -elseif(SR_LINK == Shared) +if(SR_LINK STREQUAL "Static") + set(SMARTREDIS_LINK_MODE STATIC) + set(SMARTREDIS_LINK_LIBRARY_SUFFIX .a) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) +elseif(SR_LINK STREQUAL "Shared") set(SMARTREDIS_LINK_MODE SHARED) set(SMARTREDIS_LINK_LIBRARY_SUFFIX .so) else() From 0098ec0897393ef100cb53cb536ea3ee97c31315 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 17 May 2023 17:35:48 -0500 Subject: [PATCH 09/50] Examples are now built with specified build mode and link type --- examples/parallel/CMakeLists.txt | 5 +++- examples/parallel/cpp/CMakeLists.txt | 29 ++++++++++++++++---- examples/parallel/fortran/CMakeLists.txt | 34 ++++++++++++++++------- examples/serial/CMakeLists.txt | 5 +++- examples/serial/c/CMakeLists.txt | 29 ++++++++++++++++---- examples/serial/cpp/CMakeLists.txt | 28 +++++++++++++++---- examples/serial/fortran/CMakeLists.txt | 35 +++++++++++++++++------- 7 files changed, 126 insertions(+), 39 deletions(-) diff --git a/examples/parallel/CMakeLists.txt b/examples/parallel/CMakeLists.txt index 600ceeeff..ea38611ae 100644 --- a/examples/parallel/CMakeLists.txt +++ b/examples/parallel/CMakeLists.txt @@ -24,11 +24,14 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(ParallelExamples) +# Project definition for the SmartRedis-Examples-Parallel project +project(SmartRedis-Examples-Parallel) cmake_minimum_required(VERSION 3.13) +# Enable language support for the examples enable_language(CXX) enable_language(Fortran) +# Bring in subdirectories add_subdirectory(cpp) add_subdirectory(fortran) diff --git a/examples/parallel/cpp/CMakeLists.txt b/examples/parallel/cpp/CMakeLists.txt index 07a847401..1340c1a6f 100644 --- a/examples/parallel/cpp/CMakeLists.txt +++ b/examples/parallel/cpp/CMakeLists.txt @@ -24,29 +24,46 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientParallelExamples) +# Project definition for the SmartRedis-Examples-Parallel-Cpp project +project(SmartRedis-Examples-Parallel-Cpp) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the examples enable_language(CXX) -set(CMAKE_BUILD_TYPE Release) +# Configure the build set(CMAKE_CXX_STANDARD 17) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) -find_package(MPI) +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") +endif() -find_library(SR_LIB smartredis PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_package(MPI) +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${MPI_INCLUDE_PATH} - ../../../install/include + ${SMARTREDIS_INSTALL_PATH}/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES smartredis_put_get_3D smartredis_mnist ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_cpp_parallel ${EXECUTABLE}.cpp diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index 5399c7be1..8974ccef7 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -24,44 +24,58 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientParallelExamples) +# Project definition for the SmartRedis-Examples-Parallel-Fortran project +project(SmartRedis-Examples-Parallel-Fortran) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the examples enable_language(Fortran) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE Debug) +# Configure the build set(CMAKE_CXX_STANDARD 17) -set(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) # Assume by default that users should link against the install directory in this repository if(NOT DEFINED SMARTREDIS_INSTALL_PATH) set(SMARTREDIS_INSTALL_PATH "../../../install/") endif() -# Specify all pre-processor and library dependencies +# Locate dependencies find_package(MPI REQUIRED) IF(NOT MPI_Fortran_FOUND) message(FATAL_ERROR "Could not find Fortran MPI components") endif() -find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) set(SMARTREDIS_LIBRARIES - ${SMARTREDIS_LIBRARY} - ${SMARTREDIS_FORTRAN_LIBRARY} + ${SR_LIB} + ${SR_FTN_LIB} ) + +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${MPI_INCLUDE_PATH} ${SMARTREDIS_INSTALL_PATH}/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES smartredis_dataset smartredis_mnist smartredis_put_get_3D ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_fortran_parallel ${EXECUTABLE}.F90 diff --git a/examples/serial/CMakeLists.txt b/examples/serial/CMakeLists.txt index b7bea33b5..4ec64edc3 100644 --- a/examples/serial/CMakeLists.txt +++ b/examples/serial/CMakeLists.txt @@ -24,13 +24,16 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(SerialExamples) +# Project definition for the SmartRedis-Examples-Serial project +project(SmartRedis-Examples-Serial) cmake_minimum_required(VERSION 3.13) +# Enable language support for the examples enable_language(C) enable_language(CXX) enable_language(Fortran) +# Bring in subdirectories add_subdirectory(c) add_subdirectory(cpp) add_subdirectory(fortran) diff --git a/examples/serial/c/CMakeLists.txt b/examples/serial/c/CMakeLists.txt index e2dd09d98..94f1afb59 100644 --- a/examples/serial/c/CMakeLists.txt +++ b/examples/serial/c/CMakeLists.txt @@ -24,28 +24,45 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CClientSerialExamples) +# Project definition for the SmartRedis-Examples-Serial-C project +project(SmartRedis-Examples-Serial-C) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the examples enable_language(C) -set(CMAKE_BUILD_TYPE RELEASE) +# Configure the build set(CMAKE_CXX_STANDARD 17) SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) + +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") +endif() -find_library(SR_LIB smartredis PATHS ../../../install/lib - NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include - ../../../install/include + ${SMARTREDIS_INSTALL_PATH}/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES example_put_unpack_1D example_put_get_3D ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_c_serial ${EXECUTABLE}.c diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index c1dbfd0c6..c6cefcd41 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -24,22 +24,39 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientSerialExamples) +# Project definition for the SmartRedis-Examples-Serial-Cpp project +project(SmartRedis-Examples-Serial-Cpp) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the examples enable_language(CXX) -set(CMAKE_BUILD_TYPE RELEASE) +# Configure the build set(CMAKE_CXX_STANDARD 17) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) + +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") +endif() -find_library(SR_LIB smartredis PATHS ../../../install/lib - NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include ../../../install/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES smartredis_put_get_3D smartredis_dataset @@ -47,6 +64,7 @@ list(APPEND EXECUTABLES smartredis_mnist ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_cpp_serial ${EXECUTABLE}.cpp diff --git a/examples/serial/fortran/CMakeLists.txt b/examples/serial/fortran/CMakeLists.txt index 2502539ad..02c1632c3 100644 --- a/examples/serial/fortran/CMakeLists.txt +++ b/examples/serial/fortran/CMakeLists.txt @@ -24,40 +24,55 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientSerialExamples) +# Project definition for the SmartRedis-Examples-Serial-Fortran project +project(SmartRedis-Examples-Serial-Fortran) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the examples enable_language(Fortran) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE Debug) +# Configure the build set(CMAKE_CXX_STANDARD 17) -set(CMAKE_C_STANDARD 99) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) -# Assume by default that users should link against the install directory in this repository +# Assume by default that users should link against the +# install directory in this repository if(NOT DEFINED SMARTREDIS_INSTALL_PATH) set(SMARTREDIS_INSTALL_PATH "../../../install/") endif() -# Specify all pre-processor and library dependencies -find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +find_library(SR_FTN_LIB SMARTREDIS_FORTRAN_LIB + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) set(SMARTREDIS_LIBRARIES ${SMARTREDIS_LIBRARY} ${SMARTREDIS_FORTRAN_LIBRARY} ) + +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${SMARTREDIS_INSTALL_PATH}/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES smartredis_dataset smartredis_put_get_3D ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) - add_executable(${EXECUTABLE}_fortran_serial ${EXECUTABLE}.F90 ) From 4427348489ad9990f1d0c4d36122a564e187ba52 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 17 May 2023 18:34:02 -0500 Subject: [PATCH 10/50] Tests are now built with specified build mode and link type --- examples/serial/fortran/CMakeLists.txt | 6 ++-- tests/CMakeLists.txt | 10 +++--- tests/c/CMakeLists.txt | 28 +++++++++++------ tests/cpp/CMakeLists.txt | 30 ++++++++++++------ tests/cpp/unit-tests/CMakeLists.txt | 36 ++++++++++++++------- tests/fortran/CMakeLists.txt | 43 +++++++++++++++----------- 6 files changed, 98 insertions(+), 55 deletions(-) diff --git a/examples/serial/fortran/CMakeLists.txt b/examples/serial/fortran/CMakeLists.txt index 02c1632c3..4b3ad4d90 100644 --- a/examples/serial/fortran/CMakeLists.txt +++ b/examples/serial/fortran/CMakeLists.txt @@ -49,14 +49,14 @@ find_library(SR_LIB ${SMARTREDIS_LIB} REQUIRED ${SMARTREDIS_LINK_MODE} ) -find_library(SR_FTN_LIB SMARTREDIS_FORTRAN_LIB +find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED ${SMARTREDIS_LINK_MODE} ) set(SMARTREDIS_LIBRARIES - ${SMARTREDIS_LIBRARY} - ${SMARTREDIS_FORTRAN_LIBRARY} + ${SR_LIB} + ${SR_FTN_LIB} ) # Define include directories for header files diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a372b8821..57835b6e4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,19 +24,21 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Project definition for the SmartRedis-Tests project project(SmartSim-Tests) cmake_minimum_required(VERSION 3.13) +# Enable language support for the examples enable_language(C) enable_language(CXX) enable_language(Fortran) -set(ALLOW_DUPLICATE_CUSTOM_TARGETS true) - +# Bring in subdirectories add_subdirectory(c) add_subdirectory(cpp) add_subdirectory(fortran) -# The docker subdirectory is designed to be built within a container, not as part -# of the normal test build. We therefore do not include it in a test build +# NOTE: The docker subdirectory is designed to be built within a container, +# not as part of the normal test build. We therefore do not include it in a +# test build #add_subdirectory(docker) diff --git a/tests/c/CMakeLists.txt b/tests/c/CMakeLists.txt index e2d3ce640..6a4b71f91 100644 --- a/tests/c/CMakeLists.txt +++ b/tests/c/CMakeLists.txt @@ -24,24 +24,33 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CClientTester) +# Project definition for the SmartRedis-Tests-C project +project(SmartSim-Tests-C) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the tests enable_language(C) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE DEBUG) +# Configure the build set(CMAKE_CXX_STANDARD 17) SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../..") +include(smartredis_defs) -if(COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - set(CMAKE_CXX_FLAGS --coverage) - add_link_options(--coverage) - endif() +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../install/") endif() -find_library(SR_LIB smartredis PATHS ../../install/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include ../../install/include @@ -58,6 +67,7 @@ list(APPEND EXECUTABLES client_test_put_get_3D ) +# Build the tests foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_c_test ${EXECUTABLE}.c diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index a47c0592d..5765c4ab8 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -24,25 +24,36 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientTester) +# Project definition for the SmartRedis-Tests-Cpp project +project(SmartSim-Tests-Cpp) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the tests enable_language(CXX) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE DEBUG) +# Configure the build set(CMAKE_CXX_STANDARD 17) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../..") +include(smartredis_defs) +# Bring in the Catch2 unit-tests add_subdirectory(unit-tests) -if(COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - set(CMAKE_CXX_FLAGS --coverage) - add_link_options(--coverage) - endif() +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../install/") endif() -find_library(SR_LIB smartredis PATHS ../../install/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include ../../install/include @@ -75,6 +86,7 @@ list(APPEND EXECUTABLES client_test_ensemble_dataset ) +# Build the tests foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_cpp_test ${EXECUTABLE}.cpp diff --git a/tests/cpp/unit-tests/CMakeLists.txt b/tests/cpp/unit-tests/CMakeLists.txt index 8557e9fba..d0006bec7 100644 --- a/tests/cpp/unit-tests/CMakeLists.txt +++ b/tests/cpp/unit-tests/CMakeLists.txt @@ -24,33 +24,45 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientUnitTester) - +# Project definition for the SmartRedis-Tests-Cpp-UnitTests project +project(SmartSim-Tests-Cpp-UnitTests) cmake_minimum_required(VERSION 3.13) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE DEBUG) + +# Enable language support for the tests +enable_language(CXX) + +# Configure the build set(CMAKE_CXX_STANDARD 17) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) +set(THREADS_PREFER_PTHREAD_FLAG ON) -if(COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - set(CMAKE_CXX_FLAGS --coverage) - add_link_options(--coverage) - endif() +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") endif() -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_library(SRLIB smartredis PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) find_package(Threads REQUIRED) +# Define include directories for header files include_directories(SYSTEM /usr/local/include ../../../include ../../../install/include ) +# Identify source files to be built into the CPP Catch2 unit tests file(GLOB UNIT_TESTS CONFIGURE_DEPENDS ./*.cpp) +# Build the CPP Catch2 unit tests add_executable(cpp_unit_tests ${UNIT_TESTS}) - target_link_libraries(cpp_unit_tests PUBLIC ${SRLIB} PRIVATE Threads::Threads) diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index d246f4652..255fa7931 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -24,36 +24,42 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientTester) +# Project definition for the SmartRedis-Tests-Fortran project +project(SmartRedis-Tests-Fortran) cmake_minimum_required(VERSION 3.13) + +# Enable language support for the tests enable_language(Fortran) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE DEBUG) +# Configure the build set(CMAKE_CXX_STANDARD 17) -set(CMAKE_C_STANDARD 99) - -if(COVERAGE) - if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") - add_compile_options(--coverage) - add_link_options(--coverage) - else() - message(WARNING "COVERAGE was specified, but the CMAKE compiler is not GCC") - endif() -endif() +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../..") +include(smartredis_defs) -# Assume by default that users should link against the install directory in this repository +# Assume by default that users should link against the +# install directory in this repository if(NOT DEFINED SMARTREDIS_INSTALL_PATH) - set(SMARTREDIS_INSTALL_PATH "../../install/") + set(SMARTREDIS_INSTALL_PATH "../../install/") endif() -# Specify all pre-processor and library dependencies -find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) set(SMARTREDIS_LIBRARIES ${SMARTREDIS_LIBRARY} ${SMARTREDIS_FORTRAN_LIBRARY} ) + +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${SMARTREDIS_INSTALL_PATH}/include @@ -78,6 +84,7 @@ list(APPEND EXECUTABLES client_test_configoptions ) +# Build the tests foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_fortran_test ${EXECUTABLE}.F90 From cf38a4e79387594130e5e20d2bb7f80e1fbf830e Mon Sep 17 00:00:00 2001 From: billschereriii Date: Thu, 18 May 2023 19:20:27 -0500 Subject: [PATCH 11/50] Interrim checkin --- CMakeLists.txt | 15 ++++--- Makefile | 64 +++++++++++++++-------------- setup.py | 6 +++ smartredis_defs.cmake | 8 ++++ tests/c/CMakeLists.txt | 2 +- tests/c/client_test_put_get_2D.c | 20 ++++----- tests/cpp/CMakeLists.txt | 5 ++- tests/cpp/unit-tests/CMakeLists.txt | 6 +-- 8 files changed, 75 insertions(+), 51 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1537dd00..049056847 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install) set(CMAKE_CXX_VISIBILITY_PRESET default) set(THREADS_PREFER_PTHREAD_FLAG ON) -set(CMAKE_MODULE_PATH .) +set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}) include(smartredis_defs) # If we want to use Fortran, we have to tell CMake to use it @@ -62,8 +62,14 @@ if (SR_PEDANTIC) endif() # Bring in third-party libaries needed for the SmartRedis library -find_library(REDISPP redis++ PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) +find_library(REDISPP redis++ + PATHS ${CMAKE_SOURCE_DIR}/third-party/install/lib NO_DEFAULT_PATH + REQUIRED +) +find_library(HIREDIS hiredis + PATHS ${CMAKE_SOURCE_DIR}/third-party/install/lib NO_DEFAULT_PATH + REQUIRED +) find_package(Threads REQUIRED) set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS}) @@ -172,9 +178,6 @@ if(BUILD_PYTHON) add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind ${CMAKE_SOURCE_DIR}/third-party/pybind/build) - # Note that this library is always static - add_library(smartredis_static STATIC ${CLIENT_SRC}) - pybind11_add_module(smartredisPy src/python/src/pyclient.cpp src/python/src/pyconfigoptions.cpp diff --git a/Makefile b/Makefile index 802180a0a..798d9beaa 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ SR_WLM_FLAGS := SR_DEVICE := cpu SR_PEDANTIC := OFF SR_FORTRAN := OFF +SR_PIPINSTALL := ON # Params for third-party software HIREDIS_URL := https://github.com/redis/hiredis.git @@ -54,15 +55,17 @@ deps: # help: pip-install - Register the SmartRedis library with pip .PHONY: pip-install pip-install: +ifeq ($(SR_PIPINSTALL),ON) @python -c "import smartredis" >& /dev/null || pip install -e. +endif # help: lib - Build SmartRedis C/C++/Python clients into a dynamic library .PHONY: lib lib: pip-install lib: deps - cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) - cmake --build build/$(SR_BUILD) - cmake --install build/$(SR_BUILD) + @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) + @cmake --build build/$(SR_BUILD) + @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran @@ -94,55 +97,55 @@ test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) .PHONY: build-tests build-tests: test-lib-with-fortran - cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/tests + @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests # help: build-test-cpp - build the C++ tests .PHONY: build-test-cpp build-test-cpp: test-lib - cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/tests/cpp + @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/cpp # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib - cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests + @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib - cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/tests/c + @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/c # help: build-test-fortran - build the Fortran tests .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran - cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/tests/fortran + @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/fortran # help: build-examples - build all examples (serial, parallel) .PHONY: build-examples build-examples: lib-with-fortran - cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/examples + @cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/examples # help: build-example-serial - buld serial examples .PHONY: build-example-serial build-example-serial: lib-with-fortran - cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/examples/serial + @cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/examples/serial # help: build-example-parallel - build parallel examples (requires MPI) .PHONY: build-example-parallel build-example-parallel: lib-with-fortran - cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - cmake --build build/$(SR_BUILD)/examples/parellel + @cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/examples/parellel # help: clean-deps - remove third-party deps @@ -238,7 +241,7 @@ test: test-verbose: test-deps test-verbose: build-tests test-verbose: - PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) + @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) # help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] .PHONY: test-verbose-with-coverage @@ -246,7 +249,7 @@ test-verbose-with-coverage: SR_BUILD=Coverage test-verbose-with-coverage: test-deps test-verbose-with-coverage: build-tests test-verbose-with-coverage: - PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) + @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) # help: test-c - Build and run all C tests .PHONY: test-c @@ -273,6 +276,7 @@ unit-test-cpp: # help: test-py - run python tests .PHONY: test-py test-py: test-deps +test-py: lib test-py: @PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/ --build $(SR_BUILD) @@ -301,23 +305,23 @@ test-examples: # Hiredis (hidden build target) .phony: hiredis -hiredis: install/lib/libhiredis.a -install/lib/libhiredis.a: +hiredis: third-party/install/lib/libhiredis.a +third-party/install/lib/libhiredis.a: @rm -rf third-party/hiredis @mkdir -p third-party @cd third-party && \ git clone $(HIREDIS_URL) hiredis --branch $(HIREDIS_VER) --depth=1 @cd third-party/hiredis && \ - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" static -j && \ - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" install && \ - rm -f ../../install/lib/libhiredis*.so && \ - rm -f ../../install/lib/libhiredis*.dylib && \ + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../install" static -j && \ + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../install" install && \ + rm -f ../install/lib/libhiredis*.so && \ + rm -f ../install/lib/libhiredis*.dylib && \ echo "Finished installing Hiredis" # Redis-plus-plus (hidden build target) .phony: redis-plus-plus -redis-plus-plus: install/lib/libredis++.a -install/lib/libredis++.a: +redis-plus-plus: third-party/install/lib/libredis++.a +third-party/install/lib/libredis++.a: @rm -rf third-party/redis-plus-plus @mkdir -p third-party @cd third-party && \ @@ -325,7 +329,7 @@ install/lib/libredis++.a: @cd third-party/redis-plus-plus && \ mkdir -p compile && \ cd compile && \ - cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ + cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../install" -DCMAKE_CXX_STANDARD=17 .. && \ CC=gcc CXX=g++ make -j && \ CC=gcc CXX=g++ make install && \ echo "Finished installing Redis-plus-plus" diff --git a/setup.py b/setup.py index dd19c6c0e..9adad206a 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,12 @@ def run(self): if not build_directory.is_dir(): os.makedirs(self.build_temp) + # make install dir + setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() + install_directory = setup_path.joinpath("install") + if not install_directory.is_dir(): + os.makedirs(install_directory) + print('-'*10, 'Building C dependencies', '-'*40) make_cmd = shutil.which("make") setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() diff --git a/smartredis_defs.cmake b/smartredis_defs.cmake index 15ca49e39..df9607b32 100644 --- a/smartredis_defs.cmake +++ b/smartredis_defs.cmake @@ -24,6 +24,14 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Set defaults if variables are undefined +if(NOT DEFINED SR_BUILD) + set(SR_BUILD "Release") +endif() +if(NOT DEFINED SR_LINK) + set(SR_LINK "Shared") +endif() + # Configure the CMake build based on the SR_BUILD selection if(SR_BUILD STREQUAL "Release") set(CMAKE_BUILD_TYPE RELEASE) diff --git a/tests/c/CMakeLists.txt b/tests/c/CMakeLists.txt index 6a4b71f91..e7d9c71b4 100644 --- a/tests/c/CMakeLists.txt +++ b/tests/c/CMakeLists.txt @@ -53,7 +53,7 @@ find_library(SR_LIB ${SMARTREDIS_LIB} # Define include directories for header files include_directories(SYSTEM /usr/local/include - ../../install/include + ${SMARTREDIS_INSTALL_PATH}/include ) # Define all the tests to be built diff --git a/tests/c/client_test_put_get_2D.c b/tests/c/client_test_put_get_2D.c index bc6f06c28..32746ed47 100644 --- a/tests/c/client_test_put_get_2D.c +++ b/tests/c/client_test_put_get_2D.c @@ -114,7 +114,7 @@ int put_get_2D_tensor(void* client, values. If the sent and received tensors do not match, a non-zero value is returned. */ -int put_get_2D_tensor_double(size_t* dims, int n_dims, +int put_get_2D_tensor_double(size_t* dims, size_t n_dims, char* key_suffix, int key_suffix_length) { void* client = NULL; @@ -162,7 +162,7 @@ int put_get_2D_tensor_double(size_t* dims, int n_dims, values. If the sent and received tensors do not match, a non-zero value is returned. */ -int put_get_2D_tensor_float(size_t* dims, int n_dims, +int put_get_2D_tensor_float(size_t* dims, size_t n_dims, char* key_suffix, int key_suffix_length) { @@ -211,7 +211,7 @@ int put_get_2D_tensor_float(size_t* dims, int n_dims, values. If the sent and received tensors do not match, a non-zero value is returned. */ -int put_get_2D_tensor_i8(size_t* dims, int n_dims, +int put_get_2D_tensor_i8(size_t* dims, size_t n_dims, char* key_suffix, int key_suffix_length) { @@ -264,7 +264,7 @@ for(int i=0; i Date: Thu, 18 May 2023 19:37:44 -0500 Subject: [PATCH 12/50] Fix link errrors for Fortran and C++ unit-test --- tests/cpp/unit-tests/CMakeLists.txt | 2 +- tests/fortran/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cpp/unit-tests/CMakeLists.txt b/tests/cpp/unit-tests/CMakeLists.txt index 3808dec73..f2162f6c4 100644 --- a/tests/cpp/unit-tests/CMakeLists.txt +++ b/tests/cpp/unit-tests/CMakeLists.txt @@ -65,4 +65,4 @@ file(GLOB UNIT_TESTS CONFIGURE_DEPENDS ./*.cpp) # Build the CPP Catch2 unit tests add_executable(cpp_unit_tests ${UNIT_TESTS}) -target_link_libraries(cpp_unit_tests PUBLIC ${SRLIB} PRIVATE Threads::Threads) +target_link_libraries(cpp_unit_tests PUBLIC ${SR_LIB} PRIVATE Threads::Threads) diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 255fa7931..09f812c54 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -55,8 +55,8 @@ find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} ${SMARTREDIS_LINK_MODE} ) set(SMARTREDIS_LIBRARIES - ${SMARTREDIS_LIBRARY} - ${SMARTREDIS_FORTRAN_LIBRARY} + ${SR_LIB} + ${SR_FTN_LIB} ) # Define include directories for header files From dbc3eaf835068ce6deb28e5e9b77e19a8266d763 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Thu, 18 May 2023 20:03:56 -0500 Subject: [PATCH 13/50] Parallel build support --- Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 798d9beaa..68ba2a596 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ endif lib: pip-install lib: deps @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) - @cmake --build build/$(SR_BUILD) + @cmake --build build/$(SR_BUILD) -- -j @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library @@ -95,6 +95,7 @@ test-deps-gpu: SR_DEVICE=gpu test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) +# NOTE: Fortran tests cannot be built in parallel .PHONY: build-tests build-tests: test-lib-with-fortran @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) @@ -105,22 +106,23 @@ build-tests: test-lib-with-fortran .PHONY: build-test-cpp build-test-cpp: test-lib @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/cpp + @cmake --build build/$(SR_BUILD)/tests/cpp -- -j # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests + @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests -- -j # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/c + @cmake --build build/$(SR_BUILD)/tests/c -- -j # help: build-test-fortran - build the Fortran tests +# NOTE: Fortran tests cannot be built in parallel .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) From eac9d211be428f50cf7d43f1723aed532ffedebc Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 19 May 2023 12:29:35 -0500 Subject: [PATCH 14/50] Very slightly less pedantic for Fortran optimized builds --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 049056847..4c514da13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,9 @@ if (SR_PEDANTIC) else() message(WARNING "SR_PEDANTIC was specified, but the CMAKE compiler is not GCC") endif() + if((CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") AND (SR_BUILD STREQUAL "Release")) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Wno-maybe-uninitialized") + endif() endif() # Bring in third-party libaries needed for the SmartRedis library From 2cd36d7689964d458369a8ceca3e12f068f7a9c4 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 19 May 2023 13:34:59 -0500 Subject: [PATCH 15/50] Clobber bogus uninitialized variable use warnings in Fortran --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c514da13..68506f3c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,7 +59,7 @@ if (SR_PEDANTIC) else() message(WARNING "SR_PEDANTIC was specified, but the CMAKE compiler is not GCC") endif() - if((CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") AND (SR_BUILD STREQUAL "Release")) + if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Wno-maybe-uninitialized") endif() endif() From 53f432ac299945a974287fb6202b6a675f7b8bf0 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 19 May 2023 15:15:53 -0500 Subject: [PATCH 16/50] Tell the clustered and UDS tests where to look for test executables --- .github/workflows/run_tests.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ed67f122c..b11d742d4 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -120,12 +120,13 @@ jobs: - name: Build SmartRedis python and install run: python -m pip install -e .[dev,xarray] - - name: Build and install dependencies - run: | - mkdir -p ./third-party && - cd ./third-party && - bash ../build-scripts/build-lcov.sh && - bash ../build-scripts/build-catch.sh +# These are now installed via the Makefile +# - name: Build and install dependencies +# run: | +# mkdir -p ./third-party && +# cd ./third-party && +# bash ../build-scripts/build-lcov.sh && +# bash ../build-scripts/build-catch.sh - name: Install docker, redis-server, and RedisAI run: | @@ -181,7 +182,7 @@ jobs: redis-cli --cluster create $(echo $SSDB_CLUSTERED | tr "," " ") --cluster-yes && export SSDB=$SSDB_CLUSTERED SMARTREDIS_TEST_CLUSTER=True && python -m pytest --cov=./src/python/module/smartredis/ --cov-report=xml --cov-append \ - --ignore ./tests/docker -vv -s ./tests + --ignore ./tests/docker -vv -s ./tests --build Coverage - name: UDS DB tests run: | @@ -198,7 +199,7 @@ jobs: utils/check_redis.sh $SOCKET export SSDB=$SSDB_UDS SMARTREDIS_TEST_CLUSTER=False && python -m pytest --cov=./src/python/module/smartredis/ --cov-report=xml --cov-append \ - --ignore ./tests/docker -vv -s ./tests + --ignore ./tests/docker -vv -s ./tests --build Coverage # Process and upload code coverage (Python was collected during pytest) - name: Collect coverage from C/C++/Fortran testers From 146b170b095eea5e1156562e5bdcb9edbf1a00ef Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 19 May 2023 15:35:39 -0500 Subject: [PATCH 17/50] Test --- .github/workflows/run_tests.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b11d742d4..6067e04c4 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -120,13 +120,12 @@ jobs: - name: Build SmartRedis python and install run: python -m pip install -e .[dev,xarray] -# These are now installed via the Makefile -# - name: Build and install dependencies -# run: | -# mkdir -p ./third-party && -# cd ./third-party && -# bash ../build-scripts/build-lcov.sh && -# bash ../build-scripts/build-catch.sh + - name: Build and install dependencies + run: | + mkdir -p ./third-party && + cd ./third-party && + bash ../build-scripts/build-lcov.sh && + bash ../build-scripts/build-catch.sh - name: Install docker, redis-server, and RedisAI run: | From 3aa7a9024960398c42675b710fe0ae6652431584 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 11:35:15 -0500 Subject: [PATCH 18/50] Test --- .github/workflows/run_tests.yml | 17 ++++++++++------- images/Dockerfile | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 6067e04c4..ba8723304 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -63,9 +63,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04] # cannot test on macOS as docker isn't supported on Mac - rai_v: [1.2.4, 1.2.5] # versions of RedisAI - py_v: ['3.7.x', '3.8.x', '3.9.x'] # versions of Python - compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler +# rai_v: [1.2.4, 1.2.5] # versions of RedisAI + rai_v: [1.2.4] # versions of RedisAI +# py_v: ['3.7.x', '3.8.x', '3.9.x'] # versions of Python + py_v: ['3.8.x'] # versions of Python +# compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler + compiler: [10] # intel compiler, and versions of GNU compiler env: FC: gfortran-${{ matrix.compiler }} GCC_V: ${{ matrix.compiler }} # used when the compiler is gcc/gfortran @@ -131,18 +134,18 @@ jobs: run: | sudo apt-get -y update && sudo apt-get -y install curl gnupg lsb-release software-properties-common ca-certificates && \ # Add latest redis to apt sources - echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list && \ - curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg && \ + sudo echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list && \ + sudo curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg && \ # Add latest docker to apt sources sudo mkdir -m 0755 -p /etc/apt/keyrings && - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null && # Install and update sudo apt-get -y update && \ sudo apt-get -y install iputils-ping docker-ce docker-ce-cli containerd.io redis-tools=6:6.2.5-1rl1~focal1 redis-server=6:6.2.5-1rl1~focal1 && CONTAINER_NAME="redisai_$RANDOM" && - docker create --name $CONTAINER_NAME --rm redislabs/redisai:${{ matrix.rai_v }}-cpu-bionic && \ + sudo docker create --name $CONTAINER_NAME --rm redislabs/redisai:${{ matrix.rai_v }}-cpu-bionic && \ sudo mkdir -p /usr/lib/redis/modules/ && sudo docker cp $CONTAINER_NAME:/usr/lib/redis/modules/redisai.so /usr/lib/redis/modules && sudo docker cp $CONTAINER_NAME:/usr/lib/redis/modules/backends/ /usr/lib/redis/modules/ && diff --git a/images/Dockerfile b/images/Dockerfile index ae7331cbb..d5beb6560 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -49,6 +49,7 @@ COPY . /usr/local/src/SmartRedis # Compile and install WORKDIR /usr/local/src/SmartRedis +RUN pip3 install --upgrade -i RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ && rm -rf build tests examples images utils third-party doc images From 02becd3017733aa1c1f29445e38365f1b93f6186 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 11:39:32 -0500 Subject: [PATCH 19/50] Test --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index d5beb6560..4c47b8842 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -49,7 +49,7 @@ COPY . /usr/local/src/SmartRedis # Compile and install WORKDIR /usr/local/src/SmartRedis -RUN pip3 install --upgrade -i +RUN pip3 install --upgrade RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ && rm -rf build tests examples images utils third-party doc images From 8b76c9efa889b9a7b132ab7db4a5e6d08bb1e97b Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 11:43:35 -0500 Subject: [PATCH 20/50] Test --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index 4c47b8842..fde64428e 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -49,7 +49,7 @@ COPY . /usr/local/src/SmartRedis # Compile and install WORKDIR /usr/local/src/SmartRedis -RUN pip3 install --upgrade +RUN pip3 install --upgrade pip RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ && rm -rf build tests examples images utils third-party doc images From 12bb8771db66319fa6377f2959d826cb9cee6987 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 12:57:55 -0500 Subject: [PATCH 21/50] Fix? Docker workflow --- include/redisserver.h | 1 - src/cpp/redis.cpp | 1 + src/cpp/rediscluster.cpp | 1 + src/cpp/redisserver.cpp | 1 + 4 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/redisserver.h b/include/redisserver.h index 0701c28f2..f3ab81d8c 100644 --- a/include/redisserver.h +++ b/include/redisserver.h @@ -33,7 +33,6 @@ #include #include #include -#include #include "command.h" #include "commandreply.h" diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index 22302312c..b84cc77d1 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -26,6 +26,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include "redis.h" #include "srexception.h" #include "utility.h" diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index 04c3e76a1..7f649e331 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -26,6 +26,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include "rediscluster.h" #include "nonkeyedcommand.h" #include "keyedcommand.h" diff --git a/src/cpp/redisserver.cpp b/src/cpp/redisserver.cpp index b8e9b020c..e3a206e84 100644 --- a/src/cpp/redisserver.cpp +++ b/src/cpp/redisserver.cpp @@ -27,6 +27,7 @@ */ #include +#include #include "redisserver.h" #include "srexception.h" #include "utility.h" From c79f9a24367e957108c794a6c93041dfe4294db9 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 13:42:20 -0500 Subject: [PATCH 22/50] Test --- tests/docker/cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/docker/cpp/CMakeLists.txt b/tests/docker/cpp/CMakeLists.txt index 19b3e9e3c..2637ed1e1 100644 --- a/tests/docker/cpp/CMakeLists.txt +++ b/tests/docker/cpp/CMakeLists.txt @@ -38,6 +38,7 @@ find_library(SR_LIB smartredis) include_directories(SYSTEM /usr/local/include/smartredis + /usr/local/src/SmartRedis/third-party/install ) # Build executables From f4f5bc2648580f6ede2df1d7d33f3c95f20627f1 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 13:51:10 -0500 Subject: [PATCH 23/50] Test --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index fde64428e..e5b5b09c8 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -51,7 +51,7 @@ COPY . /usr/local/src/SmartRedis WORKDIR /usr/local/src/SmartRedis RUN pip3 install --upgrade pip RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ - && rm -rf build tests examples images utils third-party doc images + && rm -rf build tests examples images utils doc images # Copy install files and directories to /usr/local/lib and # usr/local/include and delete unnecessary build files From 32fbc1010958116fddf388062ee0b99982ffcf28 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 14:23:40 -0500 Subject: [PATCH 24/50] Test --- images/Dockerfile | 4 +++- tests/docker/cpp/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index e5b5b09c8..d9ee713b5 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -57,4 +57,6 @@ RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ # usr/local/include and delete unnecessary build files RUN cp -r /usr/local/src/SmartRedis/install/lib/* /usr/local/lib/ && \ mkdir /usr/local/include/smartredis && \ - cp -r /usr/local/src/SmartRedis/install/include/* /usr/local/include/smartredis/ + cp -r /usr/local/src/SmartRedis/install/include/* /usr/local/include/smartredis/ && \ + mkdir /usr/local/include/smartredis/third-party && \ + cp -r /usr/local/src/SmartRedis/third-party/install/include/* /usr/local/include/smartredis/third-party/ diff --git a/tests/docker/cpp/CMakeLists.txt b/tests/docker/cpp/CMakeLists.txt index 2637ed1e1..b9648bc77 100644 --- a/tests/docker/cpp/CMakeLists.txt +++ b/tests/docker/cpp/CMakeLists.txt @@ -38,7 +38,7 @@ find_library(SR_LIB smartredis) include_directories(SYSTEM /usr/local/include/smartredis - /usr/local/src/SmartRedis/third-party/install + /usr/local/include/smartredis/third-party ) # Build executables From de5602258dbb61f75383c0edb13669d8fa943e99 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 15:05:16 -0500 Subject: [PATCH 25/50] CMake min version is supposed to be before the project() definition --- CMakeLists.txt | 2 +- examples/CMakeLists.txt | 2 +- examples/parallel/CMakeLists.txt | 2 +- examples/parallel/cpp/CMakeLists.txt | 2 +- examples/parallel/fortran/CMakeLists.txt | 2 +- examples/serial/CMakeLists.txt | 2 +- examples/serial/c/CMakeLists.txt | 2 +- examples/serial/cpp/CMakeLists.txt | 2 +- examples/serial/fortran/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 2 +- tests/c/CMakeLists.txt | 2 +- tests/cpp/CMakeLists.txt | 2 +- tests/cpp/unit-tests/CMakeLists.txt | 2 +- tests/docker/CMakeLists.txt | 2 +- tests/docker/c/CMakeLists.txt | 2 +- tests/docker/cpp/CMakeLists.txt | 2 +- tests/docker/fortran/CMakeLists.txt | 2 +- tests/fortran/CMakeLists.txt | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68506f3c6..16ef0d11f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,8 +30,8 @@ if (POLICY CMP0048) endif (POLICY CMP0048) # Project definition for the SmartRedis project -project(SmartRedis VERSION "0.4.0") cmake_minimum_required(VERSION 3.13) +project(SmartRedis VERSION "0.4.0") # Configure options for the SmartRedis project option(BUILD_PYTHON "Build the python module" ON) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6fbbb21ef..f17d0cf50 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples project -project(SmartRedis-Examples) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples) # Enable language support for the examples enable_language(C) diff --git a/examples/parallel/CMakeLists.txt b/examples/parallel/CMakeLists.txt index ea38611ae..07f6c0d40 100644 --- a/examples/parallel/CMakeLists.txt +++ b/examples/parallel/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Parallel project -project(SmartRedis-Examples-Parallel) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Parallel) # Enable language support for the examples enable_language(CXX) diff --git a/examples/parallel/cpp/CMakeLists.txt b/examples/parallel/cpp/CMakeLists.txt index 1340c1a6f..bf2c24e3f 100644 --- a/examples/parallel/cpp/CMakeLists.txt +++ b/examples/parallel/cpp/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Parallel-Cpp project -project(SmartRedis-Examples-Parallel-Cpp) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Parallel-Cpp) # Enable language support for the examples enable_language(CXX) diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index 8974ccef7..e427cbc0b 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Parallel-Fortran project -project(SmartRedis-Examples-Parallel-Fortran) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Parallel-Fortran) # Enable language support for the examples enable_language(Fortran) diff --git a/examples/serial/CMakeLists.txt b/examples/serial/CMakeLists.txt index 4ec64edc3..aa0de409c 100644 --- a/examples/serial/CMakeLists.txt +++ b/examples/serial/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Serial project -project(SmartRedis-Examples-Serial) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial) # Enable language support for the examples enable_language(C) diff --git a/examples/serial/c/CMakeLists.txt b/examples/serial/c/CMakeLists.txt index 94f1afb59..5544a5ef4 100644 --- a/examples/serial/c/CMakeLists.txt +++ b/examples/serial/c/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Serial-C project -project(SmartRedis-Examples-Serial-C) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial-C) # Enable language support for the examples enable_language(C) diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index c6cefcd41..c8b97e10d 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Serial-Cpp project -project(SmartRedis-Examples-Serial-Cpp) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial-Cpp) # Enable language support for the examples enable_language(CXX) diff --git a/examples/serial/fortran/CMakeLists.txt b/examples/serial/fortran/CMakeLists.txt index 4b3ad4d90..a8b127733 100644 --- a/examples/serial/fortran/CMakeLists.txt +++ b/examples/serial/fortran/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Examples-Serial-Fortran project -project(SmartRedis-Examples-Serial-Fortran) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial-Fortran) # Enable language support for the examples enable_language(Fortran) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 57835b6e4..792f31906 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Tests project -project(SmartSim-Tests) cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests) # Enable language support for the examples enable_language(C) diff --git a/tests/c/CMakeLists.txt b/tests/c/CMakeLists.txt index e7d9c71b4..0491543e6 100644 --- a/tests/c/CMakeLists.txt +++ b/tests/c/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Tests-C project -project(SmartSim-Tests-C) cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests-C) # Enable language support for the tests enable_language(C) diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 54b07e9cf..3058cb2ac 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Tests-Cpp project -project(SmartSim-Tests-Cpp) cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests-Cpp) # Enable language support for the tests enable_language(CXX) diff --git a/tests/cpp/unit-tests/CMakeLists.txt b/tests/cpp/unit-tests/CMakeLists.txt index f2162f6c4..94f23f9a3 100644 --- a/tests/cpp/unit-tests/CMakeLists.txt +++ b/tests/cpp/unit-tests/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Tests-Cpp-UnitTests project -project(SmartSim-Tests-Cpp-UnitTests) cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests-Cpp-UnitTests) # Enable language support for the tests diff --git a/tests/docker/CMakeLists.txt b/tests/docker/CMakeLists.txt index 7e5a8174d..81e978e8a 100644 --- a/tests/docker/CMakeLists.txt +++ b/tests/docker/CMakeLists.txt @@ -24,8 +24,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(SmartSim-Tests-Docker) cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests-Docker) enable_language(C) enable_language(CXX) diff --git a/tests/docker/c/CMakeLists.txt b/tests/docker/c/CMakeLists.txt index a44364b57..3ccb77ed7 100644 --- a/tests/docker/c/CMakeLists.txt +++ b/tests/docker/c/CMakeLists.txt @@ -26,8 +26,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -project(DockerTesterC) cmake_minimum_required(VERSION 3.13) +project(DockerTesterC) enable_language(C) set(CMAKE_VERBOSE_MAKEFILE ON) diff --git a/tests/docker/cpp/CMakeLists.txt b/tests/docker/cpp/CMakeLists.txt index b9648bc77..b2acaf645 100644 --- a/tests/docker/cpp/CMakeLists.txt +++ b/tests/docker/cpp/CMakeLists.txt @@ -26,8 +26,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -project(DockerTesterCpp) cmake_minimum_required(VERSION 3.13) +project(DockerTesterCpp) enable_language(CXX) set(CMAKE_VERBOSE_MAKEFILE ON) diff --git a/tests/docker/fortran/CMakeLists.txt b/tests/docker/fortran/CMakeLists.txt index 86e28588f..5e3217a3a 100644 --- a/tests/docker/fortran/CMakeLists.txt +++ b/tests/docker/fortran/CMakeLists.txt @@ -26,8 +26,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -project(DockerTesterFortran) cmake_minimum_required(VERSION 3.13) +project(DockerTesterFortran) enable_language(Fortran) set(CMAKE_VERBOSE_MAKEFILE ON) diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 09f812c54..2a45902ab 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -25,8 +25,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Project definition for the SmartRedis-Tests-Fortran project -project(SmartRedis-Tests-Fortran) cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Tests-Fortran) # Enable language support for the tests enable_language(Fortran) From db135be58cccc8d4653ea5c0fbd319c65d1d52d5 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 17:36:13 -0500 Subject: [PATCH 26/50] Revert moving third-party libraries to live under third-party/install --- CMakeLists.txt | 9 ++++----- Makefile | 18 +++++++++--------- examples/parallel/fortran/CMakeLists.txt | 2 ++ examples/serial/cpp/CMakeLists.txt | 2 +- images/Dockerfile | 4 +--- tests/cpp/CMakeLists.txt | 2 +- tests/cpp/unit-tests/CMakeLists.txt | 1 - tests/docker/cpp/CMakeLists.txt | 1 - 8 files changed, 18 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16ef0d11f..fae65e972 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,12 +66,12 @@ endif() # Bring in third-party libaries needed for the SmartRedis library find_library(REDISPP redis++ - PATHS ${CMAKE_SOURCE_DIR}/third-party/install/lib NO_DEFAULT_PATH - REQUIRED + PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH + REQUIRED STATIC ) find_library(HIREDIS hiredis - PATHS ${CMAKE_SOURCE_DIR}/third-party/install/lib NO_DEFAULT_PATH - REQUIRED + PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH + REQUIRED STATIC ) find_package(Threads REQUIRED) set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS}) @@ -122,7 +122,6 @@ set(CLIENT_SRC include_directories(SYSTEM include install/include - third-party/install/include ) # Build the Fortran library diff --git a/Makefile b/Makefile index 68ba2a596..608e7a0bf 100644 --- a/Makefile +++ b/Makefile @@ -307,23 +307,23 @@ test-examples: # Hiredis (hidden build target) .phony: hiredis -hiredis: third-party/install/lib/libhiredis.a -third-party/install/lib/libhiredis.a: +hiredis: install/lib/libhiredis.a +install/lib/libhiredis.a: @rm -rf third-party/hiredis @mkdir -p third-party @cd third-party && \ git clone $(HIREDIS_URL) hiredis --branch $(HIREDIS_VER) --depth=1 @cd third-party/hiredis && \ - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../install" static -j && \ - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../install" install && \ - rm -f ../install/lib/libhiredis*.so && \ - rm -f ../install/lib/libhiredis*.dylib && \ + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" static -j && \ + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" install && \ + rm -f ../../install/lib/libhiredis*.so && \ + rm -f ../../install/lib/libhiredis*.dylib && \ echo "Finished installing Hiredis" # Redis-plus-plus (hidden build target) .phony: redis-plus-plus -redis-plus-plus: third-party/install/lib/libredis++.a -third-party/install/lib/libredis++.a: +redis-plus-plus: install/lib/libredis++.a +install/lib/libredis++.a: @rm -rf third-party/redis-plus-plus @mkdir -p third-party @cd third-party && \ @@ -331,7 +331,7 @@ third-party/install/lib/libredis++.a: @cd third-party/redis-plus-plus && \ mkdir -p compile && \ cd compile && \ - cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../install" -DCMAKE_CXX_STANDARD=17 .. && \ + cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ CC=gcc CXX=g++ make -j && \ CC=gcc CXX=g++ make install && \ echo "Finished installing Redis-plus-plus" diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index e427cbc0b..6b135da7f 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -59,6 +59,8 @@ find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} set(SMARTREDIS_LIBRARIES ${SR_LIB} ${SR_FTN_LIB} + ${HIREDIS} + ${REDISPP} ) # Define include directories for header files diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index c8b97e10d..bca479b47 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -53,7 +53,7 @@ find_library(SR_LIB ${SMARTREDIS_LIB} # Define include directories for header files include_directories(SYSTEM /usr/local/include - ../../../install/include + ${SMARTREDIS_INSTALL_PATH}/include ) # Define all the examples to be built diff --git a/images/Dockerfile b/images/Dockerfile index d9ee713b5..e5b5b09c8 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -57,6 +57,4 @@ RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ # usr/local/include and delete unnecessary build files RUN cp -r /usr/local/src/SmartRedis/install/lib/* /usr/local/lib/ && \ mkdir /usr/local/include/smartredis && \ - cp -r /usr/local/src/SmartRedis/install/include/* /usr/local/include/smartredis/ && \ - mkdir /usr/local/include/smartredis/third-party && \ - cp -r /usr/local/src/SmartRedis/third-party/install/include/* /usr/local/include/smartredis/third-party/ + cp -r /usr/local/src/SmartRedis/install/include/* /usr/local/include/smartredis/ diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 3058cb2ac..73aaecdc3 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -57,7 +57,7 @@ find_library(SR_LIB ${SMARTREDIS_LIB} include_directories(SYSTEM /usr/local/include ${SMARTREDIS_INSTALL_PATH}/include - ${SMARTREDIS_INSTALL_PATH}/../third-party/install/include +# ${SMARTREDIS_INSTALL_PATH}/../third-party/install/include ) diff --git a/tests/cpp/unit-tests/CMakeLists.txt b/tests/cpp/unit-tests/CMakeLists.txt index 94f23f9a3..d0e5ab8c7 100644 --- a/tests/cpp/unit-tests/CMakeLists.txt +++ b/tests/cpp/unit-tests/CMakeLists.txt @@ -57,7 +57,6 @@ find_package(Threads REQUIRED) include_directories(SYSTEM /usr/local/include ${SMARTREDIS_INSTALL_PATH}/include - ${SMARTREDIS_INSTALL_PATH}/../third-party/install/include ) # Identify source files to be built into the CPP Catch2 unit tests diff --git a/tests/docker/cpp/CMakeLists.txt b/tests/docker/cpp/CMakeLists.txt index b2acaf645..6de60d519 100644 --- a/tests/docker/cpp/CMakeLists.txt +++ b/tests/docker/cpp/CMakeLists.txt @@ -38,7 +38,6 @@ find_library(SR_LIB smartredis) include_directories(SYSTEM /usr/local/include/smartredis - /usr/local/include/smartredis/third-party ) # Build executables From ffe4f7b3d7c867c7ba7245bc7ebc8e2e1f83df25 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 18:06:36 -0500 Subject: [PATCH 27/50] Minor cleanup --- examples/parallel/fortran/CMakeLists.txt | 2 -- tests/cpp/CMakeLists.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index 6b135da7f..e427cbc0b 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -59,8 +59,6 @@ find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} set(SMARTREDIS_LIBRARIES ${SR_LIB} ${SR_FTN_LIB} - ${HIREDIS} - ${REDISPP} ) # Define include directories for header files diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 73aaecdc3..b10954bff 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -57,7 +57,6 @@ find_library(SR_LIB ${SMARTREDIS_LIB} include_directories(SYSTEM /usr/local/include ${SMARTREDIS_INSTALL_PATH}/include -# ${SMARTREDIS_INSTALL_PATH}/../third-party/install/include ) From 7d13a0091f7737b48b87edba8dff964fb77204c2 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 18:25:16 -0500 Subject: [PATCH 28/50] Build library before pip install --- Makefile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 608e7a0bf..9470220c7 100644 --- a/Makefile +++ b/Makefile @@ -59,14 +59,22 @@ ifeq ($(SR_PIPINSTALL),ON) @python -c "import smartredis" >& /dev/null || pip install -e. endif -# help: lib - Build SmartRedis C/C++/Python clients into a dynamic library -.PHONY: lib -lib: pip-install -lib: deps +# build-lib: hidden make target to build the library (needed so we can pip install AFTER building) +.PHONY: build-lib +build-lib: deps @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD) -- -j @cmake --install build/$(SR_BUILD) +# help: lib - Build SmartRedis C/C++/Python clients into a dynamic library +.PHONY: lib +lib: build-lib +lib: pip-install +#lib: deps +# @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) +# @cmake --build build/$(SR_BUILD) -- -j +# @cmake --install build/$(SR_BUILD) + # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran lib-with-fortran: SR_FORTRAN=ON From 9bd0c3bde5aba500eb14402869c3cccff64aa141 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 18:44:28 -0500 Subject: [PATCH 29/50] Moar happy? --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ba8723304..19d5f1583 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -205,7 +205,7 @@ jobs: # Process and upload code coverage (Python was collected during pytest) - name: Collect coverage from C/C++/Fortran testers - run: third-party/lcov/install/usr/local/bin/lcov -c -d build/CMakeFiles -o coverage.info + run: third-party/lcov/install/usr/local/bin/lcov -c -d build/Coverage/CMakeFiles -o coverage.info - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 379d220bbe6637011fe193ce0fed67f18dff3d7e Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 19:00:30 -0500 Subject: [PATCH 30/50] Reenable matrix --- .github/workflows/run_tests.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 19d5f1583..c70b0971c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -63,12 +63,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04] # cannot test on macOS as docker isn't supported on Mac -# rai_v: [1.2.4, 1.2.5] # versions of RedisAI - rai_v: [1.2.4] # versions of RedisAI -# py_v: ['3.7.x', '3.8.x', '3.9.x'] # versions of Python - py_v: ['3.8.x'] # versions of Python -# compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler - compiler: [10] # intel compiler, and versions of GNU compiler + rai_v: [1.2.4, 1.2.5] # versions of RedisAI + py_v: ['3.7.x', '3.8.x', '3.9.x'] # versions of Python + compiler: [intel, 8, 9, 10, 11] # intel compiler, and versions of GNU compiler env: FC: gfortran-${{ matrix.compiler }} GCC_V: ${{ matrix.compiler }} # used when the compiler is gcc/gfortran From 4c5212c075d79050a64000e9060379bc579599cc Mon Sep 17 00:00:00 2001 From: billschereriii Date: Mon, 22 May 2023 20:42:10 -0500 Subject: [PATCH 31/50] Update to current Intel compiler --- .github/workflows/run_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index c70b0971c..d8fbcef26 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -108,8 +108,8 @@ jobs: sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic source /opt/intel/oneapi/setvars.sh && printenv >> $GITHUB_ENV && - echo "CC=icc" >> $GITHUB_ENV && - echo "CXX=icpc" >> $GITHUB_ENV && + echo "CC=icx" >> $GITHUB_ENV && + echo "CXX=icpx" >> $GITHUB_ENV && echo "FC=ifort" >> $GITHUB_ENV # Install additional dependencies From 8b1db619b1c100a29ef5ccc4b118b8af19a144ed Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 10:39:03 -0500 Subject: [PATCH 32/50] Don't allow unlimited process use --- Makefile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 9470220c7..40d557d62 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ MAKEFLAGS += --no-print-directory COV_FLAGS := SHELL:=/bin/bash +NPROC:=$(shell python -c "import multiprocessing as mp; print(mp.cpu_count())") # Build variables SR_BUILD := Release @@ -63,7 +64,7 @@ endif .PHONY: build-lib build-lib: deps @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) - @cmake --build build/$(SR_BUILD) -- -j + @cmake --build build/$(SR_BUILD) -- -j $(NPROC) @cmake --install build/$(SR_BUILD) # help: lib - Build SmartRedis C/C++/Python clients into a dynamic library @@ -114,19 +115,19 @@ build-tests: test-lib-with-fortran .PHONY: build-test-cpp build-test-cpp: test-lib @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/cpp -- -j + @cmake --build build/$(SR_BUILD)/tests/cpp -- -j $(NPROC) # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests -- -j + @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests -- -j $(NPROC) # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/c -- -j + @cmake --build build/$(SR_BUILD)/tests/c -- -j $(NPROC) # help: build-test-fortran - build the Fortran tests @@ -340,7 +341,7 @@ install/lib/libredis++.a: mkdir -p compile && \ cd compile && \ cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ - CC=gcc CXX=g++ make -j && \ + CC=gcc CXX=g++ make -j $(NPROC) && \ CC=gcc CXX=g++ make install && \ echo "Finished installing Redis-plus-plus" @@ -362,7 +363,7 @@ third-party/redis/src/redis-server: @cd third-party && \ git clone $(REDIS_URL) redis --branch $(REDIS_VER) --depth=1 @cd third-party/redis && \ - CC=gcc CXX=g++ make MALLOC=libc -j && \ + CC=gcc CXX=g++ make MALLOC=libc -j $(NPROC) && \ echo "Finished installing redis" # cudann-check (hidden test target) @@ -398,7 +399,7 @@ third-party/RedisAI/install-cpu/redisai.so: GIT_LFS_SKIP_SMUDGE=1 git clone --recursive $(REDISAI_URL) RedisAI --branch $(REDISAI_VER) --depth=1 -@cd third-party/RedisAI && \ CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh $(SR_DEVICE) && \ - CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j -C opt clean build && \ + CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j $(NPROC) -C opt clean build && \ echo "Finished installing RedisAI" # Catch2 (hidden test target) From 4273cde304e3748b6876507f6d010c8a8eeec2ef Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 12:17:13 -0500 Subject: [PATCH 33/50] Enable parallel build for Fortran tests --- Makefile | 9 ++------- setup.cfg | 2 +- setup.py | 33 ++++++++++++++++----------------- tests/fortran/CMakeLists.txt | 6 ++++-- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 40d557d62..be2661da0 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,6 @@ build-lib: deps .PHONY: lib lib: build-lib lib: pip-install -#lib: deps -# @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) -# @cmake --build build/$(SR_BUILD) -- -j -# @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran @@ -104,11 +100,10 @@ test-deps-gpu: SR_DEVICE=gpu test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) -# NOTE: Fortran tests cannot be built in parallel .PHONY: build-tests build-tests: test-lib-with-fortran @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests + @cmake --build build/$(SR_BUILD)/tests -- -j $(NPROC) # help: build-test-cpp - build the C++ tests @@ -135,7 +130,7 @@ build-test-c: test-lib .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/fortran + @cmake --build build/$(SR_BUILD)/tests/fortran -- -j $(NPROC) # help: build-examples - build all examples (serial, parallel) diff --git a/setup.cfg b/setup.cfg index ab617dff3..a1d6e5949 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ package_dir= =src/python/module packages=find: setup_requires = - setuptools>=39.2 + setuptools>=42 include_package_data = True install_requires = numpy>=1.18.2 diff --git a/setup.py b/setup.py index 9adad206a..a28a73fd6 100644 --- a/setup.py +++ b/setup.py @@ -88,25 +88,24 @@ def run(self): make_cmd = shutil.which("make") setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() - # build dependencies - subprocess.check_call([f"{make_cmd} deps"], + # build library and dependencies + subprocess.check_call([f"{make_cmd} lib"], cwd=setup_path, shell=True) - # run cmake prep step - print('-'*10, 'Running CMake prepare', '-'*40) - subprocess.check_call([self.cmake, setup_path] + cmake_args, - cwd=build_directory, - env=env) - - - print('-'*10, 'Building extensions', '-'*40) - cmake_cmd = [self.cmake, '--build', '.'] + self.build_args - subprocess.check_call(cmake_cmd, - cwd=build_directory) - - shutil.copytree(setup_path.joinpath("install"), - build_directory.joinpath("install")) +# # run cmake prep step +# print('-'*10, 'Running CMake prepare', '-'*40) +# subprocess.check_call([self.cmake, setup_path] + cmake_args, +# cwd=build_directory, +# env=env) +# +# +# print('-'*10, 'Building extensions', '-'*40) +# cmake_cmd = [self.cmake, '--build', '.'] + self.build_args +# subprocess.check_call(cmake_cmd, cwd=build_directory) +# +# shutil.copytree(setup_path.joinpath("install"), +# build_directory.joinpath("install")) # Move from build temp to final position for ext in self.extensions: @@ -125,7 +124,7 @@ def move_output(self, ext): # TODO: Check versions for compatible versions def check_prereq(command): try: - out = subprocess.check_output([command, '--version']) + _ = subprocess.check_output([command, '--version']) except OSError: raise RuntimeError( f"{command} must be installed to build SmartRedis") diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 2a45902ab..40eca9dc3 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -65,6 +65,9 @@ include_directories(SYSTEM ${SMARTREDIS_INSTALL_PATH}/include ) +# Stuff the test_utils into a library to enable parallel builds +add_library(test-utils STATIC test_utils.F90) + # Define all the tests to be built list(APPEND EXECUTABLES client_test_dataset_aggregation @@ -88,12 +91,11 @@ list(APPEND EXECUTABLES foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_fortran_test ${EXECUTABLE}.F90 - test_utils.F90 ) set_target_properties(${EXECUTABLE}_fortran_test PROPERTIES OUTPUT_NAME ${EXECUTABLE} ) target_link_libraries(${EXECUTABLE}_fortran_test - ${SMARTREDIS_LIBRARIES} + ${SMARTREDIS_LIBRARIES} test-utils ) endforeach() From 2d8ea5a56823af66630a3ee695356f575b10239d Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 12:25:23 -0500 Subject: [PATCH 34/50] Don't mess with setup.py --- .github/workflows/run_tests.yml | 12 ++++++------ setup.py | 31 ++++++++++++++++--------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d8fbcef26..13c85e475 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -121,12 +121,12 @@ jobs: run: python -m pip install -e .[dev,xarray] - name: Build and install dependencies - run: | - mkdir -p ./third-party && - cd ./third-party && - bash ../build-scripts/build-lcov.sh && - bash ../build-scripts/build-catch.sh - + run: make test-deps +# mkdir -p ./third-party && +# cd ./third-party && +# bash ../build-scripts/build-lcov.sh && +# bash ../build-scripts/build-catch.sh +# - name: Install docker, redis-server, and RedisAI run: | sudo apt-get -y update && sudo apt-get -y install curl gnupg lsb-release software-properties-common ca-certificates && \ diff --git a/setup.py b/setup.py index a28a73fd6..998cb30a7 100644 --- a/setup.py +++ b/setup.py @@ -88,24 +88,25 @@ def run(self): make_cmd = shutil.which("make") setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() - # build library and dependencies - subprocess.check_call([f"{make_cmd} lib"], + # build dependencies + subprocess.check_call([f"{make_cmd} deps"], cwd=setup_path, shell=True) -# # run cmake prep step -# print('-'*10, 'Running CMake prepare', '-'*40) -# subprocess.check_call([self.cmake, setup_path] + cmake_args, -# cwd=build_directory, -# env=env) -# -# -# print('-'*10, 'Building extensions', '-'*40) -# cmake_cmd = [self.cmake, '--build', '.'] + self.build_args -# subprocess.check_call(cmake_cmd, cwd=build_directory) -# -# shutil.copytree(setup_path.joinpath("install"), -# build_directory.joinpath("install")) + # run cmake prep step + print('-'*10, 'Running CMake prepare', '-'*40) + subprocess.check_call([self.cmake, setup_path] + cmake_args, + cwd=build_directory, + env=env) + + + print('-'*10, 'Building extensions', '-'*40) + cmake_cmd = [self.cmake, '--build', '.'] + self.build_args + subprocess.check_call(cmake_cmd, + cwd=build_directory) + + shutil.copytree(setup_path.joinpath("install"), + build_directory.joinpath("install")) # Move from build temp to final position for ext in self.extensions: From 4f43da7faa16b1dfb7415b340bdfc2eca5d89123 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 13:35:00 -0500 Subject: [PATCH 35/50] Revert back to working --- .github/workflows/run_tests.yml | 12 ++++++------ Makefile | 9 +++++++-- setup.cfg | 2 +- setup.py | 2 +- tests/fortran/CMakeLists.txt | 6 ++---- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 13c85e475..d8fbcef26 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -121,12 +121,12 @@ jobs: run: python -m pip install -e .[dev,xarray] - name: Build and install dependencies - run: make test-deps -# mkdir -p ./third-party && -# cd ./third-party && -# bash ../build-scripts/build-lcov.sh && -# bash ../build-scripts/build-catch.sh -# + run: | + mkdir -p ./third-party && + cd ./third-party && + bash ../build-scripts/build-lcov.sh && + bash ../build-scripts/build-catch.sh + - name: Install docker, redis-server, and RedisAI run: | sudo apt-get -y update && sudo apt-get -y install curl gnupg lsb-release software-properties-common ca-certificates && \ diff --git a/Makefile b/Makefile index be2661da0..40d557d62 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,10 @@ build-lib: deps .PHONY: lib lib: build-lib lib: pip-install +#lib: deps +# @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) +# @cmake --build build/$(SR_BUILD) -- -j +# @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran @@ -100,10 +104,11 @@ test-deps-gpu: SR_DEVICE=gpu test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) +# NOTE: Fortran tests cannot be built in parallel .PHONY: build-tests build-tests: test-lib-with-fortran @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests -- -j $(NPROC) + @cmake --build build/$(SR_BUILD)/tests # help: build-test-cpp - build the C++ tests @@ -130,7 +135,7 @@ build-test-c: test-lib .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/fortran -- -j $(NPROC) + @cmake --build build/$(SR_BUILD)/tests/fortran # help: build-examples - build all examples (serial, parallel) diff --git a/setup.cfg b/setup.cfg index a1d6e5949..ab617dff3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ package_dir= =src/python/module packages=find: setup_requires = - setuptools>=42 + setuptools>=39.2 include_package_data = True install_requires = numpy>=1.18.2 diff --git a/setup.py b/setup.py index 998cb30a7..9adad206a 100644 --- a/setup.py +++ b/setup.py @@ -125,7 +125,7 @@ def move_output(self, ext): # TODO: Check versions for compatible versions def check_prereq(command): try: - _ = subprocess.check_output([command, '--version']) + out = subprocess.check_output([command, '--version']) except OSError: raise RuntimeError( f"{command} must be installed to build SmartRedis") diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 40eca9dc3..2a45902ab 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -65,9 +65,6 @@ include_directories(SYSTEM ${SMARTREDIS_INSTALL_PATH}/include ) -# Stuff the test_utils into a library to enable parallel builds -add_library(test-utils STATIC test_utils.F90) - # Define all the tests to be built list(APPEND EXECUTABLES client_test_dataset_aggregation @@ -91,11 +88,12 @@ list(APPEND EXECUTABLES foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_fortran_test ${EXECUTABLE}.F90 + test_utils.F90 ) set_target_properties(${EXECUTABLE}_fortran_test PROPERTIES OUTPUT_NAME ${EXECUTABLE} ) target_link_libraries(${EXECUTABLE}_fortran_test - ${SMARTREDIS_LIBRARIES} test-utils + ${SMARTREDIS_LIBRARIES} ) endforeach() From 9ae2f0c1ab217088e21f7ecafc038c848d020367 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 13:57:28 -0500 Subject: [PATCH 36/50] Nuke dead code in Makefile --- Makefile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile b/Makefile index 40d557d62..0672be8eb 100644 --- a/Makefile +++ b/Makefile @@ -71,10 +71,6 @@ build-lib: deps .PHONY: lib lib: build-lib lib: pip-install -#lib: deps -# @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) -# @cmake --build build/$(SR_BUILD) -- -j -# @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran From c1b514bdd951c45a0f627cdfa82047eb22c4194d Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 14:18:24 -0500 Subject: [PATCH 37/50] Allow Fortran tests to be built in parallel --- Makefile | 5 ++--- tests/fortran/CMakeLists.txt | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0672be8eb..be2661da0 100644 --- a/Makefile +++ b/Makefile @@ -100,11 +100,10 @@ test-deps-gpu: SR_DEVICE=gpu test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) -# NOTE: Fortran tests cannot be built in parallel .PHONY: build-tests build-tests: test-lib-with-fortran @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests + @cmake --build build/$(SR_BUILD)/tests -- -j $(NPROC) # help: build-test-cpp - build the C++ tests @@ -131,7 +130,7 @@ build-test-c: test-lib .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) - @cmake --build build/$(SR_BUILD)/tests/fortran + @cmake --build build/$(SR_BUILD)/tests/fortran -- -j $(NPROC) # help: build-examples - build all examples (serial, parallel) diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index 2a45902ab..f6c47383f 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -65,6 +65,9 @@ include_directories(SYSTEM ${SMARTREDIS_INSTALL_PATH}/include ) +# Stuff the test_utils into a library to enable parallel builds +add_library(test-utils STATIC test_utils.F90) + # Define all the tests to be built list(APPEND EXECUTABLES client_test_dataset_aggregation @@ -94,6 +97,6 @@ foreach(EXECUTABLE ${EXECUTABLES}) OUTPUT_NAME ${EXECUTABLE} ) target_link_libraries(${EXECUTABLE}_fortran_test - ${SMARTREDIS_LIBRARIES} + ${SMARTREDIS_LIBRARIES} test-utils ) endforeach() From 1b6d8d62c4279e88a0758d76c0d3c12b1d7123f2 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 14:38:02 -0500 Subject: [PATCH 38/50] Test --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ab617dff3..a1d6e5949 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ package_dir= =src/python/module packages=find: setup_requires = - setuptools>=39.2 + setuptools>=42 include_package_data = True install_requires = numpy>=1.18.2 From 29379eedd72ad627eabbbff0cbe9213c74bd3d1a Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 14:53:39 -0500 Subject: [PATCH 39/50] Remove test tuils.F90 from Fortran test builds since we have it in the static library now --- doc/installation.rst | 4 ++-- tests/fortran/CMakeLists.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/installation.rst b/doc/installation.rst index 4bfab0e2b..73e665896 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -31,10 +31,9 @@ SmartRedis is tested with the following compilers on a daily basis: * - Compilers (tested daily) * - GNU (GCC/GFortran) - * - Intel (icc/icpc/ifort) + * - Intel LLVM (icx/icpx/ifort) * - Apple Clang - SmartRedis has been tested with the following compiler in the past, but on a less regular basis as the compilers listed above: .. list-table:: @@ -44,6 +43,7 @@ SmartRedis has been tested with the following compiler in the past, but on a les * - Compilers (irregularly tested in the past) * - Cray Clang + * - Intel Classic (icc/icpc) SmartRedis has been used with the following compilers in the past, but they have not been tested. We do not imply that these compilers work for certain: diff --git a/tests/fortran/CMakeLists.txt b/tests/fortran/CMakeLists.txt index f6c47383f..40eca9dc3 100644 --- a/tests/fortran/CMakeLists.txt +++ b/tests/fortran/CMakeLists.txt @@ -91,7 +91,6 @@ list(APPEND EXECUTABLES foreach(EXECUTABLE ${EXECUTABLES}) add_executable(${EXECUTABLE}_fortran_test ${EXECUTABLE}.F90 - test_utils.F90 ) set_target_properties(${EXECUTABLE}_fortran_test PROPERTIES OUTPUT_NAME ${EXECUTABLE} From 2b509e87a3049992129e0e76b72a164e0defd4d4 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 15:49:21 -0500 Subject: [PATCH 40/50] Docs update --- doc/install/lib.rst | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/doc/install/lib.rst b/doc/install/lib.rst index 4b480dc6f..d02f0fd08 100644 --- a/doc/install/lib.rst +++ b/doc/install/lib.rst @@ -25,6 +25,58 @@ installed in ``SmartRedis/install/include/``. The library installation can be used to easily include SmartRedis capabilities in C++, C, and Fortran applications. +Customizing the library build +----------------------------- + +By default, the SmartRedis library is built as a shared library. For some +applications, however, it is preferable to link to a statically compiled +library. This can be done easily with the command: + +.. code-block:: bash + + cd SmartRedis + # Static build + make lib SR_LINK=Static + # Release build + make lib SR_LINK=Shared #or skip the SR_LINK variable as this is the default + +Linked statically, the SmartRedis library will have a ``.a`` file extension. When +linked dynamically, the SmartRedis library will have a ``.so`` file extension. + +It is also possible to adjust compilation settings for the SmartRedis libary. +By default, the library compiles in an optimized build (Release), but debug builds +with full symbols (Debug) can be created as can debug builds with extensions enabled +for code coverage metrics (Coverage; this build type is only available with GNU +compilers). Similar to configuring a link type, selecting the build mode can be done +via a variable supplied to make: + +.. code-block:: bash + + cd SmartRedis + # Release build + make lib SR_BUILD=Release #or skip the SR_BUILD variable as this is the default + # Debug build + make lib SR_BUILD=Debug + # Code coverage build + make lib SR_BUILD=Coverage + +The name of the library produced for a Debug mode build is ``smartredis-debug``. +The name of the library produced for a Coverage mode build is ``smartredis-debug``. +The name of the library produced for a Release mode build is ``smartredis``. +In each case, the file extension is dependent on the link type, ``.so`` or ``.a``. + +The build mode and link type settings are fully orthogonal; any combination of the +two is supported. For example, a statically linked debug build may be achieved via +the following command: + +.. code-block:: bash + + cd SmartRedis + make lib SR_LINK=Static SR_BUILD=Debug + +The SR_LINK and SR_BUILD variables are supported for all test and build targets in +the Makefile. + Linking instructions using compiler flags ----------------------------------------- From e135d45e1363d21decda7d5449915d144f094cb4 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 16:33:41 -0500 Subject: [PATCH 41/50] Doc update --- doc/install/lib.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/install/lib.rst b/doc/install/lib.rst index d02f0fd08..27a1ae1fc 100644 --- a/doc/install/lib.rst +++ b/doc/install/lib.rst @@ -44,9 +44,9 @@ Linked statically, the SmartRedis library will have a ``.a`` file extension. Wh linked dynamically, the SmartRedis library will have a ``.so`` file extension. It is also possible to adjust compilation settings for the SmartRedis libary. -By default, the library compiles in an optimized build (Release), but debug builds -with full symbols (Debug) can be created as can debug builds with extensions enabled -for code coverage metrics (Coverage; this build type is only available with GNU +By default, the library compiles in an optimized build (``Release``), but debug builds +with full symbols (``Debug``) can be created as can debug builds with extensions enabled +for code coverage metrics (``Coverage``; this build type is only available with GNU compilers). Similar to configuring a link type, selecting the build mode can be done via a variable supplied to make: @@ -74,8 +74,8 @@ the following command: cd SmartRedis make lib SR_LINK=Static SR_BUILD=Debug -The SR_LINK and SR_BUILD variables are supported for all test and build targets in -the Makefile. +The SR_LINK and SR_BUILD variables are fully supported for all test and build targets +in the Makefile. Linking instructions using compiler flags ----------------------------------------- From 931338f816c37dab683d823a22be31e00025e2a5 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Tue, 23 May 2023 16:38:50 -0500 Subject: [PATCH 42/50] stop using build scripts in .yml file --- .github/workflows/run_tests.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d8fbcef26..f94a3104f 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -120,12 +120,8 @@ jobs: - name: Build SmartRedis python and install run: python -m pip install -e .[dev,xarray] - - name: Build and install dependencies - run: | - mkdir -p ./third-party && - cd ./third-party && - bash ../build-scripts/build-lcov.sh && - bash ../build-scripts/build-catch.sh + - name: Build and install test dependencies + run: make lcov && make catch2 - name: Install docker, redis-server, and RedisAI run: | From dfe812f348bcfac9c778c78fbefbff65dd7a3872 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 31 May 2023 15:36:23 -0500 Subject: [PATCH 43/50] Address most feedback received so far --- .github/workflows/run_post_merge_tests.yml | 2 +- .github/workflows/run_tests.yml | 2 +- CMakeLists.txt | 4 +- Makefile | 129 +++++++++++++++------ doc/install/lib.rst | 20 +++- doc/installation.rst | 2 +- setup.py | 1 - 7 files changed, 113 insertions(+), 47 deletions(-) diff --git a/.github/workflows/run_post_merge_tests.yml b/.github/workflows/run_post_merge_tests.yml index 549a61c20..30f58a7d2 100644 --- a/.github/workflows/run_post_merge_tests.yml +++ b/.github/workflows/run_post_merge_tests.yml @@ -170,5 +170,5 @@ jobs: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/install/lib && export SR_LOG_FILE=smartredis_examples_log.txt && export SR_LOG_LEVEL=INFO && - make test-examples + make test-examples SR_FORTRAN=ON SR_PYTHON=ON diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index f94a3104f..e45fe9a0b 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -158,7 +158,7 @@ jobs: popd && export SSDB=$SSDB_SINGLE SMARTREDIS_TEST_CLUSTER=False && utils/check_redis.sh $PORT && - make test-verbose-with-coverage COV_FLAGS="--cov=./src/python/module/smartredis/ --cov-report=xml --cov-append" + make test-verbose-with-coverage COV_FLAGS="--cov=./src/python/module/smartredis/ --cov-report=xml --cov-append" SR_FORTRAN=ON SR_PYTHON=ON - name: Clustered DB tests run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index fae65e972..ca88a4d44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,7 @@ cmake_minimum_required(VERSION 3.13) project(SmartRedis VERSION "0.4.0") # Configure options for the SmartRedis project -option(BUILD_PYTHON "Build the python module" ON) +option(SR_PYTHON "Build the python module" OFF) option(SR_FORTRAN "Build the fortran client library" OFF) option(SR_PEDANTIC "Build with pickiest compiler settings" OFF) @@ -175,7 +175,7 @@ install(TARGETS smartredis LIBRARY DESTINATION lib) # Build the Python library for SmartRedis -if(BUILD_PYTHON) +if(SR_PYTHON) message("-- Python client build enabled") add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind ${CMAKE_SOURCE_DIR}/third-party/pybind/build) diff --git a/Makefile b/Makefile index be2661da0..1a2c53031 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,48 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# General settings MAKEFLAGS += --no-print-directory -COV_FLAGS := SHELL:=/bin/bash -NPROC:=$(shell python -c "import multiprocessing as mp; print(mp.cpu_count())") # Build variables +NPROC := $(shell nproc 2>/dev/null || python -c "import multiprocessing as mp; print (mp.cpu_count())" 2>/dev/null || echo 4) SR_BUILD := Release SR_LINK := Shared -SR_TEST_REDIS_MODE := Clustered -SR_TEST_RAI_VER := 1.2.7 -SR_WLM := Local -SR_WLM_FLAGS := -SR_DEVICE := cpu SR_PEDANTIC := OFF SR_FORTRAN := OFF -SR_PIPINSTALL := ON +SR_PYTHON := ON + +# Test variables +COV_FLAGS := +SR_TEST_REDIS_MODE := Clustered +SR_TEST_RAI_VER := 1.2.7 +SR_TEST_WLM := Local +SR_TEST_WLM_FLAGS := +SR_TEST_DEVICE := cpu # Params for third-party software HIREDIS_URL := https://github.com/redis/hiredis.git @@ -43,8 +72,44 @@ help: @grep "^# help\:" Makefile | grep -v grep | sed 's/\# help\: //' | sed 's/\# help\://' # help: -# help: Build -# help: ------- +# help: Build variables +# help: --------------- +# help: +# help: These variables affect the way that the SmartRedis library is built. Each +# help: has several options; the first listed is the default. Use by appending +# help: the variable name and setting after the make target, e.g. +# help: make lib SR_BUILD=Debug SR_LINK=Static SR_FORTRAN=ON +# help: +# help: SR_BUILD {Release, Debug, Coverage} -- optimization level for the build +# help: SR_LINK {Shared, Static} -- linkage for the SmartRedis library +# help: SR_PEDANTIC {OFF, ON} -- GNU only; enable pickiest compiler settings +# help: SR_FORTRAN {OFF, ON} -- Enable/disable build of Fortran library +# help: SR_PYTHON {OFF, ON} -- Enable/disable build of Python library + +# (NOTE: Test variables are not yet implemented. The following interface +# is planned for an upcoming version of the SmartRedis test system.) +# halp: +# halp: Test variables +# halp: -------------- +# halp: +# halp: These variables affect the way that the SmartRedis library is tested. Each +# halp: has several options; the first listed is the default. Use by appending +# halp: the variable name and setting after the make target, e.g. +# halp: make test SR_BUILD=Debug SR_LINK=Static SR_FORTRAN=ON +# halp: +# halp: SR_TEST_REDIS_MODE {Clustered, Standalone, Colocated} -- type of Redis backend launched for tests +# halp: SR_TEST_RAI_VER {1.2.7, 1.2.5} -- version of RedisAI to use for tests +# halp: SR_TEST_WLM {Local, Slurm, PBS} -- workload manager to use for launching tests +# halp: SR_TEST_WLM_ALLOC_COMMAND {default is none} -- command to use to request an allocation +# halp: SR_TEST_WLM_ALLOC_FLAGS {default is none} -- flags to use when requesting an allocation from the WLM +# halp: SR_TEST_WLM_LAUNCH_COMMAND {default is none} -- command to use to launch a test process +# halp: SR_TEST_WLM_LAUNCH_FLAGS {default is none} -- flags to use when launching a test process +# halp: SR_TEST_DEVICE {cpu, gpu} -- device type to test on + + +# help: +# help: Build targets +# help: ------------- # help: deps - Make SmartRedis dependencies .PHONY: deps @@ -53,24 +118,13 @@ deps: redis-plus-plus deps: pybind deps: -# help: pip-install - Register the SmartRedis library with pip -.PHONY: pip-install -pip-install: -ifeq ($(SR_PIPINSTALL),ON) - @python -c "import smartredis" >& /dev/null || pip install -e. -endif - -# build-lib: hidden make target to build the library (needed so we can pip install AFTER building) -.PHONY: build-lib -build-lib: deps - @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) - @cmake --build build/$(SR_BUILD) -- -j $(NPROC) - @cmake --install build/$(SR_BUILD) - # help: lib - Build SmartRedis C/C++/Python clients into a dynamic library .PHONY: lib -lib: build-lib -lib: pip-install +lib: deps +lib: + @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) -DSR_PYTHON=$(SR_PYTHON) + @cmake --build build/$(SR_BUILD) -- -j $(NPROC) + @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran @@ -96,7 +150,7 @@ test-deps: lcov # help: test-deps-gpu - Make SmartRedis GPU testing dependencies .PHONY: test-deps-gpu -test-deps-gpu: SR_DEVICE=gpu +test-deps-gpu: SR_TEST_DEVICE=gpu test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) @@ -126,7 +180,6 @@ build-test-c: test-lib # help: build-test-fortran - build the Fortran tests -# NOTE: Fortran tests cannot be built in parallel .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) @@ -172,8 +225,8 @@ clobber: clean clean-deps # help: -# help: Style -# help: ------- +# help: Style targets +# help: ------------- # help: style - Sort imports and format with black .PHONY: style @@ -216,8 +269,8 @@ check-lint: # help: -# help: Documentation -# help: ------- +# help: Documentation targets +# help: --------------------- # help: docs - generate project documentation .PHONY: docs @@ -231,8 +284,8 @@ cov: @echo if data was present, coverage report is in htmlcov/index.html # help: -# help: Test -# help: ------- +# help: Test targets +# help: ------------ # help: test - Build and run all tests (C, C++, Fortran, Python) .PHONY: test @@ -318,7 +371,7 @@ install/lib/libhiredis.a: @cd third-party && \ git clone $(HIREDIS_URL) hiredis --branch $(HIREDIS_VER) --depth=1 @cd third-party/hiredis && \ - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" static -j && \ + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" static -j $(NPROC) && \ LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" install && \ rm -f ../../install/lib/libhiredis*.so && \ rm -f ../../install/lib/libhiredis*.dylib && \ @@ -365,7 +418,7 @@ third-party/redis/src/redis-server: # checks cuda dependencies for GPU build .phony: cudann-check cudann-check: -ifeq ($(SR_DEVICE),gpu) +ifeq ($(SR_TEST_DEVICE),gpu) ifndef CUDA_HOME $(error ERROR: CUDA_HOME is not set) endif @@ -388,12 +441,12 @@ endif redisAI: cudann-check redisAI: third-party/RedisAI/install-cpu/redisai.so third-party/RedisAI/install-cpu/redisai.so: - $(eval DEVICE_IS_GPU := $(shell test $(SR_DEVICE) == "cpu"; echo $$?)) + $(eval DEVICE_IS_GPU := $(shell test $(SR_TEST_DEVICE) == "cpu"; echo $$?)) @mkdir -p third-party @cd third-party && \ GIT_LFS_SKIP_SMUDGE=1 git clone --recursive $(REDISAI_URL) RedisAI --branch $(REDISAI_VER) --depth=1 -@cd third-party/RedisAI && \ - CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh $(SR_DEVICE) && \ + CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh $(SR_TEST_DEVICE) && \ CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j $(NPROC) -C opt clean build && \ echo "Finished installing RedisAI" diff --git a/doc/install/lib.rst b/doc/install/lib.rst index 27a1ae1fc..287c22606 100644 --- a/doc/install/lib.rst +++ b/doc/install/lib.rst @@ -65,9 +65,16 @@ The name of the library produced for a Coverage mode build is ``smartredis-debug The name of the library produced for a Release mode build is ``smartredis``. In each case, the file extension is dependent on the link type, ``.so`` or ``.a``. -The build mode and link type settings are fully orthogonal; any combination of the -two is supported. For example, a statically linked debug build may be achieved via -the following command: +Finally, it is possible to build SmartRedis without including Python support: + +.. code-block:: bash + + cd SmartRedis + make lib SR_PYTHON=OFF + +The build mode, link type, and Python support settings are fully orthogonal; any +combination of them is supported. For example, a statically linked debug build +may be achieved via the following command: .. code-block:: bash @@ -77,6 +84,13 @@ the following command: The SR_LINK and SR_BUILD variables are fully supported for all test and build targets in the Makefile. +Additional make variables may be seen in the ``help`` make target: + +.. code-block:: bash + + cd SmartRedis + make help + Linking instructions using compiler flags ----------------------------------------- diff --git a/doc/installation.rst b/doc/installation.rst index 73e665896..5fea9b190 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -31,7 +31,7 @@ SmartRedis is tested with the following compilers on a daily basis: * - Compilers (tested daily) * - GNU (GCC/GFortran) - * - Intel LLVM (icx/icpx/ifort) + * - Intel oneAPI (icx/icpx/ifort) * - Apple Clang SmartRedis has been tested with the following compiler in the past, but on a less regular basis as the compilers listed above: diff --git a/setup.py b/setup.py index 9adad206a..959580727 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,6 @@ def run(self): print('-'*10, 'Building C dependencies', '-'*40) make_cmd = shutil.which("make") - setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() # build dependencies subprocess.check_call([f"{make_cmd} deps"], From 34e5242962db598ace15d2ad5953dac28c77f455 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 31 May 2023 15:39:18 -0500 Subject: [PATCH 44/50] Enable python for python installation builds --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 959580727..477c86ee9 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def run(self): build_directory = Path(self.build_temp).resolve() cmake_args = [ '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(build_directory), - '-DPYTHON_EXECUTABLE=' + sys.executable + '-DPYTHON_EXECUTABLE=' + sys.executable, + '-DSR_PYTHON=ON' ] cfg = 'Debug' if self.debug else 'Release' From 79eabab9275167d84ac88c4b2f68eaba51c0bbde Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 31 May 2023 17:47:59 -0500 Subject: [PATCH 45/50] Address remaining feedback --- Makefile | 68 +++++++++++++++++++++++--------- conftest.py | 6 +++ examples/CMakeLists.txt | 4 +- examples/parallel/CMakeLists.txt | 8 +++- examples/serial/CMakeLists.txt | 8 +++- examples/test_examples.py | 29 ++++++++------ tests/CMakeLists.txt | 8 +++- 7 files changed, 92 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 1a2c53031..96a5fb01b 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ SR_BUILD := Release SR_LINK := Shared SR_PEDANTIC := OFF SR_FORTRAN := OFF -SR_PYTHON := ON +SR_PYTHON := OFF # Test variables COV_FLAGS := @@ -122,7 +122,8 @@ deps: .PHONY: lib lib: deps lib: - @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) -DSR_PYTHON=$(SR_PYTHON) + @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) \ + -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) -DSR_PYTHON=$(SR_PYTHON) @cmake --build build/$(SR_BUILD) -- -j $(NPROC) @cmake --install build/$(SR_BUILD) @@ -156,54 +157,62 @@ test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) .PHONY: build-tests build-tests: test-lib-with-fortran - @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/tests -- -j $(NPROC) # help: build-test-cpp - build the C++ tests .PHONY: build-test-cpp build-test-cpp: test-lib - @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/cpp -- -j $(NPROC) # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib - @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests \ + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests -- -j $(NPROC) # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib - @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/c -- -j $(NPROC) # help: build-test-fortran - build the Fortran tests .PHONY: build-test-fortran build-test-fortran: test-lib-with-fortran - @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/fortran -- -j $(NPROC) # help: build-examples - build all examples (serial, parallel) .PHONY: build-examples build-examples: lib-with-fortran - @cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples # help: build-example-serial - buld serial examples .PHONY: build-example-serial build-example-serial: lib-with-fortran - @cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial \ + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples/serial # help: build-example-parallel - build parallel examples (requires MPI) .PHONY: build-example-parallel build-example-parallel: lib-with-fortran - @cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel \ + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples/parellel @@ -287,12 +296,20 @@ cov: # help: Test targets # help: ------------ +ifeq ($(SR_PYTHON),OFF) +SKIP_PYTHON = --ignore ./tests/python +endif +ifeq ($(SR_FORTRAN),OFF) +SKIP_FORTRAN = --ignore ./tests/fortran +endif + # help: test - Build and run all tests (C, C++, Fortran, Python) .PHONY: test test: test-deps test: build-tests test: - @PYTHONFAULTHANDLER=1 python -m pytest --ignore ./tests/docker -vv ./tests --build $(SR_BUILD) + @PYTHONFAULTHANDLER=1 python -m pytest --ignore ./tests/docker \ + $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv ./tests --build $(SR_BUILD) # help: test-verbose - Build and run all tests [verbosely] @@ -300,7 +317,8 @@ test: test-verbose: test-deps test-verbose: build-tests test-verbose: - @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) + @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker \ + $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv -s ./tests --build $(SR_BUILD) # help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] .PHONY: test-verbose-with-coverage @@ -308,7 +326,8 @@ test-verbose-with-coverage: SR_BUILD=Coverage test-verbose-with-coverage: test-deps test-verbose-with-coverage: build-tests test-verbose-with-coverage: - @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests --build $(SR_BUILD) + @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker \ + $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv -s ./tests --build $(SR_BUILD) # help: test-c - Build and run all C tests .PHONY: test-c @@ -335,6 +354,7 @@ unit-test-cpp: # help: test-py - run python tests .PHONY: test-py test-py: test-deps +test-py: SR_PYTHON=ON test-py: lib test-py: @PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/ --build $(SR_BUILD) @@ -348,15 +368,17 @@ test-fortran: build-test-fortran # help: testpy-cov - run python tests with coverage .PHONY: testpy-cov testpy-cov: test-deps +testpy-cov: SR_PYTHON=ON testpy-cov: - @PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ -vv ./tests/python/ --build $(SR_BUILD) + @PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ \ + -vv ./tests/python/ --build $(SR_BUILD) # help: test-examples - Build and run all examples .PHONY: test-examples test-examples: test-deps test-examples: build-examples test-examples: - @python -m pytest -vv -s ./examples --build $(SR_BUILD) + @python -m pytest -vv -s ./examples --build $(SR_BUILD) --sr_fortran $(SR_FORTRAN) ############################################################################ @@ -388,9 +410,13 @@ install/lib/libredis++.a: @cd third-party/redis-plus-plus && \ mkdir -p compile && \ cd compile && \ - cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ + cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF \ + -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" \ + -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ CC=gcc CXX=g++ make -j $(NPROC) && \ CC=gcc CXX=g++ make install && \ + cd ../../../ && \ + cp install/lib64/libredis++.a install/lib 2&> /dev/null || true && \ echo "Finished installing Redis-plus-plus" # Pybind11 (hidden build target) @@ -444,10 +470,14 @@ third-party/RedisAI/install-cpu/redisai.so: $(eval DEVICE_IS_GPU := $(shell test $(SR_TEST_DEVICE) == "cpu"; echo $$?)) @mkdir -p third-party @cd third-party && \ - GIT_LFS_SKIP_SMUDGE=1 git clone --recursive $(REDISAI_URL) RedisAI --branch $(REDISAI_VER) --depth=1 + rm -rf RedisAI && \ + GIT_LFS_SKIP_SMUDGE=1 git clone --recursive $(REDISAI_URL) RedisAI \ + --branch $(REDISAI_VER) --depth=1 -@cd third-party/RedisAI && \ - CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh $(SR_TEST_DEVICE) && \ - CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j $(NPROC) -C opt clean build && \ + CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh \ + $(SR_TEST_DEVICE) && \ + CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 \ + WITH_UNIT_TESTS=0 make -j $(NPROC) -C opt clean build && \ echo "Finished installing RedisAI" # Catch2 (hidden test target) diff --git a/conftest.py b/conftest.py index b272145e9..44cc351a8 100644 --- a/conftest.py +++ b/conftest.py @@ -140,8 +140,14 @@ def create_torch_cnn(filepath=None): # Add a build type option to pytest command lines def pytest_addoption(parser): parser.addoption("--build", action="store", default="Release") + parser.addoption("--sr_fortran", action="store", default="OFF") # Fixture to retrieve the build type setting @pytest.fixture(scope="session") def build(request): return request.config.getoption("--build") + +# Fixture to retrieve the Fortran enablement setting +@pytest.fixture(scope="session") +def sr_fortran(request): + return request.config.getoption("--sr_fortran") diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f17d0cf50..0d58cd50e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -31,7 +31,9 @@ project(SmartRedis-Examples) # Enable language support for the examples enable_language(C) enable_language(CXX) -enable_language(Fortran) +if (SR_FORTRAN) + enable_language(Fortran) +endif() # Bring in subdirectories add_subdirectory(parallel) diff --git a/examples/parallel/CMakeLists.txt b/examples/parallel/CMakeLists.txt index 07f6c0d40..293129686 100644 --- a/examples/parallel/CMakeLists.txt +++ b/examples/parallel/CMakeLists.txt @@ -30,8 +30,12 @@ project(SmartRedis-Examples-Parallel) # Enable language support for the examples enable_language(CXX) -enable_language(Fortran) +if (SR_FORTRAN) + enable_language(Fortran) +endif() # Bring in subdirectories add_subdirectory(cpp) -add_subdirectory(fortran) +if (SR_FORTRAN) + add_subdirectory(fortran) +endif() diff --git a/examples/serial/CMakeLists.txt b/examples/serial/CMakeLists.txt index aa0de409c..6e62c20a8 100644 --- a/examples/serial/CMakeLists.txt +++ b/examples/serial/CMakeLists.txt @@ -31,9 +31,13 @@ project(SmartRedis-Examples-Serial) # Enable language support for the examples enable_language(C) enable_language(CXX) -enable_language(Fortran) +if (SR_FORTRAN) + enable_language(Fortran) +endif() # Bring in subdirectories add_subdirectory(c) add_subdirectory(cpp) -add_subdirectory(fortran) +if (SR_FORTRAN) + add_subdirectory(fortran) +endif() diff --git a/examples/test_examples.py b/examples/test_examples.py index b155a8265..0755a20b1 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -48,19 +48,22 @@ def get_test_names(): return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_example(test, use_cluster, build): - # Build the path to the test executable from the source file name - # . keep only the last three parts of the path: (parallel/serial, language, basename) - test = "/".join(test.split("/")[-3:]) - # . drop the file extension - test = ".".join(test.split(".")[:-1]) - # . prepend the path to the built test executable - test = f"{getcwd()}/build/{build}/examples/{test}" - cmd = [test] - print(f"Running test: {osp.basename(test)}") - print(f"Using cluster: {use_cluster}") - execute_cmd(cmd) - time.sleep(1) +def test_example(test, use_cluster, build, sr_fortran): + if (sr_fortran == "ON" or ".F90" not in test): + # Build the path to the test executable from the source file name + # . keep only the last three parts of the path: (parallel/serial, language, basename) + test = "/".join(test.split("/")[-3:]) + # . drop the file extension + test = ".".join(test.split(".")[:-1]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/examples/{test}" + cmd = [test] + print(f"Running test: {osp.basename(test)}") + print(f"Using cluster: {use_cluster}") + execute_cmd(cmd) + time.sleep(1) + else: + print (f"Skipping Fortran test {test}") def find_path(executable_path): if executable_path.find('/') == -1: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 792f31906..771d3950d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -31,12 +31,16 @@ project(SmartSim-Tests) # Enable language support for the examples enable_language(C) enable_language(CXX) -enable_language(Fortran) +if (SR_FORTRAN) + enable_language(Fortran) +endif() # Bring in subdirectories add_subdirectory(c) add_subdirectory(cpp) -add_subdirectory(fortran) +if (SR_FORTRAN) + add_subdirectory(fortran) +endif() # NOTE: The docker subdirectory is designed to be built within a container, # not as part of the normal test build. We therefore do not include it in a From 97f1a80d70d714cfa21e7d9a65d84bd72b97719c Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 31 May 2023 20:56:40 -0500 Subject: [PATCH 46/50] Make SR_BUILD and SR_LINK case insensitive --- Makefile | 2 +- smartredis_defs.cmake | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 96a5fb01b..92cc81a73 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ help: # halp: SR_TEST_WLM_ALLOC_FLAGS {default is none} -- flags to use when requesting an allocation from the WLM # halp: SR_TEST_WLM_LAUNCH_COMMAND {default is none} -- command to use to launch a test process # halp: SR_TEST_WLM_LAUNCH_FLAGS {default is none} -- flags to use when launching a test process -# halp: SR_TEST_DEVICE {cpu, gpu} -- device type to test on +# halp: SR_TEST_DEVICE {cpu, gpu} -- device type to test on. Warning, this variable is CASE SENSITIVE! # help: diff --git a/smartredis_defs.cmake b/smartredis_defs.cmake index df9607b32..e8d0ada6a 100644 --- a/smartredis_defs.cmake +++ b/smartredis_defs.cmake @@ -33,13 +33,14 @@ if(NOT DEFINED SR_LINK) endif() # Configure the CMake build based on the SR_BUILD selection -if(SR_BUILD STREQUAL "Release") +string(TOLOWER "${SR_BUILD}" srbuild_lowercase) +if(srbuild_lowercase STREQUAL "release") set(CMAKE_BUILD_TYPE RELEASE) set(SRLIB_NAME_SUFFIX "") -elseif(SR_BUILD STREQUAL "Debug") +elseif(srbuild_lowercase STREQUAL "debug") set(CMAKE_BUILD_TYPE DEBUG) set(SRLIB_NAME_SUFFIX "-debug") -elseif(SR_BUILD STREQUAL "Coverage") +elseif(srbuild_lowercase STREQUAL "coverage") set(CMAKE_BUILD_TYPE DEBUG) set(SRLIB_NAME_SUFFIX "-coverage") if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) @@ -53,11 +54,12 @@ else() endif() # Configure CMake linkage on the SR_LINK selection -if(SR_LINK STREQUAL "Static") +string(TOLOWER "${SR_LINK}" srlink_lowercase) +if(srlink_lowercase STREQUAL "static") set(SMARTREDIS_LINK_MODE STATIC) set(SMARTREDIS_LINK_LIBRARY_SUFFIX .a) set(CMAKE_POSITION_INDEPENDENT_CODE ON) -elseif(SR_LINK STREQUAL "Shared") +elseif(srlink_lowercase STREQUAL "shared") set(SMARTREDIS_LINK_MODE SHARED) set(SMARTREDIS_LINK_LIBRARY_SUFFIX .so) else() From b5b0d4fdf529d9c2aee2c1a11b9a9de973d870ae Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 31 May 2023 21:03:34 -0500 Subject: [PATCH 47/50] Don't build Fortran library unless needed --- Makefile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 92cc81a73..90acefc3b 100644 --- a/Makefile +++ b/Makefile @@ -156,9 +156,9 @@ test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) .PHONY: build-tests -build-tests: test-lib-with-fortran +build-tests: test-lib @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) \ - -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/tests -- -j $(NPROC) @@ -166,37 +166,38 @@ build-tests: test-lib-with-fortran .PHONY: build-test-cpp build-test-cpp: test-lib @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) \ - -DSR_LINK=$(SR_LINK) + -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/cpp -- -j $(NPROC) # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests \ - -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests -- -j $(NPROC) # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) \ - -DSR_LINK=$(SR_LINK) + -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/c -- -j $(NPROC) # help: build-test-fortran - build the Fortran tests .PHONY: build-test-fortran -build-test-fortran: test-lib-with-fortran +build-test-fortran: SR_FORTRAN=ON +build-test-fortran: test-lib @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) \ - -DSR_LINK=$(SR_LINK) + -DSR_LINK=$(SR_LINK) @cmake --build build/$(SR_BUILD)/tests/fortran -- -j $(NPROC) # help: build-examples - build all examples (serial, parallel) .PHONY: build-examples -build-examples: lib-with-fortran +build-examples: lib @cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) \ - -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples @@ -204,7 +205,7 @@ build-examples: lib-with-fortran .PHONY: build-example-serial build-example-serial: lib-with-fortran @cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial \ - -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples/serial @@ -212,7 +213,7 @@ build-example-serial: lib-with-fortran .PHONY: build-example-parallel build-example-parallel: lib-with-fortran @cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel \ - -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples/parellel From c3f236f2b74156158144a5f1b87a07cee14f3f4c Mon Sep 17 00:00:00 2001 From: billschereriii Date: Wed, 31 May 2023 21:10:19 -0500 Subject: [PATCH 48/50] Cleaner solution for Redis++ --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 90acefc3b..e695073a7 100644 --- a/Makefile +++ b/Makefile @@ -203,7 +203,7 @@ build-examples: lib # help: build-example-serial - buld serial examples .PHONY: build-example-serial -build-example-serial: lib-with-fortran +build-example-serial: lib @cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial \ -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples/serial @@ -211,7 +211,7 @@ build-example-serial: lib-with-fortran # help: build-example-parallel - build parallel examples (requires MPI) .PHONY: build-example-parallel -build-example-parallel: lib-with-fortran +build-example-parallel: lib @cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel \ -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) @cmake --build build/$(SR_BUILD)/examples/parellel @@ -413,11 +413,10 @@ install/lib/libredis++.a: cd compile && \ cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF \ -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" \ - -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ + -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_LIBDIR="lib" .. && \ CC=gcc CXX=g++ make -j $(NPROC) && \ CC=gcc CXX=g++ make install && \ - cd ../../../ && \ - cp install/lib64/libredis++.a install/lib 2&> /dev/null || true && \ echo "Finished installing Redis-plus-plus" # Pybind11 (hidden build target) From f6ba3270b66385eababaaadf9f58ac0b06bf2d1b Mon Sep 17 00:00:00 2001 From: billschereriii Date: Thu, 1 Jun 2023 10:00:21 -0500 Subject: [PATCH 49/50] Address MattE feedback --- Makefile | 27 +-------------------------- doc/changelog.rst | 4 ++-- doc/install/lib.rst | 35 +++++++++++++++++++++-------------- images/Dockerfile | 2 +- 4 files changed, 25 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index e695073a7..1aaee0e89 100644 --- a/Makefile +++ b/Makefile @@ -38,10 +38,6 @@ SR_PYTHON := OFF # Test variables COV_FLAGS := -SR_TEST_REDIS_MODE := Clustered -SR_TEST_RAI_VER := 1.2.7 -SR_TEST_WLM := Local -SR_TEST_WLM_FLAGS := SR_TEST_DEVICE := cpu # Params for third-party software @@ -86,27 +82,6 @@ help: # help: SR_FORTRAN {OFF, ON} -- Enable/disable build of Fortran library # help: SR_PYTHON {OFF, ON} -- Enable/disable build of Python library -# (NOTE: Test variables are not yet implemented. The following interface -# is planned for an upcoming version of the SmartRedis test system.) -# halp: -# halp: Test variables -# halp: -------------- -# halp: -# halp: These variables affect the way that the SmartRedis library is tested. Each -# halp: has several options; the first listed is the default. Use by appending -# halp: the variable name and setting after the make target, e.g. -# halp: make test SR_BUILD=Debug SR_LINK=Static SR_FORTRAN=ON -# halp: -# halp: SR_TEST_REDIS_MODE {Clustered, Standalone, Colocated} -- type of Redis backend launched for tests -# halp: SR_TEST_RAI_VER {1.2.7, 1.2.5} -- version of RedisAI to use for tests -# halp: SR_TEST_WLM {Local, Slurm, PBS} -- workload manager to use for launching tests -# halp: SR_TEST_WLM_ALLOC_COMMAND {default is none} -- command to use to request an allocation -# halp: SR_TEST_WLM_ALLOC_FLAGS {default is none} -- flags to use when requesting an allocation from the WLM -# halp: SR_TEST_WLM_LAUNCH_COMMAND {default is none} -- command to use to launch a test process -# halp: SR_TEST_WLM_LAUNCH_FLAGS {default is none} -- flags to use when launching a test process -# halp: SR_TEST_DEVICE {cpu, gpu} -- device type to test on. Warning, this variable is CASE SENSITIVE! - - # help: # help: Build targets # help: ------------- @@ -321,7 +296,7 @@ test-verbose: @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker \ $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv -s ./tests --build $(SR_BUILD) -# help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] +# help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] .PHONY: test-verbose-with-coverage test-verbose-with-coverage: SR_BUILD=Coverage test-verbose-with-coverage: test-deps diff --git a/doc/changelog.rst b/doc/changelog.rst index f0b8036c4..3e2c6e9c5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,7 +8,7 @@ To be released at some future point in time Description -- Revamp build and test systems for SmartRedis +- Major revamo of build and test systems for SmartRedis - Refactor Fortran methods to return default logical kind - Update CI/CD tests to use a modern version of MacOS - Fix the spelling of the Dataset destructor's C interface (now DeallocateDataSet) @@ -19,7 +19,7 @@ Description Detailed Notes -- Rework the build and test system to improve maintainability of the library (PR341_) +- Rework the build and test system to improve maintainability of the library. There have been several significant changes, including that Python and Fortran clients are no longer built by defaults and that there are Make variables that customize the build process. Please review the build documentation and ``make help`` to see all that has changed. (PR341_) - Many Fortran routines were returning logical kind = c_bool which turns out not to be the same default kind of most Fortran compilers. These have now been refactored so that users need not import `iso_c_binding` in their own applications (PR340_) diff --git a/doc/install/lib.rst b/doc/install/lib.rst index 287c22606..ea1b844d0 100644 --- a/doc/install/lib.rst +++ b/doc/install/lib.rst @@ -9,10 +9,10 @@ The release tarball can also be used instead of cloning the git repository, but the preferred method is a repository clone. The ```Makefile`` included in the top level of the SmartRedis repository has two -main targets: ``lib`` which will create a dynamic library for C, C++, and Python -clients and ``lib-with-fortran`` which will also additionally build a library -for Fortran applications. ``make help`` will list additional targets that are -used for SmartRedis development. +main targets: ``lib`` which will create a dynamic library for C, C++, and +(optionally) Fortran and Python clients; and ``lib-with-fortran`` which will also +unconditionally builds a library for Fortran applications. ``make help`` will list +additional targets that are used for SmartRedis development. .. code-block:: bash @@ -61,30 +61,37 @@ via a variable supplied to make: make lib SR_BUILD=Coverage The name of the library produced for a Debug mode build is ``smartredis-debug``. -The name of the library produced for a Coverage mode build is ``smartredis-debug``. +The name of the library produced for a Coverage mode build is ``smartredis-coverage``. The name of the library produced for a Release mode build is ``smartredis``. In each case, the file extension is dependent on the link type, ``.so`` or ``.a``. +All libraries will be located in the ``install/lib`` folder. -Finally, it is possible to build SmartRedis without including Python support: +Finally, it is possible to build SmartRedis to include Python and/or Fortran support +(both are omitted by default): .. code-block:: bash cd SmartRedis - make lib SR_PYTHON=OFF + # Build support for Python + make lib SR_PYTHON=ON + # Build support for Fortran + make lib SR_FORTRAN=ON # equivalent to make lib-with-fortran + # Build support for Python and Fortran + make lib SR_PYTHON=ON SR_FORTRAN=ON # or make lib-with-fortran SR_PYTHON=ON -The build mode, link type, and Python support settings are fully orthogonal; any -combination of them is supported. For example, a statically linked debug build -may be achieved via the following command: +The build mode, link type, and Fortran/Python support settings are fully orthogonal; +any combination of them is supported. For example, a statically linked debug build +with Python support may be achieved via the following command: .. code-block:: bash cd SmartRedis - make lib SR_LINK=Static SR_BUILD=Debug + make lib SR_LINK=Static SR_BUILD=Debug SR_PYTHON=ON -The SR_LINK and SR_BUILD variables are fully supported for all test and build targets -in the Makefile. +The SR_LINK, SR_BUILD, SR_PYTHON, and SR_FORTRAN variables are fully supported for all +test and build targets in the Makefile. -Additional make variables may be seen in the ``help`` make target: +Additional make variables are described in the ``help`` make target: .. code-block:: bash diff --git a/images/Dockerfile b/images/Dockerfile index e5b5b09c8..fde64428e 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -51,7 +51,7 @@ COPY . /usr/local/src/SmartRedis WORKDIR /usr/local/src/SmartRedis RUN pip3 install --upgrade pip RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ - && rm -rf build tests examples images utils doc images + && rm -rf build tests examples images utils third-party doc images # Copy install files and directories to /usr/local/lib and # usr/local/include and delete unnecessary build files From 4168ce7c84af458ed6d4f34e2a88cf92312a88c1 Mon Sep 17 00:00:00 2001 From: billschereriii Date: Fri, 2 Jun 2023 10:24:21 -0500 Subject: [PATCH 50/50] typo --- doc/install/lib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/lib.rst b/doc/install/lib.rst index ea1b844d0..afc72e0dd 100644 --- a/doc/install/lib.rst +++ b/doc/install/lib.rst @@ -11,7 +11,7 @@ the preferred method is a repository clone. The ```Makefile`` included in the top level of the SmartRedis repository has two main targets: ``lib`` which will create a dynamic library for C, C++, and (optionally) Fortran and Python clients; and ``lib-with-fortran`` which will also -unconditionally builds a library for Fortran applications. ``make help`` will list +unconditionally build a library for Fortran applications. ``make help`` will list additional targets that are used for SmartRedis development. .. code-block:: bash