From f2eb9ad82e3fad44b2d2e0f9230599e1ea255b65 Mon Sep 17 00:00:00 2001 From: Bill Scherer <36514047+billschereriii@users.noreply.github.com> Date: Fri, 2 Jun 2023 10:44:06 -0500 Subject: [PATCH] Refactor SmartRedis build system (#341) Refactor the SmartRedis build system as planned in the design document: . add support for SR_BUILD and SR_LINK Make variables . add missing make dependencies . eliminate use of scripts for building the SmartRedis libraries . eliminate unnecessary deletion of SmartRedis builds, allowing repeat builds to be much faster . update examples, test cases, CI/CD to work with new build system . update Intel compiler in CI/CD to move away from deprecated compiler that will EOL in the next few months . make Python and Fortran optional parts of the build, not built by default [ committed by @billschereriii ] [ reviewed by @ashao @MattToast @mellis13 ] --- .github/workflows/run_post_merge_tests.yml | 2 +- .github/workflows/run_tests.yml | 28 +- CMakeLists.txt | 126 +++++---- MANIFEST.in | 1 + Makefile | 204 ++++++++++----- conftest.py | 15 ++ doc/changelog.rst | 3 + doc/install/lib.rst | 81 +++++- doc/installation.rst | 4 +- examples/CMakeLists.txt | 40 +++ examples/parallel/CMakeLists.txt | 41 +++ examples/parallel/cpp/CMakeLists.txt | 68 ++--- examples/parallel/cpp/smartredis_mnist.cpp | 6 +- examples/parallel/fortran/CMakeLists.txt | 46 ++-- .../parallel/fortran/smartredis_mnist.F90 | 4 +- examples/serial/CMakeLists.txt | 43 ++++ examples/serial/c/CMakeLists.txt | 56 ++-- examples/serial/cpp/CMakeLists.txt | 74 +++--- examples/serial/cpp/smartredis_mnist.cpp | 11 +- examples/serial/cpp/smartredis_model.cpp | 6 +- examples/serial/fortran/CMakeLists.txt | 52 ++-- examples/test_examples.py | 48 ++-- images/Dockerfile | 1 + include/redisserver.h | 1 - setup.cfg | 2 +- setup.py | 10 +- smartredis_defs.cmake | 71 +++++ src/cpp/redis.cpp | 1 + src/cpp/rediscluster.cpp | 1 + src/cpp/redisserver.cpp | 1 + tests/CMakeLists.txt | 48 ++++ tests/c/CMakeLists.txt | 103 +++----- tests/c/client_test_put_get_2D.c | 20 +- tests/c/test_c_client.py | 19 +- tests/cpp/CMakeLists.txt | 243 +++++------------- tests/cpp/client_test_ensemble.cpp | 6 +- tests/cpp/client_test_mnist.cpp | 6 +- tests/cpp/client_test_mnist_dataset.cpp | 6 +- tests/cpp/test_cpp_client.py | 20 +- tests/cpp/unit-tests/CMakeLists.txt | 47 ++-- tests/cpp/unit-tests/test_client.cpp | 6 +- tests/cpp/unit-tests/test_client_ensemble.cpp | 6 +- tests/cpp/unit-tests/test_unit_cpp_client.py | 20 +- tests/docker/CMakeLists.txt | 38 +++ tests/docker/c/CMakeLists.txt | 12 +- tests/docker/cpp/CMakeLists.txt | 11 +- tests/docker/fortran/CMakeLists.txt | 12 +- tests/fortran/CMakeLists.txt | 63 ++--- tests/fortran/client_test_ensemble.F90 | 4 +- tests/fortran/client_test_mnist.F90 | 4 +- tests/fortran/client_test_mnist_multigpu.F90 | 4 +- tests/fortran/test_fortran_client.py | 32 ++- 52 files changed, 1111 insertions(+), 666 deletions(-) create mode 100644 examples/CMakeLists.txt create mode 100644 examples/parallel/CMakeLists.txt create mode 100644 examples/serial/CMakeLists.txt create mode 100644 smartredis_defs.cmake create mode 100644 tests/CMakeLists.txt create mode 100644 tests/docker/CMakeLists.txt diff --git a/.github/workflows/run_post_merge_tests.yml b/.github/workflows/run_post_merge_tests.yml index 549a61c20..30f58a7d2 100644 --- a/.github/workflows/run_post_merge_tests.yml +++ b/.github/workflows/run_post_merge_tests.yml @@ -170,5 +170,5 @@ jobs: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/install/lib && export SR_LOG_FILE=smartredis_examples_log.txt && export SR_LOG_LEVEL=INFO && - make test-examples + make test-examples SR_FORTRAN=ON SR_PYTHON=ON diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index ed67f122c..e45fe9a0b 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -108,8 +108,8 @@ jobs: sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic source /opt/intel/oneapi/setvars.sh && printenv >> $GITHUB_ENV && - echo "CC=icc" >> $GITHUB_ENV && - echo "CXX=icpc" >> $GITHUB_ENV && + echo "CC=icx" >> $GITHUB_ENV && + echo "CXX=icpx" >> $GITHUB_ENV && echo "FC=ifort" >> $GITHUB_ENV # Install additional dependencies @@ -120,29 +120,25 @@ jobs: - name: Build SmartRedis python and install run: python -m pip install -e .[dev,xarray] - - name: Build and install dependencies - run: | - mkdir -p ./third-party && - cd ./third-party && - bash ../build-scripts/build-lcov.sh && - bash ../build-scripts/build-catch.sh + - name: Build and install test dependencies + run: make lcov && make catch2 - name: Install docker, redis-server, and RedisAI run: | sudo apt-get -y update && sudo apt-get -y install curl gnupg lsb-release software-properties-common ca-certificates && \ # Add latest redis to apt sources - echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list && \ - curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg && \ + sudo echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list && \ + sudo curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg && \ # Add latest docker to apt sources sudo mkdir -m 0755 -p /etc/apt/keyrings && - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null && # Install and update sudo apt-get -y update && \ sudo apt-get -y install iputils-ping docker-ce docker-ce-cli containerd.io redis-tools=6:6.2.5-1rl1~focal1 redis-server=6:6.2.5-1rl1~focal1 && CONTAINER_NAME="redisai_$RANDOM" && - docker create --name $CONTAINER_NAME --rm redislabs/redisai:${{ matrix.rai_v }}-cpu-bionic && \ + sudo docker create --name $CONTAINER_NAME --rm redislabs/redisai:${{ matrix.rai_v }}-cpu-bionic && \ sudo mkdir -p /usr/lib/redis/modules/ && sudo docker cp $CONTAINER_NAME:/usr/lib/redis/modules/redisai.so /usr/lib/redis/modules && sudo docker cp $CONTAINER_NAME:/usr/lib/redis/modules/backends/ /usr/lib/redis/modules/ && @@ -162,7 +158,7 @@ jobs: popd && export SSDB=$SSDB_SINGLE SMARTREDIS_TEST_CLUSTER=False && utils/check_redis.sh $PORT && - make test-verbose-with-coverage COV_FLAGS="--cov=./src/python/module/smartredis/ --cov-report=xml --cov-append" + make test-verbose-with-coverage COV_FLAGS="--cov=./src/python/module/smartredis/ --cov-report=xml --cov-append" SR_FORTRAN=ON SR_PYTHON=ON - name: Clustered DB tests run: | @@ -181,7 +177,7 @@ jobs: redis-cli --cluster create $(echo $SSDB_CLUSTERED | tr "," " ") --cluster-yes && export SSDB=$SSDB_CLUSTERED SMARTREDIS_TEST_CLUSTER=True && python -m pytest --cov=./src/python/module/smartredis/ --cov-report=xml --cov-append \ - --ignore ./tests/docker -vv -s ./tests + --ignore ./tests/docker -vv -s ./tests --build Coverage - name: UDS DB tests run: | @@ -198,11 +194,11 @@ jobs: utils/check_redis.sh $SOCKET export SSDB=$SSDB_UDS SMARTREDIS_TEST_CLUSTER=False && python -m pytest --cov=./src/python/module/smartredis/ --cov-report=xml --cov-append \ - --ignore ./tests/docker -vv -s ./tests + --ignore ./tests/docker -vv -s ./tests --build Coverage # Process and upload code coverage (Python was collected during pytest) - name: Collect coverage from C/C++/Fortran testers - run: third-party/lcov/install/usr/local/bin/lcov -c -d build/CMakeFiles -o coverage.info + run: third-party/lcov/install/usr/local/bin/lcov -c -d build/Coverage/CMakeFiles -o coverage.info - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/CMakeLists.txt b/CMakeLists.txt index 039dd1452..ca88a4d44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,47 +24,59 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(SmartRedis) +# Enable setting version in the project statement +if (POLICY CMP0048) + cmake_policy(SET CMP0048 NEW) +endif (POLICY CMP0048) +# Project definition for the SmartRedis project cmake_minimum_required(VERSION 3.13) +project(SmartRedis VERSION "0.4.0") -option(BUILD_PYTHON "Build the python module" ON) -option(BUILD_FORTRAN "Build the fortran client library" OFF) -option(COVERAGE "Build the fortran client library" OFF) +# Configure options for the SmartRedis project +option(SR_PYTHON "Build the python module" OFF) +option(SR_FORTRAN "Build the fortran client library" OFF) +option(SR_PEDANTIC "Build with pickiest compiler settings" OFF) -set(CMAKE_BUILD_TYPE RELEASE) set(CMAKE_CXX_STANDARD 17) set(CMAKE_INSTALL_PREFIX ${CMAKE_SOURCE_DIR}/install) set(CMAKE_CXX_VISIBILITY_PRESET default) set(THREADS_PREFER_PTHREAD_FLAG ON) +set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}) +include(smartredis_defs) -if (BUILD_FORTRAN) +# If we want to use Fortran, we have to tell CMake to use it +if (SR_FORTRAN) enable_language(Fortran) endif() -if (WERROR) +# For now, we only support Pedantic on the main library build. +# If/when we fine-tune the examples and test cases, move this block +# to smartredis_defs.cmake +if (SR_PEDANTIC) if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) add_compile_options(-Wall -Werror) else() - message(WARNING "WERROR was specified, but the CMAKE compiler is not GCC") + message(WARNING "SR_PEDANTIC was specified, but the CMAKE compiler is not GCC") endif() -endif() - -if (COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - add_compile_options(--coverage) - add_link_options(--coverage) - else() - message(WARNING "COVERAGE was specified, but the CMAKE compiler is not GCC") + if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Wno-maybe-uninitialized") endif() endif() -find_library(REDISPP redis++ PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH REQUIRED) +# Bring in third-party libaries needed for the SmartRedis library +find_library(REDISPP redis++ + PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH + REQUIRED STATIC +) +find_library(HIREDIS hiredis + PATHS ${CMAKE_SOURCE_DIR}/install/lib NO_DEFAULT_PATH + REQUIRED STATIC +) find_package(Threads REQUIRED) - set(EXT_CLIENT_LIBRARIES ${REDISPP} ${HIREDIS}) +# Define source code that goes into the SmartRedis library set(CLIENT_SRC src/c/c_client.cpp src/c/c_configoptions.cpp @@ -73,45 +85,47 @@ set(CLIENT_SRC src/c/c_logcontext.cpp src/c/c_logger.cpp src/cpp/address.cpp + src/cpp/addressallcommand.cpp + src/cpp/addressanycommand.cpp + src/cpp/addressatcommand.cpp src/cpp/client.cpp - src/cpp/dataset.cpp + src/cpp/clusterinfocommand.cpp src/cpp/command.cpp - src/cpp/keyedcommand.cpp - src/cpp/nonkeyedcommand.cpp - src/cpp/multikeycommand.cpp - src/cpp/singlekeycommand.cpp + src/cpp/commandlist.cpp + src/cpp/commandreply.cpp src/cpp/compoundcommand.cpp - src/cpp/addressatcommand.cpp - src/cpp/addressanycommand.cpp - src/cpp/addressallcommand.cpp - src/cpp/clusterinfocommand.cpp + src/cpp/configoptions.cpp + src/cpp/dataset.cpp src/cpp/dbinfocommand.cpp + src/cpp/dbnode.cpp src/cpp/gettensorcommand.cpp - src/cpp/commandlist.cpp + src/cpp/keyedcommand.cpp + src/cpp/logger.cpp src/cpp/metadata.cpp - src/cpp/tensorbase.cpp - src/cpp/tensorpack.cpp - src/cpp/dbnode.cpp - src/cpp/commandreply.cpp - src/cpp/redisserver.cpp - src/cpp/rediscluster.cpp - src/cpp/redis.cpp src/cpp/metadatafield.cpp - src/cpp/stringfield.cpp + src/cpp/multikeycommand.cpp + src/cpp/nonkeyedcommand.cpp src/cpp/pipelinereply.cpp + src/cpp/redis.cpp + src/cpp/rediscluster.cpp + src/cpp/redisserver.cpp + src/cpp/singlekeycommand.cpp + src/cpp/srobject.cpp + src/cpp/stringfield.cpp + src/cpp/tensorbase.cpp + src/cpp/tensorpack.cpp src/cpp/threadpool.cpp src/cpp/utility.cpp - src/cpp/logger.cpp - src/cpp/srobject.cpp - src/cpp/configoptions.cpp ) +# Define include directories for header files include_directories(SYSTEM include install/include ) -if (BUILD_FORTRAN) +# Build the Fortran library +if (SR_FORTRAN) set(FORTRAN_SRC src/fortran/errors.F90 src/fortran/client.F90 @@ -125,8 +139,13 @@ if (BUILD_FORTRAN) # Note the following has to be before ANY add_library command) set(CMAKE_Fortran_MODULE_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include") # Fortran library - add_library(smartredis-fortran SHARED ${FORTRAN_SRC}) - set_target_properties(smartredis-fortran PROPERTIES SUFFIX ".so") + add_library(smartredis-fortran ${SMARTREDIS_LINK_MODE} ${FORTRAN_SRC}) + set_target_properties(smartredis-fortran PROPERTIES + SUFFIX ${SMARTREDIS_LINK_LIBRARY_SUFFIX} + ) + set_target_properties(smartredis-fortran PROPERTIES + OUTPUT_NAME ${SMARTREDIS_FORTRAN_LIB} + ) target_link_libraries(smartredis-fortran PUBLIC smartredis ${EXT_CLIENT_LIBRARIES}) # Install dynamic library and headers install(TARGETS smartredis-fortran @@ -134,11 +153,17 @@ if (BUILD_FORTRAN) endif() -# Build dynamic library -add_library(smartredis SHARED ${CLIENT_SRC}) -set_target_properties(smartredis PROPERTIES SUFFIX ".so") +# Build the main SmartRedis library +add_library(smartredis ${SMARTREDIS_LINK_MODE} ${CLIENT_SRC}) +set_target_properties(smartredis PROPERTIES + SUFFIX ${SMARTREDIS_LINK_LIBRARY_SUFFIX} +) +set_target_properties(smartredis PROPERTIES + OUTPUT_NAME ${SMARTREDIS_LIB} +) target_link_libraries(smartredis PUBLIC ${EXT_CLIENT_LIBRARIES} PRIVATE Threads::Threads) +# Install SmartRedis header files install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION "include" FILES_MATCHING @@ -149,13 +174,12 @@ install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" install(TARGETS smartredis LIBRARY DESTINATION lib) -if(BUILD_PYTHON) +# Build the Python library for SmartRedis +if(SR_PYTHON) message("-- Python client build enabled") add_subdirectory(${CMAKE_SOURCE_DIR}/third-party/pybind ${CMAKE_SOURCE_DIR}/third-party/pybind/build) - add_library(smartredis_static STATIC ${CLIENT_SRC}) - pybind11_add_module(smartredisPy src/python/src/pyclient.cpp src/python/src/pyconfigoptions.cpp @@ -166,8 +190,8 @@ if(BUILD_PYTHON) src/python/bindings/bind.cpp) target_link_libraries(smartredisPy PUBLIC ${EXT_CLIENT_LIBRARIES}) - install(TARGETS smartredisPy - LIBRARY DESTINATION lib) + install(TARGETS smartredisPy LIBRARY DESTINATION lib) + install(TARGETS smartredisPy LIBRARY DESTINATION ../src/python/module/smartredis) else() message("-- Skipping Python client build") endif() diff --git a/MANIFEST.in b/MANIFEST.in index 5fb01f159..496dc070b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,7 @@ include requirements.txt include requirements-dev.txt include Makefile include CMakeLists.txt +include smartredis_defs.cmake graft src graft build-scripts graft include diff --git a/Makefile b/Makefile index c9561703e..1aaee0e89 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,44 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# General settings MAKEFLAGS += --no-print-directory -COV_FLAGS := SHELL:=/bin/bash -SR_DEVICE := cpu + +# Build variables +NPROC := $(shell nproc 2>/dev/null || python -c "import multiprocessing as mp; print (mp.cpu_count())" 2>/dev/null || echo 4) +SR_BUILD := Release +SR_LINK := Shared +SR_PEDANTIC := OFF +SR_FORTRAN := OFF +SR_PYTHON := OFF + +# Test variables +COV_FLAGS := +SR_TEST_DEVICE := cpu # Params for third-party software HIREDIS_URL := https://github.com/redis/hiredis.git @@ -31,8 +68,23 @@ help: @grep "^# help\:" Makefile | grep -v grep | sed 's/\# help\: //' | sed 's/\# help\://' # help: -# help: Build -# help: ------- +# help: Build variables +# help: --------------- +# help: +# help: These variables affect the way that the SmartRedis library is built. Each +# help: has several options; the first listed is the default. Use by appending +# help: the variable name and setting after the make target, e.g. +# help: make lib SR_BUILD=Debug SR_LINK=Static SR_FORTRAN=ON +# help: +# help: SR_BUILD {Release, Debug, Coverage} -- optimization level for the build +# help: SR_LINK {Shared, Static} -- linkage for the SmartRedis library +# help: SR_PEDANTIC {OFF, ON} -- GNU only; enable pickiest compiler settings +# help: SR_FORTRAN {OFF, ON} -- Enable/disable build of Fortran library +# help: SR_PYTHON {OFF, ON} -- Enable/disable build of Python library + +# help: +# help: Build targets +# help: ------------- # help: deps - Make SmartRedis dependencies .PHONY: deps @@ -41,30 +93,28 @@ deps: redis-plus-plus deps: pybind deps: -# help: pip-install - Register the SmartRedis library with pip -.PHONY: pip-install -pip-install: - @if ! python -c "import smartredis" >& /dev/null; then pip install -e.; fi - # help: lib - Build SmartRedis C/C++/Python clients into a dynamic library .PHONY: lib -lib: pip-install lib: deps - @bash ./build-scripts/build_lib.sh $(LIB_BUILD_ARGS) +lib: + @cmake -S . -B build/$(SR_BUILD) -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) \ + -DSR_PEDANTIC=$(SR_PEDANTIC) -DSR_FORTRAN=$(SR_FORTRAN) -DSR_PYTHON=$(SR_PYTHON) + @cmake --build build/$(SR_BUILD) -- -j $(NPROC) + @cmake --install build/$(SR_BUILD) # help: lib-with-fortran - Build SmartRedis C/C++/Python and Fortran clients into a dynamic library .PHONY: lib-with-fortran -lib-with-fortran: deps - @bash ./build-scripts/build_lib.sh $(LIB_BUILD_ARGS) $(CMAKE_ARGS) -DBUILD_FORTRAN=ON +lib-with-fortran: SR_FORTRAN=ON +lib-with-fortran: lib # help: test-lib - Build SmartRedis clients into a dynamic library with least permissive compiler settings .PHONY: test-lib -test-lib: LIB_BUILD_ARGS="-DWERROR=ON" +test-lib: SR_PEDANTIC=ON test-lib: lib # help: test-lib-with-fortran - Build SmartRedis clients into a dynamic library with least permissive compiler settings .PHONY: test-lib-with-fortran -test-lib-with-fortran: LIB_BUILD_ARGS="-DWERROR=ON" +test-lib-with-fortran: SR_PEDANTIC=ON test-lib-with-fortran: lib-with-fortran # help: test-deps - Make SmartRedis testing dependencies @@ -76,57 +126,70 @@ test-deps: lcov # help: test-deps-gpu - Make SmartRedis GPU testing dependencies .PHONY: test-deps-gpu -test-deps-gpu: - $(error To build test-deps for GPU, please execute "make test-deps SR_DEVICE=gpu") +test-deps-gpu: SR_TEST_DEVICE=gpu +test-deps-gpu: test-deps # help: build-tests - build all tests (C, C++, Fortran) .PHONY: build-tests -build-tests: test-lib-with-fortran - ./build-scripts/build_cpp_tests.sh $(CMAKE_ARGS) - ./build-scripts/build_cpp_unit_tests.sh $(CMAKE_ARGS) - ./build-scripts/build_c_tests.sh $(CMAKE_ARGS) - ./build-scripts/build_fortran_tests.sh $(CMAKE_ARGS) +build-tests: test-lib + @cmake -S tests -B build/$(SR_BUILD)/tests -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + @cmake --build build/$(SR_BUILD)/tests -- -j $(NPROC) # help: build-test-cpp - build the C++ tests .PHONY: build-test-cpp build-test-cpp: test-lib - ./build-scripts/build_cpp_tests.sh - ./build-scripts/build_cpp_unit_tests.sh + @cmake -S tests/cpp -B build/$(SR_BUILD)/tests/cpp -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/cpp -- -j $(NPROC) # help: build-unit-test-cpp - build the C++ unit tests .PHONY: build-unit-test-cpp build-unit-test-cpp: test-lib - ./build-scripts/build_cpp_unit_tests.sh + @cmake -S tests/cpp/unit-tests -B build/$(SR_BUILD)/tests/cpp/unit-tests \ + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/cpp/unit-tests -- -j $(NPROC) # help: build-test-c - build the C tests .PHONY: build-test-c build-test-c: test-lib - ./build-scripts/build_c_tests.sh + @cmake -S tests/c -B build/$(SR_BUILD)/tests/c -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/c -- -j $(NPROC) # help: build-test-fortran - build the Fortran tests .PHONY: build-test-fortran -build-test-fortran: test-lib-with-fortran - ./build-scripts/build_fortran_tests.sh +build-test-fortran: SR_FORTRAN=ON +build-test-fortran: test-lib + @cmake -S tests/fortran -B build/$(SR_BUILD)/tests/fortran -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) + @cmake --build build/$(SR_BUILD)/tests/fortran -- -j $(NPROC) # help: build-examples - build all examples (serial, parallel) .PHONY: build-examples -build-examples: lib-with-fortran - ./build-scripts/build_serial_examples.sh - ./build-scripts/build_parallel_examples.sh +build-examples: lib + @cmake -S examples -B build/$(SR_BUILD)/examples -DSR_BUILD=$(SR_BUILD) \ + -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + @cmake --build build/$(SR_BUILD)/examples + # help: build-example-serial - buld serial examples .PHONY: build-example-serial build-example-serial: lib - ./build-scripts/build_serial_examples.sh + @cmake -S examples/serial -B build/$(SR_BUILD)/examples/serial \ + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + @cmake --build build/$(SR_BUILD)/examples/serial # help: build-example-parallel - build parallel examples (requires MPI) .PHONY: build-example-parallel build-example-parallel: lib - ./build-scripts/build_parallel_examples.sh + @cmake -S examples/parallel -B build/$(SR_BUILD)/examples/parallel \ + -DSR_BUILD=$(SR_BUILD) -DSR_LINK=$(SR_LINK) -DSR_FORTRAN=$(SR_FORTRAN) + @cmake --build build/$(SR_BUILD)/examples/parellel # help: clean-deps - remove third-party deps @@ -147,8 +210,8 @@ clobber: clean clean-deps # help: -# help: Style -# help: ------- +# help: Style targets +# help: ------------- # help: style - Sort imports and format with black .PHONY: style @@ -191,8 +254,8 @@ check-lint: # help: -# help: Documentation -# help: ------- +# help: Documentation targets +# help: --------------------- # help: docs - generate project documentation .PHONY: docs @@ -206,15 +269,23 @@ cov: @echo if data was present, coverage report is in htmlcov/index.html # help: -# help: Test -# help: ------- +# help: Test targets +# help: ------------ + +ifeq ($(SR_PYTHON),OFF) +SKIP_PYTHON = --ignore ./tests/python +endif +ifeq ($(SR_FORTRAN),OFF) +SKIP_FORTRAN = --ignore ./tests/fortran +endif # help: test - Build and run all tests (C, C++, Fortran, Python) .PHONY: test test: test-deps test: build-tests test: - @PYTHONFAULTHANDLER=1 python -m pytest --ignore ./tests/docker -vv ./tests + @PYTHONFAULTHANDLER=1 python -m pytest --ignore ./tests/docker \ + $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv ./tests --build $(SR_BUILD) # help: test-verbose - Build and run all tests [verbosely] @@ -222,22 +293,24 @@ test: test-verbose: test-deps test-verbose: build-tests test-verbose: - PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests + @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker \ + $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv -s ./tests --build $(SR_BUILD) -# help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] +# help: test-verbose-with-coverage - Build and run all tests [verbose-with-coverage] .PHONY: test-verbose-with-coverage +test-verbose-with-coverage: SR_BUILD=Coverage test-verbose-with-coverage: test-deps test-verbose-with-coverage: build-tests -test-verbose-with-coverage: CMAKE_ARGS="-DCOVERAGE=on" test-verbose-with-coverage: - PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker -vv -s ./tests + @PYTHONFAULTHANDLER=1 python -m pytest $(COV_FLAGS) --ignore ./tests/docker \ + $(SKIP_PYTHON) $(SKIP_FORTRAN) -vv -s ./tests --build $(SR_BUILD) # help: test-c - Build and run all C tests .PHONY: test-c test-c: test-deps test-c: build-test-c test-c: - @python -m pytest -vv -s ./tests/c/ + @python -m pytest -vv -s ./tests/c/ --build $(SR_BUILD) # help: test-cpp - Build and run all C++ tests .PHONY: test-cpp @@ -245,39 +318,43 @@ test-cpp: test-deps test-cpp: build-test-cpp test-cpp: build-unit-test-cpp test-cpp: - @python -m pytest -vv -s ./tests/cpp/ + @python -m pytest -vv -s ./tests/cpp/ --build $(SR_BUILD) # help: unit-test-cpp - Build and run unit tests for C++ .PHONY: unit-test-cpp unit-test-cpp: test-deps unit-test-cpp: build-unit-test-cpp unit-test-cpp: - @python -m pytest -vv -s ./tests/cpp/unit-tests/ + @python -m pytest -vv -s ./tests/cpp/unit-tests/ --build $(SR_BUILD) # help: test-py - run python tests .PHONY: test-py test-py: test-deps +test-py: SR_PYTHON=ON +test-py: lib test-py: - @PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/ + @PYTHONFAULTHANDLER=1 python -m pytest -vv ./tests/python/ --build $(SR_BUILD) # help: test-fortran - run fortran tests .PHONY: test-fortran test-fortran: test-deps test-fortran: build-test-fortran - @python -m pytest -vv ./tests/fortran/ + @python -m pytest -vv ./tests/fortran/ --build $(SR_BUILD) # help: testpy-cov - run python tests with coverage .PHONY: testpy-cov testpy-cov: test-deps +testpy-cov: SR_PYTHON=ON testpy-cov: - @PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ -vv ./tests/python/ + @PYTHONFAULTHANDLER=1 python -m pytest --cov=./src/python/module/smartredis/ \ + -vv ./tests/python/ --build $(SR_BUILD) # help: test-examples - Build and run all examples .PHONY: test-examples test-examples: test-deps test-examples: build-examples test-examples: - @python -m pytest -vv -s ./examples + @python -m pytest -vv -s ./examples --build $(SR_BUILD) --sr_fortran $(SR_FORTRAN) ############################################################################ @@ -292,7 +369,7 @@ install/lib/libhiredis.a: @cd third-party && \ git clone $(HIREDIS_URL) hiredis --branch $(HIREDIS_VER) --depth=1 @cd third-party/hiredis && \ - LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" static -j && \ + LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" static -j $(NPROC) && \ LIBRARY_PATH=lib CC=gcc CXX=g++ make PREFIX="../../install" install && \ rm -f ../../install/lib/libhiredis*.so && \ rm -f ../../install/lib/libhiredis*.dylib && \ @@ -309,8 +386,11 @@ install/lib/libredis++.a: @cd third-party/redis-plus-plus && \ mkdir -p compile && \ cd compile && \ - cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 .. && \ - CC=gcc CXX=g++ make -j && \ + cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_PLUS_PLUS_BUILD_TEST=OFF \ + -DREDIS_PLUS_PLUS_BUILD_SHARED=OFF -DCMAKE_PREFIX_PATH="../../../install/lib/" \ + -DCMAKE_INSTALL_PREFIX="../../../install" -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_LIBDIR="lib" .. && \ + CC=gcc CXX=g++ make -j $(NPROC) && \ CC=gcc CXX=g++ make install && \ echo "Finished installing Redis-plus-plus" @@ -332,14 +412,14 @@ third-party/redis/src/redis-server: @cd third-party && \ git clone $(REDIS_URL) redis --branch $(REDIS_VER) --depth=1 @cd third-party/redis && \ - CC=gcc CXX=g++ make MALLOC=libc -j && \ + CC=gcc CXX=g++ make MALLOC=libc -j $(NPROC) && \ echo "Finished installing redis" # cudann-check (hidden test target) # checks cuda dependencies for GPU build .phony: cudann-check cudann-check: -ifeq ($(SR_DEVICE),gpu) +ifeq ($(SR_TEST_DEVICE),gpu) ifndef CUDA_HOME $(error ERROR: CUDA_HOME is not set) endif @@ -362,13 +442,17 @@ endif redisAI: cudann-check redisAI: third-party/RedisAI/install-cpu/redisai.so third-party/RedisAI/install-cpu/redisai.so: - $(eval DEVICE_IS_GPU := $(shell test $(SR_DEVICE) == "cpu"; echo $$?)) + $(eval DEVICE_IS_GPU := $(shell test $(SR_TEST_DEVICE) == "cpu"; echo $$?)) @mkdir -p third-party @cd third-party && \ - GIT_LFS_SKIP_SMUDGE=1 git clone --recursive $(REDISAI_URL) RedisAI --branch $(REDISAI_VER) --depth=1 + rm -rf RedisAI && \ + GIT_LFS_SKIP_SMUDGE=1 git clone --recursive $(REDISAI_URL) RedisAI \ + --branch $(REDISAI_VER) --depth=1 -@cd third-party/RedisAI && \ - CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh $(SR_DEVICE) && \ - CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 WITH_UNIT_TESTS=0 make -j -C opt clean build && \ + CC=gcc CXX=g++ WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 bash get_deps.sh \ + $(SR_TEST_DEVICE) && \ + CC=gcc CXX=g++ GPU=$(DEVICE_IS_GPU) WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0 \ + WITH_UNIT_TESTS=0 make -j $(NPROC) -C opt clean build && \ echo "Finished installing RedisAI" # Catch2 (hidden test target) diff --git a/conftest.py b/conftest.py index 8b4eca3a6..44cc351a8 100644 --- a/conftest.py +++ b/conftest.py @@ -136,3 +136,18 @@ def create_torch_cnn(filepath=None): torch.jit.save(module, buffer) str_model = buffer.getvalue() return str_model + +# Add a build type option to pytest command lines +def pytest_addoption(parser): + parser.addoption("--build", action="store", default="Release") + parser.addoption("--sr_fortran", action="store", default="OFF") + +# Fixture to retrieve the build type setting +@pytest.fixture(scope="session") +def build(request): + return request.config.getoption("--build") + +# Fixture to retrieve the Fortran enablement setting +@pytest.fixture(scope="session") +def sr_fortran(request): + return request.config.getoption("--sr_fortran") diff --git a/doc/changelog.rst b/doc/changelog.rst index d119fff7f..3e2c6e9c5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -8,6 +8,7 @@ To be released at some future point in time Description +- Major revamo of build and test systems for SmartRedis - Refactor Fortran methods to return default logical kind - Update CI/CD tests to use a modern version of MacOS - Fix the spelling of the Dataset destructor's C interface (now DeallocateDataSet) @@ -18,6 +19,7 @@ Description Detailed Notes +- Rework the build and test system to improve maintainability of the library. There have been several significant changes, including that Python and Fortran clients are no longer built by defaults and that there are Make variables that customize the build process. Please review the build documentation and ``make help`` to see all that has changed. (PR341_) - Many Fortran routines were returning logical kind = c_bool which turns out not to be the same default kind of most Fortran compilers. These have now been refactored so that users need not import `iso_c_binding` in their own applications (PR340_) @@ -28,6 +30,7 @@ users need not import `iso_c_binding` in their own applications (PR340_) - New pip-install target in Makefile will be a dependency of the lib target going forward so that users don't have to manually pip install SmartRedis in the future (PR330_) - Added ConfigOptions class and API, which will form the backbone of multiDB support (PR303_) +.. _PR341: https://github.com/CrayLabs/SmartRedis/pull/341 .. _PR340: https://github.com/CrayLabs/SmartRedis/pull/340 .. _PR339: https://github.com/CrayLabs/SmartRedis/pull/339 .. _PR338: https://github.com/CrayLabs/SmartRedis/pull/338 diff --git a/doc/install/lib.rst b/doc/install/lib.rst index 4b480dc6f..afc72e0dd 100644 --- a/doc/install/lib.rst +++ b/doc/install/lib.rst @@ -9,10 +9,10 @@ The release tarball can also be used instead of cloning the git repository, but the preferred method is a repository clone. The ```Makefile`` included in the top level of the SmartRedis repository has two -main targets: ``lib`` which will create a dynamic library for C, C++, and Python -clients and ``lib-with-fortran`` which will also additionally build a library -for Fortran applications. ``make help`` will list additional targets that are -used for SmartRedis development. +main targets: ``lib`` which will create a dynamic library for C, C++, and +(optionally) Fortran and Python clients; and ``lib-with-fortran`` which will also +unconditionally build a library for Fortran applications. ``make help`` will list +additional targets that are used for SmartRedis development. .. code-block:: bash @@ -25,6 +25,79 @@ installed in ``SmartRedis/install/include/``. The library installation can be used to easily include SmartRedis capabilities in C++, C, and Fortran applications. +Customizing the library build +----------------------------- + +By default, the SmartRedis library is built as a shared library. For some +applications, however, it is preferable to link to a statically compiled +library. This can be done easily with the command: + +.. code-block:: bash + + cd SmartRedis + # Static build + make lib SR_LINK=Static + # Release build + make lib SR_LINK=Shared #or skip the SR_LINK variable as this is the default + +Linked statically, the SmartRedis library will have a ``.a`` file extension. When +linked dynamically, the SmartRedis library will have a ``.so`` file extension. + +It is also possible to adjust compilation settings for the SmartRedis libary. +By default, the library compiles in an optimized build (``Release``), but debug builds +with full symbols (``Debug``) can be created as can debug builds with extensions enabled +for code coverage metrics (``Coverage``; this build type is only available with GNU +compilers). Similar to configuring a link type, selecting the build mode can be done +via a variable supplied to make: + +.. code-block:: bash + + cd SmartRedis + # Release build + make lib SR_BUILD=Release #or skip the SR_BUILD variable as this is the default + # Debug build + make lib SR_BUILD=Debug + # Code coverage build + make lib SR_BUILD=Coverage + +The name of the library produced for a Debug mode build is ``smartredis-debug``. +The name of the library produced for a Coverage mode build is ``smartredis-coverage``. +The name of the library produced for a Release mode build is ``smartredis``. +In each case, the file extension is dependent on the link type, ``.so`` or ``.a``. +All libraries will be located in the ``install/lib`` folder. + +Finally, it is possible to build SmartRedis to include Python and/or Fortran support +(both are omitted by default): + +.. code-block:: bash + + cd SmartRedis + # Build support for Python + make lib SR_PYTHON=ON + # Build support for Fortran + make lib SR_FORTRAN=ON # equivalent to make lib-with-fortran + # Build support for Python and Fortran + make lib SR_PYTHON=ON SR_FORTRAN=ON # or make lib-with-fortran SR_PYTHON=ON + +The build mode, link type, and Fortran/Python support settings are fully orthogonal; +any combination of them is supported. For example, a statically linked debug build +with Python support may be achieved via the following command: + +.. code-block:: bash + + cd SmartRedis + make lib SR_LINK=Static SR_BUILD=Debug SR_PYTHON=ON + +The SR_LINK, SR_BUILD, SR_PYTHON, and SR_FORTRAN variables are fully supported for all +test and build targets in the Makefile. + +Additional make variables are described in the ``help`` make target: + +.. code-block:: bash + + cd SmartRedis + make help + Linking instructions using compiler flags ----------------------------------------- diff --git a/doc/installation.rst b/doc/installation.rst index 4bfab0e2b..5fea9b190 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -31,10 +31,9 @@ SmartRedis is tested with the following compilers on a daily basis: * - Compilers (tested daily) * - GNU (GCC/GFortran) - * - Intel (icc/icpc/ifort) + * - Intel oneAPI (icx/icpx/ifort) * - Apple Clang - SmartRedis has been tested with the following compiler in the past, but on a less regular basis as the compilers listed above: .. list-table:: @@ -44,6 +43,7 @@ SmartRedis has been tested with the following compiler in the past, but on a les * - Compilers (irregularly tested in the past) * - Cray Clang + * - Intel Classic (icc/icpc) SmartRedis has been used with the following compilers in the past, but they have not been tested. We do not imply that these compilers work for certain: diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 000000000..0d58cd50e --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,40 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Project definition for the SmartRedis-Examples project +cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples) + +# Enable language support for the examples +enable_language(C) +enable_language(CXX) +if (SR_FORTRAN) + enable_language(Fortran) +endif() + +# Bring in subdirectories +add_subdirectory(parallel) +add_subdirectory(serial) diff --git a/examples/parallel/CMakeLists.txt b/examples/parallel/CMakeLists.txt new file mode 100644 index 000000000..293129686 --- /dev/null +++ b/examples/parallel/CMakeLists.txt @@ -0,0 +1,41 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Project definition for the SmartRedis-Examples-Parallel project +cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Parallel) + +# Enable language support for the examples +enable_language(CXX) +if (SR_FORTRAN) + enable_language(Fortran) +endif() + +# Bring in subdirectories +add_subdirectory(cpp) +if (SR_FORTRAN) + add_subdirectory(fortran) +endif() diff --git a/examples/parallel/cpp/CMakeLists.txt b/examples/parallel/cpp/CMakeLists.txt index 2dba75e3a..bf2c24e3f 100644 --- a/examples/parallel/cpp/CMakeLists.txt +++ b/examples/parallel/cpp/CMakeLists.txt @@ -24,45 +24,55 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientExamples) - +# Project definition for the SmartRedis-Examples-Parallel-Cpp project cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Parallel-Cpp) + +# Enable language support for the examples +enable_language(CXX) -set(CMAKE_BUILD_TYPE Release) +# Configure the build set(CMAKE_CXX_STANDARD 17) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) -find_package(MPI) +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") +endif() -find_library(SR_LIB smartredis PATHS ../../../install/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_package(MPI) +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${MPI_INCLUDE_PATH} - ../../../install/include + ${SMARTREDIS_INSTALL_PATH}/include ) -set(ftn_client_src - ../../../src/fortran/fortran_c_interop.F90 - ../../../src/fortran/dataset.F90 - ../../../src/fortran/client.F90 +# Define all the examples to be built +list(APPEND EXECUTABLES + smartredis_put_get_3D + smartredis_mnist ) -# Build executables - -add_executable(smartredis_put_get_3D - smartredis_put_get_3D.cpp - ${ftn_client_src} -) -target_link_libraries(smartredis_put_get_3D - MPI::MPI_CXX - ${SR_LIB} -) - -add_executable(smartredis_mnist - smartredis_mnist.cpp - ${ftn_client_src} -) -target_link_libraries(smartredis_mnist - MPI::MPI_CXX - ${SR_LIB} -) +# Build the examples +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_cpp_parallel + ${EXECUTABLE}.cpp + ) + set_target_properties(${EXECUTABLE}_cpp_parallel PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_cpp_parallel + MPI::MPI_CXX + ${SR_LIB} + ) +endforeach() diff --git a/examples/parallel/cpp/smartredis_mnist.cpp b/examples/parallel/cpp/smartredis_mnist.cpp index 05266e006..974eeca01 100644 --- a/examples/parallel/cpp/smartredis_mnist.cpp +++ b/examples/parallel/cpp/smartredis_mnist.cpp @@ -43,7 +43,7 @@ void run_mnist(const std::string& model_name, // Load the mnist image from a file using MPI rank 0 if (rank == 0) { - std::string image_file = "../../../common/mnist_data/one.raw"; + std::string image_file = "../../../../../examples/common/mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -113,7 +113,7 @@ int main(int argc, char* argv[]) { // Build model key, file name, and then set model // from file using client API std::string model_key = "mnist_model"; - std::string model_file = "../../../"\ + std::string model_file = "../../../../../examples/"\ "common/mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU", 20); @@ -121,7 +121,7 @@ int main(int argc, char* argv[]) { // Build script key, file name, and then set script // from file using client API std::string script_key = "mnist_script"; - std::string script_file = "../../../common/mnist_data/" + std::string script_file = "../../../../../examples/common/mnist_data/" "data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); diff --git a/examples/parallel/fortran/CMakeLists.txt b/examples/parallel/fortran/CMakeLists.txt index 8cf05c2bd..e427cbc0b 100644 --- a/examples/parallel/fortran/CMakeLists.txt +++ b/examples/parallel/fortran/CMakeLists.txt @@ -24,57 +24,67 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientExamples) - +# Project definition for the SmartRedis-Examples-Parallel-Fortran project cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Parallel-Fortran) +# Enable language support for the examples enable_language(Fortran) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE Debug) +# Configure the build set(CMAKE_CXX_STANDARD 17) -set(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) # Assume by default that users should link against the install directory in this repository if(NOT DEFINED SMARTREDIS_INSTALL_PATH) set(SMARTREDIS_INSTALL_PATH "../../../install/") endif() -# Specify all pre-processor and library dependencies +# Locate dependencies find_package(MPI REQUIRED) IF(NOT MPI_Fortran_FOUND) message(FATAL_ERROR "Could not find Fortran MPI components") endif() -find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(REDISPP redis++ PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) set(SMARTREDIS_LIBRARIES - ${SMARTREDIS_LIBRARY} - ${SMARTREDIS_FORTRAN_LIBRARY} - ${HIREDIS} - ${REDISPP} + ${SR_LIB} + ${SR_FTN_LIB} ) + +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${MPI_INCLUDE_PATH} ${SMARTREDIS_INSTALL_PATH}/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES smartredis_dataset smartredis_mnist smartredis_put_get_3D ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) - - add_executable(${EXECUTABLE} + add_executable(${EXECUTABLE}_fortran_parallel ${EXECUTABLE}.F90 example_utils.F90 ) - target_link_libraries(${EXECUTABLE} + set_target_properties(${EXECUTABLE}_fortran_parallel PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_fortran_parallel ${SMARTREDIS_LIBRARIES} MPI::MPI_Fortran ) diff --git a/examples/parallel/fortran/smartredis_mnist.F90 b/examples/parallel/fortran/smartredis_mnist.F90 index 780c24159..940cd23fb 100644 --- a/examples/parallel/fortran/smartredis_mnist.F90 +++ b/examples/parallel/fortran/smartredis_mnist.F90 @@ -35,9 +35,9 @@ program mnist_example #include "enum_fortran.inc" character(len=*), parameter :: model_key = "mnist_model" - character(len=*), parameter :: model_file = "../../../common/mnist_data/mnist_cnn.pt" + character(len=*), parameter :: model_file = "../../../../../examples/common/mnist_data/mnist_cnn.pt" character(len=*), parameter :: script_key = "mnist_script" - character(len=*), parameter :: script_file = "../../../common/mnist_data/data_processing_script.txt" + character(len=*), parameter :: script_file = "../../../../../examples/common/mnist_data/data_processing_script.txt" type(client_type) :: client integer :: err_code, pe_id, result diff --git a/examples/serial/CMakeLists.txt b/examples/serial/CMakeLists.txt new file mode 100644 index 000000000..6e62c20a8 --- /dev/null +++ b/examples/serial/CMakeLists.txt @@ -0,0 +1,43 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Project definition for the SmartRedis-Examples-Serial project +cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial) + +# Enable language support for the examples +enable_language(C) +enable_language(CXX) +if (SR_FORTRAN) + enable_language(Fortran) +endif() + +# Bring in subdirectories +add_subdirectory(c) +add_subdirectory(cpp) +if (SR_FORTRAN) + add_subdirectory(fortran) +endif() diff --git a/examples/serial/c/CMakeLists.txt b/examples/serial/c/CMakeLists.txt index 4184cfecd..5544a5ef4 100644 --- a/examples/serial/c/CMakeLists.txt +++ b/examples/serial/c/CMakeLists.txt @@ -24,35 +24,53 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CClientExamples) - +# Project definition for the SmartRedis-Examples-Serial-C project cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial-C) + +# Enable language support for the examples +enable_language(C) -set(CMAKE_BUILD_TYPE RELEASE) +# Configure the build set(CMAKE_CXX_STANDARD 17) SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) -find_library(SR_LIB smartredis PATHS ../../../install/lib - NO_DEFAULT_PATH REQUIRED) - -include_directories(SYSTEM - /usr/local/include - ../../../install/include -) +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") +endif() -add_executable(example_put_unpack_1D - example_put_unpack_1D.c +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} ) -target_link_libraries(example_put_unpack_1D - ${SR_LIB} +# Define include directories for header files +include_directories(SYSTEM + /usr/local/include + ${SMARTREDIS_INSTALL_PATH}/include ) -add_executable(example_put_get_3D - example_put_get_3D.c +# Define all the examples to be built +list(APPEND EXECUTABLES + example_put_unpack_1D + example_put_get_3D ) -target_link_libraries(example_put_get_3D +# Build the examples +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_c_serial + ${EXECUTABLE}.c + ) + set_target_properties(${EXECUTABLE}_c_serial PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_c_serial ${SR_LIB} -) - + ) +endforeach() diff --git a/examples/serial/cpp/CMakeLists.txt b/examples/serial/cpp/CMakeLists.txt index d3f436973..bca479b47 100644 --- a/examples/serial/cpp/CMakeLists.txt +++ b/examples/serial/cpp/CMakeLists.txt @@ -24,47 +24,55 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CppClientExamples) - +# Project definition for the SmartRedis-Examples-Serial-Cpp project cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial-Cpp) -set(CMAKE_BUILD_TYPE RELEASE) -set(CMAKE_CXX_STANDARD 17) - -find_library(SR_LIB smartredis PATHS ../../../install/lib - NO_DEFAULT_PATH REQUIRED) +# Enable language support for the examples +enable_language(CXX) -include_directories(SYSTEM - /usr/local/include - ../../../install/include -) +# Configure the build +set(CMAKE_CXX_STANDARD 17) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) -# Build executables +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../../install/") +endif() -add_executable(smartredis_put_get_3D - smartredis_put_get_3D.cpp -) -target_link_libraries(smartredis_put_get_3D - ${SR_LIB} +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} ) -add_executable(smartredis_dataset - smartredis_dataset.cpp -) -target_link_libraries(smartredis_dataset - ${SR_LIB} +# Define include directories for header files +include_directories(SYSTEM + /usr/local/include + ${SMARTREDIS_INSTALL_PATH}/include ) -add_executable(smartredis_model - smartredis_model.cpp -) -target_link_libraries(smartredis_model - ${SR_LIB} +# Define all the examples to be built +list(APPEND EXECUTABLES + smartredis_put_get_3D + smartredis_dataset + smartredis_model + smartredis_mnist ) -add_executable(smartredis_mnist - smartredis_mnist.cpp -) -target_link_libraries(smartredis_mnist - ${SR_LIB} -) +# Build the examples +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_cpp_serial + ${EXECUTABLE}.cpp + ) + set_target_properties(${EXECUTABLE}_cpp_serial PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_cpp_serial + ${SR_LIB} + ) +endforeach() diff --git a/examples/serial/cpp/smartredis_mnist.cpp b/examples/serial/cpp/smartredis_mnist.cpp index 0a3b05296..0c85e4e9d 100644 --- a/examples/serial/cpp/smartredis_mnist.cpp +++ b/examples/serial/cpp/smartredis_mnist.cpp @@ -37,7 +37,7 @@ void run_mnist(const std::string& model_name, std::vector img(n_values, 0); // Load the MNIST image from a file - std::string image_file = "../../../common/mnist_data/one.raw"; + std::string image_file = "../../../../../examples/common/mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -83,15 +83,14 @@ int main(int argc, char* argv[]) { // Build model key, file name, and then set model // from file using client API std::string model_key = "mnist_model"; - std::string model_file = "../../../"\ - "common/mnist_data/mnist_cnn.pt"; - client.set_model_from_file(model_key, model_file, - "TORCH", "CPU", 20); + std::string model_file = "../../../../../examples/common/mnist_data/mnist_cnn.pt"; + client.set_model_from_file( + model_key, model_file, "TORCH", "CPU", 20); // Build script key, file name, and then set script // from file using client API std::string script_key = "mnist_script"; - std::string script_file = "../../../common/mnist_data/" + std::string script_file = "../../../../../examples/common/mnist_data/" "data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); diff --git a/examples/serial/cpp/smartredis_model.cpp b/examples/serial/cpp/smartredis_model.cpp index 0c520f21d..1fab8ad59 100644 --- a/examples/serial/cpp/smartredis_model.cpp +++ b/examples/serial/cpp/smartredis_model.cpp @@ -37,7 +37,7 @@ int main(int argc, char* argv[]) { std::vector img(n_values, 0); // Load the mnist image from a file - std::string image_file = "../../../common/mnist_data/one.raw"; + std::string image_file = "../../../../../examples/common/mnist_data/one.raw"; std::ifstream fin(image_file, std::ios::binary); std::ostringstream ostream; ostream << fin.rdbuf(); @@ -52,12 +52,12 @@ int main(int argc, char* argv[]) { // Use the client to set a model in the database from a file std::string model_key = "mnist_model"; - std::string model_file = "../../../common/mnist_data/mnist_cnn.pt"; + std::string model_file = "../../../../../examples/common/mnist_data/mnist_cnn.pt"; client.set_model_from_file(model_key, model_file, "TORCH", "CPU", 20); // Use the client to set a script from the database form a file std::string script_key = "mnist_script"; - std::string script_file = "../../../common/mnist_data/data_processing_script.txt"; + std::string script_file = "../../../../../examples/common/mnist_data/data_processing_script.txt"; client.set_script_from_file(script_key, "CPU", script_file); // Declare keys that we will use in forthcoming client commands diff --git a/examples/serial/fortran/CMakeLists.txt b/examples/serial/fortran/CMakeLists.txt index 59008918d..a8b127733 100644 --- a/examples/serial/fortran/CMakeLists.txt +++ b/examples/serial/fortran/CMakeLists.txt @@ -24,50 +24,62 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(FortranClientExamples) - +# Project definition for the SmartRedis-Examples-Serial-Fortran project cmake_minimum_required(VERSION 3.13) +project(SmartRedis-Examples-Serial-Fortran) +# Enable language support for the examples enable_language(Fortran) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE Debug) +# Configure the build set(CMAKE_CXX_STANDARD 17) -set(CMAKE_C_STANDARD 99) +SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../../..") +include(smartredis_defs) -# Assume by default that users should link against the install directory in this repository +# Assume by default that users should link against the +# install directory in this repository if(NOT DEFINED SMARTREDIS_INSTALL_PATH) set(SMARTREDIS_INSTALL_PATH "../../../install/") endif() -# Specify all pre-processor and library dependencies -find_library(SMARTREDIS_LIBRARY smartredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(SMARTREDIS_FORTRAN_LIBRARY smartredis-fortran PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(HIREDIS hiredis PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) -find_library(REDISPP redis++ PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +find_library(SR_FTN_LIB ${SMARTREDIS_FORTRAN_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) set(SMARTREDIS_LIBRARIES - ${SMARTREDIS_LIBRARY} - ${SMARTREDIS_FORTRAN_LIBRARY} - ${HIREDIS} - ${REDISPP} + ${SR_LIB} + ${SR_FTN_LIB} ) + +# Define include directories for header files include_directories(SYSTEM /usr/local/include ${SMARTREDIS_INSTALL_PATH}/include ) -# Define all the tests to be built +# Define all the examples to be built list(APPEND EXECUTABLES smartredis_dataset smartredis_put_get_3D ) +# Build the examples foreach(EXECUTABLE ${EXECUTABLES}) - - add_executable(${EXECUTABLE} + add_executable(${EXECUTABLE}_fortran_serial ${EXECUTABLE}.F90 ) - target_link_libraries(${EXECUTABLE} + set_target_properties(${EXECUTABLE}_fortran_serial PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_fortran_serial ${SMARTREDIS_LIBRARIES} ) -endforeach() \ No newline at end of file +endforeach() diff --git a/examples/test_examples.py b/examples/test_examples.py index ef0ebe818..0755a20b1 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -26,8 +26,8 @@ import pytest from os import path as osp +from os import getcwd from glob import glob -from shutil import which from subprocess import Popen, PIPE, TimeoutExpired import time @@ -38,41 +38,43 @@ def get_test_names(): """Obtain test names by globbing for client_test Add tests manually if necessary """ - glob_path_1 = osp.join(TEST_PATH, "*/*/build/example*") - glob_path_2 = osp.join(TEST_PATH, "*/*/build/smartredis*") + glob_path_1 = osp.join(TEST_PATH, "*/*/example*") + glob_path_2 = osp.join(TEST_PATH, "*/*/smartredis*") test_names = glob(glob_path_1) + glob(glob_path_2) - test_names = list(filter(lambda test: test.find('.mod') == -1, test_names)) + test_names = list(filter(lambda test: test.find('example_utils') == -1, test_names)) + test_names = list(filter(lambda test: test.find('.py') == -1, test_names)) + test_names = [(pytest.param(test, + id=osp.basename(test))) for test in test_names] return test_names @pytest.mark.parametrize("test", get_test_names()) -def test_cpp_client(test, use_cluster): - cmd = [] - cmd.append(test) - print(f"Running test: {osp.basename(test)}") - print(f"Test command {' '.join(cmd)}") - print(f"Using cluster: {use_cluster}") - execute_cmd(cmd) - time.sleep(1) +def test_example(test, use_cluster, build, sr_fortran): + if (sr_fortran == "ON" or ".F90" not in test): + # Build the path to the test executable from the source file name + # . keep only the last three parts of the path: (parallel/serial, language, basename) + test = "/".join(test.split("/")[-3:]) + # . drop the file extension + test = ".".join(test.split(".")[:-1]) + # . prepend the path to the built test executable + test = f"{getcwd()}/build/{build}/examples/{test}" + cmd = [test] + print(f"Running test: {osp.basename(test)}") + print(f"Using cluster: {use_cluster}") + execute_cmd(cmd) + time.sleep(1) + else: + print (f"Skipping Fortran test {test}") def find_path(executable_path): - if(executable_path.find('/') == -1): + if executable_path.find('/') == -1: return '.' - - start = 0 - while True: - slash = executable_path.find('/', start) - if (slash == -1): - return executable_path[0:start] - else: - start = slash + 1 + return "/".join(executable_path.split("/")[:-1]) def execute_cmd(cmd_list): """Execute a command """ # spawning the subprocess and connecting to its output run_path = find_path(cmd_list[0]) - print("cmd_list", cmd_list) - print("cwd", run_path) proc = Popen( cmd_list, stderr=PIPE, stdout=PIPE, stdin=PIPE, cwd=run_path) try: diff --git a/images/Dockerfile b/images/Dockerfile index ae7331cbb..fde64428e 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -49,6 +49,7 @@ COPY . /usr/local/src/SmartRedis # Compile and install WORKDIR /usr/local/src/SmartRedis +RUN pip3 install --upgrade pip RUN make clobber && make lib && pip install . && rm -rf ~/.cache/pip \ && rm -rf build tests examples images utils third-party doc images diff --git a/include/redisserver.h b/include/redisserver.h index 0701c28f2..f3ab81d8c 100644 --- a/include/redisserver.h +++ b/include/redisserver.h @@ -33,7 +33,6 @@ #include #include #include -#include #include "command.h" #include "commandreply.h" diff --git a/setup.cfg b/setup.cfg index ab617dff3..a1d6e5949 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ package_dir= =src/python/module packages=find: setup_requires = - setuptools>=39.2 + setuptools>=42 include_package_data = True install_requires = numpy>=1.18.2 diff --git a/setup.py b/setup.py index dd19c6c0e..477c86ee9 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def run(self): build_directory = Path(self.build_temp).resolve() cmake_args = [ '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + str(build_directory), - '-DPYTHON_EXECUTABLE=' + sys.executable + '-DPYTHON_EXECUTABLE=' + sys.executable, + '-DSR_PYTHON=ON' ] cfg = 'Debug' if self.debug else 'Release' @@ -78,9 +79,14 @@ def run(self): if not build_directory.is_dir(): os.makedirs(self.build_temp) + # make install dir + setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() + install_directory = setup_path.joinpath("install") + if not install_directory.is_dir(): + os.makedirs(install_directory) + print('-'*10, 'Building C dependencies', '-'*40) make_cmd = shutil.which("make") - setup_path = Path(os.path.abspath(os.path.dirname(__file__))).resolve() # build dependencies subprocess.check_call([f"{make_cmd} deps"], diff --git a/smartredis_defs.cmake b/smartredis_defs.cmake new file mode 100644 index 000000000..e8d0ada6a --- /dev/null +++ b/smartredis_defs.cmake @@ -0,0 +1,71 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Set defaults if variables are undefined +if(NOT DEFINED SR_BUILD) + set(SR_BUILD "Release") +endif() +if(NOT DEFINED SR_LINK) + set(SR_LINK "Shared") +endif() + +# Configure the CMake build based on the SR_BUILD selection +string(TOLOWER "${SR_BUILD}" srbuild_lowercase) +if(srbuild_lowercase STREQUAL "release") + set(CMAKE_BUILD_TYPE RELEASE) + set(SRLIB_NAME_SUFFIX "") +elseif(srbuild_lowercase STREQUAL "debug") + set(CMAKE_BUILD_TYPE DEBUG) + set(SRLIB_NAME_SUFFIX "-debug") +elseif(srbuild_lowercase STREQUAL "coverage") + set(CMAKE_BUILD_TYPE DEBUG) + set(SRLIB_NAME_SUFFIX "-coverage") + if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) + add_compile_options(--coverage) + add_link_options(--coverage) + else() + message(WARNING "A coverage build was specified, but the CMAKE compiler is not GCC") + endif() +else() + message(FATAL_ERROR "Unrecognized build type (${SR_BUILD}) specified in SR_BUILD") +endif() + +# Configure CMake linkage on the SR_LINK selection +string(TOLOWER "${SR_LINK}" srlink_lowercase) +if(srlink_lowercase STREQUAL "static") + set(SMARTREDIS_LINK_MODE STATIC) + set(SMARTREDIS_LINK_LIBRARY_SUFFIX .a) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) +elseif(srlink_lowercase STREQUAL "shared") + set(SMARTREDIS_LINK_MODE SHARED) + set(SMARTREDIS_LINK_LIBRARY_SUFFIX .so) +else() + message(FATAL_ERROR "Unrecognized link type (${SR_LINK}) specified in SR_LINK") +endif() + +# Identify the SmartRedis library names based on the build and link +set(SMARTREDIS_LIB smartredis${SRLIB_NAME_SUFFIX}) +set(SMARTREDIS_FORTRAN_LIB smartredis-fortran${SRLIB_NAME_SUFFIX}) diff --git a/src/cpp/redis.cpp b/src/cpp/redis.cpp index 22302312c..b84cc77d1 100644 --- a/src/cpp/redis.cpp +++ b/src/cpp/redis.cpp @@ -26,6 +26,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include "redis.h" #include "srexception.h" #include "utility.h" diff --git a/src/cpp/rediscluster.cpp b/src/cpp/rediscluster.cpp index 04c3e76a1..7f649e331 100644 --- a/src/cpp/rediscluster.cpp +++ b/src/cpp/rediscluster.cpp @@ -26,6 +26,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include "rediscluster.h" #include "nonkeyedcommand.h" #include "keyedcommand.h" diff --git a/src/cpp/redisserver.cpp b/src/cpp/redisserver.cpp index b8e9b020c..e3a206e84 100644 --- a/src/cpp/redisserver.cpp +++ b/src/cpp/redisserver.cpp @@ -27,6 +27,7 @@ */ #include +#include #include "redisserver.h" #include "srexception.h" #include "utility.h" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..771d3950d --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,48 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2023, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Project definition for the SmartRedis-Tests project +cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests) + +# Enable language support for the examples +enable_language(C) +enable_language(CXX) +if (SR_FORTRAN) + enable_language(Fortran) +endif() + +# Bring in subdirectories +add_subdirectory(c) +add_subdirectory(cpp) +if (SR_FORTRAN) + add_subdirectory(fortran) +endif() + +# NOTE: The docker subdirectory is designed to be built within a container, +# not as part of the normal test build. We therefore do not include it in a +# test build +#add_subdirectory(docker) diff --git a/tests/c/CMakeLists.txt b/tests/c/CMakeLists.txt index 5926beb68..0491543e6 100644 --- a/tests/c/CMakeLists.txt +++ b/tests/c/CMakeLists.txt @@ -24,81 +24,58 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -project(CClientTester) - +# Project definition for the SmartRedis-Tests-C project cmake_minimum_required(VERSION 3.13) +project(SmartSim-Tests-C) + +# Enable language support for the tests +enable_language(C) -set(CMAKE_VERBOSE_MAKEFILE ON) -set(CMAKE_BUILD_TYPE DEBUG) +# Configure the build set(CMAKE_CXX_STANDARD 17) SET(CMAKE_C_STANDARD 99) +set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/../..") +include(smartredis_defs) -if(COVERAGE) - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_C_COMPILER_ID STREQUAL "GNU")) - set(CMAKE_CXX_FLAGS --coverage) - add_link_options(--coverage) - endif() +# Assume by default that users should link against the +# install directory in this repository +if(NOT DEFINED SMARTREDIS_INSTALL_PATH) + set(SMARTREDIS_INSTALL_PATH "../../install/") endif() -find_library(SR_LIB smartredis PATHS ../../install/lib NO_DEFAULT_PATH REQUIRED) +# Locate dependencies +find_library(SR_LIB ${SMARTREDIS_LIB} + PATHS ${SMARTREDIS_INSTALL_PATH}/lib NO_DEFAULT_PATH + REQUIRED + ${SMARTREDIS_LINK_MODE} +) +# Define include directories for header files include_directories(SYSTEM /usr/local/include - ../../install/include + ${SMARTREDIS_INSTALL_PATH}/include ) -add_executable(client_test_dataset_aggregation -client_test_dataset_aggregation.c +# Define all the tests to be built +list(APPEND EXECUTABLES + client_test_dataset_aggregation + client_test_dataset_exists + client_test_logging + client_test_put_unpack_1D + client_test_put_get_1D + client_test_put_get_2D + client_test_put_get_3D ) -target_link_libraries(client_test_dataset_aggregation +# Build the tests +foreach(EXECUTABLE ${EXECUTABLES}) + add_executable(${EXECUTABLE}_c_test + ${EXECUTABLE}.c + ) + set_target_properties(${EXECUTABLE}_c_test PROPERTIES + OUTPUT_NAME ${EXECUTABLE} + ) + target_link_libraries(${EXECUTABLE}_c_test ${SR_LIB} -) - -add_executable(client_test_dataset_exists - client_test_dataset_exists.c -) - -target_link_libraries(client_test_dataset_exists - ${SR_LIB} -) - -add_executable(client_test_logging - client_test_logging.c -) - -target_link_libraries(client_test_logging - ${SR_LIB} -) - -add_executable(client_test_put_unpack_1D - client_test_put_unpack_1D.c -) - -target_link_libraries(client_test_put_unpack_1D - ${SR_LIB} -) - -add_executable(client_test_put_get_1D - client_test_put_get_1D.c -) - -target_link_libraries(client_test_put_get_1D - ${SR_LIB} -) - -add_executable(client_test_put_get_2D - client_test_put_get_2D.c -) - -target_link_libraries(client_test_put_get_2D - ${SR_LIB} -) - -add_executable(client_test_put_get_3D - client_test_put_get_3D.c -) - -target_link_libraries(client_test_put_get_3D - ${SR_LIB} -) + ) +endforeach() diff --git a/tests/c/client_test_put_get_2D.c b/tests/c/client_test_put_get_2D.c index bc6f06c28..32746ed47 100644 --- a/tests/c/client_test_put_get_2D.c +++ b/tests/c/client_test_put_get_2D.c @@ -114,7 +114,7 @@ int put_get_2D_tensor(void* client, values. If the sent and received tensors do not match, a non-zero value is returned. */ -int put_get_2D_tensor_double(size_t* dims, int n_dims, +int put_get_2D_tensor_double(size_t* dims, size_t n_dims, char* key_suffix, int key_suffix_length) { void* client = NULL; @@ -162,7 +162,7 @@ int put_get_2D_tensor_double(size_t* dims, int n_dims, values. If the sent and received tensors do not match, a non-zero value is returned. */ -int put_get_2D_tensor_float(size_t* dims, int n_dims, +int put_get_2D_tensor_float(size_t* dims, size_t n_dims, char* key_suffix, int key_suffix_length) { @@ -211,7 +211,7 @@ int put_get_2D_tensor_float(size_t* dims, int n_dims, values. If the sent and received tensors do not match, a non-zero value is returned. */ -int put_get_2D_tensor_i8(size_t* dims, int n_dims, +int put_get_2D_tensor_i8(size_t* dims, size_t n_dims, char* key_suffix, int key_suffix_length) { @@ -264,7 +264,7 @@ for(int i=0; i